chore(main): release 1.722.0 (#9493)

* chore(main): release 1.722.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com>

.agents/skills/pr/SKILL.md

@@ -55,18 +55,55 @@ The body MUST be explicit about what changed. Structure:
 
 The harness/tooling that invoked the skill may add its own attribution trailer; the skill itself does not prescribe one.
 
+## Screenshots (required for frontend changes)
+
+If `git diff main...HEAD --name-only` matches `^frontend/`, the PR body **must** include
+screenshots of the affected UI. Skip only when there is no visible UI effect (types,
+tests, build config) — and say so in the body.
+
+1. Verify the change in the browser (AGENTS.md → "Verifying Frontend Changes").
+2. Screenshot each affected page with `mcp__playwright__browser_take_screenshot` (save to a file).
+3. Host each image and get its Markdown embed by pushing to the public
+   `windmill-labs/agent-screenshots-internal` repo. **Pipe base64 through stdin** —
+   passing it as `-f content=…` fails with `argument list too long` on real images:
+
+   ```bash
+   REPO=windmill-labs/agent-screenshots-internal
+   IMG=screenshot.png                                          # repeat per page
+   DEST="shots/$(git branch --show-current)/$(date +%s)-$(basename "$IMG")"
+   base64 -w0 "$IMG" | jq -Rs --arg m "add $DEST" '{message:$m, content:.}' \
+     | gh api -X PUT "repos/$REPO/contents/$DEST" --input - >/dev/null
+   echo "![$(basename "$IMG" .png)](https://raw.githubusercontent.com/$REPO/main/$DEST)"
+   ```
+   Derive `$DEST` from the file name (as above) so distinct pages never collide — a
+   fixed name would make same-second uploads reuse one path, and the second `PUT`
+   then 422s (the Contents API needs the existing file's `sha` to overwrite).
+4. Put the printed `![…](…)` lines under a `## Screenshots` heading in the PR body.
+
+Requires `gh` (`repo` scope), `jq`, `base64` — all in the devShell. The host repo is
+public (so the raw URLs render for reviewers without a token) and its history is
+permanent — **never screenshot pages that show secrets or sensitive values** (workspace
+variables, resource values, instance settings, OAuth/SMTP config); deleting the file
+can't undo an accidental capture. (GitHub's drag-and-drop uploader needs a browser
+session and can't be driven from a token.)
+
+If `gh` can't push to the host repo (e.g. a CI token scoped only to `windmill`), do
+**not** fail the PR or skip silently — hand the upload to the user, who has push access,
+and continue once they confirm it's done.
+
 ## Execution Steps
 
 1. Run `git status` to check for uncommitted changes
 2. Run `git log main..HEAD --oneline` to see all commits in this branch
 3. Run `git diff main...HEAD` to see the full diff against main
 4. **Invoke the `local-review` skill** before creating the PR (`/local-review` in Claude Code, `$local-review` in Codex, `pi --skill local-review` / `/skill:local-review` in Pi). If issues are found, fix them and commit before proceeding. Do not skip this step.
-5. Check if remote branch exists and is up to date:
+5. **Screenshots for frontend changes**: if `git diff main...HEAD --name-only` matches `^frontend/`, capture and embed screenshots of the affected UI per "Screenshots" above before writing the PR body (skip only if there is no visible UI effect).
+6. Check if remote branch exists and is up to date:
    ```bash
    git rev-parse --abbrev-ref --symbolic-full-name @{u} 2>/dev/null || echo "no upstream"
    ```
-6. Push to remote if needed: `git push -u origin HEAD`
-7. Create draft PR using gh CLI:
+7. Push to remote if needed: `git push -u origin HEAD`
+8. Create draft PR using gh CLI:
    ```bash
    gh pr create --draft --title "<type>: <description>" --body "$(cat <<'EOF'
    ## Summary
@@ -82,7 +119,7 @@ The harness/tooling that invoked the skill may add its own attribution trailer;
    EOF
    )"
    ```
-8. Return the PR URL to the user
+9. Return the PR URL to the user
 
 ## EE Companion PR (when `*_ee.rs` files were modified)
 

.agents/skills/svelte-frontend/SKILL.md

@@ -78,3 +78,7 @@ Use the Svelte MCP tools when working on Svelte code:
 2. **get-documentation**: Fetch relevant sections based on use_cases
 3. **svelte-autofixer**: MUST use on all Svelte code before finalizing — keep calling until no issues
 4. **playground-link**: Only after user confirms and code was NOT written to project files
+
+## Verifying in the Browser
+
+After changing Svelte code, use the **Playwright MCP** (`mcp__playwright__*`) to drive the running frontend and confirm the change works. See AGENTS.md → "Verifying Frontend Changes" for the full flow. Use `playwright` (headless) on devboxes; `playwright-headed` when a display is available.

.claude/hooks/guard-main-branch.sh

@@ -16,6 +16,23 @@ command="$(echo "$input" | jq -r '.tool_input.command // empty')"
 if [[ "$command" =~ ^git\ (push|reset|revert|checkout|merge|rebase|commit|add) ]]; then
   branch="$(git rev-parse --abbrev-ref HEAD 2>/dev/null || true)"
   if [[ "$branch" == "main" ]]; then
-    echo "BLOCK: You are on the main branch. Create or switch to a feature branch first."
+    echo "BLOCK: You are on the main branch. Create or switch to a feature branch first." >&2
+    exit 2
+  fi
+fi
+
+# Block force-push targeting main from any branch.
+if [[ "$command" =~ ^git[[:space:]]+push([[:space:]]|$) ]]; then
+  has_force=false
+  if [[ "$command" =~ (--force([[:space:]]|=|$)|--force-with-lease|[[:space:]]-f([[:space:]]|$)) ]]; then
+    has_force=true
+  fi
+  # `+ref` refspec syntax is also a force push.
+  if [[ "$command" =~ [[:space:]]\+[A-Za-z] ]]; then
+    has_force=true
+  fi
+  if $has_force && [[ "$command" =~ (^|[[:space:]:])\+?main([[:space:]]|$) ]]; then
+    echo "BLOCK: Force-push to main is not allowed via Claude. Run it yourself if you really mean to." >&2
+    exit 2
   fi
 fi

.claude/settings.json

@@ -44,7 +44,25 @@
       "Bash(git merge:*)",
       "Bash(git rebase:*)",
       "Bash(git add:*)",
-      "Bash(git commit:*)"
+      "Bash(git commit:*)",
+      "Read(/tmp/**)",
+      "Write(/tmp/**)",
+      "Edit(/tmp/**)",
+      "Bash(rm:/tmp/*)",
+      "Bash(rm:/tmp/**)",
+      "Bash(rmdir:/tmp/*)",
+      "Bash(mkdir:/tmp/*)",
+      "Bash(mkdir:/tmp/**)",
+      "Bash(cp:/tmp/*)",
+      "Bash(cp:/tmp/**)",
+      "Bash(mv:/tmp/*)",
+      "Bash(mv:/tmp/**)",
+      "Bash(touch:/tmp/*)",
+      "Bash(touch:/tmp/**)",
+      "Bash(chmod:/tmp/*)",
+      "Bash(chmod:/tmp/**)",
+      "Bash(tar * /tmp/*)",
+      "Bash(unzip * /tmp/*)"
     ],
     "deny": [
       "Read(.env)",
@@ -72,7 +90,13 @@
       "Bash(chown:*)",
       "Bash(truncate:*)",
       "Bash(shred:*)",
-      "Bash(unlink:*)"
+      "Bash(unlink:*)",
+      "mcp__claude_ai_Stripe",
+      "mcp__claude_ai_Gmail",
+      "mcp__claude_ai_Google_Calendar",
+      "mcp__claude_ai_Google_Drive",
+      "mcp__claude_ai_Slack",
+      "mcp__claude_ai_Linear"
     ]
   },
   "enableAllProjectMcpServers": true,

.github/DockerfileBackendTests

@@ -28,7 +28,7 @@ ENV PATH="${PATH}:/usr/local/go/bin"
 ENV GO_PATH=/usr/local/go/bin/go
 
 # UV
-RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.9.24/uv-installer.sh | sh && mv /usr/local/cargo/bin/uv /usr/local/bin/uv
+RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.9.25/uv-installer.sh | sh && mv /usr/local/cargo/bin/uv /usr/local/bin/uv
 
 ENV TZ=Etc/UTC
 

.github/change-versions-mac.sh

@@ -7,7 +7,7 @@ VERSION=$1
 echo "Updating versions to: $VERSION"
 
 sed -i '' -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/backend/Cargo.toml
-sed -i '' -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/"  ${root_dirpath}/cli/src/main.ts
+sed -i '' -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/"  ${root_dirpath}/cli/src/core/constants.ts
 sed -i '' -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" ${root_dirpath}/benchmarks/lib.ts
 sed -i '' -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/backend/windmill-api/openapi.yaml
 sed -i '' -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/openflow.openapi.yaml

.github/change-versions.sh

@@ -7,7 +7,7 @@ VERSION=$1
 echo "Updating versions to: $VERSION"
 
 sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/backend/Cargo.toml
-sed -i -e "/^export const VERSION =/s/= .*/= \"$VERSION\";/" ${root_dirpath}/cli/src/main.ts
+sed -i -e "/^export const VERSION =/s/= .*/= \"$VERSION\";/" ${root_dirpath}/cli/src/core/constants.ts
 sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" ${root_dirpath}/benchmarks/lib.ts
 sed -i -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/backend/windmill-api/openapi.yaml
 sed -i -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/openflow.openapi.yaml

.github/workflows/backend-test-windows.yml

@@ -74,7 +74,7 @@ jobs:
 
       - uses: astral-sh/setup-uv@v6.2.1
         with:
-          version: "0.9.24"
+          version: "0.9.25"
 
       - uses: shivammathur/setup-php@v2
         with:
@@ -98,6 +98,21 @@ jobs:
           vcpkg.exe install openssl:x64-windows-static
           vcpkg.exe integrate install
 
+      - name: Free disk space (post-vcpkg)
+        shell: pwsh
+        run: |
+          # vcpkg leaves multi-GB of buildtrees/downloads after installing openssl;
+          # we only need the installed/ dir for linking.
+          $vcpkgRoot = $env:VCPKG_INSTALLATION_ROOT
+          foreach ($sub in @("buildtrees", "downloads", "packages")) {
+            $path = Join-Path $vcpkgRoot $sub
+            if (Test-Path $path) {
+              Write-Host "Removing $path"
+              Remove-Item -Recurse -Force -ErrorAction SilentlyContinue $path
+            }
+          }
+          Get-PSDrive C | Select-Object Used,Free | Format-Table -AutoSize
+
       - name: Get runtime paths
         id: runtime-paths
         shell: pwsh
@@ -119,6 +134,10 @@ jobs:
           cargo build --release -p windmill_duckdb_ffi_internal
           New-Item -ItemType Directory -Path ..\target\debug -Force
           Copy-Item target\release\windmill_duckdb_ffi_internal.dll ..\target\debug\
+          # duckdb is bundled (~2GB of build artifacts); the DLL is the only
+          # thing we need from this excluded-crate target dir.
+          Remove-Item -Recurse -Force -ErrorAction SilentlyContinue target
+          Get-PSDrive C | Select-Object Used,Free | Format-Table -AutoSize
 
       - name: Print runtime versions and env
         shell: pwsh
@@ -136,6 +155,10 @@ jobs:
           echo "USERPROFILE=$env:USERPROFILE"
           echo "HOME=$env:HOME"
 
+      - name: Disk space before cargo test
+        shell: pwsh
+        run: Get-PSDrive C | Select-Object Used,Free | Format-Table -AutoSize
+
       - name: cargo test
         working-directory: backend
         timeout-minutes: 60
@@ -144,7 +167,18 @@ jobs:
           RUST_LOG: "off"
           RUST_LOG_STYLE: never
           CARGO_NET_GIT_FETCH_WITH_CLI: true
-          CARGO_BUILD_JOBS: 12
+          # 16-vcpu runners with disabled PDB still hit LNK1180 ("insufficient
+          # disk space") at link time with 12 parallel link jobs: each test
+          # binary link spikes several hundred MB of transient I/O. Capping at
+          # 8 trades ~25% wall time for headroom on the ~75GB runner disk.
+          CARGO_BUILD_JOBS: 8
+          # backend/Cargo.toml sets split-debuginfo = "unpacked", which on
+          # windows-msvc is coerced to "packed": every test-binary link spawns
+          # the mspdbsrv.exe PDB type server and writes a large .pdb. CI needs
+          # no debug info, so disable PDB generation for the dev/test profiles
+          # here (avoids both LNK1318 type-server limit and PDB disk usage).
+          CARGO_PROFILE_DEV_SPLIT_DEBUGINFO: "off"
+          CARGO_PROFILE_TEST_SPLIT_DEBUGINFO: "off"
           # Tests' poll-time stack frames (deep nested async fn chains in
           # debug builds) reach ~1.8MB. 4MB gives ~2x headroom against flaky
           # overflows under parallel-test contention.

.github/workflows/backend-test.yml

@@ -62,7 +62,7 @@ jobs:
           node-version: "20"
       - uses: astral-sh/setup-uv@v6.2.1
         with:
-          version: "0.9.24"
+          version: "0.9.25"
       - uses: shivammathur/setup-php@v2
         with:
           php-version: "8.3"

.github/workflows/check-empty-fixture.yml

@@ -0,0 +1,19 @@
+name: Check fixture is empty
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "fixtures/**"
+  pull_request:
+    paths:
+      - "fixtures/**"
+
+jobs:
+  check-empty-fixture:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Ensure fixtures/cli-sync/ has no committed snapshot
+        run: bash fixtures/check-empty.sh

.github/workflows/claude-plan.yml

@@ -45,7 +45,7 @@ jobs:
           allowed_bots: 'windmill-internal-app[bot]'
           trigger_phrase: '/plan'
           claude_args: |
-            --model opus
+            --model claude-fable-5
             --system-prompt "# Claude Planning Mode
 
             You are operating in PLANNING MODE ONLY. Your role is to create detailed, structured plans without making any code changes.

.github/workflows/claude.yml

@@ -51,4 +51,4 @@ jobs:
             }
           claude_args: |
             --allowedTools "Bash,WebFetch,WebSearch"
-            --model opus
+            --model claude-fable-5

.github/workflows/codex-pr-review.yml

@@ -20,6 +20,8 @@ on:
         type: string
         default: ''
     secrets:
+      OPENAI_API_KEY:
+        required: false
       CODEX_AUTH_JSON:
         required: false
       WINDMILL_EE_PRIVATE_ACCESS:
@@ -60,13 +62,18 @@ jobs:
       - name: Check Codex configuration
         id: codex_config
         env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           CODEX_AUTH_JSON: ${{ secrets.CODEX_AUTH_JSON }}
         run: |
-          if [ -n "$CODEX_AUTH_JSON" ]; then
+          if [ -n "$OPENAI_API_KEY" ]; then
             echo "enabled=true" >> "$GITHUB_OUTPUT"
+            echo "auth_mode=api_key" >> "$GITHUB_OUTPUT"
+          elif [ -n "$CODEX_AUTH_JSON" ]; then
+            echo "enabled=true" >> "$GITHUB_OUTPUT"
+            echo "auth_mode=oauth_json" >> "$GITHUB_OUTPUT"
           else
             echo "enabled=false" >> "$GITHUB_OUTPUT"
-            echo "CODEX_AUTH_JSON is not configured; skipping Codex review."
+            echo "Codex auth is not configured; set OPENAI_API_KEY or CODEX_AUTH_JSON to enable Codex review."
           fi
 
       - name: Resolve PR metadata
@@ -169,9 +176,10 @@ jobs:
         if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true'
         run: npm install --global @openai/codex@0.128.0
 
-      - name: Configure file-backed Codex auth
+      - name: Configure Codex auth
         if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true'
         env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           CODEX_AUTH_JSON: ${{ secrets.CODEX_AUTH_JSON }}
         run: |
           CODEX_HOME="$HOME/.codex"
@@ -181,9 +189,13 @@ jobs:
           cat > "$CODEX_HOME/config.toml" <<'EOF'
           cli_auth_credentials_store = "file"
           EOF
-          printf '%s' "$CODEX_AUTH_JSON" > "$CODEX_HOME/auth.json"
-          chmod 600 "$CODEX_HOME/auth.json"
-          node -e 'JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"))' "$CODEX_HOME/auth.json"
+          if [ -n "$OPENAI_API_KEY" ]; then
+            printf '%s' "$OPENAI_API_KEY" | codex login --with-api-key
+          else
+            printf '%s' "$CODEX_AUTH_JSON" > "$CODEX_HOME/auth.json"
+            chmod 600 "$CODEX_HOME/auth.json"
+            node -e 'JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"))' "$CODEX_HOME/auth.json"
+          fi
 
       - name: Pre-fetch base and head refs for the PR
         if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true'

.github/workflows/git-sync-test.yml

@@ -8,6 +8,7 @@ on:
       - "backend/windmill-git-sync/**"
       - "backend/windmill-api-integration-tests/tests/git_sync*"
       - "backend/ee-repo-ref.txt"
+      - "backend/windmill-common/src/workspaces.rs"
       - "integration_tests/test/git_sync_test.py"
       - ".github/workflows/git-sync-test.yml"
   pull_request:
@@ -16,6 +17,7 @@ on:
       - "backend/windmill-git-sync/**"
       - "backend/windmill-api-integration-tests/tests/git_sync*"
       - "backend/ee-repo-ref.txt"
+      - "backend/windmill-common/src/workspaces.rs"
       - "integration_tests/test/git_sync_test.py"
       - ".github/workflows/git-sync-test.yml"
 
@@ -49,7 +51,7 @@ jobs:
           echo "$CHANGED_FILES"
 
           # Direct git sync file changes — always relevant
-          if echo "$CHANGED_FILES" | grep -qE '^(backend/windmill-git-sync/|backend/windmill-api-integration-tests/tests/git_sync|integration_tests/test/git_sync|\.github/workflows/git-sync-test\.yml)'; then
+          if echo "$CHANGED_FILES" | grep -qE '^(backend/windmill-git-sync/|backend/windmill-api-integration-tests/tests/git_sync|backend/windmill-common/src/workspaces\.rs|integration_tests/test/git_sync|\.github/workflows/git-sync-test\.yml)'; then
             echo "should_run=true" >> "$GITHUB_OUTPUT"
             echo "Relevant: direct git sync file changes"
             exit 0

.github/workflows/pr-ready-review.yml

@@ -160,4 +160,4 @@ jobs:
             ${{ env.REVIEW_PROMPT }}
           claude_args: |
             --allowedTools "mcp__github_inline_comment__create_inline_comment,Bash(gh pr comment:*),Bash(gh pr diff:*),Bash(gh pr view:*)"
-            --model opus
+            --model claude-fable-5

.github/workflows/pr-review-commands.yml

@@ -100,6 +100,7 @@ jobs:
       extra_prompt: ${{ needs.parse.outputs.extra_prompt }}
       triggered_by: ${{ github.event.comment.user.login }}
     secrets:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       CODEX_AUTH_JSON: ${{ secrets.CODEX_AUTH_JSON }}
       WINDMILL_EE_PRIVATE_ACCESS: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
 

.github/workflows/spawn-ephemeral-backend.yml

@@ -1,126 +0,0 @@
-name: Spawn Ephemeral Backend
-
-on:
-  issue_comment:
-    types: [created]
-  pull_request_review_comment:
-    types: [created]
-  workflow_dispatch:
-    inputs:
-      pr_number:
-        description: "PR number"
-        required: true
-        type: number
-
-jobs:
-  check-membership:
-    if: |
-      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '/spawnbackend')) ||
-      (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '/spawnbackend'))
-    uses: ./.github/workflows/check-org-membership.yml
-    secrets:
-      access_token: ${{ secrets.ORG_ACCESS_TOKEN }}
-
-  spawn-backend:
-    needs: check-membership
-    # Only run on PR comments that contain /spawn-backend, or manual dispatch
-    if: |
-      github.event_name == 'workflow_dispatch' ||
-      (github.event.issue.pull_request && needs.check-membership.outputs.is_member == 'true')
-    runs-on: ubuntu-latest
-    permissions:
-      pull-requests: write
-      contents: read
-
-    steps:
-      - name: Get PR details
-        id: pr-details
-        uses: actions/github-script@v7
-        with:
-          script: |
-            const prNumber = context.eventName === 'workflow_dispatch'
-              ? context.payload.inputs.pr_number
-              : context.issue.number;
-
-            const pr = await github.rest.pulls.get({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              pull_number: prNumber
-            });
-
-            // Get branch name and format it for Cloudflare Pages
-            // Replace '/' with '-' for the URL
-            const branchName = pr.data.head.ref;
-            const formattedBranch = branchName.replace(/\//g, '-');
-            const cfFrontendUrl = `https://${formattedBranch}.windmill.pages.dev`;
-
-            core.setOutput('commit_hash', pr.data.head.sha);
-            core.setOutput('pr_number', prNumber);
-            core.setOutput('branch_name', branchName);
-            core.setOutput('cf_frontend_url', cfFrontendUrl);
-
-      - name: Check manager URL
-        id: check-manager-url
-        run: |
-          if [ -z "${{ secrets.EPHEMERAL_BACKEND_QUEUE_URL }}" ]; then
-            echo "manager_url_set=false" >> $GITHUB_OUTPUT
-          else
-            echo "manager_url_set=true" >> $GITHUB_OUTPUT
-          fi
-
-      - name: Post error comment if manager not running
-        if: steps.check-manager-url.outputs.manager_url_set == 'false'
-        uses: actions/github-script@v7
-        with:
-          script: |
-            const prNumber = context.eventName === 'workflow_dispatch'
-              ? Number(context.payload.inputs.pr_number)
-              : context.issue.number;
-
-            await github.rest.issues.createComment({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              issue_number: prNumber,
-              body: `❌ Manager URL not set (did you start the ephemeral backend manager?)\n\nThe ephemeral backend manager needs to be running to spawn backends. Please start the manager first.`
-            });
-
-      - name: Fail if manager not running
-        if: steps.check-manager-url.outputs.manager_url_set == 'false'
-        run: |
-          echo "Error: EPHEMERAL_BACKEND_QUEUE_URL secret is not set"
-          exit 1
-
-      - name: Trigger Windmill flow
-        if: steps.check-manager-url.outputs.manager_url_set == 'true'
-        id: trigger-flow
-        run: |
-          JOB_UUID=$(curl -s -X POST "https://app.windmill.dev/api/w/windmill-labs/jobs/run/f/f/all/run_ephemeral_backend" \
-            -H "Authorization: Bearer ${{ secrets.WINDMILL_RUN_FLOW_TOKEN }}" \
-            -H "Content-Type: application/json" \
-            -d '{
-              "manager_url": "${{ secrets.EPHEMERAL_BACKEND_QUEUE_URL }}",
-              "commit_hash": "${{ steps.pr-details.outputs.commit_hash }}",
-              "pr_number": ${{ steps.pr-details.outputs.pr_number }},
-              "cf_frontend_url": "${{ steps.pr-details.outputs.cf_frontend_url }}"
-            }' | tr -d '"')
-
-          echo "Job UUID: $JOB_UUID"
-          echo "job_uuid=$JOB_UUID" >> $GITHUB_OUTPUT
-
-      - name: Post comment with job link
-        if: steps.check-manager-url.outputs.manager_url_set == 'true'
-        uses: actions/github-script@v7
-        with:
-          script: |
-            const jobUuid = '${{ steps.trigger-flow.outputs.job_uuid }}';
-            const appUrl = `https://app.windmill.dev/public/windmill-labs/a106bad0256c1dfa7a4f9279c42b1a4b#${jobUuid}`;
-            const prNumber = context.eventName === 'workflow_dispatch'
-              ? Number(context.payload.inputs.pr_number)
-              : context.issue.number;
-
-            await github.rest.issues.createComment({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              issue_number: prNumber,
-              body: `🚀 Spawning new ephemeral backend!\n\n${appUrl}`
-            });

.gitignore

@@ -20,6 +20,7 @@ rust-client/Cargo.toml
 
 # Worktree-specific Claude Code settings (generated by scripts/worktree-env)
 .claude/settings.local.json
+.claude/worktrees/
 
 # Symlinked cache directories (for git worktrees)
 backend/target

.mcp.json

@@ -3,6 +3,16 @@
     "svelte": {
       "type": "http",
       "url": "https://mcp.svelte.dev/mcp"
+    },
+    "playwright": {
+      "type": "stdio",
+      "command": "npx",
+      "args": ["-y", "@playwright/mcp@latest", "--browser", "chromium", "--headless"]
+    },
+    "playwright-headed": {
+      "type": "stdio",
+      "command": "npx",
+      "args": ["-y", "@playwright/mcp@latest", "--browser", "chromium"]
     }
   }
 }

.webmux.yaml

@@ -47,6 +47,7 @@ profiles:
       For this window specifically, backend is running on: ${BACKEND_PORT} and frontend is running on: ${FRONTEND_PORT}.
       To connect to the database, use this connection string: ${DATABASE_URL}
       Because we are running backend with cargo watch, to verify your changes, just check the logs in the backend pane. No need for cargo check.
+      For UI verification, use the Playwright MCP (`mcp__playwright__*`) — the `playwright` server is headless and works without a display. Navigate to http://localhost:${FRONTEND_PORT}, log in as admin@windmill.dev / changeme.
       IMPORTANT: Read docs/autonomous-mode.md before starting any work.
     panes:
       - id: agent
@@ -76,6 +77,7 @@ profiles:
       On this window specifically, frontend is running on: ${FRONTEND_PORT}.
       To connect to the database, use this connection string: ${DATABASE_URL}
       Because we are running frontend with npm run dev, to verify your changes, just check the logs in the frontend pane. No need for npm run build.
+      For UI verification, use the Playwright MCP (`mcp__playwright__*`) — the `playwright` server is headless and works without a display. Navigate to http://localhost:${FRONTEND_PORT}, log in as admin@windmill.dev / changeme.
       IMPORTANT: Read docs/autonomous-mode.md before starting any work.
     panes:
       - id: agent
@@ -100,9 +102,55 @@ profiles:
 
 integrations:
   github:
+    autoRemoveOnMerge: true
     linkedRepos:
       - repo: windmill-labs/windmill-ee-private
         alias: ee-private
         dir: ../windmill-ee-private__worktrees
   linear:
     enabled: true
+    autoCreateWorktrees: true
+    watchTeams: [WIN,GIT]
+
+oneshot:
+  systemPrompt: |
+    You are running in webmux ONESHOT mode.
+
+    # No interactive user
+    There is NO interactive user — nobody is watching the chat or will respond
+    to questions, approvals, or status checks. Any message asking the user to
+    review, approve, confirm, take a look, or "let you know" is wasted output:
+    it will not be answered.
+
+    # Your job
+    Take the task to its real conclusion without pausing:
+      1. Make the change.
+      2. Validate it (run the relevant tests, typecheck, build, or quick
+         manual check). For UI changes, drive the running frontend with
+         the Playwright MCP (`mcp__playwright__*`, headless) and confirm
+         the change works end-to-end before moving on.
+      3. Commit.
+      4. Push.
+      5. Open a pull request.
+    Only then are you done.
+
+    # Decisions
+    When something is ambiguous, pick the most reasonable default and proceed.
+    When you would normally ask "should I X or Y?", just pick one and continue
+    — note the choice in the PR description if it matters.
+
+    # PR readiness
+    Default to opening the PR as a draft. If you are highly confident in the
+    change — the scope is small and well-understood, validation passed
+    cleanly, and you would not change anything if a reviewer pushed back —
+    open the PR as ready-for-review directly (omit `--draft` when invoking
+    `gh pr create`, or call `gh pr ready <number>` after creation). Err on
+    the side of draft when validation was partial, the change touches
+    public APIs or shared infrastructure, or you made a non-obvious judgment
+    call.
+
+    # Ending your turn
+    Never end your turn with a question, a suggestion to "take a look", or a
+    request for approval. Stop only when the PR is open, or when you hit a
+    technical error you cannot recover from yourself (in which case clearly
+    state the blocker).

AGENTS.md

@@ -15,6 +15,7 @@ Open-source platform for internal tools, workflows, API integrations, background
 - **Enterprise**: `docs/enterprise.md` — EE file conventions and PR workflow
 - **Backend patterns**: use the `rust-backend` skill when writing Rust code
 - **Frontend patterns**: use the `svelte-frontend` skill when writing Svelte code. Do NOT edit svelte files unless you have read that skill.
+- **Frontend UUIDs**: do not call `crypto.randomUUID()` in frontend code. Import `randomUUID` from `$lib/utils/uuid` instead.
 - **Code review**: review the current PR or branch against the shared review policy in `REVIEW.md` (severity triage, public-surface checklist, AGENTS.md compliance, test-coverage assessment). The skill at `.agents/skills/local-review/SKILL.md` orchestrates it. All three CLIs auto-discover the same SKILL — Claude reads `.claude/skills/` (symlinked to the canonical `.agents/skills/` file), Codex and Pi read `.agents/skills/` directly. Invoke with `/local-review` in Claude Code, `$local-review` (or `/skills` selector) in Codex, or `pi --skill local-review` / `/skill:local-review` in Pi.
 - **Domain guides**: `.claude/skills/native-trigger/` and `frontend/tutorial-system-guide.mdc`
 - **Brand/UI guidelines**: `frontend/brand-guidelines.md`
@@ -29,6 +30,28 @@ Open-source platform for internal tools, workflows, API integrations, background
 - **Instance settings**: navigate to `/#superadmin-settings`
 - **Migrations**: use `cargo sqlx migrate add -r <name>` from `backend/` to create new migrations (never generate timestamps manually)
 
+## Verifying Frontend Changes
+
+After modifying frontend code, drive the running dev server with the **Playwright MCP** to verify the change in a real browser — don't claim a UI change works without exercising it.
+
+Two MCP servers are registered in `.mcp.json`:
+- `playwright` — headless Chromium, default for devboxes (no display required)
+- `playwright-headed` — windowed Chromium, when a display is available
+
+**One-time setup:** run `npx playwright install chromium` to download the browser binary (Playwright won't fetch it automatically on first use).
+
+Typical flow:
+1. Ensure backend (`cargo run`) and frontend (`REMOTE=http://localhost:8000 npm run dev`) are running
+2. `mcp__playwright__browser_navigate` to the relevant page (login at `admin@windmill.dev` / `changeme`)
+3. `mcp__playwright__browser_snapshot` to inspect the accessibility tree (preferred over screenshots for reading the DOM)
+4. `mcp__playwright__browser_click` / `browser_fill_form` / `browser_type` to interact
+5. `mcp__playwright__browser_take_screenshot` for visual confirmation
+6. `mcp__playwright__browser_console_messages` / `browser_network_requests` to surface errors
+
+**Attach the screenshots to the PR.** For any change under `frontend/`, embed screenshots of the affected UI in the PR body — the `pr` skill requires this and carries the upload recipe.
+
+If you cannot exercise a UI change (no dev server, etc.), say so explicitly rather than claiming success.
+
 ## Banned Patterns
 
 ### `$bindable(default_value)` on optional props
@@ -86,3 +109,4 @@ $NAV --root backend callees "X"                           # what does X call?
 - Search for existing code to reuse before writing new code
 - Follow established patterns in the codebase
 - Keep changes focused — don't refactor beyond what's asked
+- **Never attribute work to a specific customer, account, or "requested by a customer" in repo-tracked content** (PR descriptions, commit messages, code comments, docs). Describe changes by their technical motivation instead.

CHANGELOG.md

@@ -1,5 +1,500 @@
 # Changelog
 
+## [1.722.0](https://github.com/windmill-labs/windmill/compare/v1.721.0...v1.722.0) (2026-06-11)
+
+
+### Features
+
+* add reasoning effort control and thinking display to AI chat ([#9511](https://github.com/windmill-labs/windmill/issues/9511)) ([7f987e8](https://github.com/windmill-labs/windmill/commit/7f987e8c9807b72d9cc3901b6e4d02a24c423f50))
+* **ai-chat:** collapse big pastes, cap input height, escape HTML ([#9487](https://github.com/windmill-labs/windmill/issues/9487)) ([365e204](https://github.com/windmill-labs/windmill/commit/365e20410ed528d5b4e967b64fb282bcc1e03ccd))
+* **ai-chat:** quick access to AI prompt settings from chat ([#9508](https://github.com/windmill-labs/windmill/issues/9508)) ([b894f78](https://github.com/windmill-labs/windmill/commit/b894f783f183ab795eab6f57da8654275d9ad82d))
+* **ai:** add list_runs and get_job_logs tools to global chat mode ([#9488](https://github.com/windmill-labs/windmill/issues/9488)) ([cfe5119](https://github.com/windmill-labs/windmill/commit/cfe51190356a9e922f6398dd875abc368accbd15))
+* clear conflict error + force delete when reusing a fork workspace id ([#9499](https://github.com/windmill-labs/windmill/issues/9499)) ([fddabe9](https://github.com/windmill-labs/windmill/commit/fddabe9c5c6f178b4b09854dc11e50adafd44c87))
+* **flow:** support worker tag override on AI agent steps ([#9513](https://github.com/windmill-labs/windmill/issues/9513)) ([a6a5600](https://github.com/windmill-labs/windmill/commit/a6a5600833063e36b35aac7f90dcbdcd3db7c627))
+* folder-level label inheritance for scripts, flows and jobs ([#9524](https://github.com/windmill-labs/windmill/issues/9524)) ([765f50c](https://github.com/windmill-labs/windmill/commit/765f50c474f8abf76550664ed15418ddb3c0b221))
+* **frontend:** show AI sessions in narrow-screen burger menu ([#9523](https://github.com/windmill-labs/windmill/issues/9523)) ([a8f1062](https://github.com/windmill-labs/windmill/commit/a8f1062f37228e7a8257aac1cd3295bd5084ff90))
+* prefer idle worker pods on k8s autoscaling scale-in via pod-deletion-cost ([#9515](https://github.com/windmill-labs/windmill/issues/9515)) ([3c3f157](https://github.com/windmill-labs/windmill/commit/3c3f15722fd22713cc8baa744d9a81207943a33c))
+* prompt browser confirmation on page exit with unsaved changes ([#9503](https://github.com/windmill-labs/windmill/issues/9503)) ([3119e16](https://github.com/windmill-labs/windmill/commit/3119e16ed8df0daec0e019ae2efb2176d7e9e953))
+* **worker:** #ssh directive to run a bash script on a remote SSH host ([#9479](https://github.com/windmill-labs/windmill/issues/9479)) ([afddfe8](https://github.com/windmill-labs/windmill/commit/afddfe84452357b06f4fb815566d4c8269ceafbf))
+* workspace protection rule to restrict anonymous app deployment ([#9509](https://github.com/windmill-labs/windmill/issues/9509)) ([cf9ad54](https://github.com/windmill-labs/windmill/commit/cf9ad54181d38c5d3c3aa05208e17d8d97eae4ef))
+
+
+### Bug Fixes
+
+* **cli:** include lock-relevant script content in lock cache key ([#9528](https://github.com/windmill-labs/windmill/issues/9528)) ([dc60e1a](https://github.com/windmill-labs/windmill/commit/dc60e1aa174f2a7616d2d4a37117a463ef25a9d6))
+* **frontend:** allow copy/paste shortcuts inside ConfirmationModal ([#9505](https://github.com/windmill-labs/windmill/issues/9505)) ([7fc5340](https://github.com/windmill-labs/windmill/commit/7fc5340da39c049d059aa0ae15ceb6a98ca58053))
+* **frontend:** clarify trigger filters match the message parsed as JSON ([#9516](https://github.com/windmill-labs/windmill/issues/9516)) ([0b17843](https://github.com/windmill-labs/windmill/commit/0b178437ce8cd8295be21c0c6077c39e957a3337))
+* **frontend:** enable Apply button when env vars change in worker group config ([#9501](https://github.com/windmill-labs/windmill/issues/9501)) ([c80c6d8](https://github.com/windmill-labs/windmill/commit/c80c6d8fcdfc1de8fe841e9699452f67a25cad23))
+* **frontend:** improve AI chat markdown and typing dots in dark mode ([#9497](https://github.com/windmill-labs/windmill/issues/9497)) ([4e86806](https://github.com/windmill-labs/windmill/commit/4e868062d4c22c5574149d35235b7ad1abe805f3))
+* **frontend:** stop echoing draft values in global chat write tool results ([#9530](https://github.com/windmill-labs/windmill/issues/9530)) ([ce6e2f7](https://github.com/windmill-labs/windmill/commit/ce6e2f7ade25ca91c375f0de5f1be3f1c82ccb47))
+* inherit container NO_PROXY into MITM tracing proxy job exclusions ([#9492](https://github.com/windmill-labs/windmill/issues/9492)) ([4c22e3b](https://github.com/windmill-labs/windmill/commit/4c22e3b712a74828cf654ea7d89aeab5b50cfbd7))
+* make default chat model optional in AI settings ([#9514](https://github.com/windmill-labs/windmill/issues/9514)) ([1d43288](https://github.com/windmill-labs/windmill/commit/1d4328877fcc87352fb56de63f0570739d5a9dc4))
+* **nsjail:** make ansible collections mount non-mandatory ([#9510](https://github.com/windmill-labs/windmill/issues/9510)) ([08da7a1](https://github.com/windmill-labs/windmill/commit/08da7a121b4b835500dfc2bd943c4bdce912c63e))
+* **nsjail:** make ansible uv tools mount non-mandatory ([#9507](https://github.com/windmill-labs/windmill/issues/9507)) ([dc368a9](https://github.com/windmill-labs/windmill/commit/dc368a9669e812c593ad848266f645b6223501e6))
+
+## [1.721.0](https://github.com/windmill-labs/windmill/compare/v1.720.0...v1.721.0) (2026-06-09)
+
+
+### Features
+
+* deployed↔draft compare + AI-session draft bar ([#9435](https://github.com/windmill-labs/windmill/issues/9435)) ([b0b330c](https://github.com/windmill-labs/windmill/commit/b0b330c7864d0159af4b0f17dbb3c09bd015145b))
+
+
+### Bug Fixes
+
+* **cli:** reconcile case-only path drift during sync on case-insensitive filesystems (WIN-2020) ([#9485](https://github.com/windmill-labs/windmill/issues/9485)) ([c258928](https://github.com/windmill-labs/windmill/commit/c258928ab62adc1327913c21556520dfd1e5c24c))
+* drop archived items from fork compare (spurious 'not visible' warning) ([#9481](https://github.com/windmill-labs/windmill/issues/9481)) ([92c21bb](https://github.com/windmill-labs/windmill/commit/92c21bbe6586f3c285796a98692e976515a629d5))
+* require auth to view approval details when user_auth_required ([#9482](https://github.com/windmill-labs/windmill/issues/9482)) ([5f41ddd](https://github.com/windmill-labs/windmill/commit/5f41ddd3a592bcd504f94fc99060ca5d79c36190))
+
+## [1.720.0](https://github.com/windmill-labs/windmill/compare/v1.719.0...v1.720.0) (2026-06-08)
+
+
+### Features
+
+* allow private MCP server URLs ([#9470](https://github.com/windmill-labs/windmill/issues/9470)) ([3bc5800](https://github.com/windmill-labs/windmill/commit/3bc5800197db383ae6f708415701a4bdbe2e3345))
+* **api:** add endpoint to update token label ([#9474](https://github.com/windmill-labs/windmill/issues/9474)) ([e8e0701](https://github.com/windmill-labs/windmill/commit/e8e0701a360d0614c4c5a74f6410ba6ac0638caa))
+* **frontend:** use unified drill picker for AI chat @-mention dropdown ([#9159](https://github.com/windmill-labs/windmill/issues/9159)) ([64b089c](https://github.com/windmill-labs/windmill/commit/64b089cd23cca4601abb09f092a32becb80d9394))
+
+
+### Bug Fixes
+
+* center auth0/okta icons and respect currentColor ([#9457](https://github.com/windmill-labs/windmill/issues/9457)) ([5d0ef7d](https://github.com/windmill-labs/windmill/commit/5d0ef7dfd91b3021d125a1b34f81f0788f173786))
+* **forks:** keep trigger/schedule operational state owned by the parent - WIN-2019 ([#9476](https://github.com/windmill-labs/windmill/issues/9476)) ([192574a](https://github.com/windmill-labs/windmill/commit/192574ab8f98d9521a232fc8a4935d407b00cb3a))
+* **frontend:** respect forced column order for numeric column names ([#9463](https://github.com/windmill-labs/windmill/issues/9463)) ([44f5dd6](https://github.com/windmill-labs/windmill/commit/44f5dd6636d4b23aa55383b8b8abe4c2f73bc88d))
+* **frontend:** use ban icon for canceled jobs instead of hourglass ([#9478](https://github.com/windmill-labs/windmill/issues/9478)) ([fa86c62](https://github.com/windmill-labs/windmill/commit/fa86c62b6600e7d47dadf4706d7002706333d919))
+* gate native integration pickers behind non-operator check ([#9465](https://github.com/windmill-labs/windmill/issues/9465)) ([6156e23](https://github.com/windmill-labs/windmill/commit/6156e2372a785ccd0c6f29cb74e90bee69e76483))
+* **oauth:** persist refreshed token through configured secret backend ([#9471](https://github.com/windmill-labs/windmill/issues/9471)) ([76c0d97](https://github.com/windmill-labs/windmill/commit/76c0d970a18bf28ddc48dd746570e73486542606))
+* refresh session editor preview on breadcrumb target switch ([#9475](https://github.com/windmill-labs/windmill/issues/9475)) ([6d522b3](https://github.com/windmill-labs/windmill/commit/6d522b3989ace1f214bd049d910bc0d2a2a6893e))
+
+## [1.719.0](https://github.com/windmill-labs/windmill/compare/v1.718.0...v1.719.0) (2026-06-06)
+
+
+### Features
+
+* **otel:** connect jobs to the inbound distributed trace ([#9456](https://github.com/windmill-labs/windmill/issues/9456)) ([fad1a54](https://github.com/windmill-labs/windmill/commit/fad1a549d95c00d0746a48163c4f95fc69733e1a))
+
+
+### Bug Fixes
+
+* authenticate slack callback payload with per-workspace hmac ([#9461](https://github.com/windmill-labs/windmill/issues/9461)) ([fbdf81b](https://github.com/windmill-labs/windmill/commit/fbdf81ba5f77d282c025360ecee14138dd4cb4a2))
+* prevent token label collision bypassing job read access control ([#9462](https://github.com/windmill-labs/windmill/issues/9462)) ([e1e7af6](https://github.com/windmill-labs/windmill/commit/e1e7af6a25a44eb06b67332ce1efeae2a21e0c6d))
+* **python:** escape reserved-keyword step ids in wrapper codegen ([#9460](https://github.com/windmill-labs/windmill/issues/9460)) ([6a15a9b](https://github.com/windmill-labs/windmill/commit/6a15a9b152ad20be4b5c3de6000516da231e41e0)), closes [#8893](https://github.com/windmill-labs/windmill/issues/8893)
+
+## [1.718.0](https://github.com/windmill-labs/windmill/compare/v1.717.1...v1.718.0) (2026-06-05)
+
+
+### Features
+
+* **flows:** opt-in to include the stopping step's result in early-stop errors ([#9446](https://github.com/windmill-labs/windmill/issues/9446)) ([f2f0812](https://github.com/windmill-labs/windmill/commit/f2f0812a04c9256cfc8eba5e0dcf38d71d971410))
+* make C# dotnet target framework configurable via DOTNET_TARGET_FRAMEWORK ([#9454](https://github.com/windmill-labs/windmill/issues/9454)) ([9a609bf](https://github.com/windmill-labs/windmill/commit/9a609bf08ac1b6157dbdfb827fc01e771d71262e))
+* sandboxed daemonless container runtime via '# sandbox <image>' ([#9453](https://github.com/windmill-labs/windmill/issues/9453)) ([1727271](https://github.com/windmill-labs/windmill/commit/1727271e197b34026efeaf1b6561bb404a440baa))
+* **sandbox:** pull/extract images with crane instead of podman ([#9455](https://github.com/windmill-labs/windmill/issues/9455)) ([7590b28](https://github.com/windmill-labs/windmill/commit/7590b281085afd1fc2774e8fb37a4c0af3aedbad))
+
+
+### Bug Fixes
+
+* distinguish canceled jobs in runs ([#9452](https://github.com/windmill-labs/windmill/issues/9452)) ([9067787](https://github.com/windmill-labs/windmill/commit/90677872f6185eb0c81e0e84a426a54653818457))
+
+## [1.717.1](https://github.com/windmill-labs/windmill/compare/v1.717.0...v1.717.1) (2026-06-04)
+
+
+### Bug Fixes
+
+* invalidate relative-import cache when imported script changes ([#9443](https://github.com/windmill-labs/windmill/issues/9443)) ([f595787](https://github.com/windmill-labs/windmill/commit/f595787409a3fcda9278bbcf2cfcc80092f16460))
+
+## [1.717.0](https://github.com/windmill-labs/windmill/compare/v1.716.0...v1.717.0) (2026-06-04)
+
+
+### Features
+
+* let flow AI chat create and edit sticky notes ([#9412](https://github.com/windmill-labs/windmill/issues/9412)) ([e4e0984](https://github.com/windmill-labs/windmill/commit/e4e0984e55afd3c73f1c365cd0608493a9fd87ed))
+
+
+### Bug Fixes
+
+* **cli:** push whole raw app instead of treating frontend files as scripts ([#9442](https://github.com/windmill-labs/windmill/issues/9442)) ([b5a6a1e](https://github.com/windmill-labs/windmill/commit/b5a6a1eeab663c2d6aaec2c89eab7a550cb0bb6b))
+* read latest db draft for scripts/flows in global mode read tool ([#9441](https://github.com/windmill-labs/windmill/issues/9441)) ([819ba5e](https://github.com/windmill-labs/windmill/commit/819ba5e150ec9f5199919fbea50874fc156d0189))
+
+## [1.716.0](https://github.com/windmill-labs/windmill/compare/v1.715.0...v1.716.0) (2026-06-03)
+
+
+### Features
+
+* add metadata generation model setting ([#9418](https://github.com/windmill-labs/windmill/issues/9418)) ([cf5fefb](https://github.com/windmill-labs/windmill/commit/cf5fefb521479170b9dc64b884630c4dac789931))
+* auto-generate AI session names ([#9399](https://github.com/windmill-labs/windmill/issues/9399)) ([26b7270](https://github.com/windmill-labs/windmill/commit/26b727041830c9b741668a9ab73e2eb90c7cec74))
+* support $f/ and $u/ import path aliases for scripts ([#9378](https://github.com/windmill-labs/windmill/issues/9378)) ([220cd35](https://github.com/windmill-labs/windmill/commit/220cd35cf799c42ebf588bc97a6d8e6f4e97c2e3))
+* use metadata model for small AI tasks ([#9431](https://github.com/windmill-labs/windmill/issues/9431)) ([79178f6](https://github.com/windmill-labs/windmill/commit/79178f6f5a7c606a2e05677c6efcbdd84c608325))
+
+
+### Bug Fixes
+
+* **apps:** relock no longer reverts raw app to a stale version ([#9432](https://github.com/windmill-labs/windmill/issues/9432)) ([073857a](https://github.com/windmill-labs/windmill/commit/073857ac0a9ed54bdeac8f373f7c855fe34eb0ac))
+* **security:** scope variable and resource value caches by caller identity ([#9427](https://github.com/windmill-labs/windmill/issues/9427)) ([0ba128a](https://github.com/windmill-labs/windmill/commit/0ba128afe797bd016da60563949ac3abbbfe1978))
+
+## [1.715.0](https://github.com/windmill-labs/windmill/compare/v1.714.1...v1.715.0) (2026-06-03)
+
+
+### Features
+
+* **frontend:** add rebuild dependency map button to workspace settings ([#9424](https://github.com/windmill-labs/windmill/issues/9424)) ([3b2e748](https://github.com/windmill-labs/windmill/commit/3b2e748daf0a8ec4447c30423068df803f3f9ca2))
+
+
+### Bug Fixes
+
+* **auth:** filter script/flow listings by token scope (GHSA-2ppx-66jv-wpw5) ([#9426](https://github.com/windmill-labs/windmill/issues/9426)) ([7edf3f0](https://github.com/windmill-labs/windmill/commit/7edf3f02122e20fde1e95e0252e7bda641075326))
+* **backend:** authorize single-job read endpoints by job/flow visibility ([#9416](https://github.com/windmill-labs/windmill/issues/9416)) ([89a7a37](https://github.com/windmill-labs/windmill/commit/89a7a377764086911db18252f2478f42f0e1e3ea))
+* **mcp:** resolve MCP resource token via caller RLS + SSRF-guard url ([#9428](https://github.com/windmill-labs/windmill/issues/9428)) ([8053266](https://github.com/windmill-labs/windmill/commit/8053266f88bd4c94fc86278412df5a0beeed5e77))
+* **nsjail:** precompile python stdlib + raise download rlimit_as ([#9429](https://github.com/windmill-labs/windmill/issues/9429)) ([7031744](https://github.com/windmill-labs/windmill/commit/7031744a199f0bf8b8e35043afa959977e5ecdbd))
+* omit temperature for gpt-5+ and o-series models on all providers ([#9422](https://github.com/windmill-labs/windmill/issues/9422)) ([11d1ad9](https://github.com/windmill-labs/windmill/commit/11d1ad9a872d2ec2f14cde35708c84a0c7bdc172))
+
+## [1.714.1](https://github.com/windmill-labs/windmill/compare/v1.714.0...v1.714.1) (2026-06-02)
+
+
+### Bug Fixes
+
+* **backend:** route //native TypeScript previews to native workers (WIN-2007) ([#9407](https://github.com/windmill-labs/windmill/issues/9407)) ([73edebc](https://github.com/windmill-labs/windmill/commit/73edebc833a981488a8ea116f4f13c020a011a6f))
+* **nsjail:** raise python download fd limit for --compile-bytecode (WIN-2009) ([#9414](https://github.com/windmill-labs/windmill/issues/9414)) ([9e6559a](https://github.com/windmill-labs/windmill/commit/9e6559a6f688cc8d982277b19920219ea6d0fd8e))
+* **triggers:** prevent Zoom challenge handler from being used as a signing oracle ([#9413](https://github.com/windmill-labs/windmill/issues/9413)) ([ab2a15b](https://github.com/windmill-labs/windmill/commit/ab2a15b2a859096eabde718bf6e60289ae187118))
+
+## [1.714.0](https://github.com/windmill-labs/windmill/compare/v1.713.1...v1.714.0) (2026-06-02)
+
+
+### Features
+
+* add global ai chat test tools ([#9391](https://github.com/windmill-labs/windmill/issues/9391)) ([5c20d6b](https://github.com/windmill-labs/windmill/commit/5c20d6b4f79f2ccc1987ce7fdaf74e6b8f697846))
+* add workspace datatable tools to global AI chat mode ([#9395](https://github.com/windmill-labs/windmill/issues/9395)) ([943ef6e](https://github.com/windmill-labs/windmill/commit/943ef6eb2089f4b744cfa7945ce47f7f3b361ec7))
+* **flow-ai:** constrain flow-group colors to the NoteColor palette ([#9343](https://github.com/windmill-labs/windmill/issues/9343)) ([e4213c1](https://github.com/windmill-labs/windmill/commit/e4213c1ab8c448f492f372580f5c9df37e33fffc))
+* **frontend:** surface local drafts in drawer editors with an unsaved-changes banner ([#9335](https://github.com/windmill-labs/windmill/issues/9335)) ([075faab](https://github.com/windmill-labs/windmill/commit/075faabf3bba16a10a02ae3973008e5a13473085))
+* handle CTRL_BREAK_EVENT for graceful shutdown on Windows ([#9400](https://github.com/windmill-labs/windmill/issues/9400)) ([2e14456](https://github.com/windmill-labs/windmill/commit/2e1445616a412c5112ad2247b4087c7ddc218845))
+* refine ask-user-question chat display and keyboard nav ([#9392](https://github.com/windmill-labs/windmill/issues/9392)) ([1275487](https://github.com/windmill-labs/windmill/commit/1275487f028d4c74a9eeb18981ed05c225505be0))
+* sessions page with isolated AI chat + flow editor ([#9034](https://github.com/windmill-labs/windmill/issues/9034)) ([eadeac2](https://github.com/windmill-labs/windmill/commit/eadeac248bd022c2796cfe638eb617c6143b8fc4))
+
+
+### Bug Fixes
+
+* **cli:** make encryption key push non-interactive-safe + add --skip-reencrypt-on-key-change ([#9402](https://github.com/windmill-labs/windmill/issues/9402)) ([e356bb1](https://github.com/windmill-labs/windmill/commit/e356bb1f5df92eca3fbb0ca2114b9f4c32d4c496))
+* **cli:** stop git-sync promotion deploys from dropping triggers/schedules ([#9403](https://github.com/windmill-labs/windmill/issues/9403)) ([24e3ef2](https://github.com/windmill-labs/windmill/commit/24e3ef27be8498fb820c228a52febf6a0a91b487))
+* **frontend:** align Monaco editor font size with text-xs ([#9161](https://github.com/windmill-labs/windmill/issues/9161)) ([de76668](https://github.com/windmill-labs/windmill/commit/de76668c10c04abe8771a8ca7bba7b2259819a1c))
+* resolve username rename failing on apps with runnable deps ([#9401](https://github.com/windmill-labs/windmill/issues/9401)) ([e8ad53d](https://github.com/windmill-labs/windmill/commit/e8ad53dae92597f5a1a8b76f38a7d8c24f578a47))
+
+
+### Performance Improvements
+
+* **python:** add --compile-bytecode to uv pip install ([#9393](https://github.com/windmill-labs/windmill/issues/9393)) ([c19441b](https://github.com/windmill-labs/windmill/commit/c19441bc8cb2da064e4ad44d77dc04ab8bbb22ec))
+
+## [1.713.1](https://github.com/windmill-labs/windmill/compare/v1.713.0...v1.713.1) (2026-06-01)
+
+
+### Bug Fixes
+
+* **api:** handle multi-version scripts when removing granular ACL ([#9388](https://github.com/windmill-labs/windmill/issues/9388)) ([9d9c503](https://github.com/windmill-labs/windmill/commit/9d9c5038ce8b0016320a670c434ef9063cb40441))
+
+## [1.713.0](https://github.com/windmill-labs/windmill/compare/v1.712.0...v1.713.0) (2026-05-31)
+
+
+### Features
+
+* **flows:** preserve step/subflow worker tags under a custom-tagged flow ([#9375](https://github.com/windmill-labs/windmill/issues/9375)) ([f0301b1](https://github.com/windmill-labs/windmill/commit/f0301b1605cee5fba4024803555333e6fa5c40ee))
+* **oauth:** support per-provider sandbox URLs ([#9358](https://github.com/windmill-labs/windmill/issues/9358)) ([2bf11dc](https://github.com/windmill-labs/windmill/commit/2bf11dcb15540c538ea2ac3cf70dcbe589060b4e))
+
+
+### Bug Fixes
+
+* **ai:** validate token_url for SSRF in OAuth credentials flow ([#9385](https://github.com/windmill-labs/windmill/issues/9385)) ([4b06881](https://github.com/windmill-labs/windmill/commit/4b06881918b76c5a411cc70b318e46efcc1393a7))
+* **api:** authorize and harden log-file reading endpoints ([#9368](https://github.com/windmill-labs/windmill/issues/9368)) ([bb90f4c](https://github.com/windmill-labs/windmill/commit/bb90f4ce83a0e60af219b11c12ab4fe1d13f47a4))
+* **apps:** make public apps opt into cross-origin isolation via wm_coep (GIT-884) ([#9374](https://github.com/windmill-labs/windmill/issues/9374)) ([2c0c2c4](https://github.com/windmill-labs/windmill/commit/2c0c2c467f163cd24c14c7be2db07af9cf2ce020))
+* **auth:** enforce monotonic privilege on user token lifecycle endpoints ([#9371](https://github.com/windmill-labs/windmill/issues/9371)) ([2ddf93d](https://github.com/windmill-labs/windmill/commit/2ddf93de96622b2a1b2b6f59398a7a1f59360efd))
+* batch encryption-key rotation into one git-sync job ([#9355](https://github.com/windmill-labs/windmill/issues/9355)) ([04a0897](https://github.com/windmill-labs/windmill/commit/04a08976aec4ba9b0516350316df303e9f96bfd3))
+* **cli:** preserve user drafts on sync push and permissioned-as ([#9381](https://github.com/windmill-labs/windmill/issues/9381)) ([b0c3b01](https://github.com/windmill-labs/windmill/commit/b0c3b01d31b0ab3a6566e1f5fec60e3e230cfadb))
+* **frontend:** sanitize user markdown to prevent stored XSS ([#9386](https://github.com/windmill-labs/windmill/issues/9386)) ([def01b8](https://github.com/windmill-labs/windmill/commit/def01b8ff6f331cc36ce02b947adc31c766042c4))
+* **security:** re-pin cached hub scripts to CVE-patched versions (+ HUB_BASE_URL override for cache mode) ([#9387](https://github.com/windmill-labs/windmill/issues/9387)) ([edf340c](https://github.com/windmill-labs/windmill/commit/edf340c4d4f18b16b142cb7deb67afa586f10946))
+
+## [1.712.0](https://github.com/windmill-labs/windmill/compare/v1.711.0...v1.712.0) (2026-05-28)
+
+
+### Features
+
+* add deepseek fim support ([#9365](https://github.com/windmill-labs/windmill/issues/9365)) ([2553fbf](https://github.com/windmill-labs/windmill/commit/2553fbfe31417bd985e7994eac695bf918f97ce2))
+* deploy raw apps from global chat ([#9349](https://github.com/windmill-labs/windmill/issues/9349)) ([dec58e6](https://github.com/windmill-labs/windmill/commit/dec58e6c4f55062b42a752c43c89ef05903e713a))
+* inject active editor into global chat ([#9361](https://github.com/windmill-labs/windmill/issues/9361)) ([9e7eaf3](https://github.com/windmill-labs/windmill/commit/9e7eaf36847ad3a004ec84e8b7d4784771b7b451))
+* **queue:** duration-weighted fairness admission ([#9334](https://github.com/windmill-labs/windmill/issues/9334)) ([045d120](https://github.com/windmill-labs/windmill/commit/045d12043e7c99830ef90bc0da798c94e2094711))
+* warn when custom instance db is shared across workspaces ([#9359](https://github.com/windmill-labs/windmill/issues/9359)) ([a9e5140](https://github.com/windmill-labs/windmill/commit/a9e514099585e5ee72df21bd551a223cceb20fb0))
+
+
+### Bug Fixes
+
+* **cli:** redact encryption_key diff in stdout by default ([#9347](https://github.com/windmill-labs/windmill/issues/9347)) ([88056f8](https://github.com/windmill-labs/windmill/commit/88056f8d4c91c1d14d85a08851ecf0bd97e2260d))
+* **cli:** stop re-prompting on wmill refresh prompts ([#9357](https://github.com/windmill-labs/windmill/issues/9357)) ([c2b5ba8](https://github.com/windmill-labs/windmill/commit/c2b5ba8871abbbcff6de69c90e2f09fee70586c1))
+* **frontend:** close other sidebar menus when hovering Help ([#9354](https://github.com/windmill-labs/windmill/issues/9354)) ([da882c5](https://github.com/windmill-labs/windmill/commit/da882c54b21e3eaf2c1d1abccd0996b243d96dce))
+* **frontend:** prevent duplicate asset node ids crashing flow graph ([#9367](https://github.com/windmill-labs/windmill/issues/9367)) ([9a659b6](https://github.com/windmill-labs/windmill/commit/9a659b636d713ee8fdfbdad41c58bb3d7c79e0d9))
+* **frontend:** prevent MultiSelect crash on undefined value ([#9364](https://github.com/windmill-labs/windmill/issues/9364)) ([aea0061](https://github.com/windmill-labs/windmill/commit/aea00611c41379be2afdad0eedd608c9537d03f7))
+* **git-sync:** publish fork branch on only_create_branch from the CLI ([#9366](https://github.com/windmill-labs/windmill/issues/9366)) ([2fdc51e](https://github.com/windmill-labs/windmill/commit/2fdc51e62985fc755884436130bdd58e294247c8))
+* infer script arg schema when deploying via AI chat ([#9356](https://github.com/windmill-labs/windmill/issues/9356)) ([4efc372](https://github.com/windmill-labs/windmill/commit/4efc37212a98571214aba135b0fbb10dc263fd4f))
+* **monitor:** cleanup stale server_heartbeat background_task_state rows ([#9338](https://github.com/windmill-labs/windmill/issues/9338)) ([59ab038](https://github.com/windmill-labs/windmill/commit/59ab038d7718d8a4c25efa5928f42e1393ebbf40))
+
+## [1.711.0](https://github.com/windmill-labs/windmill/compare/v1.710.1...v1.711.0) (2026-05-26)
+
+
+### Features
+
+* **cli:** add object-storage commands and flow test-step ([#9326](https://github.com/windmill-labs/windmill/issues/9326)) ([36f574f](https://github.com/windmill-labs/windmill/commit/36f574ff951198a4d40ee068a27d74c41ce32154))
+
+
+### Bug Fixes
+
+* **cli:** handle __flow suffix when deriving the flow's Windmill path ([#9333](https://github.com/windmill-labs/windmill/issues/9333)) ([6f77034](https://github.com/windmill-labs/windmill/commit/6f770346fb330997a836c39fba347df4c088a83c))
+* **queue:** duration-weighted workspace fairness signal ([#9329](https://github.com/windmill-labs/windmill/issues/9329)) ([42d2121](https://github.com/windmill-labs/windmill/commit/42d2121af925de50f549ecb72ffb5132f5c41079))
+
+## [1.710.1](https://github.com/windmill-labs/windmill/compare/v1.710.0...v1.710.1) (2026-05-26)
+
+
+### Bug Fixes
+
+* improve workspace fairness ([896add0](https://github.com/windmill-labs/windmill/commit/896add0350f4de31f5674d6be0907a582c5ec17e))
+
+## [1.710.0](https://github.com/windmill-labs/windmill/compare/v1.709.0...v1.710.0) (2026-05-26)
+
+
+### Features
+
+* **queue:** stochastic admission + EE availability of workspace fairness algorithm ([#9321](https://github.com/windmill-labs/windmill/issues/9321)) ([8bf7fd2](https://github.com/windmill-labs/windmill/commit/8bf7fd2c921c48861b71731a085b18ea8f72fb68))
+
+
+### Bug Fixes
+
+* **websocket-trigger:** honor HTTPS_PROXY/HTTP_PROXY/NO_PROXY ([#9324](https://github.com/windmill-labs/windmill/issues/9324)) ([6f36316](https://github.com/windmill-labs/windmill/commit/6f363163df9cd15f5af7d56cf34a01b70d236830))
+
+## [1.709.0](https://github.com/windmill-labs/windmill/compare/v1.708.0...v1.709.0) (2026-05-25)
+
+
+### Features
+
+* add copy button to Path component ([#9311](https://github.com/windmill-labs/windmill/issues/9311)) ([98bd5e7](https://github.com/windmill-labs/windmill/commit/98bd5e7f2a437b8b534028838b6ed0d7c59f7011))
+* **ai-chat:** align footer bar + DropdownV2 mode/autonomy selectors ([#9308](https://github.com/windmill-labs/windmill/issues/9308)) ([2f50e8b](https://github.com/windmill-labs/windmill/commit/2f50e8bab0b5ae9ae297c79abfe96df441f405e2))
+* **ai-chat:** expand chat question answers ([#9310](https://github.com/windmill-labs/windmill/issues/9310)) ([3f219ae](https://github.com/windmill-labs/windmill/commit/3f219aed98d93158aefce01bb51ed12dcb4711a1))
+* plug global chat drafts into userdraft ([#9291](https://github.com/windmill-labs/windmill/issues/9291)) ([1eef531](https://github.com/windmill-labs/windmill/commit/1eef53170b1b2afb75b9812e33787d1f28cf50dd))
+* **raw_apps:** surface UI Builder build errors over the preview pane ([#9316](https://github.com/windmill-labs/windmill/issues/9316)) ([90a196d](https://github.com/windmill-labs/windmill/commit/90a196d8d81993ffc2377d7088ab98f7b0f5ddcc))
+* **raw_apps:** tab-based editor surface with split-with-preview ([#9273](https://github.com/windmill-labs/windmill/issues/9273)) ([368e677](https://github.com/windmill-labs/windmill/commit/368e6774194a58058f28d1b4a42f8f4a7ec4ab63))
+* **service-accounts:** allow choosing role at creation time ([#9307](https://github.com/windmill-labs/windmill/issues/9307)) ([b125eca](https://github.com/windmill-labs/windmill/commit/b125eca7628b07c071bd102b161d389259fd6c62))
+
+
+### Bug Fixes
+
+* **auth:** filter resource/variable listings by token scope (WIN-1981) ([#9302](https://github.com/windmill-labs/windmill/issues/9302)) ([b5a0d46](https://github.com/windmill-labs/windmill/commit/b5a0d46695fdfe692d64573d1cfa06511e3b33f5))
+* **jobs:** authorization bypass in only_result job updates (WIN-1980) ([#9301](https://github.com/windmill-labs/windmill/issues/9301)) ([108a88a](https://github.com/windmill-labs/windmill/commit/108a88a1801548c8570d56aa3e1eb80246367bf4))
+
+## [1.708.0](https://github.com/windmill-labs/windmill/compare/v1.707.0...v1.708.0) (2026-05-24)
+
+
+### Features
+
+* **queue:** per-workspace fairness cap on the shared cloud worker pool ([#9303](https://github.com/windmill-labs/windmill/issues/9303)) ([de2e243](https://github.com/windmill-labs/windmill/commit/de2e243313ee34348675dec600cb412b475d1b4b))
+
+## [1.707.0](https://github.com/windmill-labs/windmill/compare/v1.706.1...v1.707.0) (2026-05-22)
+
+
+### Features
+
+* add wmill job rerun subcommand ([#9275](https://github.com/windmill-labs/windmill/issues/9275)) ([e0ffea2](https://github.com/windmill-labs/windmill/commit/e0ffea2deb5acf30815edd3669f4fc4c818b6e19))
+* **github-app:** hide cloud-only UI on self-managed + admin assignment UI ([#9299](https://github.com/windmill-labs/windmill/issues/9299)) ([dcee8cc](https://github.com/windmill-labs/windmill/commit/dcee8cc0d3dd71c3a12f1720e3ce4eb86cdacf4f))
+* **typescript-client:** add deleteS3File + optional workspace arg on S3 helpers ([#9300](https://github.com/windmill-labs/windmill/issues/9300)) ([daab561](https://github.com/windmill-labs/windmill/commit/daab561ec0763468d93e42e8f7f0796dc77be74d))
+
+
+### Bug Fixes
+
+* **auth:** tighten token-owner fallback for unscoped tokens (WIN-1978) ([#9293](https://github.com/windmill-labs/windmill/issues/9293)) ([7003998](https://github.com/windmill-labs/windmill/commit/7003998a575d76c272c6abd0789a1d1f7b722076))
+* **cli:** wmill sync pull updates wmill-lock.yaml for raw apps ([#9289](https://github.com/windmill-labs/windmill/issues/9289)) ([486e5f9](https://github.com/windmill-labs/windmill/commit/486e5f947b1649c17d32e3b214c50d4be701a4e8))
+* flow recording teardown crash + rename package to @windmill-labs/components ([#9288](https://github.com/windmill-labs/windmill/issues/9288)) ([13a2fae](https://github.com/windmill-labs/windmill/commit/13a2fae745ba4862006db5ee0811475c1d27fd1d))
+* **flows:** restore Variables and Resources in flow editor prop picker ([#9290](https://github.com/windmill-labs/windmill/issues/9290)) ([5566c7b](https://github.com/windmill-labs/windmill/commit/5566c7b3ff2d5a6b15cb9187aa15ce1c7245b3fb))
+* **ResourceEditor:** don't reset state when `selected` reverts to undefined ([#9295](https://github.com/windmill-labs/windmill/issues/9295)) ([1f2d2c1](https://github.com/windmill-labs/windmill/commit/1f2d2c11493db20b87615d41c21e5e1c35564739))
+* **secret-backend:** pass DB to Vault migrations + show failure details ([#9292](https://github.com/windmill-labs/windmill/issues/9292)) ([ace2291](https://github.com/windmill-labs/windmill/commit/ace22910c40585a6a2c9abd0c46f7e5e0214e78e))
+
+## [1.706.1](https://github.com/windmill-labs/windmill/compare/v1.706.0...v1.706.1) (2026-05-22)
+
+
+### Bug Fixes
+
+* fork compare visibility for non-admins and stale-token superadmins ([#9283](https://github.com/windmill-labs/windmill/issues/9283)) ([8272244](https://github.com/windmill-labs/windmill/commit/82722449e79da0b4b0ad4142aec7e7965e9ff236))
+* **git-sync:** bump to hub/28234 with stateless gpg.program wrapper (WIN-1974) ([#9282](https://github.com/windmill-labs/windmill/issues/9282)) ([89a2f07](https://github.com/windmill-labs/windmill/commit/89a2f07218818b95238b4a4484deab3138099672))
+* **nsjail:** gate unix-symlink test behind cfg(unix) for Windows build ([#9280](https://github.com/windmill-labs/windmill/issues/9280)) ([72e2c3a](https://github.com/windmill-labs/windmill/commit/72e2c3a6b3e0cb0f5bddf8291ae18bb8cf55ec28))
+
+## [1.706.0](https://github.com/windmill-labs/windmill/compare/v1.705.0...v1.706.0) (2026-05-21)
+
+
+### Features
+
+* add userdraft listing primitives ([#9268](https://github.com/windmill-labs/windmill/issues/9268)) ([d0ee697](https://github.com/windmill-labs/windmill/commit/d0ee697e8b8de58085ea0b2ecde1af2b2441428d))
+* add UV_PYTHON_INSTALL_MIRROR env and instance setting ([#9271](https://github.com/windmill-labs/windmill/issues/9271)) ([1169371](https://github.com/windmill-labs/windmill/commit/1169371d4885bdc18c76d03c6caae71f0e440235))
+* add yolo mode for ai chat tools ([#9258](https://github.com/windmill-labs/windmill/issues/9258)) ([ac26aa4](https://github.com/windmill-labs/windmill/commit/ac26aa4e4c7cc2d493f136b59738c0708803cc6d))
+* CLI datatable serve / psql ([#9267](https://github.com/windmill-labs/windmill/issues/9267)) ([28c8b5c](https://github.com/windmill-labs/windmill/commit/28c8b5c60fd46f961ae11b363b9be834fad6ee68))
+* **cli:** add `wmill init prompts` and custom override slot ([#9266](https://github.com/windmill-labs/windmill/issues/9266)) ([1ba8ed8](https://github.com/windmill-labs/windmill/commit/1ba8ed8abd827313ce0f7728d9f84357417206ee))
+* **nsjail:** optional disk-backed /tmp via instance setting ([#9272](https://github.com/windmill-labs/windmill/issues/9272)) ([b656dc6](https://github.com/windmill-labs/windmill/commit/b656dc6cdc8c50ef9740240447f119cceed18547))
+
+
+### Bug Fixes
+
+* **ai:** enforce RLS and scope check on user-supplied X-Resource-Path ([#9276](https://github.com/windmill-labs/windmill/issues/9276)) ([0692b97](https://github.com/windmill-labs/windmill/commit/0692b97c8a3818549d7050ea3e057e9cbf1ddb44))
+* **debugger:** add non-root user support to Dockerfile ([#9277](https://github.com/windmill-labs/windmill/issues/9277)) ([0bdb6a9](https://github.com/windmill-labs/windmill/commit/0bdb6a9d5d5fb28a27af1b6eda9fde7172308faf))
+* **indexer:** tell admins when ingress routes search to wrong pod ([#9274](https://github.com/windmill-labs/windmill/issues/9274)) ([d29a561](https://github.com/windmill-labs/windmill/commit/d29a5612fcd17eb4197468289e955a1209127cc1))
+
+## [1.705.0](https://github.com/windmill-labs/windmill/compare/v1.704.1...v1.705.0) (2026-05-20)
+
+
+### Features
+
+* add flow_user_state(key) to QuickJS input transform sandbox (WIN-1947) ([#9093](https://github.com/windmill-labs/windmill/issues/9093)) ([88c1493](https://github.com/windmill-labs/windmill/commit/88c149314576789f46feb5c7e1af3225b061f0c6))
+* add wmill protection-rules pull/push CLI commands ([#9240](https://github.com/windmill-labs/windmill/issues/9240)) ([01bad16](https://github.com/windmill-labs/windmill/commit/01bad16c0cc40fa64b2a72ccb8ded487c729cf35))
+* **chat:** visual redesign — input, streaming indicator, scroll polish ([#9232](https://github.com/windmill-labs/windmill/issues/9232)) ([31a0469](https://github.com/windmill-labs/windmill/commit/31a046973af960764ee4e153b68c020cbd4690ce))
+* **chat:** waiting-for-user indicator + scroll-to-latest polish ([#9252](https://github.com/windmill-labs/windmill/issues/9252)) ([7909878](https://github.com/windmill-labs/windmill/commit/790987831380611b5bd19a760b0a5433492d7796))
+* **cli:** add datatable and ducklake list/run commands ([#9257](https://github.com/windmill-labs/windmill/issues/9257)) ([1d04904](https://github.com/windmill-labs/windmill/commit/1d04904a47245062e50c2cc6362bbbb21a6987aa))
+* **debug:** show ghost breakpoint and tooltip on gutter hover ([#9150](https://github.com/windmill-labs/windmill/issues/9150)) ([271f0cb](https://github.com/windmill-labs/windmill/commit/271f0cbd087851fca86ed1530618ddbf728f13f3))
+* **editors:** responsive top-bars + collapsible raw-app sidebar ([#9237](https://github.com/windmill-labs/windmill/issues/9237)) ([b0ed270](https://github.com/windmill-labs/windmill/commit/b0ed27096d9e918e946cf8a8a04af8ff1892b50f))
+* export audit logs to a dedicated object store folder ([#9207](https://github.com/windmill-labs/windmill/issues/9207)) ([ba6fb70](https://github.com/windmill-labs/windmill/commit/ba6fb7021b5a720bff8e86b4741031902cf1c267))
+* **frontend:** new path component ([#9017](https://github.com/windmill-labs/windmill/issues/9017)) ([9c28bbf](https://github.com/windmill-labs/windmill/commit/9c28bbfd694a5047b4a8a9fe5cc2e54309f8f067))
+* **frontend:** sync home search bar state to URL ([#9256](https://github.com/windmill-labs/windmill/issues/9256)) ([31b7810](https://github.com/windmill-labs/windmill/commit/31b781000e62384af6b8e1ba0172e45ac6ab591f))
+* **git-sync:** hidden `sync git-deploy` owns wm_deploy branch + e2e regression tests ([#9230](https://github.com/windmill-labs/windmill/issues/9230)) ([07202fd](https://github.com/windmill-labs/windmill/commit/07202fd048c999c9d32f3feee94a08625050283d))
+* **indexer:** observability for unavailable search index (WIN-1956) ([#9239](https://github.com/windmill-labs/windmill/issues/9239)) ([285a787](https://github.com/windmill-labs/windmill/commit/285a78752a23aa467f9a82868d784599793d3a1f))
+* **nsjail:** make tmpfs size configurable via instance setting ([#9261](https://github.com/windmill-labs/windmill/issues/9261)) ([9111f89](https://github.com/windmill-labs/windmill/commit/9111f8908de82e9032a63711158dff9c6bca255b))
+* open ai chat path links in drawers ([#9220](https://github.com/windmill-labs/windmill/issues/9220)) ([f6fcdb5](https://github.com/windmill-labs/windmill/commit/f6fcdb5599c28b4890d6f775f657bfafeea1d380))
+* persistent in-editor drafts via UserDraft ([#9121](https://github.com/windmill-labs/windmill/issues/9121)) ([0f7dd86](https://github.com/windmill-labs/windmill/commit/0f7dd86e5c3a43bc62c4c0501efec34226b6e279))
+* resolve relative imports from local content in script/flow preview ([#9233](https://github.com/windmill-labs/windmill/issues/9233)) ([2a780ad](https://github.com/windmill-labs/windmill/commit/2a780ad87af69358f241536697f694612fe92d93))
+* **snowflake:** derive public key from private key when omitted (WIN-1959) ([#9251](https://github.com/windmill-labs/windmill/issues/9251)) ([aa12c66](https://github.com/windmill-labs/windmill/commit/aa12c66c25e68eefec22c213e8f228fd0699d8ce))
+* **vault:** optional KV secret path prefix setting (WIN-1960) ([#9249](https://github.com/windmill-labs/windmill/issues/9249)) ([d08f72b](https://github.com/windmill-labs/windmill/commit/d08f72b3e1ef194b5d656cafa15bc88e8b6ba731))
+
+
+### Bug Fixes
+
+* **autoscaling:** count custom worker groups by row, divide only native by NUM_WORKERS ([#9255](https://github.com/windmill-labs/windmill/issues/9255)) ([76d949e](https://github.com/windmill-labs/windmill/commit/76d949e7bc30eb8cadfdc52e031fcdf5ad97d2ed))
+* **autoscaling:** full-scale below min_workers on large backlog ([#9234](https://github.com/windmill-labs/windmill/issues/9234)) ([a4d59a8](https://github.com/windmill-labs/windmill/commit/a4d59a81dfb6fffbd185a3aa009eb90c160bf42b))
+* bound resource/variable interpolation recursion depth (WIN-1957) ([#9243](https://github.com/windmill-labs/windmill/issues/9243)) ([26f3cbe](https://github.com/windmill-labs/windmill/commit/26f3cbef259e643c6d79be893701eec70b7c0501))
+* cgroup-aware DuckDB memory_limit + allocator memory release ([#9245](https://github.com/windmill-labs/windmill/issues/9245)) ([0022112](https://github.com/windmill-labs/windmill/commit/00221128cbf0801a45bad40246e50beceaba0a7e))
+* collapse successful ai tool details ([#9265](https://github.com/windmill-labs/windmill/issues/9265)) ([413404a](https://github.com/windmill-labs/windmill/commit/413404a788bbe6b5c9df387a2db3000ffec74083))
+* early return should consider failure_module result ([#9241](https://github.com/windmill-labs/windmill/issues/9241)) ([2db1c0a](https://github.com/windmill-labs/windmill/commit/2db1c0a1fcfdcad94cae97dcffa090ffb91494f7))
+* enable jemalloc background_thread to prevent worker RSS growth ([#9236](https://github.com/windmill-labs/windmill/issues/9236)) ([a974ff6](https://github.com/windmill-labs/windmill/commit/a974ff68e00278ccaf441b0567cd46e2b5067fdd))
+* enforce auth guards on app component preview execution ([#9235](https://github.com/windmill-labs/windmill/issues/9235)) ([4b1bea8](https://github.com/windmill-labs/windmill/commit/4b1bea8aed51eb9e24940d89d984ce32f375ab0c))
+* **flows:** flag noLogs jobs and lazily resolve them in log panel ([#9099](https://github.com/windmill-labs/windmill/issues/9099)) ([740a35b](https://github.com/windmill-labs/windmill/commit/740a35bf7b20f0bd8cb94c3d703dd353f0711b0a))
+* **frontend:** flow progress bar for early-stop completion and error handler (WIN-1961) ([#9254](https://github.com/windmill-labs/windmill/issues/9254)) ([cc141ef](https://github.com/windmill-labs/windmill/commit/cc141effa3b1019f70f4b7230fe7ebc7017632a0))
+* **frontend:** open customer portal in popup synchronously to bypass Safari blocker ([#9242](https://github.com/windmill-labs/windmill/issues/9242)) ([f51b51a](https://github.com/windmill-labs/windmill/commit/f51b51a9a1aee5183fa597cf93ff14fbaddaffa9))
+* prevent undefined user flickering in multiplayer presence list ([#9231](https://github.com/windmill-labs/windmill/issues/9231)) ([8c1f6cc](https://github.com/windmill-labs/windmill/commit/8c1f6ccc5d22e657a83831eb5f37a9516cdec10a))
+* **s3:** sandbox stored XSS via download response headers ([#9263](https://github.com/windmill-labs/windmill/issues/9263)) ([bb78b1c](https://github.com/windmill-labs/windmill/commit/bb78b1c06de5b73b951691460f81a3a2ec6e7f80))
+* **saml:** preserve deep links from /a/[...path] across SAML round-trip ([#9259](https://github.com/windmill-labs/windmill/issues/9259)) ([78cf6c7](https://github.com/windmill-labs/windmill/commit/78cf6c7f8181ad431cffebd18c45a3a802b3a601))
+* scope VSCode webview clipboard paste to focused editor ([#9221](https://github.com/windmill-labs/windmill/issues/9221)) ([bd06282](https://github.com/windmill-labs/windmill/commit/bd062825a255da364c1590820ead65172e835d13))
+
+## [1.704.1](https://github.com/windmill-labs/windmill/compare/v1.704.0...v1.704.1) (2026-05-19)
+
+
+### Bug Fixes
+
+* fix git sync ([ff1deaa](https://github.com/windmill-labs/windmill/commit/ff1deaa7e2f3f1f650861c1e2a0f663e597d501c))
+* honor SAML RelayState to redirect to deep link after SSO login ([#9225](https://github.com/windmill-labs/windmill/issues/9225)) ([89306d7](https://github.com/windmill-labs/windmill/commit/89306d7dbc96d0c7dfe2c6025cefc2d72e4f224e))
+* revert git sync script bump ([0f54ecd](https://github.com/windmill-labs/windmill/commit/0f54ecd34cf1ac86ded9305bc84a044eb6a86e72))
+
+## [1.704.0](https://github.com/windmill-labs/windmill/compare/v1.703.3...v1.704.0) (2026-05-18)
+
+
+### Features
+
+* add global ask user question tool ([#9217](https://github.com/windmill-labs/windmill/issues/9217)) ([f965512](https://github.com/windmill-labs/windmill/commit/f965512c7a9aca32c252ca0cda7ec00ab08a38e0))
+* add global chat selected context ([#9216](https://github.com/windmill-labs/windmill/issues/9216)) ([49ebf6f](https://github.com/windmill-labs/windmill/commit/49ebf6f8ba0ea55ea7987f40ecdd32738241a3f2))
+* show job status in favicon on the run page ([#9206](https://github.com/windmill-labs/windmill/issues/9206)) ([2e05bdd](https://github.com/windmill-labs/windmill/commit/2e05bdd73a664ddeec513653ab74e8c696ea1cfd))
+
+
+### Bug Fixes
+
+* don't fail flow on AlreadyCompleted after zombie restart ([#9214](https://github.com/windmill-labs/windmill/issues/9214)) ([8b7f7b3](https://github.com/windmill-labs/windmill/commit/8b7f7b37bdb91449cbd868bd3ee33a0ccbaf288f))
+* **git-sync:** bump default sync script to hub/28229 for extra_perms support ([#9223](https://github.com/windmill-labs/windmill/issues/9223)) ([0538412](https://github.com/windmill-labs/windmill/commit/0538412f1c370981be1915d8c724879d2c54fb83))
+* preserve ai reasoning content ([#9208](https://github.com/windmill-labs/windmill/issues/9208)) ([fec4008](https://github.com/windmill-labs/windmill/commit/fec40086961174fea25b4e1f796991152b84b211))
+* reject path traversal in MCP endpoint path parameters ([#9211](https://github.com/windmill-labs/windmill/issues/9211)) ([ad5ec29](https://github.com/windmill-labs/windmill/commit/ad5ec293b5a189135faea21e0d9c93637b77670f))
+* resolve absolute-path imports in monaco ts editor ([#9213](https://github.com/windmill-labs/windmill/issues/9213)) ([156eb0b](https://github.com/windmill-labs/windmill/commit/156eb0b045171e8d6990af9eeab752071bf7097b))
+
+## [1.703.3](https://github.com/windmill-labs/windmill/compare/v1.703.2...v1.703.3) (2026-05-18)
+
+
+### Bug Fixes
+
+* constrain unauthenticated get_public_resource to app_theme resources ([#9203](https://github.com/windmill-labs/windmill/issues/9203)) ([24eedef](https://github.com/windmill-labs/windmill/commit/24eedef918376d9d401335b6fada577916f8cc0e))
+* enforce folder ACL on flow run-by-version routes ([#9202](https://github.com/windmill-labs/windmill/issues/9202)) ([ab11c77](https://github.com/windmill-labs/windmill/commit/ab11c7747a9076e8121fcea6eafb8e88079ac987))
+* enforce jobs:run scope on job preview and inline endpoints ([#9198](https://github.com/windmill-labs/windmill/issues/9198)) ([664edcd](https://github.com/windmill-labs/windmill/commit/664edcdfb746f6c8513e2b487383b5d9ab9f5434))
+* **mcp:** validate oauth dynamic client registration redirect_uris ([#9197](https://github.com/windmill-labs/windmill/issues/9197)) ([8bc2295](https://github.com/windmill-labs/windmill/commit/8bc2295b94df159a7c8630cdbe02953b8b7c13a1))
+* validate entrypoint override to prevent worker code injection (GHSA-wxjq-w5pj-jqhx) ([#9204](https://github.com/windmill-labs/windmill/issues/9204)) ([bd05bca](https://github.com/windmill-labs/windmill/commit/bd05bcadde06b65fc4b732f576d89aae908b5a3f))
+
+## [1.703.2](https://github.com/windmill-labs/windmill/compare/v1.703.1...v1.703.2) (2026-05-17)
+
+
+### Bug Fixes
+
+* prevent cross-tenant DNS poisoning via writable /etc in nsjail ([#9194](https://github.com/windmill-labs/windmill/issues/9194)) ([f8467f3](https://github.com/windmill-labs/windmill/commit/f8467f38c8a053117ce62f96684cfb15ef792f08))
+
+## [1.703.1](https://github.com/windmill-labs/windmill/compare/v1.703.0...v1.703.1) (2026-05-16)
+
+
+### Bug Fixes
+
+* actionable error when a custom_path is taken by an app in another workspace ([#9190](https://github.com/windmill-labs/windmill/issues/9190)) ([dfeed9c](https://github.com/windmill-labs/windmill/commit/dfeed9c5c2e39bf3e10eea4f69ea140ee9e7832f))
+* atomic bundle cache writes to prevent parallel cold-load race ([#9186](https://github.com/windmill-labs/windmill/issues/9186)) ([81b5736](https://github.com/windmill-labs/windmill/commit/81b573610692b386e4861ef989fa7698b53fc861))
+* detect S3 assets passed as SDK object arg in ts parser ([#9181](https://github.com/windmill-labs/windmill/issues/9181)) ([6a334e9](https://github.com/windmill-labs/windmill/commit/6a334e9a07a7d0cffabde48be75263b0844d586c))
+* don't show ALLOW_PRIVATE_AI_BASE_URLS hint for malformed AI base URLs ([#9188](https://github.com/windmill-labs/windmill/issues/9188)) ([4e25954](https://github.com/windmill-labs/windmill/commit/4e259547225e13e5b51a166a84cdbbbfa35c3264))
+* reset parent_hash in auto_parent when all versions at path are archived ([#9172](https://github.com/windmill-labs/windmill/issues/9172)) ([52960ca](https://github.com/windmill-labs/windmill/commit/52960ca30ab9c019186a28b3ab054a1dfe72f451))
+
+## [1.703.0](https://github.com/windmill-labs/windmill/compare/v1.702.1...v1.703.0) (2026-05-15)
+
+
+### Features
+
+* **otel-tracing-proxy:** configurable tracing MITM NO_PROXY hosts ([#9169](https://github.com/windmill-labs/windmill/issues/9169)) ([d48d61c](https://github.com/windmill-labs/windmill/commit/d48d61cc79114f0b36736306d4015789be10c1f4))
+
+
+### Bug Fixes
+
+* aggregate wait time should target the true root job, not flow_innermost_root_job ([#9177](https://github.com/windmill-labs/windmill/issues/9177)) ([e181931](https://github.com/windmill-labs/windmill/commit/e1819313e15766007c959497a84fae5f5c78a46b))
+* apply pip_local_dependencies filtering to deployed scripts with populated lockfiles ([#9178](https://github.com/windmill-labs/windmill/issues/9178)) ([69b3141](https://github.com/windmill-labs/windmill/commit/69b3141e0370b95f2e13987503480d341608dbdf))
+* never mark failure/trigger/approval scripts as auto_kind=lib ([#9168](https://github.com/windmill-labs/windmill/issues/9168)) ([f414ffc](https://github.com/windmill-labs/windmill/commit/f414ffc4849cf4b92fcd5ca9611ecd246e59a7bd))
+
+## [1.702.1](https://github.com/windmill-labs/windmill/compare/v1.702.0...v1.702.1) (2026-05-14)
+
+
+### Bug Fixes
+
+* **nativets:** pass tracing-enabled OtelConfig to deno_telemetry::init ([#9163](https://github.com/windmill-labs/windmill/issues/9163)) ([bf99283](https://github.com/windmill-labs/windmill/commit/bf99283c3333bcdbc7679f4aea04ba29e41a48a5))
+
+## [1.702.0](https://github.com/windmill-labs/windmill/compare/v1.701.0...v1.702.0) (2026-05-14)
+
+
+### Features
+
+* **git-sync:** sync extra_perms for flows/scripts/apps ([#9162](https://github.com/windmill-labs/windmill/issues/9162)) ([5e909b2](https://github.com/windmill-labs/windmill/commit/5e909b2b4f2819f19deaf06d9e78e6458b324683))
+* include service accounts in instance settings users list ([#9157](https://github.com/windmill-labs/windmill/issues/9157)) ([e5286f4](https://github.com/windmill-labs/windmill/commit/e5286f46074cf2893e6ccd26175f929f16011c8f))
+
+
+### Bug Fixes
+
+* **mcp:** sanitize and enrich nested resource schemas ([#9158](https://github.com/windmill-labs/windmill/issues/9158)) ([d870edc](https://github.com/windmill-labs/windmill/commit/d870edc959481a06c894b4eda5e2be1a0269d7d0))
+
+## [1.701.0](https://github.com/windmill-labs/windmill/compare/v1.700.2...v1.701.0) (2026-05-13)
+
+
+### Features
+
+* **frontend:** unified EditorHeader with file picker for flow/script/app editors ([#9047](https://github.com/windmill-labs/windmill/issues/9047)) ([d0f23cc](https://github.com/windmill-labs/windmill/commit/d0f23cc5238b025208c61e983701894de28536d5))
+* read-only flag on API tokens ([#9144](https://github.com/windmill-labs/windmill/issues/9144)) ([d666e84](https://github.com/windmill-labs/windmill/commit/d666e8431cdbf14d9373d9ef625b5aafc50ac50a))
+
+
+### Bug Fixes
+
+* align script path existence check with deploy logic; hide Delete for non-admin ([#9152](https://github.com/windmill-labs/windmill/issues/9152)) ([c509206](https://github.com/windmill-labs/windmill/commit/c5092069cbeda2c4c18bea80dd629c7c087b30bf))
+* Allow devops role to use all_workspaces runs filter in admins workspace ([#9153](https://github.com/windmill-labs/windmill/issues/9153)) ([110bef0](https://github.com/windmill-labs/windmill/commit/110bef0a6e76615c7b371c5c0f5bc1f4e7a73a64))
+* **bun:** pass --preserve-symlinks on unbundled execution ([#9147](https://github.com/windmill-labs/windmill/issues/9147)) ([4d0f2c2](https://github.com/windmill-labs/windmill/commit/4d0f2c26a116a0f8a89a64231dc824eabda0a8c3))
+* **cli:** prevent !inline-corruption in flow push/pull ([#9142](https://github.com/windmill-labs/windmill/issues/9142)) ([79c5b7b](https://github.com/windmill-labs/windmill/commit/79c5b7b8b7676b0a06fa6480dd04b7105d39d250))
+* **operator:** refresh IAM RDS / Entra ID tokens in operator process ([#9141](https://github.com/windmill-labs/windmill/issues/9141)) ([7ebb081](https://github.com/windmill-labs/windmill/commit/7ebb08133cd4027bc00bacc4a0fc5865cd5709ec))
+* **python:** preserve strings containing Infinity/NaN in result JSON ([#9149](https://github.com/windmill-labs/windmill/issues/9149)) ([33bf01b](https://github.com/windmill-labs/windmill/commit/33bf01b627c8ea430c03dfc27a97a8f2d770582f))
+* scope promotion-mode debounce key per repo ([#9145](https://github.com/windmill-labs/windmill/issues/9145)) ([2ec1863](https://github.com/windmill-labs/windmill/commit/2ec1863340e759bba3408dbc4f41b16912b959ea))
+* send flow push-loop ping outside transaction so zombie monitor sees it ([#9136](https://github.com/windmill-labs/windmill/issues/9136)) ([818cb31](https://github.com/windmill-labs/windmill/commit/818cb31fbc731fa5c70ddf5942bb37bc4bc56e4d))
+
+
+### Performance Improvements
+
+* **dynselect:** only retrigger when helper args actually change ([#9148](https://github.com/windmill-labs/windmill/issues/9148)) ([dd19e52](https://github.com/windmill-labs/windmill/commit/dd19e52a84fb9a9f48e3ad061b084841c2ee7464))
+
 ## [1.700.2](https://github.com/windmill-labs/windmill/compare/v1.700.1...v1.700.2) (2026-05-12)
 
 

Dockerfile

@@ -66,6 +66,7 @@ RUN npm ci
 COPY frontend .
 RUN mkdir /backend
 COPY /backend/windmill-api/openapi.yaml /backend/windmill-api/openapi.yaml
+COPY /backend/oauth_connect.json /backend/oauth_connect.json
 COPY /openflow.openapi.yaml /openflow.openapi.yaml
 COPY /backend/windmill-api/build_openapi.sh /backend/windmill-api/build_openapi.sh
 COPY /system_prompts/auto-generated /system_prompts/auto-generated
@@ -232,11 +233,14 @@ ENV PATH="${PATH}:/usr/local/go/bin"
 ENV GO_PATH=/usr/local/go/bin/go
 
 # Install UV
-RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.9.24/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv
+RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.9.25/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv
 
 # Preinstall python runtimes to temp build location (will copy with world-writable perms later)
-RUN UV_CACHE_DIR=/tmp/build_cache/uv UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install 3.11
-RUN UV_CACHE_DIR=/tmp/build_cache/uv UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install $LATEST_STABLE_PY
+# --compile-bytecode precompiles the stdlib to .pyc so jobs don't recompile it on every run
+# under the read-only nsjail runtime mount (uv >= 0.9.25). The copy below MUST preserve
+# timestamps or Python's mtime-based .pyc invalidation discards these compiled files.
+RUN UV_CACHE_DIR=/tmp/build_cache/uv UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install 3.11 --compile-bytecode
+RUN UV_CACHE_DIR=/tmp/build_cache/uv UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install $LATEST_STABLE_PY --compile-bytecode
 
 
 RUN curl -sL https://deb.nodesource.com/setup_20.x | bash -
@@ -258,7 +262,7 @@ RUN export GOCACHE=/tmp/build_cache/go && \
 # chmod a+rw adds read+write WITHOUT removing execute bits (755->777, 644->666)
 # Note: uv python install only creates py_runtime, not uv cache - we create uv/go dirs for runtime
 RUN mkdir -p /tmp/windmill/cache && \
-    cp -r /tmp/build_cache/* /tmp/windmill/cache/ && \
+    cp -r --preserve=timestamps /tmp/build_cache/* /tmp/windmill/cache/ && \
     chmod -R a+rw /tmp/windmill/cache && \
     rm -rf /tmp/build_cache && \
     mkdir -p -m 777 /tmp/windmill/cache/uv /tmp/windmill/cache/go /tmp/windmill/cache/rustup /tmp/windmill/cache/cargo
@@ -299,10 +303,20 @@ ENV CARGO_HOME="/tmp/windmill/cache/cargo"
 ENV LD_LIBRARY_PATH="."
 
 # nsjail runtime deps and binary
-RUN apt-get update && apt-get install -y libprotobuf-dev libnl-route-3-dev \
+RUN apt-get update && apt-get install -y --no-install-recommends libprotobuf32 libnl-route-3-200 libnl-3-200 \
     && apt-get clean && rm -rf /var/lib/apt/lists/*
 COPY --from=nsjail /nsjail/nsjail /bin/nsjail
 
+# crane: pulls + flattens images for the sandboxed container runtime (`# sandbox <image>`).
+# Single static binary — no daemon/store/root needed. See docs/docker-v2-runtime.md.
+ARG CRANE_VERSION=v0.20.6
+RUN arch="$(dpkg --print-architecture)"; \
+    case "$arch" in amd64) crane_arch=x86_64 ;; arm64) crane_arch=arm64 ;; *) echo >&2 "error: unsupported arch '$arch' for crane"; exit 1 ;; esac; \
+    wget -O /tmp/crane.tgz "https://github.com/google/go-containerregistry/releases/download/${CRANE_VERSION}/go-containerregistry_Linux_${crane_arch}.tar.gz" \
+    && tar -xzf /tmp/crane.tgz -C /usr/local/bin crane \
+    && rm /tmp/crane.tgz \
+    && chmod +x /usr/local/bin/crane
+
 WORKDIR ${APP}
 
 RUN ln -s ${APP}/windmill /usr/local/bin/windmill

ai_evals/AGENTS.md

@@ -86,6 +86,31 @@ Global prompts should exercise workspace-level drafting behavior:
 
 Keep deterministic validation focused on the draft contract: required draft type/path, required content snippets, forbidden draft paths, and forbidden mutating tools such as deploy/delete unless the case explicitly asks for them.
 
+Datatable cases should set `skipJudge: true` and validate through tool-use
+(`requiredToolsUsed` / `forbiddenToolsUsed`) and SQL-argument assertions
+(`toolCallArgs` with `stringIncludesAnyOf`, e.g. `['select']`, `['create table']`,
+`['update', 'insert into']`). Two reasons the judge is unreliable here:
+
+- `list_datatables`, `get_datatable_table_schema`, and `exec_datatable_sql`
+  produce no drafts, and the global judge only sees the drafts artifact — it
+  scores a no-draft conversational answer as empty (same as the
+  `askUserQuestion` cases).
+- Even a case that *does* produce a draft (a script reading the data table via
+  `wmill.datatable()` at runtime) is mis-judged: the judge has no datatable SDK
+  reference and penalizes correct `wmill.datatable()` usage as wrong. Verify the
+  SDK call deterministically instead — `requiredDrafts.valueIncludes: ['wmill.datatable(']`
+  plus forbidding `exec_datatable_sql` (keeping chat-time SQL distinct from
+  runtime SDK use).
+
+`stringIncludesAnyOf` is existential over calls (at least one matching call), so a
+mutation case still passes when the model mixes its UPDATE/INSERT with
+verification SELECTs. The in-memory engine (`datatableSqlEngine.ts`) is stateful
+within a case — writes persist, so a model that re-queries to verify its
+CREATE/UPDATE sees the change and does not loop. But the engine is best-effort
+(SELECT returns all rows of the referenced/first table with no WHERE/projection),
+so still never assert specific returned row values. Seed data via
+`workspace.datatables` in the `initial` fixture (see README).
+
 ## Deterministic validation
 
 Use deterministic validation only for hard failures such as:

ai_evals/README.md

@@ -56,7 +56,7 @@ bun run cli -- run flow flow-test4-order-processing-loop --model opus
 bun run cli -- run flow flow-test0-sum-two-numbers --models haiku,opus,4o
 bun run cli -- run flow flow-test0-sum-two-numbers --runs 3 --verbose
 bun run cli -- run flow --record
-GEMINI_API_KEY=... bun run cli -- run app app-test1-counter-create --model gemini-pro
+GEMINI_API_KEY=... bun run cli -- run app app-test1-counter-create --model gemini-3-flash-preview
 WMILL_AI_EVAL_BACKEND_URL=http://127.0.0.1:8000 bun run cli -- run flow --backend-validation preview
 bun run cli -- run global global-test1-script-create
 bun run cli -- run cli bun-hello-script
@@ -88,15 +88,16 @@ Today:
 - `sonnet`
 - `opus`
 - `4o`
-- `gemini-flash`
-- `gemini-pro`
+- `gpt-5.5`
 - `gemini-3-flash-preview`
 - `gemini-3.1-pro-preview`
+- `deepseek-v4-flash`
+- `deepseek-v4-pro`
 
 Notes:
 
-- the command also prints accepted alias spellings such as `gpt-4o`, `claude-opus-4.6`, and `claude-haiku-4.5`
-- frontend modes (`flow`, `script`, `app`, `global`) can use Anthropic, OpenAI, and Gemini-backed aliases
+- the command also prints accepted alias spellings such as `gpt-4o`, `gpt-55`, `claude-opus-4.6`, and `claude-haiku-4.5`
+- frontend modes (`flow`, `script`, `app`, `global`) can use Anthropic, OpenAI, Gemini, and DeepSeek-backed aliases
 - `cli` mode always uses the Anthropic agent SDK, so only Anthropic aliases are valid there
 - the judge model is separate and currently defaults to `claude-sonnet-4-6`
 
@@ -142,6 +143,32 @@ For `global` mode, `validate` can express draft-level requirements such as:
 - required or forbidden draft counts
 - forbidden draft paths
 
+Global initial fixtures can also seed `liveEditorDrafts` with `type`,
+`storagePath`, `effectivePath`, and `value` fields. These drafts emulate the
+currently open script, flow, or raw app editor so cases can test prompts that
+refer to "this" or the "current" item.
+
+Global (and flow) initial fixtures can seed `workspace.datatables` so the
+`list_datatables`, `get_datatable_table_schema`, and `exec_datatable_sql` tools
+return seeded data during evals. Each entry is
+`{ datatable_name, schemas: { <schema>: { <table>: { columns, rows? } } } }`.
+SQL runs through a small in-memory engine (`datatableSqlEngine.ts`), not a real
+database. Writes are **stateful within a case**: `CREATE`/`DROP`/`INSERT`/`UPDATE`/
+`DELETE` mutate the seeded datatable in place, so a later `list_datatables`,
+`get_datatable_table_schema`, `SELECT`, or `information_schema` query reflects them
+— this is what stops a model from looping when it re-queries to verify a write.
+The engine is best-effort: `SELECT` returns all rows of the referenced (or first)
+table with no WHERE filtering/projection/joins, `WHERE` on UPDATE/DELETE supports
+`col = value` predicates joined by `AND`, and anything unparseable is a no-op
+success. So validate datatable cases through tool-use and SQL-argument assertions
+(`requiredToolsUsed`, `stringIncludesAnyOf`) — not through exact returned row
+values. An empty/absent `datatables` seed makes `list_datatables` return `[]`,
+which is what the "no datatable configured" blocking cases rely on.
+
+Set `WMILL_AI_EVAL_DISABLE_ACTIVE_EDITOR_CONTEXT=1` to run those cases with
+the old behavior where the live editor is only discoverable through
+`list_workspace_items`.
+
 App fixtures can also include an optional `datatables.json` file at the fixture root.
 
 For `flow` mode, an `initial` fixture can also include a benchmark workspace catalog of
@@ -189,11 +216,15 @@ If `--record` is used, the CLI also appends one compact JSON line to:
 Each recorded line contains:
 
 - run metadata (`createdAt`, `gitSha`, `mode`, `runModel`, `judgeModel`)
-- suite totals (`caseCount`, `attemptCount`, `passedAttempts`, `passRate`, `averageDurationMs`, `averageJudgeScore`)
-- average token usage (`averageTokenUsagePerAttempt`)
-- per-case metrics under `cases[]` (`averageDurationMs`, `averageJudgeScore`, `averageTokenUsagePerAttempt`, pass rate)
+- suite totals (`caseCount`, `attemptCount`, `passedAttempts`, `passRate`, `averageDurationMs`, `averagePassedDurationMs`, `averageJudgeScore`)
+- average token usage (`averageTokenUsagePerAttempt`, `averageTokenUsagePerPassedAttempt`)
+- per-case metrics under `cases[]` (`averageDurationMs`, `averagePassedDurationMs`, `averageJudgeScore`, `averageTokenUsagePerAttempt`, `averageTokenUsagePerPassedAttempt`, pass rate)
 - `failedCaseIds`
 
+The CLI headline duration and token averages use passed attempts only.
+All-attempt averages are still recorded to make failures auditable without
+letting failed attempts skew success cost comparisons.
+
 Example:
 
 - summary: `ai_evals/results/2026-04-09T09-40-33.051Z__flow.json`

ai_evals/adapters/frontend/benchmarkRunner.ts

@@ -10,10 +10,6 @@ import { runSuite } from "../../core/runSuite";
 import type { BenchmarkRunResult, ModeRunner } from "../../core/types";
 import { resolveWindmillBackendSettings } from "../../core/windmillBackendSettings";
 import { emitFrontendBenchmarkProgress } from "./progress";
-import { createAppModeRunner } from "../../modes/app";
-import { createFlowModeRunner } from "../../modes/flow";
-import { createGlobalModeRunner } from "../../modes/global";
-import { createScriptModeRunner } from "../../modes/script";
 import { DEFAULT_JUDGE_MODEL } from "../../core/judge";
 
 export type FrontendBenchmarkMode = "flow" | "app" | "script" | "global";
@@ -40,7 +36,7 @@ export async function runFrontendBenchmarkFromEnv(): Promise<BenchmarkRunResult>
   const backendSettings = resolveWindmillBackendSettings();
 
   const selectedCases = await loadSelectedCases(mode, caseIds);
-  const modeRunner = getModeRunner(
+  const modeRunner = await getModeRunner(
     mode,
     getFrontendEvalModel(model),
     backendValidation,
@@ -69,25 +65,33 @@ export async function runFrontendBenchmarkFromEnv(): Promise<BenchmarkRunResult>
   });
 }
 
-function getModeRunner(
+async function getModeRunner(
   mode: FrontendBenchmarkMode,
   model: ReturnType<typeof getFrontendEvalModel>,
   backendValidation: ReturnType<typeof resolveBackendValidationSettings>,
   backendSettings: ReturnType<typeof resolveWindmillBackendSettings>,
-): ModeRunner<any, any, any> {
+): Promise<ModeRunner<any, any, any>> {
   switch (mode) {
-    case "flow":
+    case "flow": {
+      const { createFlowModeRunner } = await import("../../modes/flow");
       return createFlowModeRunner(model, backendValidation, backendSettings);
-    case "app":
+    }
+    case "app": {
+      const { createAppModeRunner } = await import("../../modes/app");
       return createAppModeRunner(model, backendSettings);
-    case "script":
+    }
+    case "script": {
+      const { createScriptModeRunner } = await import("../../modes/script");
       return createScriptModeRunner(
         model,
         backendValidation,
         backendSettings,
       );
-    case "global":
+    }
+    case "global": {
+      const { createGlobalModeRunner } = await import("../../modes/global");
       return createGlobalModeRunner(model, backendSettings);
+    }
   }
 }
 

ai_evals/adapters/frontend/core/global/globalEvalRunner.ts

@@ -7,8 +7,12 @@ import {
   prepareGlobalSystemMessage,
   prepareGlobalUserMessage,
 } from "../../../../../frontend/src/lib/components/copilot/chat/global/core";
-import { globalDraftStore } from "../../../../../frontend/src/lib/components/copilot/chat/global/draftStore.svelte";
+import {
+  clearGlobalDrafts,
+  listGlobalDrafts,
+} from "../../../../../frontend/src/lib/components/copilot/chat/global/userDraftAdapter";
 import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared";
+import { UserDraft } from "../../../../../frontend/src/lib/userDraft.svelte";
 import type { ModeRunContext } from "../../../../core/types";
 import type { GlobalDraftState } from "../../../../core/validators";
 import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings";
@@ -24,6 +28,21 @@ const MUTATING_GLOBAL_TOOLS = new Set([
   "deploy_workspace_item",
   "delete_workspace_item",
 ]);
+const DISABLE_ACTIVE_EDITOR_CONTEXT_ENV =
+  "WMILL_AI_EVAL_DISABLE_ACTIVE_EDITOR_CONTEXT";
+
+const LIVE_EDITOR_ITEM_KINDS = {
+  script: "script",
+  flow: "flow",
+  app: "raw_app",
+} as const;
+
+export interface GlobalLiveEditorDraftFixture {
+  type: keyof typeof LIVE_EDITOR_ITEM_KINDS;
+  storagePath?: string;
+  effectivePath?: string;
+  value?: unknown;
+}
 
 export interface GlobalEvalResult {
   success: boolean;
@@ -38,6 +57,7 @@ export interface GlobalEvalResult {
 
 export interface GlobalEvalOptions {
   workspaceFixtures?: BenchmarkWorkspaceRunnables;
+  liveEditorDrafts?: GlobalLiveEditorDraftFixture[];
   model?: string;
   maxIterations?: number;
   provider?: AIProvider;
@@ -55,19 +75,26 @@ export async function runGlobalEval(
     options.workspaceRoot ??
     (await mkdtemp(join(tmpdir(), "wmill-frontend-global-benchmark-")));
 
-  globalDraftStore.clearDrafts(workspaceRoot);
+  clearGlobalDrafts(workspaceRoot);
   registerBenchmarkWorkspaceRunnables(workspaceRoot, options.workspaceFixtures ?? {});
+  seedLiveEditorDrafts(workspaceRoot, options.liveEditorDrafts ?? []);
 
   try {
     const model = options.model ?? "claude-haiku-4-5-20251001";
+    const injectActiveEditorContext =
+      process.env[DISABLE_ACTIVE_EDITOR_CONTEXT_ENV] !== "1";
     const rawResult = await runEval({
       userPrompt,
       systemMessage: prepareGlobalSystemMessage(),
-      userMessage: prepareGlobalUserMessage(userPrompt),
+      userMessage: prepareGlobalUserMessage(
+        userPrompt,
+        [],
+        injectActiveEditorContext ? { workspace: workspaceRoot } : {},
+      ),
       tools: getGlobalEvalTools(),
       helpers: {},
       apiKey,
-      getOutput: () => ({ drafts: globalDraftStore.listDrafts(workspaceRoot) }),
+      getOutput: () => ({ drafts: listGlobalDrafts(workspaceRoot) }),
       onAssistantMessageStart: options.runContext?.onAssistantMessageStart,
       onAssistantToken: options.runContext?.onAssistantChunk,
       onAssistantMessageEnd: options.runContext?.onAssistantMessageEnd,
@@ -94,7 +121,8 @@ export async function runGlobalEval(
       tokenUsage: rawResult.tokenUsage,
     };
   } finally {
-    globalDraftStore.clearDrafts(workspaceRoot);
+    clearGlobalDrafts(workspaceRoot);
+    clearLiveEditorDrafts(workspaceRoot, options.liveEditorDrafts ?? []);
     unregisterBenchmarkWorkspaceRunnables(workspaceRoot);
     if (!options.workspaceRoot) {
       await rm(workspaceRoot, { recursive: true, force: true });
@@ -102,6 +130,36 @@ export async function runGlobalEval(
   }
 }
 
+function seedLiveEditorDrafts(
+  workspace: string,
+  fixtures: GlobalLiveEditorDraftFixture[],
+): void {
+  for (const fixture of fixtures) {
+    const itemKind = LIVE_EDITOR_ITEM_KINDS[fixture.type];
+    const storagePath = fixture.storagePath ?? fixture.effectivePath ?? "";
+    if (fixture.value !== undefined) {
+      UserDraft.save(itemKind, storagePath, fixture.value, { workspace });
+    }
+    UserDraft.setLiveEditorDraft({
+      workspace,
+      itemKind,
+      storagePath,
+      effectivePath: fixture.effectivePath ?? fixture.storagePath,
+    });
+  }
+}
+
+function clearLiveEditorDrafts(
+  workspace: string,
+  fixtures: GlobalLiveEditorDraftFixture[],
+): void {
+  for (const fixture of fixtures) {
+    const itemKind = LIVE_EDITOR_ITEM_KINDS[fixture.type];
+    const storagePath = fixture.storagePath ?? fixture.effectivePath ?? "";
+    UserDraft.clearLiveEditorDraft(itemKind, { workspace, storagePath });
+  }
+}
+
 function getGlobalEvalTools(): ProductionTool<{}>[] {
   return (globalTools as ProductionTool<{}>[]).map((tool) => {
     if (!MUTATING_GLOBAL_TOOLS.has(tool.def.function.name)) {

ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts

@@ -236,7 +236,8 @@ function toFrontendEvalProvider(
   if (
     provider === "anthropic" ||
     provider === "openai" ||
-    provider === "googleai"
+    provider === "googleai" ||
+    provider === "deepseek"
   ) {
     return provider;
   }

ai_evals/adapters/frontend/core/shared/providerConfig.test.ts

@@ -21,16 +21,25 @@ describe("proxy helpers", () => {
 
 describe("resolveEvalModelProvider", () => {
   it("infers googleai from Gemini model ids", () => {
-    expect(resolveEvalModelProvider("gemini-2.5-flash")).toEqual({
+    expect(resolveEvalModelProvider("gemini-3-flash-preview")).toEqual({
       provider: "googleai",
-      model: "gemini-2.5-flash",
+      model: "gemini-3-flash-preview",
+    });
+  });
+
+  it("infers deepseek from DeepSeek model ids", () => {
+    expect(resolveEvalModelProvider("deepseek-v4-flash")).toEqual({
+      provider: "deepseek",
+      model: "deepseek-v4-flash",
     });
   });
 
   it("preserves an explicit provider", () => {
-    expect(resolveEvalModelProvider("gemini-2.5-pro", "googleai")).toEqual({
+    expect(
+      resolveEvalModelProvider("gemini-3.1-pro-preview", "googleai"),
+    ).toEqual({
       provider: "googleai",
-      model: "gemini-2.5-pro",
+      model: "gemini-3.1-pro-preview",
     });
   });
 });

ai_evals/adapters/frontend/core/shared/providerConfig.ts

@@ -83,6 +83,9 @@ export function resolveEvalModelProvider(
   if (model.startsWith("gemini")) {
     return { provider: "googleai", model };
   }
+  if (model.startsWith("deepseek")) {
+    return { provider: "deepseek", model };
+  }
   if (model.startsWith("gpt") || model.startsWith("o")) {
     return { provider: "openai", model };
   }

ai_evals/adapters/frontend/datatableSqlEngine.test.ts

@@ -0,0 +1,262 @@
+import { describe, expect, it } from 'bun:test'
+import { applyDatatableSql, type BenchmarkDatatableSeed } from './datatableSqlEngine'
+
+function makeDatatable(): BenchmarkDatatableSeed {
+	return {
+		datatable_name: 'main',
+		schemas: {
+			public: {
+				orders: {
+					columns: { id: 'int4', customer_id: 'int4', total: 'numeric', status: 'text' },
+					rows: [
+						{ id: 1, customer_id: 1, total: 42.5, status: 'shipped' },
+						{ id: 2, customer_id: 2, total: 19.99, status: 'pending' },
+						{ id: 3, customer_id: 1, total: 88, status: 'shipped' }
+					]
+				},
+				customers: {
+					columns: { id: 'int4', name: 'text' },
+					rows: [{ id: 1, name: 'Alice' }]
+				}
+			}
+		}
+	}
+}
+
+describe('SELECT', () => {
+	it('returns the referenced table rows', () => {
+		const dt = makeDatatable()
+		expect(applyDatatableSql(dt, 'SELECT id, name FROM customers').rows).toEqual([
+			{ id: 1, name: 'Alice' }
+		])
+	})
+
+	it('falls back to the first table when no known table is referenced', () => {
+		const dt = makeDatatable()
+		expect(applyDatatableSql(dt, 'select 1').rows).toHaveLength(3)
+	})
+
+	it('resolves a schema-qualified table', () => {
+		const dt = makeDatatable()
+		expect(applyDatatableSql(dt, 'SELECT * FROM public.customers').rows).toEqual([
+			{ id: 1, name: 'Alice' }
+		])
+	})
+})
+
+describe('CREATE TABLE', () => {
+	it('adds a table with parsed columns, skipping table constraints and FK clauses', () => {
+		const dt = makeDatatable()
+		const result = applyDatatableSql(
+			dt,
+			'CREATE TABLE public.refunds (\n  order_id int4 NOT NULL REFERENCES public.orders(id),\n  amount numeric(10,2),\n  PRIMARY KEY (order_id)\n)'
+		)
+		expect(result.rows).toEqual([])
+		expect(dt.schemas.public.refunds).toEqual({
+			columns: { order_id: 'int4', amount: 'numeric(10,2)' },
+			rows: []
+		})
+	})
+
+	it('defaults an unqualified table to the public schema', () => {
+		const dt = makeDatatable()
+		applyDatatableSql(dt, 'CREATE TABLE notes (id int4, body text)')
+		expect(dt.schemas.public.notes.columns).toEqual({ id: 'int4', body: 'text' })
+	})
+
+	it('is a no-op for an existing table with IF NOT EXISTS', () => {
+		const dt = makeDatatable()
+		applyDatatableSql(dt, 'CREATE TABLE IF NOT EXISTS public.orders (x int4)')
+		expect(Object.keys(dt.schemas.public.orders.columns)).toContain('status')
+	})
+})
+
+describe('DROP TABLE', () => {
+	it('removes the table', () => {
+		const dt = makeDatatable()
+		applyDatatableSql(dt, 'DROP TABLE IF EXISTS public.customers')
+		expect(dt.schemas.public.customers).toBeUndefined()
+	})
+})
+
+describe('INSERT', () => {
+	it('appends a row using an explicit column list', () => {
+		const dt = makeDatatable()
+		applyDatatableSql(dt, "INSERT INTO customers (id, name) VALUES (2, 'Bob')")
+		expect(dt.schemas.public.customers.rows).toContainEqual({ id: 2, name: 'Bob' })
+	})
+
+	it('infers columns from the table when none are given, and appends multiple tuples', () => {
+		const dt = makeDatatable()
+		applyDatatableSql(dt, "INSERT INTO customers VALUES (2, 'Bob'), (3, 'Carol')")
+		expect(dt.schemas.public.customers.rows).toHaveLength(3)
+	})
+
+	it('returns the inserted rows when RETURNING is present', () => {
+		const dt = makeDatatable()
+		const result = applyDatatableSql(
+			dt,
+			"INSERT INTO customers (id, name) VALUES (2, 'Bob') RETURNING *"
+		)
+		expect(result.rows).toEqual([{ id: 2, name: 'Bob' }])
+	})
+})
+
+describe('UPDATE', () => {
+	it('updates only the rows matching an equality WHERE', () => {
+		const dt = makeDatatable()
+		const result = applyDatatableSql(
+			dt,
+			"UPDATE public.orders SET status = 'shipped' WHERE id = 2"
+		)
+		expect(result.rows).toEqual([])
+		expect(dt.schemas.public.orders.rows?.find((r) => r.id === 2)?.status).toBe('shipped')
+		expect(dt.schemas.public.orders.rows?.find((r) => r.id === 1)?.status).toBe('shipped')
+	})
+
+	it('strips a Postgres cast in the WHERE value', () => {
+		const dt = makeDatatable()
+		applyDatatableSql(dt, "UPDATE orders SET status = 'done' WHERE id = 2::int4")
+		expect(dt.schemas.public.orders.rows?.find((r) => r.id === 2)?.status).toBe('done')
+	})
+
+	it('matches multiple AND predicates including a numeric literal', () => {
+		const dt = makeDatatable()
+		applyDatatableSql(
+			dt,
+			"UPDATE orders SET status = 'done' WHERE customer_id = 2 AND total = 19.99"
+		)
+		expect(dt.schemas.public.orders.rows?.find((r) => r.id === 2)?.status).toBe('done')
+		expect(dt.schemas.public.orders.rows?.find((r) => r.id === 1)?.status).toBe('shipped')
+	})
+
+	it('updates every row when there is no WHERE', () => {
+		const dt = makeDatatable()
+		applyDatatableSql(dt, "UPDATE orders SET status = 'archived'")
+		expect(dt.schemas.public.orders.rows?.every((r) => r.status === 'archived')).toBe(true)
+	})
+
+	it('returns the affected rows when RETURNING is present', () => {
+		const dt = makeDatatable()
+		const result = applyDatatableSql(
+			dt,
+			"UPDATE orders SET status = 'shipped' WHERE id = 2 RETURNING *"
+		)
+		expect(result.rows).toHaveLength(1)
+		expect(result.rows[0]).toMatchObject({ id: 2, status: 'shipped' })
+	})
+
+	it('affects no rows when the WHERE clause cannot be parsed', () => {
+		const dt = makeDatatable()
+		applyDatatableSql(dt, "UPDATE orders SET status = 'x' WHERE total > 20")
+		expect(dt.schemas.public.orders.rows?.some((r) => r.status === 'x')).toBe(false)
+	})
+})
+
+describe('DELETE', () => {
+	it('removes only the matching rows', () => {
+		const dt = makeDatatable()
+		applyDatatableSql(dt, 'DELETE FROM orders WHERE id = 2')
+		expect(dt.schemas.public.orders.rows?.map((r) => r.id)).toEqual([1, 3])
+	})
+
+	it('returns the removed rows when RETURNING is present', () => {
+		const dt = makeDatatable()
+		const result = applyDatatableSql(dt, 'DELETE FROM orders WHERE id = 2 RETURNING *')
+		expect(result.rows).toEqual([{ id: 2, customer_id: 2, total: 19.99, status: 'pending' }])
+	})
+})
+
+describe('writes are reflected by later reads', () => {
+	it('UPDATE then SELECT sees the new value (the verify-loop fix)', () => {
+		const dt = makeDatatable()
+		applyDatatableSql(dt, "UPDATE orders SET status = 'shipped' WHERE id = 2")
+		const seen = applyDatatableSql(dt, 'SELECT * FROM orders').rows
+		expect(seen.find((r) => r.id === 2)?.status).toBe('shipped')
+	})
+
+	it('INSERT then SELECT sees the new row', () => {
+		const dt = makeDatatable()
+		applyDatatableSql(dt, "INSERT INTO customers (id, name) VALUES (9, 'Zed')")
+		const seen = applyDatatableSql(dt, 'SELECT * FROM customers').rows
+		expect(seen).toContainEqual({ id: 9, name: 'Zed' })
+	})
+
+	it('CREATE then SELECT on the new table returns its (empty) rows', () => {
+		const dt = makeDatatable()
+		applyDatatableSql(dt, 'CREATE TABLE public.refunds (order_id int4, amount numeric)')
+		expect(applyDatatableSql(dt, 'SELECT * FROM refunds').rows).toEqual([])
+	})
+})
+
+describe('system-catalog queries reflect the current tables/columns', () => {
+	it('lists current tables (including a freshly created one) via information_schema.tables', () => {
+		const dt = makeDatatable()
+		applyDatatableSql(dt, 'CREATE TABLE public.refunds (order_id int4)')
+		const rows = applyDatatableSql(
+			dt,
+			"SELECT table_name FROM information_schema.tables WHERE table_name = 'refunds'"
+		).rows
+		expect(rows.map((r) => r.table_name)).toContain('refunds')
+	})
+
+	it('does not list a dropped table', () => {
+		const dt = makeDatatable()
+		applyDatatableSql(dt, 'DROP TABLE public.customers')
+		const rows = applyDatatableSql(dt, 'SELECT table_name FROM information_schema.tables').rows
+		expect(rows.map((r) => r.table_name)).not.toContain('customers')
+	})
+
+	it('reports columns via information_schema.columns', () => {
+		const dt = makeDatatable()
+		const rows = applyDatatableSql(
+			dt,
+			"SELECT column_name FROM information_schema.columns WHERE table_name = 'orders'"
+		).rows
+		expect(rows.map((r) => r.column_name)).toContain('status')
+	})
+})
+
+describe('parser robustness (string/paren-aware splitting)', () => {
+	it('does not treat the word "returning" inside a string value as a RETURNING clause', () => {
+		const dt = makeDatatable()
+		const result = applyDatatableSql(
+			dt,
+			"INSERT INTO customers (id, name) VALUES (5, 'is returning soon')"
+		)
+		expect(result.rows).toEqual([])
+		expect(dt.schemas.public.customers.rows).toContainEqual({ id: 5, name: 'is returning soon' })
+	})
+
+	it('does not split on the word "where" inside a SET string value', () => {
+		const dt = makeDatatable()
+		applyDatatableSql(dt, "UPDATE orders SET status = 'ship where ordered' WHERE id = 2")
+		expect(dt.schemas.public.orders.rows?.find((r) => r.id === 2)?.status).toBe('ship where ordered')
+		expect(dt.schemas.public.orders.rows?.find((r) => r.id === 1)?.status).toBe('shipped')
+	})
+
+	it('keeps INSERT tuples intact when a value contains a function call', () => {
+		const dt = makeDatatable()
+		applyDatatableSql(dt, "INSERT INTO customers (id, name) VALUES (6, coalesce(NULL, 'x'))")
+		expect(dt.schemas.public.customers.rows).toHaveLength(2)
+		expect(dt.schemas.public.customers.rows?.[1]).toMatchObject({ id: 6 })
+	})
+
+	it('CREATE TABLE ignores a trailing semicolon-separated statement', () => {
+		const dt = makeDatatable()
+		applyDatatableSql(
+			dt,
+			'CREATE TABLE public.refunds (id int4, amount numeric); INSERT INTO refunds VALUES (1, 5)'
+		)
+		expect(dt.schemas.public.refunds.columns).toEqual({ id: 'int4', amount: 'numeric' })
+		expect(dt.schemas.public.refunds.rows).toEqual([])
+	})
+})
+
+describe('unparseable statements are a safe no-op', () => {
+	it('returns [] and does not throw', () => {
+		const dt = makeDatatable()
+		expect(applyDatatableSql(dt, 'VACUUM ANALYZE').rows).toEqual([])
+		expect(applyDatatableSql(dt, 'GRANT SELECT ON orders TO someone').rows).toEqual([])
+	})
+})

ai_evals/adapters/frontend/datatableSqlEngine.ts

@@ -0,0 +1,541 @@
+/**
+ * A deliberately small, best-effort SQL engine for the benchmark datatable mock.
+ *
+ * This is NOT a real SQL implementation — it exists only so that writes a model
+ * issues during an eval (`CREATE TABLE`, `INSERT`, `UPDATE`, `DELETE`, `DROP`)
+ * become visible to its later reads (`list_datatables`, `get_datatable_table_schema`,
+ * `SELECT`). Without that, a model that re-queries to verify a write sees stale
+ * seed data, concludes the write failed, and loops until it exhausts its turns.
+ *
+ * It parses only the common statement shapes models produce. Anything it cannot
+ * parse is a no-op success (it never throws) — behavioral evals assert that the
+ * right statement was issued, not its exact data effects. Notable limits:
+ *  - `SELECT` returns all rows of the referenced (or first) table — no WHERE
+ *    filtering, projection, joins, or aggregation.
+ *  - `WHERE` supports `col = value` predicates joined by `AND` only; an
+ *    unparseable WHERE on UPDATE/DELETE affects zero rows (never the whole table).
+ */
+
+/** One seeded datatable table: its columns (col -> compact_type) and optional rows. */
+export interface BenchmarkDatatableTableSeed {
+	columns: Record<string, string>
+	rows?: Record<string, unknown>[]
+}
+
+/** A seeded datatable: `datatable_name` plus a `schema -> table -> seed` map. */
+export interface BenchmarkDatatableSeed {
+	datatable_name: string
+	schemas: {
+		[schema: string]: {
+			[table: string]: BenchmarkDatatableTableSeed
+		}
+	}
+}
+
+export interface DatatableSqlResult {
+	rows: Record<string, unknown>[]
+}
+
+const DEFAULT_SCHEMA = 'public'
+
+type ParsedRef = { schema: string; table: string }
+type Predicate = { column: string; value: unknown }
+
+/**
+ * Apply one SQL statement to `datatable` IN PLACE and return the result rows.
+ * SELECT returns the referenced/first table's rows; a mutation returns its
+ * affected rows when it has a RETURNING clause, otherwise `[]`.
+ */
+export function applyDatatableSql(
+	datatable: BenchmarkDatatableSeed,
+	sql: string
+): DatatableSqlResult {
+	const statement = stripTrailingSemicolon(sql.trim())
+	if (/^\s*(with|select)\b/i.test(statement)) {
+		return { rows: selectRows(datatable, statement) }
+	}
+	if (/^\s*create\s+table\b/i.test(statement)) {
+		return { rows: applyCreateTable(datatable, statement) }
+	}
+	if (/^\s*drop\s+table\b/i.test(statement)) {
+		return { rows: applyDropTable(datatable, statement) }
+	}
+	if (/^\s*insert\s+into\b/i.test(statement)) {
+		return { rows: applyInsert(datatable, statement) }
+	}
+	if (/^\s*update\b/i.test(statement)) {
+		return { rows: applyUpdate(datatable, statement) }
+	}
+	if (/^\s*delete\s+from\b/i.test(statement)) {
+		return { rows: applyDelete(datatable, statement) }
+	}
+	return { rows: [] }
+}
+
+// ============= Reads =============
+
+function selectRows(
+	datatable: BenchmarkDatatableSeed,
+	sql: string
+): Record<string, unknown>[] {
+	const fromRef = sql.match(/\bfrom\s+([a-zA-Z_"][\w."]*)/i)?.[1]
+	if (fromRef) {
+		const catalog = catalogRows(datatable, fromRef)
+		if (catalog) {
+			return catalog
+		}
+	}
+	const table = fromRef ? resolveTable(datatable, fromRef) : undefined
+	const seed = table ?? firstTable(datatable)
+	return seed?.rows ?? []
+}
+
+/**
+ * Synthesize rows for a system-catalog query so a model verifying a `CREATE`/`DROP`
+ * via `information_schema.tables` / `.columns` (or `pg_tables`) sees the current
+ * tables/columns instead of fallback data. WHERE is not applied, so the model gets
+ * the full set and finds (or no longer finds) the table it just changed.
+ * Returns `undefined` for non-catalog refs so normal table resolution proceeds.
+ */
+function catalogRows(
+	datatable: BenchmarkDatatableSeed,
+	ref: string
+): Record<string, unknown>[] | undefined {
+	const normalized = ref.toLowerCase().replace(/"/g, '')
+	const name = normalized.split('.').pop()
+	const isCatalog = normalized.includes('information_schema.') || normalized.startsWith('pg_')
+	if (!isCatalog) {
+		return undefined
+	}
+	const tables = allTables(datatable)
+	if (name === 'tables' || name === 'pg_tables') {
+		return tables.map(({ schema, table }) => ({
+			table_schema: schema,
+			table_name: table,
+			schemaname: schema,
+			tablename: table
+		}))
+	}
+	if (name === 'columns') {
+		return tables.flatMap(({ schema, table, seed }) =>
+			Object.entries(seed.columns).map(([column, type]) => ({
+				table_schema: schema,
+				table_name: table,
+				column_name: column,
+				data_type: type
+			}))
+		)
+	}
+	return undefined
+}
+
+function allTables(
+	datatable: BenchmarkDatatableSeed
+): { schema: string; table: string; seed: BenchmarkDatatableTableSeed }[] {
+	return Object.entries(datatable.schemas).flatMap(([schema, tables]) =>
+		Object.entries(tables).map(([table, seed]) => ({ schema, table, seed }))
+	)
+}
+
+// ============= DDL =============
+
+function applyCreateTable(
+	datatable: BenchmarkDatatableSeed,
+	sql: string
+): Record<string, unknown>[] {
+	const head = sql.match(
+		/^\s*create\s+table\s+(?:if\s+not\s+exists\s+)?([a-zA-Z_"][\w."]*)/i
+	)
+	// The first top-level paren group is the column-definition list; using it (rather
+	// than a greedy `(...)` capture) ignores any trailing `;`-separated statement.
+	const columnText = extractParenGroups(sql)[0]
+	if (!head || columnText === undefined) {
+		return []
+	}
+	const { schema, table } = parseRef(head[1])
+	const existing = datatable.schemas[schema]?.[table]
+	if (existing) {
+		return []
+	}
+	const columns: Record<string, string> = {}
+	for (const rawDef of splitTopLevel(columnText)) {
+		const def = rawDef.trim()
+		if (!def || isTableConstraint(def)) {
+			continue
+		}
+		const tokens = def.split(/\s+/)
+		const column = unquoteIdentifier(tokens[0])
+		if (!column) {
+			continue
+		}
+		columns[column] = tokens[1] ?? 'text'
+	}
+	if (!datatable.schemas[schema]) {
+		datatable.schemas[schema] = {}
+	}
+	datatable.schemas[schema][table] = { columns, rows: [] }
+	return []
+}
+
+function applyDropTable(
+	datatable: BenchmarkDatatableSeed,
+	sql: string
+): Record<string, unknown>[] {
+	const match = sql.match(
+		/^\s*drop\s+table\s+(?:if\s+exists\s+)?([a-zA-Z_"][\w."]*)/i
+	)
+	if (!match) {
+		return []
+	}
+	const { schema, table } = parseRef(match[1])
+	if (datatable.schemas[schema]?.[table]) {
+		delete datatable.schemas[schema][table]
+	}
+	return []
+}
+
+// ============= DML =============
+
+function applyInsert(
+	datatable: BenchmarkDatatableSeed,
+	sql: string
+): Record<string, unknown>[] {
+	const { body, returning } = splitOffReturning(sql)
+	const match = body.match(
+		/^\s*insert\s+into\s+([a-zA-Z_"][\w."]*)\s*(?:\(([^)]*)\))?\s*values\s*([\s\S]+)$/i
+	)
+	if (!match) {
+		return []
+	}
+	const table = resolveTable(datatable, match[1])
+	if (!table) {
+		return []
+	}
+	const columns = match[2]
+		? splitTopLevel(match[2]).map((entry) => unquoteIdentifier(entry.trim()))
+		: Object.keys(table.columns)
+	const inserted: Record<string, unknown>[] = []
+	for (const tuple of extractParenGroups(match[3])) {
+		const values = splitTopLevel(tuple).map((entry) => parseValue(entry))
+		const row: Record<string, unknown> = {}
+		columns.forEach((column, index) => {
+			row[column] = values[index]
+		})
+		inserted.push(row)
+	}
+	table.rows ??= []
+	table.rows.push(...inserted)
+	return returning ? inserted : []
+}
+
+function applyUpdate(
+	datatable: BenchmarkDatatableSeed,
+	sql: string
+): Record<string, unknown>[] {
+	const { body, returning } = splitOffReturning(sql)
+	const match = body.match(/^\s*update\s+([a-zA-Z_"][\w."]*)\s+set\s+([\s\S]+)$/i)
+	if (!match) {
+		return []
+	}
+	const table = resolveTable(datatable, match[1])
+	if (!table) {
+		return []
+	}
+	let assignmentText = match[2]
+	let whereText: string | undefined
+	const whereMatch = maskForClauseScan(assignmentText).match(/\swhere\s/i)
+	if (whereMatch && whereMatch.index !== undefined) {
+		whereText = assignmentText.slice(whereMatch.index + whereMatch[0].length)
+		assignmentText = assignmentText.slice(0, whereMatch.index)
+	}
+	const predicates = parsePredicates(whereText)
+	if (predicates === null) {
+		return []
+	}
+	const assignments: Record<string, unknown> = {}
+	for (const entry of splitTopLevel(assignmentText)) {
+		const pair = entry.match(/^\s*([a-zA-Z_"][\w."]*)\s*=\s*([\s\S]+?)\s*$/)
+		if (pair) {
+			assignments[lastIdentifier(pair[1])] = parseValue(pair[2])
+		}
+	}
+	const affected = (table.rows ?? []).filter((row) => rowMatches(row, predicates))
+	for (const row of affected) {
+		Object.assign(row, assignments)
+	}
+	return returning ? affected : []
+}
+
+function applyDelete(
+	datatable: BenchmarkDatatableSeed,
+	sql: string
+): Record<string, unknown>[] {
+	const { body, returning } = splitOffReturning(sql)
+	const match = body.match(/^\s*delete\s+from\s+([a-zA-Z_"][\w."]*)\s*([\s\S]*)$/i)
+	if (!match) {
+		return []
+	}
+	const table = resolveTable(datatable, match[1])
+	if (!table) {
+		return []
+	}
+	const whereText = match[2].replace(/^\s*where\s+/i, '').trim() || undefined
+	const predicates = parsePredicates(whereText)
+	if (predicates === null) {
+		return []
+	}
+	const rows = table.rows ?? []
+	const removed = rows.filter((row) => rowMatches(row, predicates))
+	table.rows = rows.filter((row) => !rowMatches(row, predicates))
+	return returning ? removed : []
+}
+
+// ============= Parsing helpers =============
+
+function resolveTable(
+	datatable: BenchmarkDatatableSeed,
+	ref: string
+): BenchmarkDatatableTableSeed | undefined {
+	const { schema, table } = parseRef(ref)
+	const direct = datatable.schemas[schema]?.[table]
+	if (direct) {
+		return direct
+	}
+	// Bare table name: fall back to searching every schema for a matching table.
+	if (!ref.includes('.')) {
+		for (const tables of Object.values(datatable.schemas)) {
+			if (tables[table]) {
+				return tables[table]
+			}
+		}
+	}
+	return undefined
+}
+
+function firstTable(
+	datatable: BenchmarkDatatableSeed
+): BenchmarkDatatableTableSeed | undefined {
+	for (const tables of Object.values(datatable.schemas)) {
+		for (const seed of Object.values(tables)) {
+			return seed
+		}
+	}
+	return undefined
+}
+
+function parseRef(ref: string): ParsedRef {
+	const parts = ref.split('.').map(unquoteIdentifier)
+	if (parts.length >= 2) {
+		return { schema: parts[parts.length - 2], table: parts[parts.length - 1] }
+	}
+	return { schema: DEFAULT_SCHEMA, table: parts[0] }
+}
+
+/** A WHERE clause with no parseable form returns `null`; absent WHERE returns `[]` (match all). */
+function parsePredicates(whereText: string | undefined): Predicate[] | null {
+	if (whereText === undefined || whereText.trim() === '') {
+		return []
+	}
+	const predicates: Predicate[] = []
+	for (const part of whereText.split(/\s+and\s+/i)) {
+		const match = part.match(/^\s*([a-zA-Z_"][\w."]*)\s*=\s*([\s\S]+?)\s*$/)
+		if (!match) {
+			return null
+		}
+		predicates.push({ column: lastIdentifier(match[1]), value: parseValue(match[2]) })
+	}
+	return predicates
+}
+
+function rowMatches(row: Record<string, unknown>, predicates: Predicate[]): boolean {
+	return predicates.every((predicate) => looseEquals(row[predicate.column], predicate.value))
+}
+
+function looseEquals(left: unknown, right: unknown): boolean {
+	if (left === null || left === undefined) {
+		return right === null || right === undefined
+	}
+	if (typeof left === 'number' && typeof right === 'number') {
+		return left === right
+	}
+	return String(left) === String(right)
+}
+
+function parseValue(raw: string): unknown {
+	// Drop a trailing Postgres cast (e.g. `2::int4`) before interpreting the literal.
+	const token = raw.trim().replace(/::\s*[a-zA-Z_][\w]*(\([^)]*\))?\s*$/, '').trim()
+	const stringMatch = token.match(/^'([\s\S]*)'$/)
+	if (stringMatch) {
+		return stringMatch[1].replace(/''/g, "'")
+	}
+	if (/^-?\d+(\.\d+)?$/.test(token)) {
+		return Number(token)
+	}
+	if (/^true$/i.test(token)) {
+		return true
+	}
+	if (/^false$/i.test(token)) {
+		return false
+	}
+	if (/^null$/i.test(token)) {
+		return null
+	}
+	return token
+}
+
+function splitOffReturning(sql: string): { body: string; returning: boolean } {
+	const match = maskForClauseScan(sql).match(/\sreturning\s/i)
+	if (!match || match.index === undefined) {
+		return { body: sql, returning: false }
+	}
+	return { body: sql.slice(0, match.index), returning: true }
+}
+
+/**
+ * A same-length copy of `sql` with the contents of single-quoted strings and
+ * parenthesized groups blanked to spaces, so a top-level keyword scan
+ * (WHERE / RETURNING) cannot match inside a string literal or a subquery. Index
+ * positions in the result map 1:1 back onto the original.
+ */
+function maskForClauseScan(sql: string): string {
+	let masked = ''
+	let depth = 0
+	let inString = false
+	for (let i = 0; i < sql.length; i++) {
+		const char = sql[i]
+		if (inString) {
+			if (char === "'") {
+				if (sql[i + 1] === "'") {
+					masked += '  '
+					i++
+					continue
+				}
+				inString = false
+			}
+			masked += ' '
+			continue
+		}
+		if (char === "'") {
+			inString = true
+			masked += ' '
+		} else if (char === '(') {
+			depth++
+			masked += ' '
+		} else if (char === ')') {
+			depth = Math.max(0, depth - 1)
+			masked += ' '
+		} else {
+			masked += depth > 0 ? ' ' : char
+		}
+	}
+	return masked
+}
+
+/**
+ * Inner text of each top-level `( ... )` group in `input`, honoring nested parens
+ * (e.g. `now()`, `numeric(10,2)`) and single-quoted strings. Used for the CREATE
+ * column-definition group and INSERT value tuples.
+ */
+function extractParenGroups(input: string): string[] {
+	const groups: string[] = []
+	let depth = 0
+	let inString = false
+	let current = ''
+	for (let i = 0; i < input.length; i++) {
+		const char = input[i]
+		if (inString) {
+			current += char
+			if (char === "'") {
+				if (input[i + 1] === "'") {
+					current += input[++i]
+				} else {
+					inString = false
+				}
+			}
+			continue
+		}
+		if (char === "'") {
+			inString = true
+			current += char
+		} else if (char === '(') {
+			depth++
+			if (depth === 1) {
+				current = ''
+			} else {
+				current += char
+			}
+		} else if (char === ')') {
+			depth = Math.max(0, depth - 1)
+			if (depth === 0) {
+				groups.push(current)
+				current = ''
+			} else {
+				current += char
+			}
+		} else if (depth > 0) {
+			current += char
+		}
+	}
+	return groups
+}
+
+/** Split on commas that are not inside parentheses or single-quoted strings. */
+function splitTopLevel(input: string): string[] {
+	const parts: string[] = []
+	let depth = 0
+	let inString = false
+	let current = ''
+	for (let i = 0; i < input.length; i++) {
+		const char = input[i]
+		if (inString) {
+			current += char
+			if (char === "'") {
+				if (input[i + 1] === "'") {
+					current += input[++i]
+				} else {
+					inString = false
+				}
+			}
+			continue
+		}
+		if (char === "'") {
+			inString = true
+			current += char
+		} else if (char === '(') {
+			depth++
+			current += char
+		} else if (char === ')') {
+			depth = Math.max(0, depth - 1)
+			current += char
+		} else if (char === ',' && depth === 0) {
+			parts.push(current)
+			current = ''
+		} else {
+			current += char
+		}
+	}
+	if (current.trim() !== '') {
+		parts.push(current)
+	}
+	return parts
+}
+
+function isTableConstraint(def: string): boolean {
+	return /^(primary\s+key|foreign\s+key|constraint|unique|check|exclude|like)\b/i.test(def)
+}
+
+function unquoteIdentifier(identifier: string): string {
+	const trimmed = identifier.trim()
+	const quoted = trimmed.match(/^"([\s\S]*)"$/)
+	return quoted ? quoted[1] : trimmed
+}
+
+/** For a qualified reference like `orders.id`, keep only the final identifier. */
+function lastIdentifier(reference: string): string {
+	const parts = reference.split('.')
+	return unquoteIdentifier(parts[parts.length - 1])
+}
+
+function stripTrailingSemicolon(sql: string): string {
+	return sql.replace(/;\s*$/, '')
+}

ai_evals/adapters/frontend/mockBackend.ts

@@ -1,7 +1,14 @@
 import { randomUUID } from 'node:crypto'
-import type { CompletedJob, Flow, Script } from '../../../frontend/src/lib/gen'
-import type { ScriptLang } from '../../../frontend/src/lib/gen/types.gen'
+import type { CompletedJob, Flow, Job, Script } from '../../../frontend/src/lib/gen'
+import type {
+	DataTableTables,
+	DataTableTableSchema,
+	ScriptLang
+} from '../../../frontend/src/lib/gen/types.gen'
 import { buildScriptLintResult } from './core/script/preview'
+import { applyDatatableSql, type BenchmarkDatatableSeed } from './datatableSqlEngine'
+
+export type { BenchmarkDatatableSeed, BenchmarkDatatableTableSeed } from './datatableSqlEngine'
 
 const BENCHMARK_TIMESTAMP = '1970-01-01T00:00:00.000Z'
 
@@ -22,17 +29,36 @@ export interface BenchmarkWorkspaceFlow {
 	value: Flow['value']
 }
 
+export interface BenchmarkWorkspaceJob {
+	/** Stable id so a case prompt can reference a specific run (e.g. for get_job_logs). */
+	id?: string
+	jobKind?: CompletedJob['job_kind']
+	scriptPath?: string
+	createdBy?: string
+	label?: string
+	success?: boolean
+	logs?: string
+}
+
 export interface BenchmarkWorkspaceRunnables {
 	scripts?: BenchmarkWorkspaceScript[]
 	flows?: BenchmarkWorkspaceFlow[]
+	datatables?: BenchmarkDatatableSeed[]
+	jobs?: BenchmarkWorkspaceJob[]
 }
 
 type BenchmarkCompletedJob = CompletedJob & { type: 'CompletedJob' }
 
 const benchmarkWorkspaces = new Set<string>()
 const benchmarkWorkspaceRunnables = new Map<string, BenchmarkWorkspaceRunnables>()
+// Keyed by `${workspace}::${jobId}` so concurrent attempts (or distinct cases)
+// can seed the same fixed job id without clobbering each other's entry.
 const benchmarkJobs = new Map<string, { workspace: string; job: BenchmarkCompletedJob }>()
 
+function benchmarkJobKey(workspace: string, jobId: string): string {
+	return `${workspace}::${jobId}`
+}
+
 export function resetBenchmarkMockBackend(): void {
 	benchmarkWorkspaces.clear()
 	benchmarkWorkspaceRunnables.clear()
@@ -48,7 +74,25 @@ export function registerBenchmarkWorkspaceRunnables(
 	runnables: BenchmarkWorkspaceRunnables
 ): void {
 	benchmarkWorkspaces.add(workspace)
-	benchmarkWorkspaceRunnables.set(workspace, runnables)
+	// Datatables are mutated in place by exec_datatable_sql (a write must be visible
+	// to later reads), so store an isolated deep copy — never mutate the caller's seed.
+	benchmarkWorkspaceRunnables.set(workspace, {
+		...runnables,
+		datatables: runnables.datatables ? structuredClone(runnables.datatables) : undefined
+	})
+	// Seed any fixture jobs so list_runs / get_job_logs have data to return.
+	for (const seed of runnables.jobs ?? []) {
+		createBenchmarkCompletedJob({
+			workspace,
+			id: seed.id,
+			jobKind: seed.jobKind ?? 'script',
+			success: seed.success,
+			scriptPath: seed.scriptPath,
+			createdBy: seed.createdBy,
+			label: seed.label,
+			logs: seed.logs
+		})
+	}
 }
 
 export function unregisterBenchmarkWorkspace(workspace: string): void {
@@ -118,14 +162,17 @@ export function createBenchmarkCompletedJob(input: {
 	scriptPath?: string
 	scriptHash?: string
 	args?: Record<string, unknown>
+	id?: string
+	createdBy?: string
+	label?: string
 }): string {
-	const jobId = `benchmark-job-${randomUUID()}`
+	const jobId = input.id ?? `benchmark-job-${randomUUID()}`
 	const now = new Date().toISOString()
 	const job: BenchmarkCompletedJob = {
 		type: 'CompletedJob',
 		id: jobId,
 		workspace_id: input.workspace,
-		created_by: 'ai-evals',
+		created_by: input.createdBy ?? 'ai-evals',
 		created_at: now,
 		started_at: now,
 		completed_at: now,
@@ -143,10 +190,11 @@ export function createBenchmarkCompletedJob(input: {
 		is_skipped: false,
 		email: 'ai-evals@local',
 		visible_to_owner: true,
-		tag: 'benchmark'
+		tag: 'benchmark',
+		labels: input.label ? [input.label] : undefined
 	}
 
-	benchmarkJobs.set(jobId, { workspace: input.workspace, job })
+	benchmarkJobs.set(benchmarkJobKey(input.workspace, jobId), { workspace: input.workspace, job })
 	return jobId
 }
 
@@ -154,13 +202,135 @@ export function getBenchmarkCompletedJob(
 	workspace: string,
 	jobId: string
 ): BenchmarkCompletedJob | null {
-	const entry = benchmarkJobs.get(jobId)
-	if (!entry || entry.workspace !== workspace) {
+	const entry = benchmarkJobs.get(benchmarkJobKey(workspace, jobId))
+	if (!entry) {
 		return null
 	}
 	return structuredClone(entry.job)
 }
 
+/**
+ * List seeded/recorded jobs for a benchmark workspace, most recent first —
+ * the shape `JobService.listJobs` returns. Returns `null` for a non-benchmark
+ * workspace so the caller can fall through to the real backend. Server-side
+ * filters (path/creator/status/limit) are intentionally not applied: global
+ * eval cases assert on the recorded `list_runs` tool call, not on filtering.
+ */
+export function listBenchmarkJobs(workspace: string): Job[] | null {
+	if (!hasBenchmarkWorkspace(workspace)) {
+		return null
+	}
+	return [...benchmarkJobs.values()]
+		.filter((entry) => entry.workspace === workspace)
+		.map((entry) => structuredClone(entry.job) as Job)
+		.sort((a, b) => (b.created_at ?? '').localeCompare(a.created_at ?? ''))
+}
+
+/**
+ * Mirror `JobService.getJobLogs` (response is the raw log string). Throws a
+ * "not found" error for an unknown id, matching the backend 404.
+ */
+export function getBenchmarkJobLogs(workspace: string, jobId: string): string {
+	const job = getBenchmarkCompletedJob(workspace, jobId)
+	if (!job) {
+		throw new Error(`Job Logs not found for "${jobId}"`)
+	}
+	return job.logs ?? ''
+}
+
+// ============= Datatables (best-effort in-memory SQL) =============
+
+/**
+ * Project the seeded datatables down to the `list_datatable_tables` response:
+ * `datatable_name` + `schema -> table_names`, with no column detail.
+ * Returns `null` for a non-benchmark workspace so callers can fall through to
+ * the real backend; an empty seed yields `[]`.
+ */
+export function listBenchmarkDatatables(workspace: string): DataTableTables[] | null {
+	const runnables = benchmarkWorkspaceRunnables.get(workspace)
+	if (!runnables) {
+		return null
+	}
+	return (runnables.datatables ?? []).map((datatable) => ({
+		datatable_name: datatable.datatable_name,
+		schemas: Object.fromEntries(
+			Object.entries(datatable.schemas).map(([schema, tables]) => [schema, Object.keys(tables)])
+		)
+	}))
+}
+
+export function getBenchmarkDatatableSchema(input: {
+	workspace: string
+	datatableName: string
+	schemaName: string
+	tableName: string
+}): DataTableTableSchema {
+	const runnables = benchmarkWorkspaceRunnables.get(input.workspace)
+	const datatable = (runnables?.datatables ?? []).find(
+		(entry) => entry.datatable_name === input.datatableName
+	)
+	if (!datatable) {
+		// Message MUST match the production `isDatatableNotConfiguredError` regex
+		// (/datatable\s+\S+\s+not found/i in datatableTools.ts) so the
+		// get_datatable_table_schema not-configured mapping is actually exercised.
+		throw new Error(`datatable "${input.datatableName}" not found`)
+	}
+	const table = datatable.schemas?.[input.schemaName]?.[input.tableName]
+	if (!table) {
+		throw new Error(
+			`table "${input.schemaName}.${input.tableName}" not found in datatable "${input.datatableName}"`
+		)
+	}
+	return {
+		datatable_name: input.datatableName,
+		schema_name: input.schemaName,
+		table_name: input.tableName,
+		columns: table.columns
+	}
+}
+
+/**
+ * Execute SQL against a seeded datatable through the best-effort in-memory engine
+ * (`applyDatatableSql`). Writes (CREATE/INSERT/UPDATE/DELETE/DROP) mutate the
+ * stored datatable in place so a later list/schema/SELECT reflects them; SELECT
+ * (and RETURNING) yield rows, other statements yield `[]`. Creates a benchmark
+ * completed job and returns its id, like `runBenchmarkScriptPreview`.
+ */
+export function runBenchmarkDatatableSql(input: {
+	workspace: string
+	datatableName: string
+	sql: string
+}): string {
+	const runnables = benchmarkWorkspaceRunnables.get(input.workspace)
+	const datatable = (runnables?.datatables ?? []).find(
+		(entry) => entry.datatable_name === input.datatableName
+	)
+	const rows = datatable ? applyDatatableSql(datatable, input.sql).rows : []
+	return createBenchmarkCompletedJob({
+		workspace: input.workspace,
+		jobKind: 'preview',
+		success: true,
+		args: { database: `datatable://${input.datatableName}` },
+		result: rows
+	})
+}
+
+/**
+ * Mirror `JobService.getCompletedJobResultMaybe` for benchmark workspaces — the
+ * shape `pollJobResult` consumes. The job is created synchronously before
+ * polling, so it is always present and completed.
+ */
+export function getBenchmarkCompletedJobResultMaybe(input: {
+	workspace: string
+	id: string
+}): { success: boolean; completed: boolean; result: unknown } {
+	const job = getBenchmarkCompletedJob(input.workspace, input.id)
+	if (!job) {
+		throw new Error(`Job "${input.id}" not found in benchmark workspace`)
+	}
+	return { success: job.success, completed: true, result: job.result }
+}
+
 export function runBenchmarkScriptPreview(input: {
 	workspace: string
 	requestBody: {

ai_evals/adapters/frontend/mockBackendDatatables.test.ts

@@ -0,0 +1,175 @@
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test'
+import {
+	getBenchmarkCompletedJobResultMaybe,
+	getBenchmarkDatatableSchema,
+	listBenchmarkDatatables,
+	registerBenchmarkWorkspaceRunnables,
+	resetBenchmarkMockBackend,
+	runBenchmarkDatatableSql,
+	type BenchmarkWorkspaceRunnables
+} from './mockBackend'
+
+const WORKSPACE = 'benchmark-datatable-ws'
+
+// Mirrors the production `isDatatableNotConfiguredError` regex in
+// datatableTools.ts. The schema mock's "not configured" message MUST match it,
+// otherwise the not-configured mapping in get_datatable_table_schema is silently
+// untested.
+const NOT_CONFIGURED_RE = /datatable\s+\S+\s+not found/i
+
+const SEED: BenchmarkWorkspaceRunnables = {
+	datatables: [
+		{
+			datatable_name: 'main',
+			schemas: {
+				public: {
+					orders: {
+						columns: { id: 'int', total: 'numeric' },
+						rows: [
+							{ id: 1, total: 10 },
+							{ id: 2, total: 20 }
+						]
+					},
+					customers: {
+						columns: { id: 'int', name: 'text' },
+						rows: [{ id: 1, name: 'alice' }]
+					}
+				}
+			}
+		}
+	]
+}
+
+beforeEach(() => resetBenchmarkMockBackend())
+afterEach(() => resetBenchmarkMockBackend())
+
+describe('listBenchmarkDatatables', () => {
+	it('returns null for a non-benchmark workspace (caller falls through to real backend)', () => {
+		expect(listBenchmarkDatatables('unregistered')).toBeNull()
+	})
+
+	it('returns [] for a registered workspace with no datatables seed', () => {
+		registerBenchmarkWorkspaceRunnables(WORKSPACE, {})
+		expect(listBenchmarkDatatables(WORKSPACE)).toEqual([])
+	})
+
+	it('projects seeded datatables to schema -> table names only (no columns)', () => {
+		registerBenchmarkWorkspaceRunnables(WORKSPACE, SEED)
+		expect(listBenchmarkDatatables(WORKSPACE)).toEqual([
+			{ datatable_name: 'main', schemas: { public: ['orders', 'customers'] } }
+		])
+	})
+})
+
+describe('getBenchmarkDatatableSchema', () => {
+	beforeEach(() => registerBenchmarkWorkspaceRunnables(WORKSPACE, SEED))
+
+	it('returns the columns for a seeded table', () => {
+		expect(
+			getBenchmarkDatatableSchema({
+				workspace: WORKSPACE,
+				datatableName: 'main',
+				schemaName: 'public',
+				tableName: 'orders'
+			})
+		).toEqual({
+			datatable_name: 'main',
+			schema_name: 'public',
+			table_name: 'orders',
+			columns: { id: 'int', total: 'numeric' }
+		})
+	})
+
+	it('throws a not-configured error matching the production regex for an unknown datatable', () => {
+		let error: Error | undefined
+		try {
+			getBenchmarkDatatableSchema({
+				workspace: WORKSPACE,
+				datatableName: 'ghost',
+				schemaName: 'public',
+				tableName: 'orders'
+			})
+		} catch (e) {
+			error = e as Error
+		}
+		expect(error).toBeDefined()
+		expect(error!.message).toMatch(NOT_CONFIGURED_RE)
+	})
+
+	it('throws a table-not-found error that does NOT match the datatable-not-configured regex', () => {
+		// The datatable IS configured; only the table is missing. Production maps
+		// this to a generic "error getting schema", not the blocking message.
+		let error: Error | undefined
+		try {
+			getBenchmarkDatatableSchema({
+				workspace: WORKSPACE,
+				datatableName: 'main',
+				schemaName: 'public',
+				tableName: 'ghost'
+			})
+		} catch (e) {
+			error = e as Error
+		}
+		expect(error).toBeDefined()
+		expect(error!.message).not.toMatch(NOT_CONFIGURED_RE)
+	})
+})
+
+describe('runBenchmarkDatatableSql + getBenchmarkCompletedJobResultMaybe', () => {
+	beforeEach(() => registerBenchmarkWorkspaceRunnables(WORKSPACE, SEED))
+
+	function exec(sql: string): { success: boolean; completed: boolean; result: unknown } {
+		const jobId = runBenchmarkDatatableSql({ workspace: WORKSPACE, datatableName: 'main', sql })
+		return getBenchmarkCompletedJobResultMaybe({ workspace: WORKSPACE, id: jobId })
+	}
+
+	it('returns the canned rows of the table named in a SELECT FROM clause', () => {
+		expect(exec('SELECT * FROM customers')).toEqual({
+			success: true,
+			completed: true,
+			result: [{ id: 1, name: 'alice' }]
+		})
+	})
+
+	it('falls back to the first seeded table when the SELECT references no known table', () => {
+		expect(exec('select 1').result).toEqual([
+			{ id: 1, total: 10 },
+			{ id: 2, total: 20 }
+		])
+	})
+
+	it('returns [] success for DDL and DML statements without RETURNING', () => {
+		expect(exec('CREATE TABLE foo (id int)').result).toEqual([])
+		expect(exec('INSERT INTO orders VALUES (3, 30)').result).toEqual([])
+		expect(exec('update orders set total = 0').result).toEqual([])
+	})
+
+	it('reflects a write in a later SELECT, isolated from the shared seed', () => {
+		exec('UPDATE orders SET total = 999 WHERE id = 1')
+		expect((exec('SELECT * FROM orders').result as Record<string, unknown>[])).toContainEqual({
+			id: 1,
+			total: 999
+		})
+		// Registration deep-clones the seed, so the shared SEED const stays pristine.
+		expect(SEED.datatables![0].schemas.public.orders.rows).toContainEqual({ id: 1, total: 10 })
+	})
+
+	it('reflects a CREATE in list_datatables and get_datatable_table_schema', () => {
+		exec('CREATE TABLE public.refunds (order_id int4, amount numeric)')
+		expect(listBenchmarkDatatables(WORKSPACE)?.[0].schemas.public).toContain('refunds')
+		expect(
+			getBenchmarkDatatableSchema({
+				workspace: WORKSPACE,
+				datatableName: 'main',
+				schemaName: 'public',
+				tableName: 'refunds'
+			}).columns
+		).toEqual({ order_id: 'int4', amount: 'numeric' })
+	})
+
+	it('throws for an unknown job id', () => {
+		expect(() =>
+			getBenchmarkCompletedJobResultMaybe({ workspace: WORKSPACE, id: 'does-not-exist' })
+		).toThrow()
+	})
+})

ai_evals/adapters/frontend/vitestAdapter.test.ts

@@ -34,15 +34,21 @@ vi.mock('$lib/gen', async () => {
 	const actual = await vi.importActual<any>('$lib/gen')
 	const {
 		getBenchmarkCompletedJob,
+		getBenchmarkCompletedJobResultMaybe,
+		getBenchmarkDatatableSchema,
 		getBenchmarkFlowByPath,
+		getBenchmarkJobLogs,
 		getBenchmarkScriptByHash,
 		getBenchmarkScriptByPath,
 		hasBenchmarkWorkspace,
+		listBenchmarkDatatables,
 		listBenchmarkFlows,
+		listBenchmarkJobs,
 		listBenchmarkScripts,
 		createBenchmarkHttpTrigger,
 		createBenchmarkSchedule,
 		previewBenchmarkSchedule,
+		runBenchmarkDatatableSql,
 		runBenchmarkFlowByPath,
 		runBenchmarkScriptPreview
 	} = await import('./mockBackend')
@@ -79,6 +85,16 @@ vi.mock('$lib/gen', async () => {
 				}
 				return actual.ScriptService.getScriptByPath(data)
 			},
+			getScriptByPathWithDraft: async (data: { workspace: string; path: string }) => {
+				if (hasBenchmarkWorkspace(data.workspace)) {
+					const script = getBenchmarkScriptByPath(data.workspace, data.path)
+					if (!script) {
+						throw new Error(`Script "${data.path}" not found in benchmark workspace`)
+					}
+					return script
+				}
+				return actual.ScriptService.getScriptByPathWithDraft(data)
+			},
 			getScriptByHash: async (data: { workspace: string; hash: string }) => {
 				if (hasBenchmarkWorkspace(data.workspace)) {
 					const script = getBenchmarkScriptByHash(data.workspace, data.hash)
@@ -108,6 +124,26 @@ vi.mock('$lib/gen', async () => {
 					return flow
 				}
 				return actual.FlowService.getFlowByPath(data)
+			},
+			getFlowByPathWithDraft: async (data: { workspace: string; path: string }) => {
+				if (hasBenchmarkWorkspace(data.workspace)) {
+					const flow = getBenchmarkFlowByPath(data.workspace, data.path)
+					if (!flow) {
+						throw new Error(`Flow "${data.path}" not found in benchmark workspace`)
+					}
+					return flow
+				}
+				return actual.FlowService.getFlowByPathWithDraft(data)
+			},
+			getFlowLatestVersion: async (data: { workspace: string; path: string }) => {
+				if (hasBenchmarkWorkspace(data.workspace)) {
+					const flow = getBenchmarkFlowByPath(data.workspace, data.path)
+					if (!flow) {
+						throw new Error(`Flow "${data.path}" not found in benchmark workspace`)
+					}
+					return { id: 1 }
+				}
+				return actual.FlowService.getFlowLatestVersion(data)
 			}
 		}),
 		JobService: wrapService(actual.JobService, {
@@ -119,13 +155,27 @@ vi.mock('$lib/gen', async () => {
 					args?: Record<string, unknown>
 					path?: string
 				}
-			}) =>
-				hasBenchmarkWorkspace(data.workspace)
-					? runBenchmarkScriptPreview({
-							workspace: data.workspace,
-							requestBody: data.requestBody ?? {}
-						})
-					: actual.JobService.runScriptPreview(data),
+			}) => {
+				if (!hasBenchmarkWorkspace(data.workspace)) {
+					return actual.JobService.runScriptPreview(data)
+				}
+				const requestBody = data.requestBody ?? {}
+				const database = requestBody.args?.database
+				// Datatable SQL runs as a `postgresql` preview against `datatable://<name>`.
+				// Execute it through the canned-SQL mock instead of linting it as a script.
+				if (
+					requestBody.language === 'postgresql' &&
+					typeof database === 'string' &&
+					database.startsWith('datatable://')
+				) {
+					return runBenchmarkDatatableSql({
+						workspace: data.workspace,
+						datatableName: database.slice('datatable://'.length),
+						sql: requestBody.content ?? ''
+					})
+				}
+				return runBenchmarkScriptPreview({ workspace: data.workspace, requestBody })
+			},
 			runFlowByPath: async (data: {
 				workspace: string
 				path: string
@@ -147,7 +197,39 @@ vi.mock('$lib/gen', async () => {
 					return job
 				}
 				return actual.JobService.getJob(data)
-			}
+			},
+			getCompletedJobResultMaybe: async (data: { workspace: string; id: string }) =>
+				hasBenchmarkWorkspace(data.workspace)
+					? getBenchmarkCompletedJobResultMaybe({ workspace: data.workspace, id: data.id })
+					: actual.JobService.getCompletedJobResultMaybe(data),
+			listJobs: async (data: { workspace: string }) =>
+				hasBenchmarkWorkspace(data.workspace)
+					? (listBenchmarkJobs(data.workspace) ?? [])
+					: actual.JobService.listJobs(data),
+			getJobLogs: async (data: { workspace: string; id: string }) =>
+				hasBenchmarkWorkspace(data.workspace)
+					? getBenchmarkJobLogs(data.workspace, data.id)
+					: actual.JobService.getJobLogs(data)
+		}),
+		WorkspaceService: wrapService(actual.WorkspaceService, {
+			listDataTableTables: async (data: { workspace: string }) =>
+				hasBenchmarkWorkspace(data.workspace)
+					? (listBenchmarkDatatables(data.workspace) ?? [])
+					: actual.WorkspaceService.listDataTableTables(data),
+			getDataTableTableSchema: async (data: {
+				workspace: string
+				datatableName: string
+				schemaName: string
+				tableName: string
+			}) =>
+				hasBenchmarkWorkspace(data.workspace)
+					? getBenchmarkDatatableSchema({
+							workspace: data.workspace,
+							datatableName: data.datatableName,
+							schemaName: data.schemaName,
+							tableName: data.tableName
+						})
+					: actual.WorkspaceService.getDataTableTableSchema(data)
 		}),
 		ScheduleService: wrapService(actual.ScheduleService, {
 			existsSchedule: async (data: { workspace: string; path: string }) =>

ai_evals/cases/flow.yaml

@@ -8,6 +8,9 @@
       args:
         a: 4
         b: 5
+  toolExpect:
+    requiredToolsUsed:
+      - test_run_flow
   judgeChecklist:
     - "the flow takes `a` and `b` as inputs"
     - "the main step is named `sum_numbers`"
@@ -25,6 +28,9 @@
       args:
         a: 2
         b: 3
+  toolExpect:
+    requiredToolsUsed:
+      - test_run_flow
   judgeChecklist:
     - "the flow takes `a` and `b` as inputs"
     - "the main step is named `sum_numbers`"
@@ -42,6 +48,9 @@
       args:
         a: 7
         b: 8
+  toolExpect:
+    requiredToolsUsed:
+      - test_run_flow
   judgeChecklist:
     - "the parent flow takes `a` and `b` as inputs"
     - "the main step is named `call_add_numbers`"
@@ -426,6 +435,7 @@
       - return_schedule_status
   toolExpect:
     requiredToolsUsed:
+      - test_run_flow
       - create_schedule
     toolCallArgs:
       - tool: create_schedule
@@ -453,6 +463,7 @@
       - webhook_response
   toolExpect:
     requiredToolsUsed:
+      - test_run_flow
       - create_trigger
     toolCallArgs:
       - tool: create_trigger

ai_evals/cases/global.yaml

@@ -87,3 +87,786 @@
     - the flow accepts numeric inputs a and b
     - the flow returns the sum of a and b
     - the result stays as an AI draft and is not deployed or saved to the workspace
+
+- id: global-test4-multi-artifact-notification-job
+  prompt: |-
+    Set up a draft stale-trial notification job.
+    Create a Bun script at `f/evals/global/check_stale_trials` that accepts `max_age_days`, uses mocked inline trial account data, and returns the stale trial account IDs.
+    Also create a weekday 09:00 UTC schedule at `f/evals/global/check_stale_trials_weekday` for that script with `max_age_days` set to 14.
+    Add an HTTP POST trigger at `f/evals/global/check_stale_trials_manual` with route path `evals/check-stale-trials` that runs the same script manually.
+    Leave everything as AI drafts only; do not deploy or save anything to the workspace.
+  runtime:
+    maxTurns: 12
+  validate:
+    draftCountExactly: 3
+    requiredDrafts:
+      - type: script
+        path: f/evals/global/check_stale_trials
+        language: bun
+        valueIncludes:
+          - max_age_days
+          - trial
+      - type: schedule
+        path: f/evals/global/check_stale_trials_weekday
+        valueIncludes:
+          - f/evals/global/check_stale_trials
+          - UTC
+          - "14"
+      - type: trigger
+        triggerKind: http
+        path: f/evals/global/check_stale_trials_manual
+        valueIncludes:
+          - evals/check-stale-trials
+          - f/evals/global/check_stale_trials
+  toolExpect:
+    requiredToolsUsed:
+      - write_script
+      - write_schedule
+      - write_trigger
+    forbiddenToolsUsed:
+      - deploy_workspace_item
+      - delete_workspace_item
+  judgeChecklist:
+    - creates a Bun script draft for stale trial accounts
+    - creates a weekday 09:00 UTC schedule draft for the script with max_age_days set to 14
+    - creates an HTTP POST trigger draft with route path evals/check-stale-trials for the same script
+    - leaves all artifacts as drafts only and does not deploy
+
+- id: global-test5-existing-flow-inline-code-edit
+  prompt: |-
+    Update the existing flow at `f/evals/global/process_invoice`.
+    Only change the `calculate_total` inline code so it applies 8% tax and returns an object containing `subtotal`, `tax`, and `total`.
+    Leave the updated flow as an AI draft only; do not deploy or save it.
+  initial: ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json
+  runtime:
+    maxTurns: 10
+  validate:
+    draftCountExactly: 1
+    requiredDrafts:
+      - type: flow
+        path: f/evals/global/process_invoice
+        valueIncludes:
+          - calculate_total
+          - tax
+          - total
+  toolExpect:
+    requiredToolsUsed:
+      - read_workspace_item
+    forbiddenToolsUsed:
+      - deploy_workspace_item
+      - delete_workspace_item
+  judgeChecklist:
+    - reads the existing process_invoice flow before editing it
+    - updates the calculate_total inline code to apply 8% tax
+    - returns subtotal, tax, and total from the updated flow logic
+    - leaves the result as an AI draft only
+
+- id: global-test6-secret-variable-draft
+  prompt: |-
+    Create a secret variable draft at `f/evals/global/slack_bot_token`.
+    Use the placeholder value `xoxb-redacted-test-token` and description `Slack bot token for eval notifications`.
+    Do not create any resource or deploy anything.
+  runtime:
+    maxTurns: 6
+  validate:
+    draftCountExactly: 1
+    requiredDrafts:
+      - type: variable
+        path: f/evals/global/slack_bot_token
+        valueIncludes:
+          - Slack bot token
+          - "true"
+    forbiddenDrafts:
+      - type: resource
+        path: f/evals/global/slack_bot_token
+  toolExpect:
+    requiredToolsUsed:
+      - write_variable
+    forbiddenToolsUsed:
+      - write_resource
+      - deploy_workspace_item
+      - delete_workspace_item
+    toolCallArgs:
+      - tool: write_variable
+        field: value
+        stringStartsWithAnyOf:
+          - xoxb-redacted-test-token
+  skipJudge: true
+  judgeChecklist:
+    - creates exactly one secret variable draft at f/evals/global/slack_bot_token
+    - uses the requested placeholder value and description
+    - does not create a resource or deploy anything
+
+- id: global-test7-ambiguous-app-asks-question
+  prompt: |-
+    Create a new raw app for triaging support tickets.
+  runtime:
+    maxTurns: 4
+  validate:
+    draftCountExactly: 0
+  toolExpect:
+    requiredToolsUsed:
+      - askUserQuestion
+    forbiddenToolsUsed:
+      - init_app
+      - write_app_file
+      - write_app_runnable
+      - deploy_workspace_item
+      - delete_workspace_item
+  skipJudge: true
+
+- id: global-test8-human-script-infer-path-language
+  prompt: |-
+    I need a small helper that formats a customer-facing welcome line.
+    It should take a person's name and return "Welcome aboard, <name>!".
+    Please just stage it as a draft for now.
+  runtime:
+    maxTurns: 8
+  validate:
+    draftCountExactly: 1
+    requiredDrafts:
+      - type: script
+        valueIncludes:
+          - Welcome aboard
+          - name
+  toolExpect:
+    requiredToolsUsed:
+      - write_script
+    forbiddenToolsUsed:
+      - deploy_workspace_item
+      - delete_workspace_item
+  judgeChecklist:
+    - creates a single script draft for a welcome-line helper
+    - accepts a person's name as input
+    - returns a message containing Welcome aboard, the provided name, and an exclamation mark
+    - chooses a reasonable workspace path and script language without needing the user to specify them
+    - leaves the result as an AI draft only
+
+- id: global-test9-human-weekday-trial-job
+  prompt: |-
+    Can you set up a draft daily job that checks a few hard-coded trial accounts and returns the ones whose trial has ended?
+    It should run every weekday morning around 9 in UTC with a 30 day cutoff.
+    Keep it as draft work only.
+  runtime:
+    maxTurns: 10
+  validate:
+    draftCountExactly: 2
+    requiredDrafts:
+      - type: script
+        pathIncludes:
+          - trial
+        valueIncludes:
+          - trial
+          - "30"
+      - type: schedule
+        pathIncludes:
+          - trial
+        valueIncludes:
+          - UTC
+  toolExpect:
+    requiredToolsUsed:
+      - write_script
+      - write_schedule
+    forbiddenToolsUsed:
+      - deploy_workspace_item
+      - delete_workspace_item
+  judgeChecklist:
+    - creates a script draft that checks hard-coded trial accounts
+    - returns the accounts whose trial has ended based on a 30 day cutoff
+    - creates a schedule draft for weekday mornings around 09:00 UTC
+    - links the schedule to the generated script
+    - leaves both artifacts as drafts only
+
+- id: global-test10-human-secret-variable
+  prompt: |-
+    I need a placeholder Slack bot token stored securely for future notification work.
+    Use xoxb-redacted-test-token and note that it is for eval notifications.
+    Only prepare a draft.
+  runtime:
+    maxTurns: 6
+  validate:
+    draftCountExactly: 1
+    requiredDrafts:
+      - type: variable
+        pathIncludes:
+          - slack
+        valueIncludes:
+          - eval notifications
+          - "true"
+  toolExpect:
+    requiredToolsUsed:
+      - write_variable
+    forbiddenToolsUsed:
+      - write_resource
+      - deploy_workspace_item
+      - delete_workspace_item
+    toolCallArgs:
+      - tool: write_variable
+        field: value
+        stringStartsWithAnyOf:
+          - xoxb-redacted-test-token
+  skipJudge: true
+  judgeChecklist:
+    - creates a single secret variable draft for the Slack bot token placeholder
+    - uses the requested placeholder value
+    - includes a note or description that it is for eval notifications
+    - does not create a resource or deploy anything
+
+- id: global-test11-human-existing-flow-informal-edit
+  prompt: |-
+    There is an invoice processing flow in this workspace.
+    Can you adjust its total calculation so it adds 8% tax and returns subtotal, tax, and total?
+    Keep the change as a draft.
+  initial: ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json
+  runtime:
+    maxTurns: 10
+  validate:
+    draftCountExactly: 1
+    requiredDrafts:
+      - type: flow
+        pathIncludes:
+          - invoice
+        valueIncludes:
+          - calculate_total
+          - tax
+          - total
+  toolExpect:
+    requiredToolsUsed:
+      - read_workspace_item
+    forbiddenToolsUsed:
+      - deploy_workspace_item
+      - delete_workspace_item
+  judgeChecklist:
+    - finds and edits the existing invoice processing flow without the user providing its exact path
+    - updates the total calculation to apply 8% tax
+    - returns subtotal, tax, and total from the updated flow logic
+    - leaves the result as an AI draft only
+
+- id: global-test12-current-live-script-edit
+  prompt: |-
+    The script I have open formats greetings.
+    Can you update this script so it uppercases the name before greeting them and ends with an exclamation mark?
+    Keep it as draft work.
+  initial: ai_evals/fixtures/frontend/global/initial/current_greeting_live_script.json
+  runtime:
+    maxTurns: 8
+  validate:
+    draftCountExactly: 1
+    requiredDrafts:
+      - type: script
+        path: f/evals/global/current_greeting
+        language: bun
+        valueIncludes:
+          - toUpperCase
+          - "!"
+    forbiddenDrafts:
+      - type: script
+        path: f/evals/global/format_greeting
+      - type: script
+        path: f/evals/global/format_greeting_archive
+  toolExpect:
+    requiredToolsUsed:
+      - read_workspace_item
+    forbiddenToolsUsed:
+      - deploy_workspace_item
+      - delete_workspace_item
+  judgeChecklist:
+    - resolves "this script" to the active live editor script instead of another similarly named workspace script
+    - updates the greeting logic to uppercase the provided name
+    - returns a greeting ending with an exclamation mark
+    - leaves the result as a draft only
+
+- id: global-test13-current-live-flow-edit
+  prompt: |-
+    I have the invoice flow open.
+    In the current flow, update the total calculation to add 8% tax and return subtotal, tax, and total.
+    Keep the change as a draft.
+  initial: ai_evals/fixtures/frontend/global/initial/current_invoice_live_flow.json
+  runtime:
+    maxTurns: 10
+  validate:
+    draftCountExactly: 1
+    requiredDrafts:
+      - type: flow
+        path: f/evals/global/current_invoice_flow
+        valueIncludes:
+          - calculate_total
+          - tax
+          - total
+    forbiddenDrafts:
+      - type: flow
+        path: f/evals/global/process_invoice
+      - type: flow
+        path: f/evals/global/process_refund
+  toolExpect:
+    requiredToolsUsed:
+      - read_workspace_item
+    forbiddenToolsUsed:
+      - deploy_workspace_item
+      - delete_workspace_item
+  judgeChecklist:
+    - resolves "current flow" to the active live editor flow
+    - does not edit the similarly named deployed invoice or refund flows
+    - updates the calculate_total logic to apply 8% tax
+    - returns subtotal, tax, and total from the updated flow logic
+    - leaves the result as a draft only
+
+- id: global-test14-current-without-live-editor-asks-question
+  prompt: |-
+    Please update this script so it returns `ok`.
+    Keep it as a draft.
+  runtime:
+    maxTurns: 4
+  validate:
+    draftCountExactly: 0
+  toolExpect:
+    forbiddenToolsUsed:
+      - write_script
+      - edit_script
+      - write_flow
+      - deploy_workspace_item
+      - delete_workspace_item
+  skipJudge: true
+  judgeChecklist:
+    - asks which script to update when the user refers to "this script" without selected or active editor context
+    - does not guess a path or create a new script draft
+
+- id: global-test15-human-postgres-resource
+  prompt: |-
+    I'm wiring the eval reporting database into this workspace.
+    Can you stage a Postgres connection for it in the shared evals/global folder?
+    Use host `reports-db.internal`, port 5432, database `evals_reporting`, user `report_reader`, and password `pg-redacted-reporting-password`.
+    Keep the credentials safe.
+    This is just draft work for now.
+  runtime:
+    maxTurns: 10
+  validate:
+    draftCountExactly: 2
+    requiredDrafts:
+      - type: variable
+        pathStartsWith: f/evals/global/
+        pathIncludes:
+          - evals
+          - global
+          - report
+          - password
+        valueIncludes:
+          - "true"
+          - report
+      - type: resource
+        pathStartsWith: f/evals/global/
+        pathIncludes:
+          - evals
+          - global
+          - report
+        valueIncludes:
+          - postgres
+          - reports-db.internal
+          - "5432"
+          - evals_reporting
+          - report_reader
+          - "$var:"
+        valueExcludes:
+          - pg-redacted-reporting-password
+  toolExpect:
+    requiredToolsUsed:
+      - write_variable
+      - search_resource_types
+      - write_resource
+    forbiddenToolsUsed:
+      - write_schedule
+      - write_trigger
+      - deploy_workspace_item
+      - delete_workspace_item
+    toolCallArgs:
+      - tool: write_variable
+        field: value
+        stringStartsWithAnyOf:
+          - pg-redacted-reporting-password
+  skipJudge: true
+  judgeChecklist:
+    - creates a Postgres resource draft for the eval reporting database
+    - creates a secret variable draft for the database password
+    - puts the drafts in sensible eval/global reporting-related paths
+    - uses the requested host, port, database, and user
+    - references the secret variable from the resource instead of embedding the password
+    - leaves the work as a draft only
+
+- id: global-test16-human-visible-variable
+  prompt: |-
+    We keep reusing a 30 day trial cutoff in eval notification jobs.
+    Can you stage that as a normal workspace variable in the shared evals/global folder, with a short description so people know what it controls?
+    It is not a secret.
+  runtime:
+    maxTurns: 6
+  validate:
+    draftCountExactly: 1
+    requiredDrafts:
+      - type: variable
+        pathStartsWith: f/evals/global/
+        pathIncludes:
+          - evals
+          - global
+          - trial
+        valueIncludes:
+          - "30"
+          - "false"
+          - trial
+  toolExpect:
+    requiredToolsUsed:
+      - write_variable
+    forbiddenToolsUsed:
+      - write_resource
+      - write_schedule
+      - write_trigger
+      - deploy_workspace_item
+      - delete_workspace_item
+  judgeChecklist:
+    - creates exactly one non-secret variable draft for the trial cutoff
+    - stores the value 30
+    - chooses a sensible eval/global path related to trials or notifications
+    - includes a useful description of what the value controls
+    - does not create resources, schedules, triggers, or deployed workspace changes
+
+- id: global-test17-human-schedule-existing-helper
+  prompt: |-
+    The workspace already has a report digest helper.
+    Can you stage a weekday 8:30 AM UTC run for it with `dry_run` turned on?
+    I only want the schedule draft for review.
+  initial: ai_evals/fixtures/frontend/global/initial/report_digest_script.json
+  runtime:
+    maxTurns: 8
+  validate:
+    draftCountExactly: 1
+    requiredDrafts:
+      - type: schedule
+        pathIncludes:
+          - digest
+        valueIncludes:
+          - f/evals/global/send_report_digest
+          - UTC
+          - dry_run
+          - "true"
+  toolExpect:
+    requiredToolsUsed:
+      - list_workspace_items
+      - write_schedule
+    forbiddenToolsUsed:
+      - write_script
+      - write_flow
+      - write_resource
+      - write_variable
+      - write_trigger
+      - deploy_workspace_item
+      - delete_workspace_item
+  judgeChecklist:
+    - finds the existing report digest helper rather than creating a new script or flow
+    - creates one schedule draft for that helper
+    - schedules it for weekdays around 08:30 UTC
+    - passes dry_run as true
+    - leaves only the schedule draft for review
+
+- id: global-test18-human-slack-resource-with-secret
+  prompt: |-
+    I'm preparing Slack notifications for eval failures.
+    Can you stage a Slack connection in the shared evals/global folder?
+    The bot token is `xoxb-redacted-test-token`; keep it safe.
+    Don't deploy anything yet.
+  runtime:
+    maxTurns: 8
+  validate:
+    draftCountExactly: 2
+    requiredDrafts:
+      - type: variable
+        pathStartsWith: f/evals/global/
+        pathIncludes:
+          - evals
+          - global
+          - slack
+          - token
+        valueIncludes:
+          - "true"
+      - type: resource
+        pathStartsWith: f/evals/global/
+        pathIncludes:
+          - evals
+          - global
+          - slack
+        valueIncludes:
+          - slack
+          - "$var:"
+        valueExcludes:
+          - xoxb-redacted-test-token
+  toolExpect:
+    requiredToolsUsed:
+      - write_variable
+      - search_resource_types
+      - write_resource
+    forbiddenToolsUsed:
+      - write_schedule
+      - write_trigger
+      - deploy_workspace_item
+      - delete_workspace_item
+    toolCallArgs:
+      - tool: write_variable
+        field: value
+        stringStartsWithAnyOf:
+          - xoxb-redacted-test-token
+  skipJudge: true
+  judgeChecklist:
+    - creates a secret variable draft for the Slack bot token placeholder
+    - creates a Slack resource draft that references the secret variable instead of embedding the token
+    - keeps both drafts under a sensible eval/global Slack-related path
+    - does not create schedules, triggers, or deployed workspace changes
+
+- id: global-test19-datatable-not-configured-asks-to-set-up
+  prompt: |-
+    Here are two newsletter signups: alice@example.com and bob@example.com.
+    Save them into a workspace data table for me.
+  runtime:
+    maxTurns: 6
+  validate:
+    draftCountExactly: 0
+  toolExpect:
+    requiredToolsUsed:
+      - list_datatables
+    forbiddenToolsUsed:
+      - exec_datatable_sql
+      - write_script
+      - deploy_workspace_item
+      - delete_workspace_item
+  skipJudge: true
+  judgeChecklist:
+    - checks which data tables exist in the workspace before acting
+    - recognizes that no data table is configured in this workspace
+    - explains that a data table must first be set up by the user in the workspace settings (Data Tables) and is not created via SQL
+    - does not run SQL, write a script, or invent a data table to work around the missing configuration
+    - tells the user to configure a data table and then try again
+
+- id: global-test20-datatable-no-hallucinated-main
+  prompt: |-
+    Pull the latest rows from the orders table in our data table so I can see recent orders.
+  runtime:
+    maxTurns: 6
+  validate:
+    draftCountExactly: 0
+  toolExpect:
+    requiredToolsUsed:
+      - list_datatables
+    forbiddenToolsUsed:
+      - exec_datatable_sql
+      - write_script
+      - deploy_workspace_item
+      - delete_workspace_item
+  skipJudge: true
+  judgeChecklist:
+    - checks which data tables exist in the workspace before querying
+    - recognizes that no data table is configured in this workspace
+    - does not assume a data table named "main" (or any other name) exists
+    - does not run SQL against a guessed data table or fabricate order rows
+    - tells the user they need to set up a data table in the workspace settings first
+
+- id: global-test21-datatable-list-summarize
+  prompt: |-
+    What tables do we have in our workspace data table? Just give me the list.
+  initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
+  runtime:
+    maxTurns: 6
+  validate:
+    draftCountExactly: 0
+  toolExpect:
+    requiredToolsUsed:
+      - list_datatables
+    forbiddenToolsUsed:
+      - get_datatable_table_schema
+      - exec_datatable_sql
+      - deploy_workspace_item
+      - delete_workspace_item
+  skipJudge: true
+  judgeChecklist:
+    - lists the tables available in the workspace data table (orders and customers)
+    - answers from the data table listing rather than fabricating table names
+    - does not fetch column details or run SQL just to produce a table list
+
+- id: global-test22-datatable-inspect-columns
+  prompt: |-
+    What columns does the orders table have in our workspace data table?
+  initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
+  runtime:
+    maxTurns: 6
+  validate:
+    draftCountExactly: 0
+  toolExpect:
+    requiredToolsUsed:
+      - get_datatable_table_schema
+    forbiddenToolsUsed:
+      - deploy_workspace_item
+      - delete_workspace_item
+  skipJudge: true
+  judgeChecklist:
+    - inspects the orders table schema in the workspace data table
+    - reports the orders columns (such as id, customer_id, total, status, created_at)
+    - answers from the retrieved schema rather than guessing the columns
+
+- id: global-test23-datatable-query-select
+  prompt: |-
+    Show me the orders in our workspace data table, including their status and total.
+  initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
+  runtime:
+    maxTurns: 6
+  validate:
+    draftCountExactly: 0
+  toolExpect:
+    requiredToolsUsed:
+      - exec_datatable_sql
+    forbiddenToolsUsed:
+      - write_script
+      - deploy_workspace_item
+      - delete_workspace_item
+    toolCallArgs:
+      - tool: exec_datatable_sql
+        field: sql
+        stringIncludesAnyOf:
+          - select
+  skipJudge: true
+  judgeChecklist:
+    - runs a SELECT query against the orders table in the workspace data table
+    - reports the orders returned by the query back to the user instead of fabricating data
+    - does not tell the user to set up a data table, since one already exists
+
+- id: global-test24-datatable-create-table
+  prompt: |-
+    Add a new table called refunds to our workspace data table, with an order id and a refund amount.
+  initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
+  runtime:
+    maxTurns: 6
+  validate:
+    draftCountExactly: 0
+  toolExpect:
+    requiredToolsUsed:
+      - exec_datatable_sql
+    forbiddenToolsUsed:
+      - write_script
+      - deploy_workspace_item
+      - delete_workspace_item
+    toolCallArgs:
+      - tool: exec_datatable_sql
+        field: sql
+        stringIncludesAnyOf:
+          - create table
+  skipJudge: true
+  judgeChecklist:
+    - creates the refunds table with a plain CREATE TABLE statement on the data table
+    - includes an order id and a refund amount column
+    - treats creating the table as a normal SQL statement and does not claim a separate registration step is needed
+    - does not write a script to create the table
+
+- id: global-test25-datatable-mutate-rows
+  prompt: |-
+    Mark order number 2 as shipped in our workspace data table.
+  initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
+  runtime:
+    # Headroom for inspect-schema -> UPDATE -> verify; the in-memory engine now
+    # persists the write, so verification confirms on the first try (no loop).
+    maxTurns: 12
+  validate:
+    draftCountExactly: 0
+  toolExpect:
+    requiredToolsUsed:
+      - exec_datatable_sql
+    forbiddenToolsUsed:
+      - write_script
+      - deploy_workspace_item
+      - delete_workspace_item
+    toolCallArgs:
+      - tool: exec_datatable_sql
+        field: sql
+        stringIncludesAnyOf:
+          - update
+          - insert into
+  skipJudge: true
+  judgeChecklist:
+    - runs an UPDATE on the orders table setting the status of order id 2 to shipped
+    - targets only order number 2 rather than rewriting the whole table
+    - confirms the change back to the user
+
+- id: global-test26-datatable-script-sdk
+  prompt: |-
+    Write a script that reads our workspace data table and returns the total revenue across all orders.
+    Leave it as an AI draft only; do not deploy or save it.
+  initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
+  runtime:
+    maxTurns: 10
+  validate:
+    draftCountExactly: 1
+    requiredDrafts:
+      - type: script
+        valueIncludes:
+          - wmill.datatable(
+  toolExpect:
+    requiredToolsUsed:
+      - get_instructions
+      - write_script
+    forbiddenToolsUsed:
+      - exec_datatable_sql
+      - deploy_workspace_item
+      - delete_workspace_item
+  # The judge has no datatable SDK reference and wrongly penalizes correct
+  # wmill.datatable() tagged-template usage, so rely on the deterministic checks:
+  # required get_instructions + write_script, forbidden exec_datatable_sql, and a
+  # draft that contains wmill.datatable(.
+  skipJudge: true
+  judgeChecklist:
+    - writes a script (not a chat-time SQL execution) that reads the workspace data table at runtime
+    - uses the wmill.datatable() SDK to query the orders table and sum the order totals
+    - returns the total revenue from the script
+    - leaves the result as an AI draft and does not deploy or save it
+
+- id: global-test27-list-recent-runs
+  prompt: |-
+    What are the most recent runs in this workspace? Give me a quick rundown of what ran and whether it succeeded.
+  initial: ai_evals/fixtures/frontend/global/initial/jobs_seed.json
+  runtime:
+    maxTurns: 6
+  validate:
+    draftCountExactly: 0
+  toolExpect:
+    requiredToolsUsed:
+      - list_runs
+    forbiddenToolsUsed:
+      - deploy_workspace_item
+      - delete_workspace_item
+      - write_script
+  # Read-only job inspection produces no draft, so the global judge (which only
+  # sees the drafts artifact) would score it empty — validate via tool use.
+  skipJudge: true
+  judgeChecklist:
+    - lists the recent runs from the workspace rather than fabricating them
+    - summarizes each run's path and success/failure from the returned data
+
+- id: global-test28-fetch-failed-run-logs
+  prompt: |-
+    The run with id 01920000-0000-7000-8000-0000000000f1 failed. Pull its logs and tell me what went wrong.
+  initial: ai_evals/fixtures/frontend/global/initial/jobs_seed.json
+  runtime:
+    maxTurns: 6
+  validate:
+    draftCountExactly: 0
+  toolExpect:
+    requiredToolsUsed:
+      - get_job_logs
+    forbiddenToolsUsed:
+      - deploy_workspace_item
+      - delete_workspace_item
+      - write_script
+    toolCallArgs:
+      - tool: get_job_logs
+        field: id
+        stringIncludesAnyOf:
+          - 01920000-0000-7000-8000-0000000000f1
+  # Same as above — no draft is produced, so rely on the deterministic tool-use
+  # and argument checks rather than the judge.
+  skipJudge: true
+  judgeChecklist:
+    - fetches the logs for the requested job id
+    - explains the failure from the returned logs (connection refused to the upstream API)

ai_evals/cases/script.yaml

@@ -5,6 +5,9 @@
     Keep it simple and do not add external dependencies.
   initial: ai_evals/fixtures/frontend/script/initial/test1_empty_bun.json
   expected: ai_evals/fixtures/frontend/script/expected/test1_greet_user.json
+  toolExpect:
+    requiredToolsUsed:
+      - test_run_script
   judgeChecklist:
     - uses the existing `name` input
     - returns a plain greeting string
@@ -20,6 +23,7 @@
   expected: ai_evals/fixtures/frontend/script/expected/test1_greet_user.json
   toolExpect:
     requiredToolsUsed:
+      - test_run_script
       - create_schedule
     toolCallArgs:
       - tool: create_schedule
@@ -44,6 +48,7 @@
   expected: ai_evals/fixtures/frontend/script/expected/test1_greet_user.json
   toolExpect:
     requiredToolsUsed:
+      - test_run_script
       - create_trigger
     toolCallArgs:
       - tool: create_trigger

ai_evals/cli/index.ts

@@ -211,7 +211,7 @@ async function handleRun(input: {
   const summaries: Array<{
     label: string;
     passRate: number;
-    averageDurationMs: number;
+    averagePassedDurationMs: number | null;
   }> = [];
 
   for (const [index, model] of models.entries()) {
@@ -259,7 +259,7 @@ async function handleRun(input: {
     summaries.push({
       label: `${model.id} (${runModel})`,
       passRate: result.passRate,
-      averageDurationMs: result.averageDurationMs,
+      averagePassedDurationMs: result.averagePassedDurationMs ?? null,
     });
   }
 
@@ -267,7 +267,7 @@ async function handleRun(input: {
     process.stdout.write("\nModel summary\n");
     for (const summary of summaries) {
       process.stdout.write(
-        `- ${summary.label}: ${formatPercent(summary.passRate)} | ${Math.round(summary.averageDurationMs)}ms\n`,
+        `- ${summary.label}: ${formatPercent(summary.passRate)} | passed avg ${formatNullableDuration(summary.averagePassedDurationMs)}\n`,
       );
     }
   }
@@ -351,6 +351,10 @@ function formatPercent(value: number): string {
   return `${(value * 100).toFixed(1)}%`;
 }
 
+function formatNullableDuration(value: number | null): string {
+  return value === null ? "n/a" : `${Math.round(value)}ms`;
+}
+
 void main().catch((error) => {
   const message = error instanceof Error ? error.message : String(error);
   process.stderr.write(`${message}\n`);

ai_evals/core/cases.test.ts

@@ -14,6 +14,21 @@ describe("loadCases", () => {
         },
       },
     });
+    expect(caseEntry?.toolExpect).toEqual({
+      requiredToolsUsed: ["test_run_flow"],
+    });
+  });
+
+  it("loads script and flow test tool expectations", async () => {
+    const scriptCases = await loadCases("script");
+    const flowCases = await loadCases("flow");
+
+    expect(scriptCases.find((entry) => entry.id === "script-test1-greet-user")?.toolExpect).toEqual({
+      requiredToolsUsed: ["test_run_script"],
+    });
+    expect(flowCases.find((entry) => entry.id === "flow-test0-sum-two-numbers")?.toolExpect).toEqual({
+      requiredToolsUsed: ["test_run_flow"],
+    });
   });
 
   it("loads the workspace-flow preference benchmark case", async () => {
@@ -203,6 +218,34 @@ describe("loadCases", () => {
     });
   });
 
+  it("loads global active-editor eval cases", async () => {
+    const globalCases = await loadCases("global");
+    const scriptCase = globalCases.find(
+      (entry) => entry.id === "global-test12-current-live-script-edit"
+    );
+    const flowCase = globalCases.find(
+      (entry) => entry.id === "global-test13-current-live-flow-edit"
+    );
+
+    expect(scriptCase?.initialPath).toContain(
+      "ai_evals/fixtures/frontend/global/initial/current_greeting_live_script.json"
+    );
+    expect(scriptCase?.toolExpect).toMatchObject({
+      requiredToolsUsed: ["read_workspace_item"],
+    });
+    expect(flowCase?.initialPath).toContain(
+      "ai_evals/fixtures/frontend/global/initial/current_invoice_live_flow.json"
+    );
+    expect(flowCase?.validate).toMatchObject({
+      requiredDrafts: [
+        {
+          type: "flow",
+          path: "f/evals/global/current_invoice_flow",
+        },
+      ],
+    });
+  });
+
   it("loads tool expectations for workspace mutation cases", async () => {
     const scriptCases = await loadCases("script");
     const caseEntry = scriptCases.find(
@@ -210,7 +253,7 @@ describe("loadCases", () => {
     );
 
     expect(caseEntry?.toolExpect).toEqual({
-      requiredToolsUsed: ["create_schedule"],
+      requiredToolsUsed: ["test_run_script", "create_schedule"],
       toolCallArgs: [
         {
           tool: "create_schedule",

ai_evals/core/models.test.ts

@@ -2,28 +2,50 @@ import { describe, expect, it } from "bun:test";
 import { resolveEvalModel } from "./models";
 
 describe("resolveEvalModel", () => {
+  it("supports GPT-5.5 aliases for frontend evals", () => {
+    expect(resolveEvalModel("flow", "gpt-5.5").frontend).toEqual({
+      provider: "openai",
+      model: "gpt-5.5",
+    });
+    expect(resolveEvalModel("app", "gpt-55").frontend).toEqual({
+      provider: "openai",
+      model: "gpt-5.5",
+    });
+    expect(resolveEvalModel("script", "5.5").frontend).toEqual({
+      provider: "openai",
+      model: "gpt-5.5",
+    });
+  });
+
   it("supports Gemini aliases for frontend evals", () => {
-    expect(resolveEvalModel("flow", "gemini").frontend).toEqual({
-      provider: "googleai",
-      model: "gemini-2.5-flash",
-    });
-    expect(resolveEvalModel("app", "gemini-pro").frontend).toEqual({
-      provider: "googleai",
-      model: "gemini-2.5-pro",
-    });
-    expect(resolveEvalModel("script", "gemini-3-flash-preview").frontend).toEqual({
+    expect(
+      resolveEvalModel("script", "gemini-3-flash-preview").frontend,
+    ).toEqual({
       provider: "googleai",
       model: "gemini-3-flash-preview",
     });
-    expect(resolveEvalModel("flow", "gemini-3.1-pro-preview").frontend).toEqual({
-      provider: "googleai",
-      model: "gemini-3.1-pro-preview",
+    expect(resolveEvalModel("flow", "gemini-3.1-pro-preview").frontend).toEqual(
+      {
+        provider: "googleai",
+        model: "gemini-3.1-pro-preview",
+      },
+    );
+  });
+
+  it("supports DeepSeek aliases for frontend evals", () => {
+    expect(resolveEvalModel("flow", "deepseek").frontend).toEqual({
+      provider: "deepseek",
+      model: "deepseek-v4-flash",
+    });
+    expect(resolveEvalModel("script", "deepseek-v4-pro").frontend).toEqual({
+      provider: "deepseek",
+      model: "deepseek-v4-pro",
     });
   });
 
   it("rejects Gemini aliases for cli evals", () => {
-    expect(() => resolveEvalModel("cli", "gemini")).toThrow(
-      "Model gemini-flash is not supported for cli mode"
+    expect(() => resolveEvalModel("cli", "gemini-3-flash-preview")).toThrow(
+      "Model gemini-3-flash-preview is not supported for cli mode",
     );
   });
 });

ai_evals/core/models.ts

@@ -1,7 +1,7 @@
 import type { EvalMode } from "./types";
 
 export interface FrontendEvalModelConfig {
-  provider: "anthropic" | "openai" | "googleai";
+  provider: "anthropic" | "openai" | "googleai" | "deepseek";
   model: string;
 }
 
@@ -88,21 +88,12 @@ export const EVAL_MODELS: EvalModelSpec[] = [
     },
   },
   {
-    id: "gemini-flash",
-    label: "Gemini 2.5 Flash",
-    aliases: ["gemini", "gemini-flash", "gemini-2.5-flash"],
+    id: "gpt-5.5",
+    label: "GPT-5.5",
+    aliases: ["gpt-5.5", "gpt-55", "5.5"],
     frontend: {
-      provider: "googleai",
-      model: "gemini-2.5-flash",
-    },
-  },
-  {
-    id: "gemini-pro",
-    label: "Gemini 2.5 Pro",
-    aliases: ["gemini-pro", "gemini-2.5-pro"],
-    frontend: {
-      provider: "googleai",
-      model: "gemini-2.5-pro",
+      provider: "openai",
+      model: "gpt-5.5",
     },
   },
   {
@@ -117,15 +108,40 @@ export const EVAL_MODELS: EvalModelSpec[] = [
   {
     id: "gemini-3.1-pro-preview",
     label: "Gemini 3.1 Pro Preview",
-    aliases: ["gemini-3.1-pro-preview", "gemini-3.1-pro", "gemini-3-pro-preview"],
+    aliases: [
+      "gemini-3.1-pro-preview",
+      "gemini-3.1-pro",
+      "gemini-3-pro-preview",
+    ],
     frontend: {
       provider: "googleai",
       model: "gemini-3.1-pro-preview",
     },
   },
+  {
+    id: "deepseek-v4-flash",
+    label: "DeepSeek V4 Flash",
+    aliases: ["deepseek", "deepseek-v4", "deepseek-v4-flash"],
+    frontend: {
+      provider: "deepseek",
+      model: "deepseek-v4-flash",
+    },
+  },
+  {
+    id: "deepseek-v4-pro",
+    label: "DeepSeek V4 Pro",
+    aliases: ["deepseek-pro", "deepseek-v4-pro"],
+    frontend: {
+      provider: "deepseek",
+      model: "deepseek-v4-pro",
+    },
+  },
 ];
 
-export function resolveEvalModel(mode: EvalMode, alias?: string): EvalModelSpec {
+export function resolveEvalModel(
+  mode: EvalMode,
+  alias?: string,
+): EvalModelSpec {
   const spec = alias ? findEvalModel(alias) : getDefaultEvalModel(mode);
   if (!spec) {
     throw new Error(`Unknown model: ${alias}`);
@@ -152,14 +168,19 @@ export function getEvalModelHelpText(): string {
   }).join("\n");
 }
 
-export function formatRunModelLabel(mode: EvalMode, model: EvalModelSpec): string {
+export function formatRunModelLabel(
+  mode: EvalMode,
+  model: EvalModelSpec,
+): string {
   if (mode === "cli") {
     return `${model.cli!.provider}:${model.cli!.model}`;
   }
   return `${model.frontend!.provider}:${model.frontend!.model}`;
 }
 
-export function getFrontendEvalModel(model: EvalModelSpec): FrontendEvalModelConfig {
+export function getFrontendEvalModel(
+  model: EvalModelSpec,
+): FrontendEvalModelConfig {
   if (!model.frontend) {
     throw new Error(`Model ${model.id} does not support frontend evals`);
   }
@@ -180,6 +201,8 @@ function getDefaultEvalModel(mode: EvalMode): EvalModelSpec {
 function findEvalModel(alias: string): EvalModelSpec | undefined {
   const normalized = alias.trim().toLowerCase();
   return EVAL_MODELS.find((model) =>
-    [model.id, ...model.aliases].some((candidate) => candidate.toLowerCase() === normalized)
+    [model.id, ...model.aliases].some(
+      (candidate) => candidate.toLowerCase() === normalized,
+    ),
   );
 }

ai_evals/core/results.test.ts

@@ -0,0 +1,242 @@
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { describe, expect, it } from "bun:test";
+import {
+  appendHistoryRecord,
+  buildRunResult,
+  formatRunSummary,
+} from "./results";
+import type { BenchmarkCaseResult } from "./types";
+
+function caseResult(
+  attempts: BenchmarkCaseResult["attempts"],
+): BenchmarkCaseResult {
+  return {
+    id: "case-1",
+    prompt: "Do the thing",
+    attempts,
+  };
+}
+
+describe("benchmark results", () => {
+  it("keeps success cost metrics separate from failed attempts", () => {
+    const result = buildRunResult({
+      mode: "global",
+      runs: 1,
+      runModel: "model-under-test",
+      judgeModel: "judge-model",
+      caseResults: [
+        caseResult([
+          {
+            attempt: 1,
+            passed: true,
+            durationMs: 1000,
+            assistantMessageCount: 1,
+            toolCallCount: 1,
+            toolsUsed: ["edit_script"],
+            skillsInvoked: [],
+            checks: [{ name: "edited", passed: true }],
+            judgeScore: 100,
+            judgeSummary: "ok",
+            error: null,
+            tokenUsage: { prompt: 100, completion: 20, total: 120 },
+          },
+          {
+            attempt: 2,
+            passed: false,
+            durationMs: 100,
+            assistantMessageCount: 1,
+            toolCallCount: 0,
+            toolsUsed: [],
+            skillsInvoked: [],
+            checks: [{ name: "edited", passed: false }],
+            judgeScore: 10,
+            judgeSummary: "missed",
+            error: "failed",
+            tokenUsage: { prompt: 10, completion: 5, total: 15 },
+          },
+        ]),
+      ],
+    });
+
+    expect(result.attemptCount).toBe(2);
+    expect(result.passedAttempts).toBe(1);
+    expect(result.passRate).toBe(0.5);
+    expect(result.averageDurationMs).toBe(550);
+    expect(result.averagePassedDurationMs).toBe(1000);
+    expect(result.totalTokenUsage).toEqual({
+      prompt: 110,
+      completion: 25,
+      total: 135,
+    });
+    expect(result.totalPassedTokenUsage).toEqual({
+      prompt: 100,
+      completion: 20,
+      total: 120,
+    });
+    expect(result.averageTokenUsagePerAttempt).toEqual({
+      prompt: 55,
+      completion: 12.5,
+      total: 67.5,
+    });
+    expect(result.averageTokenUsagePerPassedAttempt).toEqual({
+      prompt: 100,
+      completion: 20,
+      total: 120,
+    });
+
+    const summary = formatRunSummary(result);
+    expect(summary).toContain("Average duration (passed): 1000ms");
+    expect(summary).toContain("Average tokens (passed): 120 total");
+    expect(summary).toContain("Average duration (all attempts): 550ms");
+  });
+
+  it("reports passed averages as unavailable when no attempt passes", () => {
+    const result = buildRunResult({
+      mode: "global",
+      runs: 1,
+      runModel: "model-under-test",
+      judgeModel: "judge-model",
+      caseResults: [
+        caseResult([
+          {
+            attempt: 1,
+            passed: false,
+            durationMs: 100,
+            assistantMessageCount: 1,
+            toolCallCount: 0,
+            toolsUsed: [],
+            skillsInvoked: [],
+            checks: [{ name: "edited", passed: false }],
+            judgeScore: 10,
+            judgeSummary: "missed",
+            error: "failed",
+            tokenUsage: { prompt: 10, completion: 5, total: 15 },
+          },
+        ]),
+      ],
+    });
+
+    expect(result.averagePassedDurationMs).toBeNull();
+    expect(result.totalPassedTokenUsage).toBeNull();
+    expect(result.averageTokenUsagePerPassedAttempt).toBeNull();
+    expect(formatRunSummary(result)).toContain(
+      "Average duration (passed): n/a",
+    );
+  });
+
+  it("normalizes passed token averages by passed attempts", () => {
+    const result = buildRunResult({
+      mode: "global",
+      runs: 1,
+      runModel: "model-under-test",
+      judgeModel: "judge-model",
+      caseResults: [
+        caseResult([
+          {
+            attempt: 1,
+            passed: true,
+            durationMs: 1000,
+            assistantMessageCount: 1,
+            toolCallCount: 1,
+            toolsUsed: ["edit_script"],
+            skillsInvoked: [],
+            checks: [{ name: "edited", passed: true }],
+            judgeScore: 100,
+            judgeSummary: "ok",
+            error: null,
+            tokenUsage: { prompt: 100, completion: 20, total: 120 },
+          },
+          {
+            attempt: 2,
+            passed: true,
+            durationMs: 1200,
+            assistantMessageCount: 1,
+            toolCallCount: 1,
+            toolsUsed: ["edit_script"],
+            skillsInvoked: [],
+            checks: [{ name: "edited", passed: true }],
+            judgeScore: 100,
+            judgeSummary: "ok",
+            error: null,
+            tokenUsage: null,
+          },
+        ]),
+      ],
+    });
+
+    expect(result.passedAttempts).toBe(2);
+    expect(result.totalPassedTokenUsage).toEqual({
+      prompt: 100,
+      completion: 20,
+      total: 120,
+    });
+    expect(result.averageTokenUsagePerPassedAttempt).toEqual({
+      prompt: 50,
+      completion: 10,
+      total: 60,
+    });
+  });
+
+  it("records passed-attempt metrics in history", async () => {
+    const tempDir = await mkdtemp(join(tmpdir(), "windmill-ai-evals-"));
+    try {
+      const historyPath = join(tempDir, "history.jsonl");
+      const result = buildRunResult({
+        mode: "global",
+        runs: 1,
+        runModel: "model-under-test",
+        judgeModel: "judge-model",
+        caseResults: [
+          caseResult([
+            {
+              attempt: 1,
+              passed: true,
+              durationMs: 1000,
+              assistantMessageCount: 1,
+              toolCallCount: 1,
+              toolsUsed: ["edit_script"],
+              skillsInvoked: [],
+              checks: [{ name: "edited", passed: true }],
+              judgeScore: 100,
+              judgeSummary: "ok",
+              error: null,
+              tokenUsage: { prompt: 100, completion: 20, total: 120 },
+            },
+            {
+              attempt: 2,
+              passed: false,
+              durationMs: 100,
+              assistantMessageCount: 1,
+              toolCallCount: 0,
+              toolsUsed: [],
+              skillsInvoked: [],
+              checks: [{ name: "edited", passed: false }],
+              judgeScore: 10,
+              judgeSummary: "missed",
+              error: "failed",
+              tokenUsage: { prompt: 10, completion: 5, total: 15 },
+            },
+          ]),
+        ],
+      });
+
+      await appendHistoryRecord(result, historyPath);
+      const record = JSON.parse(await readFile(historyPath, "utf8"));
+
+      expect(record.averageDurationMs).toBe(550);
+      expect(record.averagePassedDurationMs).toBe(1000);
+      expect(record.averageTokenUsagePerAttempt.total).toBe(67.5);
+      expect(record.averageTokenUsagePerPassedAttempt.total).toBe(120);
+      expect(record.cases[0].averageDurationMs).toBe(550);
+      expect(record.cases[0].averagePassedDurationMs).toBe(1000);
+      expect(record.cases[0].averageTokenUsagePerAttempt.total).toBe(67.5);
+      expect(record.cases[0].averageTokenUsagePerPassedAttempt.total).toBe(
+        120,
+      );
+    } finally {
+      await rm(tempDir, { recursive: true, force: true });
+    }
+  });
+});

ai_evals/core/results.ts

@@ -4,12 +4,20 @@ import { execFileSync } from "node:child_process";
 import { getAiEvalsRoot, getRepoRoot } from "./cases";
 import type {
   BenchmarkArtifactFile,
+  BenchmarkAttemptResult,
   BenchmarkCaseResult,
   BenchmarkRunResult,
   BenchmarkTokenUsage,
   EvalMode,
 } from "./types";
 
+type AttemptAggregate = {
+  attemptCount: number;
+  durationTotal: number;
+  tokenUsageAttemptCount: number;
+  tokenUsageTotal: BenchmarkTokenUsage | null;
+};
+
 export async function writeRunResult(
   result: BenchmarkRunResult,
   outputPath?: string,
@@ -77,36 +85,12 @@ export function buildRunResult(input: {
   judgeModel: string | null;
   caseResults: BenchmarkCaseResult[];
 }): BenchmarkRunResult {
-  const attemptCount = input.caseResults.reduce(
-    (sum, entry) => sum + entry.attempts.length,
-    0,
-  );
-  const passedAttempts = input.caseResults.reduce(
-    (sum, entry) =>
-      sum + entry.attempts.filter((attempt) => attempt.passed).length,
-    0,
-  );
-  const durationTotal = input.caseResults.reduce(
-    (sum, entry) =>
-      sum +
-      entry.attempts.reduce((inner, attempt) => inner + attempt.durationMs, 0),
-    0,
-  );
-  const tokenUsageTotal = input.caseResults.reduce<BenchmarkTokenUsage | null>(
-    (sum, entry) => {
-      for (const attempt of entry.attempts) {
-        if (!attempt.tokenUsage) {
-          continue;
-        }
-        sum ??= { prompt: 0, completion: 0, total: 0 };
-        sum.prompt += attempt.tokenUsage.prompt;
-        sum.completion += attempt.tokenUsage.completion;
-        sum.total += attempt.tokenUsage.total;
-      }
-      return sum;
-    },
-    null,
-  );
+  const attempts = input.caseResults.flatMap((entry) => entry.attempts);
+  const passedAttemptResults = attempts.filter((attempt) => attempt.passed);
+  const attemptAggregate = aggregateAttempts(attempts);
+  const passedAttemptAggregate = aggregateAttempts(passedAttemptResults);
+  const attemptCount = attemptAggregate.attemptCount;
+  const passedAttempts = passedAttemptAggregate.attemptCount;
 
   return {
     version: 1,
@@ -120,16 +104,19 @@ export function buildRunResult(input: {
     attemptCount,
     passedAttempts,
     passRate: attemptCount === 0 ? 0 : passedAttempts / attemptCount,
-    averageDurationMs: attemptCount === 0 ? 0 : durationTotal / attemptCount,
-    totalTokenUsage: tokenUsageTotal,
+    averageDurationMs:
+      attemptCount === 0 ? 0 : attemptAggregate.durationTotal / attemptCount,
+    averagePassedDurationMs: averageDuration(passedAttemptAggregate),
+    totalTokenUsage: attemptAggregate.tokenUsageTotal,
+    totalPassedTokenUsage: passedAttemptAggregate.tokenUsageTotal,
     averageTokenUsagePerAttempt:
-      attemptCount === 0 || !tokenUsageTotal
+      attemptCount === 0
         ? null
-        : {
-            prompt: tokenUsageTotal.prompt / attemptCount,
-            completion: tokenUsageTotal.completion / attemptCount,
-            total: tokenUsageTotal.total / attemptCount,
-          },
+        : averageTokenUsage(attemptAggregate, attemptCount),
+    averageTokenUsagePerPassedAttempt: averageTokenUsage(
+      passedAttemptAggregate,
+      passedAttempts,
+    ),
     cases: input.caseResults,
   };
 }
@@ -138,9 +125,25 @@ export function formatRunSummary(result: BenchmarkRunResult): string {
   const lines = [
     `${result.mode} benchmark complete`,
     `Pass rate: ${formatPercent(result.passRate)} (${result.passedAttempts}/${result.attemptCount})`,
-    `Average duration: ${Math.round(result.averageDurationMs)}ms`,
+    `Average duration (passed): ${formatNullableDuration(result.averagePassedDurationMs ?? null)}`,
   ];
 
+  if (result.averageTokenUsagePerPassedAttempt) {
+    lines.push(
+      `Average tokens (passed): ${formatTokenUsage(result.averageTokenUsagePerPassedAttempt)}`,
+    );
+  }
+  if (result.passedAttempts < result.attemptCount) {
+    lines.push(
+      `Average duration (all attempts): ${Math.round(result.averageDurationMs)}ms`,
+    );
+    if (result.averageTokenUsagePerAttempt) {
+      lines.push(
+        `Average tokens (all attempts): ${formatTokenUsage(result.averageTokenUsagePerAttempt)}`,
+      );
+    }
+  }
+
   const failures = collectFailures(result);
   if (failures.length > 0) {
     lines.push("Failures:");
@@ -172,6 +175,60 @@ function collectFailures(result: BenchmarkRunResult): string[] {
   return failures;
 }
 
+function aggregateAttempts(attempts: BenchmarkAttemptResult[]): AttemptAggregate {
+  const aggregate: AttemptAggregate = {
+    attemptCount: attempts.length,
+    durationTotal: 0,
+    tokenUsageAttemptCount: 0,
+    tokenUsageTotal: null,
+  };
+
+  for (const attempt of attempts) {
+    aggregate.durationTotal += attempt.durationMs;
+    if (!attempt.tokenUsage) {
+      continue;
+    }
+    aggregate.tokenUsageAttemptCount += 1;
+    aggregate.tokenUsageTotal ??= { prompt: 0, completion: 0, total: 0 };
+    aggregate.tokenUsageTotal.prompt += attempt.tokenUsage.prompt;
+    aggregate.tokenUsageTotal.completion += attempt.tokenUsage.completion;
+    aggregate.tokenUsageTotal.total += attempt.tokenUsage.total;
+  }
+
+  return aggregate;
+}
+
+function averageDuration(aggregate: AttemptAggregate): number | null {
+  return aggregate.attemptCount === 0
+    ? null
+    : aggregate.durationTotal / aggregate.attemptCount;
+}
+
+function averageTokenUsage(
+  aggregate: AttemptAggregate,
+  denominator: number,
+): BenchmarkTokenUsage | null {
+  if (denominator === 0 || !aggregate.tokenUsageTotal) {
+    return null;
+  }
+  return {
+    prompt: aggregate.tokenUsageTotal.prompt / denominator,
+    completion: aggregate.tokenUsageTotal.completion / denominator,
+    total: aggregate.tokenUsageTotal.total / denominator,
+  };
+}
+
+function formatNullableDuration(value: number | null): string {
+  return value === null ? "n/a" : `${Math.round(value)}ms`;
+}
+
+function formatTokenUsage(value: BenchmarkTokenUsage): string {
+  const total = Math.round(value.total);
+  const prompt = Math.round(value.prompt);
+  const completion = Math.round(value.completion);
+  return `${total} total (${prompt} prompt, ${completion} completion)`;
+}
+
 function defaultFileName(mode: EvalMode): string {
   return `${new Date().toISOString().replaceAll(":", "-")}__${mode}.json`;
 }
@@ -252,12 +309,15 @@ function toHistoryRecord(result: BenchmarkRunResult) {
     passedAttempts: result.passedAttempts,
     passRate: result.passRate,
     averageDurationMs: result.averageDurationMs,
+    averagePassedDurationMs: result.averagePassedDurationMs ?? null,
     averageJudgeScore:
       judgeScores.length === 0
         ? null
         : judgeScores.reduce((sum, score) => sum + score, 0) /
           judgeScores.length,
     averageTokenUsagePerAttempt: result.averageTokenUsagePerAttempt ?? null,
+    averageTokenUsagePerPassedAttempt:
+      result.averageTokenUsagePerPassedAttempt ?? null,
     failedCaseIds: Array.from(
       new Set(
         result.cases
@@ -268,31 +328,15 @@ function toHistoryRecord(result: BenchmarkRunResult) {
       ),
     ),
     cases: result.cases.map((caseResult) => {
-      const attemptCount = caseResult.attempts.length;
-      const passedAttempts = caseResult.attempts.filter(
-        (attempt) => attempt.passed,
-      ).length;
-      const totalDurationMs = caseResult.attempts.reduce(
-        (sum, attempt) => sum + attempt.durationMs,
-        0,
+      const attemptAggregate = aggregateAttempts(caseResult.attempts);
+      const passedAttemptAggregate = aggregateAttempts(
+        caseResult.attempts.filter((attempt) => attempt.passed),
       );
+      const attemptCount = attemptAggregate.attemptCount;
+      const passedAttempts = passedAttemptAggregate.attemptCount;
       const judgeScores = caseResult.attempts.flatMap((attempt) =>
         typeof attempt.judgeScore === "number" ? [attempt.judgeScore] : [],
       );
-      const totalTokenUsage =
-        caseResult.attempts.reduce<BenchmarkTokenUsage | null>(
-          (sum, attempt) => {
-            if (!attempt.tokenUsage) {
-              return sum;
-            }
-            sum ??= { prompt: 0, completion: 0, total: 0 };
-            sum.prompt += attempt.tokenUsage.prompt;
-            sum.completion += attempt.tokenUsage.completion;
-            sum.total += attempt.tokenUsage.total;
-            return sum;
-          },
-          null,
-        );
 
       return {
         id: caseResult.id,
@@ -300,20 +344,23 @@ function toHistoryRecord(result: BenchmarkRunResult) {
         passedAttempts,
         passRate: attemptCount === 0 ? 0 : passedAttempts / attemptCount,
         averageDurationMs:
-          attemptCount === 0 ? 0 : totalDurationMs / attemptCount,
+          attemptCount === 0
+            ? 0
+            : attemptAggregate.durationTotal / attemptCount,
+        averagePassedDurationMs: averageDuration(passedAttemptAggregate),
         averageJudgeScore:
           judgeScores.length === 0
             ? null
             : judgeScores.reduce((sum, score) => sum + score, 0) /
               judgeScores.length,
         averageTokenUsagePerAttempt:
-          attemptCount === 0 || !totalTokenUsage
+          attemptCount === 0
             ? null
-            : {
-                prompt: totalTokenUsage.prompt / attemptCount,
-                completion: totalTokenUsage.completion / attemptCount,
-                total: totalTokenUsage.total / attemptCount,
-              },
+            : averageTokenUsage(attemptAggregate, attemptCount),
+        averageTokenUsagePerPassedAttempt: averageTokenUsage(
+          passedAttemptAggregate,
+          passedAttempts,
+        ),
       };
     }),
   };

ai_evals/core/types.ts

@@ -110,7 +110,9 @@ export interface AppValidationSpec {
 
 export interface GlobalDraftRequirement {
   type: string;
-  path: string;
+  path?: string;
+  pathIncludes?: string[];
+  pathStartsWith?: string;
   triggerKind?: string;
   language?: string;
   summaryIncludes?: string[];
@@ -153,6 +155,15 @@ export interface ToolCallArgumentRule {
   field: string;
   stringStartsWithAnyOf?: string[];
   stringMustNotStartWithAnyOf?: string[];
+  /**
+   * Case-insensitive "contains", existential over calls: at least one recorded
+   * call to `tool` must have `field` containing one of these substrings. Other
+   * calls to the same tool may do anything. Use instead of `stringStartsWithAnyOf`
+   * (which is universal over calls) when the meaningful token can appear anywhere
+   * in the value and the model may make additional, unrelated calls to the same
+   * tool — e.g. SQL where a mutation is mixed with verification SELECTs.
+   */
+  stringIncludesAnyOf?: string[];
 }
 
 export interface ToolValidationSpec {
@@ -324,8 +335,11 @@ export interface BenchmarkRunResult {
   passedAttempts: number;
   passRate: number;
   averageDurationMs: number;
+  averagePassedDurationMs?: number | null;
   totalTokenUsage?: BenchmarkTokenUsage | null;
+  totalPassedTokenUsage?: BenchmarkTokenUsage | null;
   averageTokenUsagePerAttempt?: BenchmarkTokenUsage | null;
+  averageTokenUsagePerPassedAttempt?: BenchmarkTokenUsage | null;
   artifactsPath?: string | null;
   cases: BenchmarkCaseResult[];
 }

ai_evals/core/validators.test.ts

@@ -140,6 +140,111 @@ describe("validateToolExpectations", () => {
       details: "tools used: write_script, deploy_workspace_item",
     });
   });
+
+  it("accepts a stringIncludesAnyOf substring regardless of case or position", () => {
+    const checks = validateToolExpectations({
+      run: {
+        success: true,
+        actual: {},
+        assistantMessageCount: 1,
+        toolCallCount: 1,
+        toolsUsed: ["exec_datatable_sql"],
+        toolCallDetails: [
+          {
+            name: "exec_datatable_sql",
+            arguments: {
+              sql: "WITH recent AS (SELECT * FROM orders) SELECT count(*) FROM recent",
+            },
+          },
+        ],
+        skillsInvoked: [],
+      },
+      toolExpect: {
+        requiredToolsUsed: ["exec_datatable_sql"],
+        toolCallArgs: [
+          {
+            tool: "exec_datatable_sql",
+            field: "sql",
+            stringIncludesAnyOf: ["select"],
+          },
+        ],
+      },
+    });
+
+    expect(checks.every((check) => check.passed)).toBe(true);
+  });
+
+  it("accepts stringIncludesAnyOf when only one of several calls matches", () => {
+    // Existential: a mutation mixed with verification SELECTs still passes.
+    const checks = validateToolExpectations({
+      run: {
+        success: true,
+        actual: {},
+        assistantMessageCount: 1,
+        toolCallCount: 2,
+        toolsUsed: ["exec_datatable_sql"],
+        toolCallDetails: [
+          {
+            name: "exec_datatable_sql",
+            arguments: { sql: "UPDATE orders SET status = 'shipped' WHERE id = 2" },
+          },
+          {
+            name: "exec_datatable_sql",
+            arguments: { sql: "SELECT * FROM orders WHERE id = 2" },
+          },
+        ],
+        skillsInvoked: [],
+      },
+      toolExpect: {
+        toolCallArgs: [
+          {
+            tool: "exec_datatable_sql",
+            field: "sql",
+            stringIncludesAnyOf: ["insert into", "update"],
+          },
+        ],
+      },
+    });
+
+    expect(checks.every((check) => check.passed)).toBe(true);
+  });
+
+  it("rejects stringIncludesAnyOf when no call matches any substring", () => {
+    const checks = validateToolExpectations({
+      run: {
+        success: true,
+        actual: {},
+        assistantMessageCount: 1,
+        toolCallCount: 1,
+        toolsUsed: ["exec_datatable_sql"],
+        toolCallDetails: [
+          {
+            name: "exec_datatable_sql",
+            arguments: {
+              sql: "DROP TABLE orders",
+            },
+          },
+        ],
+        skillsInvoked: [],
+      },
+      toolExpect: {
+        toolCallArgs: [
+          {
+            tool: "exec_datatable_sql",
+            field: "sql",
+            stringIncludesAnyOf: ["insert into", "update"],
+          },
+        ],
+      },
+    });
+
+    expect(checks).toContainEqual({
+      name: "exec_datatable_sql.sql includes a required substring",
+      passed: false,
+      details:
+        'accepted substrings: insert into, update; values: "DROP TABLE orders"',
+    });
+  });
 });
 
 describe("validateGlobalState", () => {
@@ -195,6 +300,69 @@ describe("validateGlobalState", () => {
     });
   });
 
+  it("accepts a required script draft without an exact path", () => {
+    const checks = validateGlobalState({
+      actual: {
+        drafts: [
+          {
+            type: "script",
+            path: "f/team_tools/friendly_greeting",
+            language: "bun",
+            summary: "Friendly greeting helper",
+            value:
+              "export async function main(name: string) {\n  return `Hello, ${name}!`\n}\n",
+            isDraft: true,
+          },
+        ],
+      },
+      validate: {
+        draftCountExactly: 1,
+        requiredDrafts: [
+          {
+            type: "script",
+            pathIncludes: ["greeting"],
+            language: "bun",
+            summaryIncludes: ["Friendly"],
+            valueIncludes: ["Hello"],
+          },
+        ],
+      },
+    });
+
+    expect(checks.every((check) => check.passed)).toBe(true);
+  });
+
+  it("reports flexible global draft path filters when no draft matches", () => {
+    const checks = validateGlobalState({
+      actual: {
+        drafts: [
+          {
+            type: "script",
+            path: "f/team_tools/friendly_greeting",
+            language: "bun",
+            value:
+              "export async function main(name: string) {\n  return `Hello, ${name}!`\n}\n",
+            isDraft: true,
+          },
+        ],
+      },
+      validate: {
+        requiredDrafts: [
+          {
+            type: "script",
+            pathIncludes: ["invoice"],
+          },
+        ],
+      },
+    });
+
+    expect(checks).toContainEqual({
+      name: "global includes script draft (path includes invoice)",
+      passed: false,
+      details: "drafts: script:f/team_tools/friendly_greeting",
+    });
+  });
+
   it("does not require a TypeScript entrypoint for non-TypeScript script drafts", () => {
     const checks = validateGlobalState({
       actual: {

ai_evals/core/validators.ts

@@ -222,6 +222,25 @@ export function validateToolExpectations(input: {
         )
       );
     }
+
+    if (rule.stringIncludesAnyOf && rule.stringIncludesAnyOf.length > 0) {
+      // Existential: at least one call must contain one of the substrings.
+      // Other calls to the same tool may do anything — this suits SQL, where a
+      // model mixes the requested statement (e.g. an UPDATE) with verification
+      // SELECTs that would otherwise fail an "all calls" check.
+      const needles = rule.stringIncludesAnyOf.map((needle) => needle.toLowerCase());
+      const hasMatch = values.some(
+        (value) =>
+          typeof value === "string" && needles.some((needle) => value.toLowerCase().includes(needle))
+      );
+      checks.push(
+        check(
+          `${rule.tool}.${rule.field} includes a required substring`,
+          hasMatch,
+          `accepted substrings: ${rule.stringIncludesAnyOf.join(", ")}; values: ${summarizeToolValues(values)}`
+        )
+      );
+    }
   }
 
   return checks;
@@ -315,10 +334,11 @@ export function validateGlobalState(input: {
   }
 
   for (const required of validate.requiredDrafts ?? []) {
-    const draft = findGlobalDraft(drafts, required.type, required.path, required.triggerKind);
+    const requirementLabel = formatGlobalDraftRequirement(required);
+    const draft = findGlobalDraft(drafts, required);
     checks.push(
       check(
-        `global includes ${required.type} draft ${required.path}`,
+        `global includes ${requirementLabel}`,
         Boolean(draft),
         summarizeGlobalDrafts(drafts)
       )
@@ -330,7 +350,7 @@ export function validateGlobalState(input: {
     if (required.language !== undefined) {
       checks.push(
         check(
-          `${required.type} draft ${required.path} uses ${required.language}`,
+          `${requirementLabel} uses ${required.language}`,
           draft.language === required.language,
           `language=${draft.language ?? "(none)"}`
         )
@@ -340,7 +360,7 @@ export function validateGlobalState(input: {
     for (const snippet of required.summaryIncludes ?? []) {
       checks.push(
         check(
-          `${required.type} draft ${required.path} summary includes '${snippet}'`,
+          `${requirementLabel} summary includes '${snippet}'`,
           normalizeText(draft.summary ?? "").includes(normalizeText(snippet)),
           `summary=${draft.summary ?? ""}`
         )
@@ -351,7 +371,7 @@ export function validateGlobalState(input: {
     for (const snippet of required.valueIncludes ?? []) {
       checks.push(
         check(
-          `${required.type} draft ${required.path} value includes '${snippet}'`,
+          `${requirementLabel} value includes '${snippet}'`,
           normalizeText(valueText).includes(normalizeText(snippet)),
           truncateForDetails(valueText)
         )
@@ -361,7 +381,7 @@ export function validateGlobalState(input: {
     for (const snippet of required.valueExcludes ?? []) {
       checks.push(
         check(
-          `${required.type} draft ${required.path} value excludes '${snippet}'`,
+          `${requirementLabel} value excludes '${snippet}'`,
           !normalizeText(valueText).includes(normalizeText(snippet)),
           truncateForDetails(valueText)
         )
@@ -373,7 +393,7 @@ export function validateGlobalState(input: {
     checks.push(
       check(
         `global does not include ${forbidden.type} draft ${forbidden.path}`,
-        !findGlobalDraft(drafts, forbidden.type, forbidden.path, forbidden.triggerKind),
+        !findGlobalDraft(drafts, forbidden),
         summarizeGlobalDrafts(drafts)
       )
     );
@@ -615,16 +635,100 @@ function summarizeProblems(problems: string[], limit = 5): string | undefined {
 
 function findGlobalDraft(
   drafts: GlobalDraft[],
-  type: string,
-  path: string,
-  triggerKind?: string
+  requirement: {
+    type: string;
+    path?: string;
+    pathIncludes?: string[];
+    pathStartsWith?: string;
+    triggerKind?: string;
+    summaryIncludes?: string[];
+    valueIncludes?: string[];
+    valueExcludes?: string[];
+  }
 ): GlobalDraft | undefined {
-  return drafts.find(
-    (draft) =>
-      draft.type === type &&
-      draft.path === path &&
-      (triggerKind === undefined || draft.triggerKind === triggerKind)
+  const candidates = drafts.filter((draft) =>
+    globalDraftMatchesLocator(draft, requirement)
   );
+  return (
+    candidates.find((draft) => globalDraftMatchesContent(draft, requirement)) ??
+    candidates[0]
+  );
+}
+
+function globalDraftMatchesLocator(
+  draft: GlobalDraft,
+  requirement: {
+    type: string;
+    path?: string;
+    pathIncludes?: string[];
+    pathStartsWith?: string;
+    triggerKind?: string;
+  }
+): boolean {
+  return (
+    draft.type === requirement.type &&
+    (requirement.path === undefined || draft.path === requirement.path) &&
+    (requirement.pathStartsWith === undefined ||
+      draft.path.startsWith(requirement.pathStartsWith)) &&
+    (requirement.pathIncludes ?? []).every((snippet) =>
+      normalizeText(draft.path).includes(normalizeText(snippet))
+    ) &&
+    (requirement.triggerKind === undefined ||
+      draft.triggerKind === requirement.triggerKind)
+  );
+}
+
+function globalDraftMatchesContent(
+  draft: GlobalDraft,
+  requirement: {
+    summaryIncludes?: string[];
+    valueIncludes?: string[];
+    valueExcludes?: string[];
+  }
+): boolean {
+  const summary = normalizeText(draft.summary ?? "");
+  const value = normalizeText(stringifyGlobalDraftValue(draft.value));
+  return (
+    (requirement.summaryIncludes ?? []).every((snippet) =>
+      summary.includes(normalizeText(snippet))
+    ) &&
+    (requirement.valueIncludes ?? []).every((snippet) =>
+      value.includes(normalizeText(snippet))
+    ) &&
+    (requirement.valueExcludes ?? []).every(
+      (snippet) => !value.includes(normalizeText(snippet))
+    )
+  );
+}
+
+function formatGlobalDraftRequirement(
+  requirement: {
+    type: string;
+    path?: string;
+    pathIncludes?: string[];
+    pathStartsWith?: string;
+    triggerKind?: string;
+  }
+): string {
+  const typeLabel =
+    requirement.triggerKind === undefined
+      ? requirement.type
+      : `${requirement.triggerKind} ${requirement.type}`;
+  if (requirement.path !== undefined) {
+    return `${typeLabel} draft ${requirement.path}`;
+  }
+
+  const filters = [
+    ...(requirement.pathStartsWith === undefined
+      ? []
+      : [`path starts with ${requirement.pathStartsWith}`]),
+    ...(requirement.pathIncludes ?? []).map(
+      (snippet) => `path includes ${snippet}`
+    ),
+  ];
+  return filters.length === 0
+    ? `${typeLabel} draft`
+    : `${typeLabel} draft (${filters.join(", ")})`;
 }
 
 function summarizeGlobalDrafts(drafts: GlobalDraft[]): string {

ai_evals/fixtures/frontend/global/initial/current_greeting_live_script.json

@@ -0,0 +1,66 @@
+{
+  "workspace": {
+    "scripts": [
+      {
+        "path": "f/evals/global/format_greeting",
+        "summary": "Format a deployed greeting",
+        "description": "Returns a plain greeting for a provided name.",
+        "language": "bun",
+        "schema": {
+          "$schema": "https://json-schema.org/draft/2020-12/schema",
+          "type": "object",
+          "properties": {
+            "name": {
+              "type": "string"
+            }
+          },
+          "required": ["name"]
+        },
+        "content": "export async function main(name: string) {\n  return `Hello, ${name}`\n}\n"
+      },
+      {
+        "path": "f/evals/global/format_greeting_archive",
+        "summary": "Archived greeting formatter",
+        "description": "Older greeting formatter kept for reference.",
+        "language": "bun",
+        "schema": {
+          "$schema": "https://json-schema.org/draft/2020-12/schema",
+          "type": "object",
+          "properties": {
+            "name": {
+              "type": "string"
+            }
+          },
+          "required": ["name"]
+        },
+        "content": "export async function main(name: string) {\n  return `Hi, ${name}`\n}\n"
+      }
+    ]
+  },
+  "liveEditorDrafts": [
+    {
+      "type": "script",
+      "storagePath": "f/evals/global/current_greeting",
+      "effectivePath": "f/evals/global/current_greeting",
+      "value": {
+        "path": "f/evals/global/current_greeting",
+        "summary": "Open greeting formatter",
+        "description": "Formats a greeting in the live editor.",
+        "language": "bun",
+        "schema": {
+          "$schema": "https://json-schema.org/draft/2020-12/schema",
+          "type": "object",
+          "properties": {
+            "name": {
+              "type": "string"
+            }
+          },
+          "required": ["name"]
+        },
+        "content": "export async function main(name: string) {\n  return `Hello, ${name}`\n}\n",
+        "is_template": false,
+        "kind": "script"
+      }
+    }
+  ]
+}

ai_evals/fixtures/frontend/global/initial/current_invoice_live_flow.json

@@ -0,0 +1,118 @@
+{
+  "workspace": {
+    "flows": [
+      {
+        "path": "f/evals/global/process_invoice",
+        "summary": "Deployed invoice processor",
+        "description": "Calculates invoice totals from a subtotal.",
+        "schema": {
+          "$schema": "https://json-schema.org/draft/2020-12/schema",
+          "type": "object",
+          "properties": {
+            "subtotal": {
+              "type": "number"
+            }
+          },
+          "required": ["subtotal"]
+        },
+        "value": {
+          "modules": [
+            {
+              "id": "calculate_total",
+              "summary": "Calculate total from subtotal",
+              "value": {
+                "type": "rawscript",
+                "language": "bun",
+                "content": "export async function main(subtotal: number) {\n  return { subtotal, total: subtotal }\n}\n",
+                "input_transforms": {
+                  "subtotal": {
+                    "type": "javascript",
+                    "expr": "flow_input.subtotal"
+                  }
+                }
+              }
+            }
+          ]
+        }
+      },
+      {
+        "path": "f/evals/global/process_refund",
+        "summary": "Refund processor",
+        "description": "Calculates refund totals.",
+        "schema": {
+          "$schema": "https://json-schema.org/draft/2020-12/schema",
+          "type": "object",
+          "properties": {
+            "subtotal": {
+              "type": "number"
+            }
+          },
+          "required": ["subtotal"]
+        },
+        "value": {
+          "modules": [
+            {
+              "id": "calculate_total",
+              "summary": "Calculate refund total",
+              "value": {
+                "type": "rawscript",
+                "language": "bun",
+                "content": "export async function main(subtotal: number) {\n  return { subtotal, total: subtotal }\n}\n",
+                "input_transforms": {
+                  "subtotal": {
+                    "type": "javascript",
+                    "expr": "flow_input.subtotal"
+                  }
+                }
+              }
+            }
+          ]
+        }
+      }
+    ]
+  },
+  "liveEditorDrafts": [
+    {
+      "type": "flow",
+      "storagePath": "f/evals/global/current_invoice_flow",
+      "effectivePath": "f/evals/global/current_invoice_flow",
+      "value": {
+        "path": "f/evals/global/current_invoice_flow",
+        "summary": "Open invoice processor",
+        "schema": {
+          "$schema": "https://json-schema.org/draft/2020-12/schema",
+          "type": "object",
+          "properties": {
+            "subtotal": {
+              "type": "number"
+            }
+          },
+          "required": ["subtotal"]
+        },
+        "value": {
+          "modules": [
+            {
+              "id": "calculate_total",
+              "summary": "Calculate total from subtotal",
+              "value": {
+                "type": "rawscript",
+                "language": "bun",
+                "content": "export async function main(subtotal: number) {\n  return { subtotal, total: subtotal }\n}\n",
+                "input_transforms": {
+                  "subtotal": {
+                    "type": "javascript",
+                    "expr": "flow_input.subtotal"
+                  }
+                }
+              }
+            }
+          ]
+        },
+        "edited_by": "",
+        "edited_at": "",
+        "archived": false,
+        "extra_perms": {}
+      }
+    }
+  ]
+}

ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json

@@ -0,0 +1,39 @@
+{
+  "workspace": {
+    "datatables": [
+      {
+        "datatable_name": "main",
+        "schemas": {
+          "public": {
+            "orders": {
+              "columns": {
+                "id": "int4",
+                "customer_id": "int4",
+                "total": "numeric",
+                "status": "text",
+                "created_at": "timestamptz"
+              },
+              "rows": [
+                { "id": 1, "customer_id": 1, "total": 42.5, "status": "shipped", "created_at": "2026-05-01T10:00:00Z" },
+                { "id": 2, "customer_id": 2, "total": 19.99, "status": "pending", "created_at": "2026-05-02T11:30:00Z" },
+                { "id": 3, "customer_id": 1, "total": 88, "status": "shipped", "created_at": "2026-05-03T09:15:00Z" }
+              ]
+            },
+            "customers": {
+              "columns": {
+                "id": "int4",
+                "name": "text",
+                "email": "text",
+                "tier": "text"
+              },
+              "rows": [
+                { "id": 1, "name": "Alice", "email": "alice@example.com", "tier": "gold" },
+                { "id": 2, "name": "Bob", "email": "bob@example.com", "tier": "silver" }
+              ]
+            }
+          }
+        }
+      }
+    ]
+  }
+}

ai_evals/fixtures/frontend/global/initial/jobs_seed.json

@@ -0,0 +1,30 @@
+{
+  "workspace": {
+    "jobs": [
+      {
+        "id": "01920000-0000-7000-8000-0000000000f1",
+        "scriptPath": "f/etl/sync_customers",
+        "jobKind": "script",
+        "createdBy": "alice",
+        "success": false,
+        "logs": "Starting customer sync...\nFetched 0 records\nERROR: connection refused to https://api.upstream.example.com\n  at fetchCustomers (sync_customers.ts:42)\nJob failed with exit code 1"
+      },
+      {
+        "id": "01920000-0000-7000-8000-0000000000f2",
+        "scriptPath": "f/reports/daily_digest",
+        "jobKind": "script",
+        "createdBy": "bob",
+        "success": true,
+        "logs": "Generating daily digest...\nDigest emailed to 12 recipients\nDone in 1.2s"
+      },
+      {
+        "id": "01920000-0000-7000-8000-0000000000f3",
+        "scriptPath": "f/billing/charge_invoices",
+        "jobKind": "flow",
+        "createdBy": "alice",
+        "success": true,
+        "logs": "Processing invoices...\nCharged 8 invoices totalling $1,240.00\nDone"
+      }
+    ]
+  }
+}

ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json

@@ -0,0 +1,40 @@
+{
+  "workspace": {
+    "flows": [
+      {
+        "path": "f/evals/global/process_invoice",
+        "summary": "Process an invoice subtotal",
+        "description": "Calculates invoice totals from a subtotal.",
+        "schema": {
+          "$schema": "https://json-schema.org/draft/2020-12/schema",
+          "type": "object",
+          "properties": {
+            "subtotal": {
+              "type": "number"
+            }
+          },
+          "required": ["subtotal"]
+        },
+        "value": {
+          "modules": [
+            {
+              "id": "calculate_total",
+              "summary": "Calculate total from subtotal",
+              "value": {
+                "type": "rawscript",
+                "language": "bun",
+                "content": "export async function main(subtotal: number) {\n  return { subtotal, total: subtotal }\n}\n",
+                "input_transforms": {
+                  "subtotal": {
+                    "type": "javascript",
+                    "expr": "flow_input.subtotal"
+                  }
+                }
+              }
+            }
+          ]
+        }
+      }
+    ]
+  }
+}

ai_evals/fixtures/frontend/global/initial/report_digest_script.json

@@ -0,0 +1,23 @@
+{
+  "workspace": {
+    "scripts": [
+      {
+        "path": "f/evals/global/send_report_digest",
+        "summary": "Build and send the eval report digest",
+        "description": "Returns a dry-run summary for eval report digest notifications.",
+        "language": "bun",
+        "schema": {
+          "$schema": "https://json-schema.org/draft/2020-12/schema",
+          "type": "object",
+          "properties": {
+            "dry_run": {
+              "type": "boolean"
+            }
+          },
+          "required": ["dry_run"]
+        },
+        "content": "export async function main(dry_run: boolean) {\n  return { dry_run, sent: !dry_run, message: dry_run ? 'Preview digest' : 'Digest sent' }\n}\n"
+      }
+    ]
+  }
+}

ai_evals/modes/frontendCommon.test.ts

@@ -5,12 +5,14 @@ const ORIGINAL_ENV = {
   ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY,
   OPENAI_API_KEY: process.env.OPENAI_API_KEY,
   GEMINI_API_KEY: process.env.GEMINI_API_KEY,
+  DEEPSEEK_API_KEY: process.env.DEEPSEEK_API_KEY,
 };
 
 afterEach(() => {
   process.env.ANTHROPIC_API_KEY = ORIGINAL_ENV.ANTHROPIC_API_KEY;
   process.env.OPENAI_API_KEY = ORIGINAL_ENV.OPENAI_API_KEY;
   process.env.GEMINI_API_KEY = ORIGINAL_ENV.GEMINI_API_KEY;
+  process.env.DEEPSEEK_API_KEY = ORIGINAL_ENV.DEEPSEEK_API_KEY;
 });
 
 describe("getFrontendApiKey", () => {
@@ -19,10 +21,15 @@ describe("getFrontendApiKey", () => {
     expect(getFrontendApiKey("googleai")).toBe("gemini-test-key");
   });
 
+  it("reads the DeepSeek API key for deepseek models", () => {
+    process.env.DEEPSEEK_API_KEY = "deepseek-test-key";
+    expect(getFrontendApiKey("deepseek")).toBe("deepseek-test-key");
+  });
+
   it("throws a provider-specific error when the key is missing", () => {
     delete process.env.GEMINI_API_KEY;
     expect(() => getFrontendApiKey("googleai")).toThrow(
-      "GEMINI_API_KEY is required for frontend evals"
+      "GEMINI_API_KEY is required for frontend evals",
     );
   });
 });

ai_evals/modes/frontendCommon.ts

@@ -1,12 +1,16 @@
 import type { FrontendEvalModelConfig } from "../core/models";
 
-export function getFrontendApiKey(provider: FrontendEvalModelConfig["provider"]): string {
+export function getFrontendApiKey(
+  provider: FrontendEvalModelConfig["provider"],
+): string {
   const envName =
     provider === "anthropic"
       ? "ANTHROPIC_API_KEY"
       : provider === "googleai"
         ? "GEMINI_API_KEY"
-        : "OPENAI_API_KEY";
+        : provider === "deepseek"
+          ? "DEEPSEEK_API_KEY"
+          : "OPENAI_API_KEY";
   const apiKey = process.env[envName];
   if (!apiKey) {
     throw new Error(`${envName} is required for frontend evals`);

ai_evals/modes/global.ts

@@ -1,5 +1,8 @@
 import { readFile } from "node:fs/promises";
-import { runGlobalEval } from "../adapters/frontend/core/global/globalEvalRunner";
+import {
+  runGlobalEval,
+  type GlobalLiveEditorDraftFixture,
+} from "../adapters/frontend/core/global/globalEvalRunner";
 import type { BenchmarkWorkspaceRunnables } from "../adapters/frontend/mockBackend";
 import type { FrontendEvalModelConfig } from "../core/models";
 import type { BenchmarkArtifactFile, GlobalValidationSpec, ModeRunner } from "../core/types";
@@ -9,6 +12,7 @@ import { getFrontendApiKey } from "./frontendCommon";
 
 export interface GlobalInitialFixture {
   workspace?: BenchmarkWorkspaceRunnables;
+  liveEditorDrafts?: GlobalLiveEditorDraftFixture[];
 }
 
 export function createGlobalModeRunner(
@@ -31,6 +35,7 @@ export function createGlobalModeRunner(
         getFrontendApiKey(modelConfig.provider),
         {
           workspaceFixtures: initial?.workspace,
+          liveEditorDrafts: initial?.liveEditorDrafts,
           maxIterations: context.evalCase?.runtime?.maxTurns,
           provider: modelConfig.provider,
           model: modelConfig.model,
@@ -73,6 +78,7 @@ async function loadGlobalInitialFixture(path: string): Promise<GlobalInitialFixt
   const parsed = JSON.parse(await readFile(path, "utf8")) as GlobalInitialFixture;
   return {
     workspace: parsed.workspace ?? {},
+    liveEditorDrafts: parsed.liveEditorDrafts ?? [],
   };
 }
 

backend/.sqlx/query-0142d9dc9c1b57487dd5709a0376794f18d33e5bd6340c0189be7818cda64328.json

@@ -0,0 +1,94 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT email, login_type::TEXT, super_admin, devops, verified, name, company, username, NULL::bool as operator_only, first_time_user, role_source, disabled, NULL::text as workspace_id FROM password WHERE email = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "email",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "login_type",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "super_admin",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 3,
+        "name": "devops",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 4,
+        "name": "verified",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 5,
+        "name": "name",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 6,
+        "name": "company",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 7,
+        "name": "username",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 8,
+        "name": "operator_only",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 9,
+        "name": "first_time_user",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 10,
+        "name": "role_source",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 11,
+        "name": "disabled",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 12,
+        "name": "workspace_id",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      null,
+      false,
+      false,
+      false,
+      true,
+      true,
+      true,
+      null,
+      false,
+      false,
+      false,
+      null
+    ]
+  },
+  "hash": "0142d9dc9c1b57487dd5709a0376794f18d33e5bd6340c0189be7818cda64328"
+}

backend/.sqlx/query-04409657066c624308954958d9dd451452efc25e57769fb94b771d4879150835.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n                                SELECT\n                                    flow_version.id AS version,\n                                    flow_version.value->>'early_return' as early_return,\n                                    flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor,\n                                    (flow_version.value->>'chat_input_enabled')::boolean as chat_input_enabled,\n                                    flow.tag,\n                                    flow.dedicated_worker,\n                                    flow.on_behalf_of_email,\n                                    flow.edited_by,\n                                    flow.labels\n                                FROM\n                                    flow_version\n                                INNER JOIN flow\n                                    ON flow.path = flow_version.path AND\n                                       flow.workspace_id = flow_version.workspace_id\n                                WHERE\n                                    flow_version.workspace_id = $1 AND\n                                    flow_version.path = $2 AND\n                                    flow_version.id = $3\n                            ",
+  "query": "\n                                SELECT\n                                    flow_version.id AS version,\n                                    flow_version.value->>'early_return' as early_return,\n                                    flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor,\n                                    flow_version.value->>'failure_module' IS NOT NULL as has_failure_module,\n                                    (flow_version.value->>'chat_input_enabled')::boolean as chat_input_enabled,\n                                    flow.tag,\n                                    flow.dedicated_worker,\n                                    flow.on_behalf_of_email,\n                                    flow.edited_by,\n                                    flow.labels\n                                FROM\n                                    flow_version\n                                INNER JOIN flow\n                                    ON flow.path = flow_version.path AND\n                                       flow.workspace_id = flow_version.workspace_id\n                                WHERE\n                                    flow_version.workspace_id = $1 AND\n                                    flow_version.path = $2 AND\n                                    flow_version.id = $3\n                            ",
   "describe": {
     "columns": [
       {
@@ -20,31 +20,36 @@
       },
       {
         "ordinal": 3,
-        "name": "chat_input_enabled",
+        "name": "has_failure_module",
         "type_info": "Bool"
       },
       {
         "ordinal": 4,
+        "name": "chat_input_enabled",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 5,
         "name": "tag",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 5,
+        "ordinal": 6,
         "name": "dedicated_worker",
         "type_info": "Bool"
       },
       {
-        "ordinal": 6,
+        "ordinal": 7,
         "name": "on_behalf_of_email",
         "type_info": "Text"
       },
       {
-        "ordinal": 7,
+        "ordinal": 8,
         "name": "edited_by",
         "type_info": "Varchar"
       },
       {
-        "ordinal": 8,
+        "ordinal": 9,
         "name": "labels",
         "type_info": "TextArray"
       }
@@ -61,6 +66,7 @@
       null,
       null,
       null,
+      null,
       true,
       true,
       true,
@@ -68,5 +74,5 @@
       true
     ]
   },
-  "hash": "6d992a933bb878733b7afd7a4295b9ad6f5276b60ce20e0378d6148976e02777"
+  "hash": "04409657066c624308954958d9dd451452efc25e57769fb94b771d4879150835"
 }

backend/.sqlx/query-0476ae2245aa678a50c5fd04cdee32cc151e29b177cc85caa088430b16336373.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT flow_version.path FROM flow_version\n         INNER JOIN flow\n            ON flow.path = flow_version.path AND\n               flow.workspace_id = flow_version.workspace_id\n         WHERE flow_version.id = $1 AND flow_version.workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "path",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "0476ae2245aa678a50c5fd04cdee32cc151e29b177cc85caa088430b16336373"
+}

backend/.sqlx/query-0a568f630e069118fe302099a709e89cc4a702158899f7fdc66d0922e8fb9b29.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT path FROM flow WHERE workspace_id = $1 AND path = ANY($2) AND archived = false",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "path",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "TextArray"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "0a568f630e069118fe302099a709e89cc4a702158899f7fdc66d0922e8fb9b29"
+}

backend/.sqlx/query-0cca68f11329cd41ab9372297b715af008ea7db408f49cf525656d6224092429.json

@@ -0,0 +1,28 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT operator, is_admin FROM usr WHERE email = $1 AND is_service_account IS true LIMIT 1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "operator",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 1,
+        "name": "is_admin",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      false
+    ]
+  },
+  "hash": "0cca68f11329cd41ab9372297b715af008ea7db408f49cf525656d6224092429"
+}

backend/.sqlx/query-0f5ec10de91deac2d40b1e8a6dad0e9341f57cd060e7f228b40634469269c407.json

@@ -0,0 +1,42 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "WITH bounds AS (\n                SELECT ($2::bigint % 4294967296)::text::xid AS prev_xid,\n                       ($3::bigint % 4294967296)::text::xid AS cur_xid,\n                       $1::timestamptz AS ts_floor\n            ),\n            batch AS (\n                SELECT workspace_id, id, timestamp, username, operation,\n                       action_kind::text AS action_kind, resource, parameters, email, span\n                FROM audit_partitioned, bounds b\n                WHERE timestamp >= b.ts_floor\n                  AND age(xmin) > age(b.cur_xid)\n                  AND age(xmin) <= age(b.prev_xid)\n                ORDER BY id\n            )\n            SELECT to_char(timestamp AT TIME ZONE 'UTC', 'YYYY-MM-DD') AS \"day!\",\n                   string_agg(row_to_json(batch)::text, E'\\n' ORDER BY id) AS \"ndjson!\",\n                   max(id) AS \"max_id!\",\n                   max(timestamp) AS \"max_ts!\"\n            FROM batch\n            GROUP BY 1\n            ORDER BY 1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "day!",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 1,
+        "name": "ndjson!",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "max_id!",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 3,
+        "name": "max_ts!",
+        "type_info": "Timestamptz"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Timestamptz",
+        "Int8",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      null,
+      null,
+      null,
+      null
+    ]
+  },
+  "hash": "0f5ec10de91deac2d40b1e8a6dad0e9341f57cd060e7f228b40634469269c407"
+}

backend/.sqlx/query-14bc9dd1d02a3d121297509beacc27f3c29d1b3877c1f2e7c206f0e36ef18701.json

@@ -0,0 +1,40 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        SELECT\n            (elem->>'installation_id')::bigint as installation_id,\n            elem->>'account_id' as account_id,\n            elem->>'github_base_url' as github_base_url,\n            COALESCE((elem->>'provisioned_by_admin')::bool, false) as \"provisioned_by_admin!\"\n        FROM workspace_settings,\n        LATERAL jsonb_array_elements(git_app_installations) AS elem\n        WHERE workspace_id = $1\n        ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "installation_id",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 1,
+        "name": "account_id",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "github_base_url",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 3,
+        "name": "provisioned_by_admin!",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null,
+      null,
+      null,
+      null
+    ]
+  },
+  "hash": "14bc9dd1d02a3d121297509beacc27f3c29d1b3877c1f2e7c206f0e36ef18701"
+}

backend/.sqlx/query-16b4496c21d0619dab4521dca22e5fe144c59156a8f06d8592291684c49b2f37.json

@@ -0,0 +1,95 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT email as \"email!\", login_type::text, verified as \"verified!\", super_admin as \"super_admin!\", devops as \"devops!\", name, company, username, NULL::bool as operator_only, first_time_user as \"first_time_user!\", role_source as \"role_source!\", disabled as \"disabled!\", NULL::text as workspace_id FROM password\n            UNION ALL\n            SELECT email as \"email!\", 'service_account'::text as login_type, true as \"verified!\", false as \"super_admin!\", false as \"devops!\", NULL::text as name, NULL::text as company, username, true as operator_only, false as \"first_time_user!\", 'service_account'::text as \"role_source!\", disabled as \"disabled!\", workspace_id\n            FROM usr\n            WHERE is_service_account IS true\n            ORDER BY \"super_admin!\" DESC, \"devops!\" DESC, \"email!\"\n            LIMIT $1 OFFSET $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "email!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "login_type",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "verified!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 3,
+        "name": "super_admin!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 4,
+        "name": "devops!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 5,
+        "name": "name",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 6,
+        "name": "company",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 7,
+        "name": "username",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 8,
+        "name": "operator_only",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 9,
+        "name": "first_time_user!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 10,
+        "name": "role_source!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 11,
+        "name": "disabled!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 12,
+        "name": "workspace_id",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      null,
+      null,
+      null,
+      null,
+      null,
+      null,
+      null,
+      null,
+      null,
+      null,
+      null,
+      null,
+      null
+    ]
+  },
+  "hash": "16b4496c21d0619dab4521dca22e5fe144c59156a8f06d8592291684c49b2f37"
+}

backend/.sqlx/query-19513c4158267cc7fe10d999ad571052c112e6bbb3cf834f16176cbb7e1ac319.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "WITH RECURSIVE chain(id, parent_job) AS (\n                SELECT id, parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2\n                UNION ALL\n                SELECT j.id, j.parent_job FROM v2_job j\n                    JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2\n            )\n            SELECT id AS \"id!\" FROM chain",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Uuid"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "19513c4158267cc7fe10d999ad571052c112e6bbb3cf834f16176cbb7e1ac319"
+}

backend/.sqlx/query-1d0341bd8de94ab8d34a4bb1bb2005305fb3b29751ec987bece183444e89e7d1.json

@@ -0,0 +1,100 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT email, login_type::TEXT, super_admin, devops, verified, name, company, username, NULL::bool as operator_only, NULL::bool as is_workspace_admin, first_time_user, role_source, disabled, NULL::text as workspace_id FROM password WHERE email = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "email",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "login_type",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "super_admin",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 3,
+        "name": "devops",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 4,
+        "name": "verified",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 5,
+        "name": "name",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 6,
+        "name": "company",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 7,
+        "name": "username",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 8,
+        "name": "operator_only",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 9,
+        "name": "is_workspace_admin",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 10,
+        "name": "first_time_user",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 11,
+        "name": "role_source",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 12,
+        "name": "disabled",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 13,
+        "name": "workspace_id",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      null,
+      false,
+      false,
+      false,
+      true,
+      true,
+      true,
+      null,
+      null,
+      false,
+      false,
+      false,
+      null
+    ]
+  },
+  "hash": "1d0341bd8de94ab8d34a4bb1bb2005305fb3b29751ec987bece183444e89e7d1"
+}

backend/.sqlx/query-295a88070e1762255cdd7680ba2e7bb2a2ffd66a7c432dd318e73e0f81ea9622.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO variable\n            (workspace_id, path, value, is_secret, description, account, is_oauth, expires_at, labels, edited_by)\n            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Bool",
+        "Varchar",
+        "Int4",
+        "Bool",
+        "Timestamptz",
+        "TextArray",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "295a88070e1762255cdd7680ba2e7bb2a2ffd66a7c432dd318e73e0f81ea9622"
+}

backend/.sqlx/query-2e35598cb9695b726ee1d2cd5c8364503371a5272d88e970b95760555ab33ce2.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO script (workspace_id, path, hash, content, summary, description, language, created_by, created_at, archived, schema_validation, ws_error_handler_muted, deleted, extra_perms)\n         VALUES ('wm-fork-stale-super', 'f/folder2/myscript', 333333, 'echo 1', '', '', 'bash', 'test-user-2', NOW(), false, false, false, false, $1)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Jsonb"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "2e35598cb9695b726ee1d2cd5c8364503371a5272d88e970b95760555ab33ce2"
+}

backend/.sqlx/query-2f166b5575a614b028c3130fc5089353bef40f1cccf31b7775d0e9a800425f4d.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        SELECT COALESCE((elem->>'provisioned_by_admin')::bool, false) as \"is_admin!\"\n        FROM workspace_settings,\n        LATERAL jsonb_array_elements(git_app_installations) AS elem\n        WHERE workspace_id = $1\n          AND (elem->>'installation_id')::bigint = $2\n        ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "is_admin!",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "2f166b5575a614b028c3130fc5089353bef40f1cccf31b7775d0e9a800425f4d"
+}

backend/.sqlx/query-31445efb75a7b706f4404c411a4ef6a9ed6d29a02bb7fd08f2ceb0551ecac640.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, summary, created_by)\n         VALUES ('wm-fork-visibility-test', 'folder2', 'folder2', ARRAY['u/test-user-2']::varchar[], $1, '', 'test-user-2')",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Jsonb"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "31445efb75a7b706f4404c411a4ef6a9ed6d29a02bb7fd08f2ceb0551ecac640"
+}

backend/.sqlx/query-31e486e3377e79bfab4e391d6789d081edf45fef34f630372815ec323544acee.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, summary, created_by)\n         VALUES ('test-workspace', 'folder1', 'folder1', ARRAY['u/test-user-2']::varchar[], $1, '', 'test-user-2')",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Jsonb"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "31e486e3377e79bfab4e391d6789d081edf45fef34f630372815ec323544acee"
+}

backend/.sqlx/query-3c42a56d0ffe39ad217f2ee603431637bcb22c6e713a21bdb83204de9cf383d7.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT 1 FROM workspace_settings WHERE workspace_id = $1)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "3c42a56d0ffe39ad217f2ee603431637bcb22c6e713a21bdb83204de9cf383d7"
+}

backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "WITH inserted_job AS (\n            INSERT INTO v2_job (\n                id, -- 1\n                workspace_id, -- 2\n                raw_code, -- 3\n                raw_lock, -- 4\n                raw_flow, -- 5\n                tag, -- 6\n                parent_job, -- 7\n                created_by, -- 8\n                permissioned_as, -- 9\n                runnable_id, -- 10\n                runnable_path, -- 11\n                args, -- 12\n                kind, -- 13\n                trigger, -- 14\n                script_lang, -- 15\n                same_worker, -- 16\n                pre_run_error, -- 17\n                permissioned_as_email, -- 18\n                visible_to_owner, -- 19\n                flow_innermost_root_job, -- 20\n                root_job, -- 38\n                concurrent_limit, -- 21\n                concurrency_time_window_s, -- 22\n                timeout, -- 23\n                flow_step_id, -- 24\n                cache_ttl, -- 25\n                priority, -- 26\n                trigger_kind, -- 39\n                script_entrypoint_override, -- 12\n                preprocessed, -- 27,\n                labels -- 44\n            ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18,\n            $19, $20, $38, $21, $22, $23, $24, $25, $26, $39::job_trigger_kind,\n            ($12::JSONB)->>'_ENTRYPOINT_OVERRIDE', $27, $44)\n        ),\n        inserted_runtime AS (\n            INSERT INTO v2_job_runtime (id, ping) VALUES ($1, null)\n        ),\n        inserted_job_perms AS (\n            INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id, end_user_email)\n            values ($1, $32, $33, $34, $35, $36, $37, $2, $41)\n            ON CONFLICT (job_id) DO UPDATE SET email = EXCLUDED.email, username = EXCLUDED.username, is_admin = EXCLUDED.is_admin, is_operator = EXCLUDED.is_operator, folders = EXCLUDED.folders, groups = EXCLUDED.groups, workspace_id = EXCLUDED.workspace_id, end_user_email = EXCLUDED.end_user_email\n        )\n        INSERT INTO v2_job_queue\n            (workspace_id, id, running, scheduled_for, started_at, tag, priority, cache_ignore_s3_path, runnable_settings_handle)\n            VALUES ($2, $1, $28, COALESCE($29, now()), CASE WHEN $27 OR $40 THEN now() END, $30, $31, $42, $43)",
+  "query": "WITH inserted_job AS (\n            INSERT INTO v2_job (\n                id, -- 1\n                workspace_id, -- 2\n                raw_code, -- 3\n                raw_lock, -- 4\n                raw_flow, -- 5\n                tag, -- 6\n                parent_job, -- 7\n                created_by, -- 8\n                permissioned_as, -- 9\n                runnable_id, -- 10\n                runnable_path, -- 11\n                args, -- 12\n                kind, -- 13\n                trigger, -- 14\n                script_lang, -- 15\n                same_worker, -- 16\n                pre_run_error, -- 17\n                permissioned_as_email, -- 18\n                visible_to_owner, -- 19\n                flow_innermost_root_job, -- 20\n                root_job, -- 38\n                concurrent_limit, -- 21\n                concurrency_time_window_s, -- 22\n                timeout, -- 23\n                flow_step_id, -- 24\n                cache_ttl, -- 25\n                priority, -- 26\n                trigger_kind, -- 39\n                script_entrypoint_override, -- 12\n                preprocessed, -- 27,\n                labels -- 44\n            ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18,\n            $19, $20, $38, $21, $22, $23, $24, $25, $26, $39::job_trigger_kind,\n            ($12::JSONB)->>'_ENTRYPOINT_OVERRIDE', $27,\n            -- $44 (payload labels) merged with the labels of the runnable's folder, if any\n            -- ($45/$46 are runnable_path/workspace_id again, kept separate to avoid parameter type conflicts)\n            (SELECT CASE WHEN fl.labels IS NULL THEN $44\n                ELSE (SELECT array_agg(DISTINCT lbl) FROM unnest(COALESCE($44, ARRAY[]::TEXT[]) || fl.labels) lbl)\n            END\n            FROM folder_labels($46, $45) AS fl(labels)))\n        ),\n        inserted_runtime AS (\n            INSERT INTO v2_job_runtime (id, ping) VALUES ($1, null)\n        ),\n        inserted_job_perms AS (\n            INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id, end_user_email)\n            values ($1, $32, $33, $34, $35, $36, $37, $2, $41)\n            ON CONFLICT (job_id) DO UPDATE SET email = EXCLUDED.email, username = EXCLUDED.username, is_admin = EXCLUDED.is_admin, is_operator = EXCLUDED.is_operator, folders = EXCLUDED.folders, groups = EXCLUDED.groups, workspace_id = EXCLUDED.workspace_id, end_user_email = EXCLUDED.end_user_email\n        )\n        INSERT INTO v2_job_queue\n            (workspace_id, id, running, scheduled_for, started_at, tag, priority, cache_ignore_s3_path, runnable_settings_handle)\n            VALUES ($2, $1, $28, COALESCE($29, now()), CASE WHEN $27 OR $40 THEN now() END, $30, $31, $42, $43)",
   "describe": {
     "columns": [],
     "parameters": {
@@ -136,10 +136,12 @@
         "Varchar",
         "Bool",
         "Int8",
-        "TextArray"
+        "TextArray",
+        "Text",
+        "Text"
       ]
     },
     "nullable": []
   },
-  "hash": "756f82b72af07fd690f37b2e16ed2d390604f4fc4cb330842a88d5764cbcf0c6"
+  "hash": "3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619"
 }

backend/.sqlx/query-3e82929b365a6aa7ccc39fc5615c4110e1d740e0cb0509b1c58bc6636d83cafd.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT policy->>'execution_mode' = 'anonymous' FROM app WHERE path = $1 AND workspace_id = $2 FOR UPDATE",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "?column?",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "3e82929b365a6aa7ccc39fc5615c4110e1d740e0cb0509b1c58bc6636d83cafd"
+}

backend/.sqlx/query-3fac8694f59803a42b635ce7dd1e60a7a4f53c3b7ed6592f70ef4fed97b2bc6d.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) AS \"count!\" FROM workspace_diff\n             WHERE source_workspace_id = 'test-workspace'\n               AND fork_workspace_id = 'wm-fork-rename-test'\n               AND kind = 'script'\n               AND path = 'f/folder2/myscript'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count!",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "3fac8694f59803a42b635ce7dd1e60a7a4f53c3b7ed6592f70ef4fed97b2bc6d"
+}

backend/.sqlx/query-42600bdeb6b86ac306ab276cb49241508776f5f998821e04bc92df917176cea6.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT workspace_id, name, display_name, owners, extra_perms, summary, created_by, edited_at, default_permissioned_as FROM folder WHERE name = $1 AND workspace_id = $2",
+  "query": "SELECT workspace_id, name, display_name, owners, extra_perms, summary, created_by, edited_at, default_permissioned_as, labels FROM folder WHERE name = $1 AND workspace_id = $2",
   "describe": {
     "columns": [
       {
@@ -47,6 +47,11 @@
         "ordinal": 8,
         "name": "default_permissioned_as",
         "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 9,
+        "name": "labels",
+        "type_info": "TextArray"
       }
     ],
     "parameters": {
@@ -64,8 +69,9 @@
       true,
       true,
       true,
-      false
+      false,
+      true
     ]
   },
-  "hash": "269197b692a1e451a31f14701c8ca324c8e40d2923dfa07b8e4b3934afc16fed"
+  "hash": "42600bdeb6b86ac306ab276cb49241508776f5f998821e04bc92df917176cea6"
 }

backend/.sqlx/query-4c81384b579bad74b64c72ca053839f316fe5412f99f3d3bbbdc0c65f55ab794.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT has_changes FROM workspace_diff\n         WHERE path = 'f/shared/renamed_away' AND kind = 'script' AND source_workspace_id = 'test-workspace'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "has_changes",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      true
+    ]
+  },
+  "hash": "4c81384b579bad74b64c72ca053839f316fe5412f99f3d3bbbdc0c65f55ab794"
+}

backend/.sqlx/query-4ca364e493dd51d5f83489dc4674eeadcf092b37a20d03d52f2160ca21f79fb0.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, summary, edited_at, created_by, default_permissioned_as)\n         SELECT $2, name, display_name, owners, extra_perms, summary, edited_at, created_by, default_permissioned_as\n         FROM folder\n         WHERE workspace_id = $1",
+  "query": "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, summary, edited_at, created_by, default_permissioned_as, labels)\n         SELECT $2, name, display_name, owners, extra_perms, summary, edited_at, created_by, default_permissioned_as, labels\n         FROM folder\n         WHERE workspace_id = $1",
   "describe": {
     "columns": [],
     "parameters": {
@@ -11,5 +11,5 @@
     },
     "nullable": []
   },
-  "hash": "ae949d9470fe6687b312fbd9d617d47813a035eb948f0312a8168bb1365607fb"
+  "hash": "4ca364e493dd51d5f83489dc4674eeadcf092b37a20d03d52f2160ca21f79fb0"
 }

backend/.sqlx/query-5104cf045dc9b7b82d0028af11cfb5c2f6fd58e518085caf8e8d189951f7c4d8.json

@@ -0,0 +1,28 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO draft\n            (workspace_id, path, value, typ)\n            VALUES ($1, $2, $3::text::json, $4)\n            ON CONFLICT (workspace_id, path, typ)\n            DO UPDATE SET value = EXCLUDED.value, created_at = now()",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Text",
+        {
+          "Custom": {
+            "name": "draft_type",
+            "kind": {
+              "Enum": [
+                "script",
+                "flow",
+                "app"
+              ]
+            }
+          }
+        }
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "5104cf045dc9b7b82d0028af11cfb5c2f6fd58e518085caf8e8d189951f7c4d8"
+}

backend/.sqlx/query-52b07f3ef0aa3ac4f6d6be0db60c732e396a79458489c564bbf2e4a28de06f1f.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT resource.workspace_id, resource.path, resource.value, resource.description,\n        resource.resource_type, resource.extra_perms, resource.created_by, resource.edited_at,\n        resource.labels,\n        (now() > account.expires_at) as is_expired, account.refresh_token != '' as is_refreshed,\n        account.refresh_error,\n        variable.path IS NOT NULL as is_linked,\n        variable.is_oauth as \"is_oauth?\",\n        variable.account,\n        ws_specific.path IS NOT NULL as ws_specific\n        FROM resource\n        LEFT JOIN variable ON variable.path = resource.path AND variable.workspace_id = $2\n        LEFT JOIN account ON variable.account = account.id AND account.workspace_id = $2\n        LEFT JOIN ws_specific ON ws_specific.path = resource.path AND ws_specific.workspace_id = $2 AND ws_specific.item_kind = 'resource'\n        WHERE resource.path = $1 AND resource.workspace_id = $2",
+  "query": "SELECT resource.workspace_id, resource.path, resource.value, resource.description,\n        resource.resource_type, resource.extra_perms, resource.created_by, resource.edited_at,\n        resource.labels,\n        folder_labels(resource.workspace_id, resource.path) as \"inherited_labels?\",\n        (now() > account.expires_at) as is_expired, account.refresh_token != '' as is_refreshed,\n        account.refresh_error,\n        variable.path IS NOT NULL as is_linked,\n        variable.is_oauth as \"is_oauth?\",\n        variable.account,\n        ws_specific.path IS NOT NULL as ws_specific\n        FROM resource\n        LEFT JOIN variable ON variable.path = resource.path AND variable.workspace_id = $2\n        LEFT JOIN account ON variable.account = account.id AND account.workspace_id = $2\n        LEFT JOIN ws_specific ON ws_specific.path = resource.path AND ws_specific.workspace_id = $2 AND ws_specific.item_kind = 'resource'\n        WHERE resource.path = $1 AND resource.workspace_id = $2",
   "describe": {
     "columns": [
       {
@@ -50,36 +50,41 @@
       },
       {
         "ordinal": 9,
+        "name": "inherited_labels?",
+        "type_info": "TextArray"
+      },
+      {
+        "ordinal": 10,
         "name": "is_expired",
         "type_info": "Bool"
       },
       {
-        "ordinal": 10,
+        "ordinal": 11,
         "name": "is_refreshed",
         "type_info": "Bool"
       },
       {
-        "ordinal": 11,
+        "ordinal": 12,
         "name": "refresh_error",
         "type_info": "Text"
       },
       {
-        "ordinal": 12,
+        "ordinal": 13,
         "name": "is_linked",
         "type_info": "Bool"
       },
       {
-        "ordinal": 13,
+        "ordinal": 14,
         "name": "is_oauth?",
         "type_info": "Bool"
       },
       {
-        "ordinal": 14,
+        "ordinal": 15,
         "name": "account",
         "type_info": "Int4"
       },
       {
-        "ordinal": 15,
+        "ordinal": 16,
         "name": "ws_specific",
         "type_info": "Bool"
       }
@@ -102,6 +107,7 @@
       true,
       null,
       null,
+      null,
       true,
       null,
       false,
@@ -109,5 +115,5 @@
       null
     ]
   },
-  "hash": "9254e2a0e1be830fa4cab660c69a659e2eb3f3d912e1c062bb8000a2e0a653e8"
+  "hash": "52b07f3ef0aa3ac4f6d6be0db60c732e396a79458489c564bbf2e4a28de06f1f"
 }

backend/.sqlx/query-5347ec9ab6de69a99e8823d2199757b244b280e1262dcbccd2fc5189c7b3d25b.json

@@ -0,0 +1,31 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT workspace_id, path FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2) AND NOT (path = $3 AND workspace_id = $4) LIMIT 1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "path",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text",
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      false
+    ]
+  },
+  "hash": "5347ec9ab6de69a99e8823d2199757b244b280e1262dcbccd2fc5189c7b3d25b"
+}

backend/.sqlx/query-5494652553c59b72ca5db4350a8ba3d8bbf1608519b08f2544c64ecfe130c537.json

@@ -0,0 +1,17 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE variable SET labels = $1, edited_at = now(), edited_by = $4 WHERE path = $2 AND workspace_id = $3",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "TextArray",
+        "Text",
+        "Text",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "5494652553c59b72ca5db4350a8ba3d8bbf1608519b08f2544c64ecfe130c537"
+}

backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json

@@ -15,7 +15,7 @@
       ]
     },
     "nullable": [
-      null
+      true
     ]
   },
   "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55"

backend/.sqlx/query-5b4214deb7677ae16df0bea7c6553461dee4bcc8d7180a9a517d5809c42e11d7.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO folder (name, workspace_id, display_name, owners, extra_perms, summary, edited_at, created_by, default_permissioned_as) SELECT name, $1, display_name, owners, extra_perms, summary, edited_at, created_by, default_permissioned_as FROM folder WHERE workspace_id = $2",
+  "query": "INSERT INTO folder (name, workspace_id, display_name, owners, extra_perms, summary, edited_at, created_by, default_permissioned_as, labels) SELECT name, $1, display_name, owners, extra_perms, summary, edited_at, created_by, default_permissioned_as, labels FROM folder WHERE workspace_id = $2",
   "describe": {
     "columns": [],
     "parameters": {
@@ -11,5 +11,5 @@
     },
     "nullable": []
   },
-  "hash": "86c196327af8705a33ab9a707b4a87a441f800c53885b30f9a9578217e87f2b7"
+  "hash": "5b4214deb7677ae16df0bea7c6553461dee4bcc8d7180a9a517d5809c42e11d7"
 }

backend/.sqlx/query-636dd6906a20b608eea34e04a81bddb79398f57a34d180d08aa00aaaaa1050c8.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT workspace_id, name, display_name, owners, extra_perms, summary, created_by, edited_at, default_permissioned_as FROM folder WHERE workspace_id = $1 ORDER BY name asc LIMIT $2 OFFSET $3",
+  "query": "SELECT workspace_id, name, display_name, owners, extra_perms, summary, created_by, edited_at, default_permissioned_as, labels FROM folder WHERE workspace_id = $1 ORDER BY name asc LIMIT $2 OFFSET $3",
   "describe": {
     "columns": [
       {
@@ -47,6 +47,11 @@
         "ordinal": 8,
         "name": "default_permissioned_as",
         "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 9,
+        "name": "labels",
+        "type_info": "TextArray"
       }
     ],
     "parameters": {
@@ -65,8 +70,9 @@
       true,
       true,
       true,
-      false
+      false,
+      true
     ]
   },
-  "hash": "b87d104a97026929983f27a7607490d08b65637a02fb7821571978812144c400"
+  "hash": "636dd6906a20b608eea34e04a81bddb79398f57a34d180d08aa00aaaaa1050c8"
 }

backend/.sqlx/query-652637b534f7d7b4c429a201247821e9f568b1976ea462a09e779e9a4c490197.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO usr (workspace_id, email, username, is_admin, role) VALUES\n         ('wm-fork-visibility-test', 'test2@windmill.dev', 'test-user-2', false, 'User')",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "652637b534f7d7b4c429a201247821e9f568b1976ea462a09e779e9a4c490197"
+}

backend/.sqlx/query-6fcdb09cdd7fedd7e54fdc0e49203f453fc1b85272fe212c0e1bf0ebd28bf58a.json

@@ -0,0 +1,38 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        WITH capped AS (\n            SELECT timestamp, operation, resource, parameters\n            FROM audit_partitioned\n            WHERE workspace_id = 'admins'\n              AND operation = 'workspace_fairness.capped'\n            UNION ALL\n            SELECT timestamp, operation, resource, parameters\n            FROM audit\n            WHERE workspace_id = 'admins'\n              AND operation = 'workspace_fairness.capped'\n            ORDER BY timestamp DESC\n            LIMIT 200\n        ), uncapped AS (\n            SELECT timestamp, operation, resource, parameters\n            FROM audit_partitioned\n            WHERE workspace_id = 'admins'\n              AND operation = 'workspace_fairness.uncapped'\n            UNION ALL\n            SELECT timestamp, operation, resource, parameters\n            FROM audit\n            WHERE workspace_id = 'admins'\n              AND operation = 'workspace_fairness.uncapped'\n            ORDER BY timestamp DESC\n            LIMIT 200\n        )\n        SELECT timestamp AS \"timestamp!\",\n               operation::text AS \"operation!\",\n               resource AS workspace_id,\n               parameters\n        FROM (SELECT * FROM capped UNION ALL SELECT * FROM uncapped) e\n        ORDER BY timestamp DESC\n        ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "timestamp!",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 1,
+        "name": "operation!",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 3,
+        "name": "parameters",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null,
+      null,
+      null,
+      null
+    ]
+  },
+  "hash": "6fcdb09cdd7fedd7e54fdc0e49203f453fc1b85272fe212c0e1bf0ebd28bf58a"
+}

backend/.sqlx/query-77d8a6d1b6d6cd8def844981a2bf87c66c266526a9fc484aa7f58cbeea6672fd.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT txid_snapshot_xmin(txid_current_snapshot())::bigint AS \"x!\"",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "x!",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "77d8a6d1b6d6cd8def844981a2bf87c66c266526a9fc484aa7f58cbeea6672fd"
+}

backend/.sqlx/query-7f832370916794ab0e5645053688c24678f1519d49ee7263a86dba71d45b8e8c.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO token\n            (token_hash, token_prefix, token, email, label, expiration, super_admin, scopes, read_only)\n            VALUES ($1, $2, $3, $4, $5, now() + ($6 || ' seconds')::interval, $7, $8, $9)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Text",
+        "Bool",
+        "TextArray",
+        "Bool"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "7f832370916794ab0e5645053688c24678f1519d49ee7263a86dba71d45b8e8c"
+}

backend/.sqlx/query-7fc4dcc5f46da2a1ac74bf80237a6ae28f1b817b2e1ff275a3f470bb88fc500c.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT DISTINCT unnest(labels) as \"label!\" FROM (\n            SELECT labels FROM script WHERE workspace_id = $1 AND labels IS NOT NULL\n            UNION ALL SELECT labels FROM flow WHERE workspace_id = $1 AND labels IS NOT NULL\n            UNION ALL SELECT labels FROM resource WHERE workspace_id = $1 AND labels IS NOT NULL\n            UNION ALL SELECT labels FROM variable WHERE workspace_id = $1 AND labels IS NOT NULL\n            UNION ALL SELECT labels FROM schedule WHERE workspace_id = $1 AND labels IS NOT NULL\n            UNION ALL SELECT labels FROM app WHERE workspace_id = $1 AND labels IS NOT NULL\n        ) t ORDER BY 1",
+  "query": "SELECT DISTINCT unnest(labels) as \"label!\" FROM (\n            SELECT labels FROM script WHERE workspace_id = $1 AND labels IS NOT NULL\n            UNION ALL SELECT labels FROM flow WHERE workspace_id = $1 AND labels IS NOT NULL\n            UNION ALL SELECT labels FROM resource WHERE workspace_id = $1 AND labels IS NOT NULL\n            UNION ALL SELECT labels FROM variable WHERE workspace_id = $1 AND labels IS NOT NULL\n            UNION ALL SELECT labels FROM schedule WHERE workspace_id = $1 AND labels IS NOT NULL\n            UNION ALL SELECT labels FROM app WHERE workspace_id = $1 AND labels IS NOT NULL\n            UNION ALL SELECT labels FROM folder WHERE workspace_id = $1 AND labels IS NOT NULL\n        ) t ORDER BY 1",
   "describe": {
     "columns": [
       {
@@ -18,5 +18,5 @@
       null
     ]
   },
-  "hash": "b782179261b4054289a1a595f6e06ee2035ae9811bc49abce903a04439baf800"
+  "hash": "7fc4dcc5f46da2a1ac74bf80237a6ae28f1b817b2e1ff275a3f470bb88fc500c"
 }

backend/.sqlx/query-813b91a338f2fe46a7d27a155c435f33faa61710388f71f04da2d4569bc3dd71.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, summary, created_by, edited_at, default_permissioned_as) VALUES ($1, $2, $3, $4, $5, $6, $7, now(), $8)",
+  "query": "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, summary, created_by, edited_at, default_permissioned_as, labels) VALUES ($1, $2, $3, $4, $5, $6, $7, now(), $8, $9)",
   "describe": {
     "columns": [],
     "parameters": {
@@ -12,10 +12,11 @@
         "Jsonb",
         "Text",
         "Varchar",
-        "Jsonb"
+        "Jsonb",
+        "TextArray"
       ]
     },
     "nullable": []
   },
-  "hash": "977e696c820d1412b56086925a700f59898edc9851e2364485a70071ece7bc1c"
+  "hash": "813b91a338f2fe46a7d27a155c435f33faa61710388f71f04da2d4569bc3dd71"
 }

backend/.sqlx/query-815d96aea4681490582b08630a30a168cc1191acaab96bed6a016c437059c2cd.json

@@ -0,0 +1,26 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n            SELECT ws.workspace_id AS \"workspace_id!\", entry->'catalog'->>'resource_path' AS dbname\n            FROM workspace_settings ws\n            CROSS JOIN LATERAL jsonb_each(\n                CASE WHEN jsonb_typeof(ws.ducklake->'ducklakes') = 'object'\n                    THEN ws.ducklake->'ducklakes'\n                    ELSE '{}'::jsonb END\n            ) AS dl(k, entry)\n            WHERE entry->'catalog'->>'resource_type' = 'instance'\n            AND entry->'catalog'->>'resource_path' IS NOT NULL\n            UNION ALL\n            SELECT ws.workspace_id AS \"workspace_id!\", entry->'database'->>'resource_path' AS dbname\n            FROM workspace_settings ws\n            CROSS JOIN LATERAL jsonb_each(\n                CASE WHEN jsonb_typeof(ws.datatable->'datatables') = 'object'\n                    THEN ws.datatable->'datatables'\n                    ELSE '{}'::jsonb END\n            ) AS dt(k, entry)\n            WHERE entry->'database'->>'resource_type' = 'instance'\n            AND entry->'database'->>'resource_path' IS NOT NULL\n            ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "workspace_id!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "dbname",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null,
+      null
+    ]
+  },
+  "hash": "815d96aea4681490582b08630a30a168cc1191acaab96bed6a016c437059c2cd"
+}

backend/.sqlx/query-8192986cd6106ed060b3d68dbc21e5bb34f5e68a5c6ac455a9d423188af77b23.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO background_task_state (name, value)\n             SELECT $1, jsonb_build_object(\n                        'last_xmin', txid_snapshot_xmin(txid_current_snapshot())::bigint,\n                        'last_ts', '1970-01-01T00:00:00+00:00')\n             WHERE NOT EXISTS (SELECT 1 FROM global_settings WHERE name = $2)\n             ON CONFLICT (name) DO NOTHING",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "8192986cd6106ed060b3d68dbc21e5bb34f5e68a5c6ac455a9d423188af77b23"
+}

backend/.sqlx/query-8597cd40f80e69edbf1bc7d7402baca32e33e871be454acb5175c11361fe1b0a.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM background_task_state\n         WHERE name LIKE $1\n           AND updated_at < NOW() - INTERVAL '7 days'",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "8597cd40f80e69edbf1bc7d7402baca32e33e871be454acb5175c11361fe1b0a"
+}

backend/.sqlx/query-8711bb7861cb3c453519a620057e1530039c09b824065027387bbb667a49fe8d.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO background_task_state\n                 (name, value, running, owner, started_at, finished_at, updated_at)\n             VALUES ($1, $2, false, $3, now(), now(), now())\n             ON CONFLICT (name) DO UPDATE SET\n                 value = $2, running = false, owner = $3,\n                 finished_at = now(), updated_at = now()",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Jsonb",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "8711bb7861cb3c453519a620057e1530039c09b824065027387bbb667a49fe8d"
+}

backend/.sqlx/query-8816bbe1ea9ea4f359e7a95857d349fa8bd7d23e4589b6936e0d000745cb3f34.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT a.path FROM app_script s JOIN app a ON a.id = s.app\n                 WHERE s.id = $1 AND a.workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "path",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "8816bbe1ea9ea4f359e7a95857d349fa8bd7d23e4589b6936e0d000745cb3f34"
+}

backend/.sqlx/query-8aae160c589adf02e20b7e6ba860b66fb08f156776b96b7f3701aed330a7d000.json

@@ -0,0 +1,101 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "WITH active_users AS (SELECT distinct username as email FROM (SELECT username, timestamp, operation FROM audit_partitioned UNION ALL SELECT username, timestamp, operation FROM audit) AS a WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n            authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n            SELECT email as \"email!\", (email NOT IN (SELECT email FROM authors)) as operator_only, NULL::bool as is_workspace_admin, login_type::text, verified as \"verified!\", super_admin as \"super_admin!\", devops as \"devops!\", name, company, username, first_time_user as \"first_time_user!\", role_source as \"role_source!\", disabled as \"disabled!\", NULL::text as workspace_id\n            FROM password\n            WHERE email IN (SELECT email FROM active_users)\n            UNION ALL\n            SELECT email as \"email!\", operator as operator_only, is_admin as is_workspace_admin, 'service_account'::text as login_type, true as \"verified!\", false as \"super_admin!\", false as \"devops!\", NULL::text as name, NULL::text as company, username, false as \"first_time_user!\", 'service_account'::text as \"role_source!\", disabled as \"disabled!\", workspace_id\n            FROM usr\n            WHERE is_service_account IS true\n            ORDER BY \"super_admin!\" DESC, \"devops!\" DESC\n            LIMIT $1 OFFSET $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "email!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "operator_only",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 2,
+        "name": "is_workspace_admin",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 3,
+        "name": "login_type",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 4,
+        "name": "verified!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 5,
+        "name": "super_admin!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 6,
+        "name": "devops!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 7,
+        "name": "name",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 8,
+        "name": "company",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 9,
+        "name": "username",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 10,
+        "name": "first_time_user!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 11,
+        "name": "role_source!",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 12,
+        "name": "disabled!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 13,
+        "name": "workspace_id",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      null,
+      null,
+      null,
+      null,
+      null,
+      null,
+      null,
+      null,
+      null,
+      null,
+      null,
+      null,
+      null,
+      null
+    ]
+  },
+  "hash": "8aae160c589adf02e20b7e6ba860b66fb08f156776b96b7f3701aed330a7d000"
+}

backend/.sqlx/query-8d64e61fad7bdf0cc4d4cad1a032e570bdfaf885b4f39ef3ab973256e0448ec7.json

@@ -0,0 +1,35 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT username, is_admin, operator FROM usr\n         WHERE workspace_id = $1 AND email = $2 AND disabled = false",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "username",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "is_admin",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 2,
+        "name": "operator",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false
+    ]
+  },
+  "hash": "8d64e61fad7bdf0cc4d4cad1a032e570bdfaf885b4f39ef3ab973256e0448ec7"
+}

backend/.sqlx/query-8e8933fc6648a88dc35cd81559a31d10678d6c68fc920c876914e71324d5e460.json

@@ -0,0 +1,24 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT 1 FROM v2_job WHERE id = $1 AND workspace_id = $2 AND tag = ANY($3))",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Text",
+        "TextArray"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "8e8933fc6648a88dc35cd81559a31d10678d6c68fc920c876914e71324d5e460"
+}

backend/.sqlx/query-8eb5866b6279cb386bbeb7c387a7c731317199b28694a9c3e04028ef1f1d7500.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM skip_workspace_diff_tally",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "8eb5866b6279cb386bbeb7c387a7c731317199b28694a9c3e04028ef1f1d7500"
+}