diff --git a/.agents/skills/pr/SKILL.md b/.agents/skills/pr/SKILL.md index 51cd2305c3..bd40b6c472 100644 --- a/.agents/skills/pr/SKILL.md +++ b/.agents/skills/pr/SKILL.md @@ -139,8 +139,10 @@ A PR leaves draft **only after a clean CI review round**. Never run `gh pr ready It comments `/review` on the PR — which runs the Codex, Claude and Pi CI reviewers even on a draft — waits for the spawned `PR Review Commands` workflow run(s) to complete, then prints one verdict line per reviewer and saves the full review comments to files. + `/review` (and `/codex`) are **idempotent per head SHA**: if a running or successful review already covers the current head, they skip that agent and post nothing new — the waiter reads the existing verdict for that head, so a skipped agent is *not* a missing one. A cancelled/failed head run is re-run in place; a fresh run is launched only when nothing covers the head. So an unchanged-head re-review is a near no-op, not a new round — push a commit to get genuinely fresh reviews. + 2. **Judge the round.** Codex is mandatory; Claude, Pi and cubic count whenever they posted. Every review starts with one of the three `REVIEW.md` verdicts: - - Codex verdict missing → the round is void: comment `/codex` on the PR, wait for it the same way, and judge again. + - Codex verdict missing → the round is void: the waiter warns only when the head has no green Codex run (cancelled/failed/absent — not merely skipped-because-already-reviewed). Comment `/codex` on the PR, which re-runs the interrupted run in place (or launches one if none exists), wait the same way, and judge again. - Any **"Should address issues before merging"** → fix the P0/P1 findings (and the nits while you're there), commit, push, and start a new round (step 1). - Only **"Mergeable, but should ideally address nits"** and/or **"Good to merge"** → fix the nits too; a nit that is wrong or genuinely not worth fixing may instead be dismissed by replying to the review comment with your reasoning. Push nit-only fixes without starting another full round. diff --git a/.agents/skills/pr/review-round.sh b/.agents/skills/pr/review-round.sh index a2dfa526a0..abd94005f6 100755 --- a/.agents/skills/pr/review-round.sh +++ b/.agents/skills/pr/review-round.sh @@ -79,10 +79,43 @@ while :; do sleep 60 done +# Head SHA at trigger time. `/review` is idempotent per head: it skips an agent a +# running/successful review already covers, re-runs a cancelled/failed one in place on a +# separate head-tied run, and launches fresh only when nothing covers the head. Verdict +# reading below therefore keys off the head, not just the trigger timestamp. +HEAD_SHA=$(retry gh api "repos/$REPO/pulls/$PR" --jq .head.sha) +echo "Reviewing head $HEAD_SHA" + +# Newest non-skipped run of tied to the head ("status conclusion"), or empty +# when none exists. A re-run-in-place or an already-covering review resolves on such a +# head-tied run — separate from the pr-review-commands run waited on above (a fresh +# launch instead runs inside it, and posts after the trigger). A `skipped` run is the +# draft/fork gate and produced no review, so it is ignored. +head_run_state() { + gh run list --repo "$REPO" --workflow "$1" --commit "$HEAD_SHA" --limit 20 \ + --json databaseId,status,conclusion \ + --jq '[.[] | select(.conclusion != "skipped")] | sort_by(.databaseId) | last | if . then "\(.status) \(.conclusion // "-")" else empty end' 2>/dev/null || true +} + +# A re-run-in-place review lands on a head-tied run that finishes after the fast +# pr-review-commands run, so let those settle before reading verdicts. +for wf in codex-pr-review.yml pi-pr-review.yml pr-ready-review.yml; do + while :; do + case "$(head_run_state "$wf")" in + ""|"completed "*) break ;; + *) if [ "$(date +%s)" -gt "$DEADLINE" ]; then break; fi; sleep 30 ;; + esac + done +done + OUT_DIR=$(mktemp -d -t review-round-XXXXXX) COMMENTS_RAW=$(retry gh api "repos/$REPO/issues/$PR/comments?per_page=100" --paginate) +# Two views: comments from THIS round (after the trigger) and the full history. A fresh +# launch posts after the trigger; an idempotent skip leaves the covering verdict in the +# earlier run's comment, so fall back to history when that agent's head run is green. jq -s --arg t "$TRIGGER_TIME" '[.[][] | select(.created_at > $t)]' \ <<<"$COMMENTS_RAW" > "$OUT_DIR/comments.json" +jq -s '[.[][]]' <<<"$COMMENTS_RAW" > "$OUT_DIR/comments-all.json" # cubic posts through the PR reviews API, not issue comments. REVIEWS_RAW=$(retry gh api "repos/$REPO/pulls/$PR/reviews?per_page=100" --paginate) jq -s --arg t "$TRIGGER_TIME" '[.[][] | select((.submitted_at // "") > $t)]' \ @@ -90,18 +123,28 @@ jq -s --arg t "$TRIGGER_TIME" '[.[][] | select((.submitted_at // "") > $t)]' \ VERDICT_RE='(Good to merge|Mergeable, but should ideally address nits|Should address issues before merging)' -body_by_header() { - jq -r --arg h "$1" '[.[] | select(.body // "" | contains($h))] | last | .body // empty' \ - "$OUT_DIR/comments.json" +body_by_header() { # + jq -r --arg h "$2" '[.[] | select(.body // "" | contains($h))] | last | .body // empty' "$1" } -body_by_login() { - jq -r --arg l "$1" '[.[] | select(.user.login == $l)] | last | .body // empty' \ - "$OUT_DIR/comments.json" +body_by_login() { # + jq -r --arg l "$2" '[.[] | select(.user.login == $l)] | last | .body // empty' "$1" +} +head_ok() { [ "$(head_run_state "$1")" = "completed success" ]; } +# Latest verdict for a reviewer: prefer this round's comment; if none and the reviewer's +# head run succeeded (an idempotent /review skipped re-reviewing an already-green head), +# fall back to the covering comment from the full history. +verdict_body() { # + local body + body=$("body_by_$1" "$OUT_DIR/comments.json" "$2") + if [ -z "$body" ] && head_ok "$3"; then + body=$("body_by_$1" "$OUT_DIR/comments-all.json" "$2") + fi + printf '%s' "$body" } report() { # local name=$1 body=$2 verdict if [ -z "$body" ]; then - echo "$name: (no review posted this round)" + echo "$name: (no review posted for this head)" return fi printf '%s\n' "$body" > "$OUT_DIR/$name.md" @@ -110,11 +153,11 @@ report() { # } echo -echo "=== Review round verdicts for $REPO#$PR (posted after $TRIGGER_TIME) ===" -CODEX_BODY=$(body_by_header '## Codex Review') +echo "=== Review round verdicts for $REPO#$PR (head $HEAD_SHA) ===" +CODEX_BODY=$(verdict_body header '## Codex Review' codex-pr-review.yml) report codex "$CODEX_BODY" -report claude "$(body_by_login 'claude[bot]')" -report pi "$(body_by_header '## Pi Review')" +report claude "$(verdict_body login 'claude[bot]' pr-ready-review.yml)" +report pi "$(verdict_body header '## Pi Review' pi-pr-review.yml)" CUBIC_BODY=$(jq -r '[.[] | select(.user.login | test("^cubic(-dev-ai)?(\\[bot\\])?$"; "i"))] | last | .body // empty' \ "$OUT_DIR/pr-reviews.json") if [ -z "$CUBIC_BODY" ]; then @@ -125,5 +168,5 @@ report cubic "$CUBIC_BODY" echo echo "Full round output: $OUT_DIR (comments.json, pr-reviews.json, one .md per reviewer)" if [ -z "$CODEX_BODY" ]; then - echo "WARNING: Codex verdict missing - the round is incomplete. Re-trigger with a '/codex' PR comment and wait again." >&2 + echo "WARNING: no Codex verdict for $HEAD_SHA - its head run is not green (cancelled/failed/absent, not merely skipped-because-already-reviewed). Re-trigger with a '/codex' PR comment (re-runs the interrupted run in place, or launches one) and wait again." >&2 fi diff --git a/.github/workflows/backend-test-windows.yml b/.github/workflows/backend-test-windows.yml index b952a85e7f..5042ed9bfe 100644 --- a/.github/workflows/backend-test-windows.yml +++ b/.github/workflows/backend-test-windows.yml @@ -51,6 +51,11 @@ jobs: with: cache-workspaces: backend toolchain: 1.97.0 + # This action defaults RUSTFLAGS to "-D warnings"; unset it so the test + # run is not failed by cross-platform dead-code (cfg(unix)-only helpers + # are unused on Windows). Warning hygiene is enforced on the Linux CI + # and the build_windows_worker_ release build, not this test job. + rustflags: "" - uses: actions/setup-dotnet@v4 with: @@ -203,9 +208,15 @@ jobs: WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES: 1 WMDEBUG_FORCE_RUNNABLE_SETTINGS_V0: 1 WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT: 1 + # Windows ships a worker-only binary, so test the crates a worker runs + # (windmill-worker/-common/-queue) via -p, not `--all`: this skips the + # disk-heavy windmill-api test binaries (LNK1180) and the server-only + # windmill-trigger-* crates (amqp does not build on Windows). Linux CI runs the rest. run: > cargo test --no-fail-fast - --features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,parquet,private,csharp,php,quickjs,mcp,run_inline - --all + -p windmill-worker + -p windmill-common + -p windmill-queue + --features private,enterprise,deno_core,duckdb,python,rust,csharp,php,quickjs,parquet,mcp,scoped_cache,windmill-git-sync/private,windmill-object-store/private,windmill-object-store/enterprise -- --nocapture --test-threads=10 diff --git a/.github/workflows/build_windows_worker_.yml b/.github/workflows/build_windows_worker_.yml index 2d573c6959..84f4366235 100644 --- a/.github/workflows/build_windows_worker_.yml +++ b/.github/workflows/build_windows_worker_.yml @@ -45,8 +45,12 @@ jobs: env: RUSTFLAGS: "-D warnings" run: | - mkdir frontend/build && cd backend + cd backend + # Stub the openapi specs to empty: they are compiled in via an ungated + # include_str! but a worker binary never serves them, so this avoids + # embedding ~2.5MB of spec. New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force + New-Item -Path . -Name "windmill-api/openapi-deref.json" -ItemType "File" -Force cargo check --features=ee_windows - name: Cargo build dynamic libraries windows diff --git a/.github/workflows/git-sync-test.yml b/.github/workflows/git-sync-test.yml index 3c4d37622a..4b99c62eae 100644 --- a/.github/workflows/git-sync-test.yml +++ b/.github/workflows/git-sync-test.yml @@ -9,6 +9,10 @@ on: - "backend/windmill-api-integration-tests/tests/git_sync*" - "backend/ee-repo-ref.txt" - "backend/windmill-common/src/workspaces.rs" + - "backend/windmill-worker/src/result_processor.rs" + - "backend/windmill-api-workspaces/**" + - "cli/src/commands/sync/**" + - "cli/src/utils/git.ts" - "integration_tests/test/git_sync_test.py" - ".github/workflows/git-sync-test.yml" pull_request: @@ -18,6 +22,10 @@ on: - "backend/windmill-api-integration-tests/tests/git_sync*" - "backend/ee-repo-ref.txt" - "backend/windmill-common/src/workspaces.rs" + - "backend/windmill-worker/src/result_processor.rs" + - "backend/windmill-api-workspaces/**" + - "cli/src/commands/sync/**" + - "cli/src/utils/git.ts" - "integration_tests/test/git_sync_test.py" - ".github/workflows/git-sync-test.yml" @@ -50,8 +58,8 @@ jobs: echo "Changed files:" echo "$CHANGED_FILES" - # Direct git sync file changes — always relevant - if echo "$CHANGED_FILES" | grep -qE '^(backend/windmill-git-sync/|backend/windmill-api-integration-tests/tests/git_sync|backend/windmill-common/src/workspaces\.rs|integration_tests/test/git_sync|\.github/workflows/git-sync-test\.yml)'; then + # Direct git sync file changes — always relevant. + if echo "$CHANGED_FILES" | grep -qE '^(backend/windmill-git-sync/|backend/windmill-worker/src/result_processor\.rs|backend/windmill-api-workspaces/|backend/windmill-api-integration-tests/tests/git_sync|backend/windmill-common/src/workspaces\.rs|cli/src/commands/sync/|cli/src/utils/git\.ts|integration_tests/test/git_sync|\.github/workflows/git-sync-test\.yml)'; then echo "should_run=true" >> "$GITHUB_OUTPUT" echo "Relevant: direct git sync file changes" exit 0 diff --git a/.github/workflows/pr-review-commands.yml b/.github/workflows/pr-review-commands.yml index ef93274d7e..3c04e4e5f2 100644 --- a/.github/workflows/pr-review-commands.yml +++ b/.github/workflows/pr-review-commands.yml @@ -75,14 +75,153 @@ jobs: "/repos/$REPO/issues/comments/$COMMENT_ID/reactions" \ -f content=eyes >/dev/null - claude: + # Decide, per agent, whether to launch a fresh run, re-run in place, or skip. A push + # already auto-triggers codex/pi (and claude on open) against the PR head. Relaunching + # via this issue_comment path both cancels those in-flight auto runs (shared concurrency + # group) AND lands the new run's status on main — issue_comment runs never attach a + # check to the PR head — leaving the PR showing only a cancelled review. So for every + # command, launch an agent only when nothing covers the head commit; if the head's run + # was cancelled/failed, re-run it in place (a re-run keeps the original pull_request + # event, so its checks re-attach to the PR head); skip when a running or successful run + # already covers it. `/review` applies this to all three agents; `/codex`, `/pi`, + # `/claude` apply the same decision to just their own agent. + plan: needs: [parse, check-access] + if: | + needs.parse.outputs.command != '' && + ( + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) || + needs.check-access.outputs.authorized == 'true' + ) + runs-on: ubuntu-latest + permissions: + contents: read + actions: write + pull-requests: read + statuses: write + outputs: + head_sha: ${{ steps.plan.outputs.head_sha }} + launch_codex: ${{ steps.plan.outputs.launch_codex }} + launch_pi: ${{ steps.plan.outputs.launch_pi }} + launch_claude: ${{ steps.plan.outputs.launch_claude }} + steps: + - name: Decide per-agent launch vs re-run for the head commit + id: plan + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.issue.number }} + COMMAND: ${{ needs.parse.outputs.command }} + run: | + set -euo pipefail + + HEAD_SHA=$(gh pr view "$PR_NUMBER" --repo "$REPO" --json headRefOid --jq '.headRefOid') + echo "PR #$PR_NUMBER head: $HEAD_SHA" + echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT" + + RUN_URL="$GITHUB_SERVER_URL/$REPO/actions/runs/$GITHUB_RUN_ID" + + # A fresh launch runs from this issue_comment workflow (associated with main), + # so it never appears in the PR-head run query below and its own check lands on + # main, not the head. To keep fresh launches idempotent per head, mark the head + # SHA with a `review-launch/` commit status at launch; the `finalize` job + # resolves it to success/failure. A prior launch's status covering the head lets + # a second comment skip instead of relaunching (which would cancel the first via + # the reviewer's shared concurrency group). All status calls are best-effort — a + # GitHub API hiccup must degrade to a relaunch, never abort the decision. + mark_launch() { + agent="$1" + gh api -X POST "repos/$REPO/statuses/$HEAD_SHA" \ + -f state=pending -f "context=review-launch/$agent" -f "target_url=$RUN_URL" \ + -f "description=Review launched via /$COMMAND" >/dev/null 2>&1 || true + } + + # Returns "covered" if a prior fresh launch (this or an earlier comment run) + # already covers the head: a success status, or a pending status whose launching + # run is still alive. A pending whose run has completed is stale (that run + # crashed before finalize) and does not count. + launch_coverage() { + agent="$1" + st_json=$(gh api "repos/$REPO/commits/$HEAD_SHA/statuses" \ + --jq "[.[] | select(.context == \"review-launch/$agent\")] | first // empty" 2>/dev/null || true) + [ -n "$st_json" ] || return 0 + state=$(jq -r '.state // empty' <<<"$st_json" 2>/dev/null || true) + [ "$state" = success ] && { echo covered; return 0; } + [ "$state" = pending ] || return 0 + target=$(jq -r '.target_url // empty' <<<"$st_json" 2>/dev/null || true) + run_id=$(printf '%s' "$target" | grep -oE '[0-9]+$' || true) + if [ -n "$run_id" ]; then + run_state=$(gh run view "$run_id" --repo "$REPO" --json status --jq '.status' 2>/dev/null || true) + [ "$run_state" = completed ] && return 0 # stale pending -> not covered + fi + echo covered + } + + decide() { + wf="$1"; key="$2"; agent="$3" + if [ "$(launch_coverage "$agent")" = covered ]; then + echo "$key: a prior launch already covers $HEAD_SHA (review-launch/$agent) -> skip" + echo "$key=false" >> "$GITHUB_OUTPUT" + return + fi + # `--commit` matches runs whose head SHA is the PR head. Auto reviews run on + # `pull_request` against that SHA; `/review` (issue_comment) runs execute on + # main, so they never match and are not counted as covering the head commit. + runs=$(gh run list --repo "$REPO" --workflow "$wf" --commit "$HEAD_SHA" --limit 40 \ + --json databaseId,status,conclusion) + # Healthy = still running, or completed successfully: a review already + # covers this commit, so skip. + healthy=$(jq -r '[.[] | select(.status != "completed" or .conclusion == "success")] | length' <<<"$runs") + if [ "$healthy" -gt 0 ]; then + echo "$key: a running or successful review already covers $HEAD_SHA -> skip" + echo "$key=false" >> "$GITHUB_OUTPUT" + return + fi + # Re-run only genuinely interrupted runs (cancelled/failed/timed out) in + # place, so their checks re-attach to the PR head instead of posting on + # main. A `skipped` run produced no review and would just skip again (it is + # the draft/fork gate), so it does not count — fall through to a fresh launch. + retry_id=$(jq -r '[.[] | select(.status == "completed" and (.conclusion == "cancelled" or .conclusion == "failure" or .conclusion == "timed_out"))] | sort_by(.databaseId) | last | .databaseId // empty' <<<"$runs") + if [ -n "$retry_id" ]; then + if gh run rerun "$retry_id" --repo "$REPO" >/dev/null 2>&1; then + echo "$key: re-ran interrupted run $retry_id (re-attaches to PR head)" + echo "$key=false" >> "$GITHUB_OUTPUT" + return + fi + echo "$key: re-run of $retry_id failed -> fresh launch" + mark_launch "$agent" + echo "$key=true" >> "$GITHUB_OUTPUT" + return + fi + echo "$key: no usable review for $HEAD_SHA -> launch" + mark_launch "$agent" + echo "$key=true" >> "$GITHUB_OUTPUT" + } + + # `/review` targets all three agents; `/codex`, `/pi`, `/claude` target only + # their own. A non-targeted agent is left untouched (no launch, no re-run). + decide_if_targeted() { + wf="$1"; key="$2"; agent="$3" + if [ "$COMMAND" = review ] || [ "$COMMAND" = "$agent" ]; then + decide "$wf" "$key" "$agent" + else + echo "$key: /$COMMAND does not target $agent -> skip" + echo "$key=false" >> "$GITHUB_OUTPUT" + fi + } + + decide_if_targeted codex-pr-review.yml launch_codex codex + decide_if_targeted pi-pr-review.yml launch_pi pi + decide_if_targeted pr-ready-review.yml launch_claude claude + + claude: + needs: [parse, check-access, plan] if: | ( contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) || needs.check-access.outputs.authorized == 'true' ) && - (needs.parse.outputs.command == 'review' || needs.parse.outputs.command == 'claude') + needs.plan.outputs.launch_claude == 'true' permissions: contents: read pull-requests: read @@ -97,13 +236,13 @@ jobs: WINDMILL_EE_PRIVATE_ACCESS: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} codex: - needs: [parse, check-access] + needs: [parse, check-access, plan] if: | ( contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) || needs.check-access.outputs.authorized == 'true' ) && - (needs.parse.outputs.command == 'review' || needs.parse.outputs.command == 'codex') + needs.plan.outputs.launch_codex == 'true' permissions: contents: read issues: write @@ -119,13 +258,13 @@ jobs: WINDMILL_EE_PRIVATE_ACCESS: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} pi: - needs: [parse, check-access] + needs: [parse, check-access, plan] if: | ( contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) || needs.check-access.outputs.authorized == 'true' ) && - (needs.parse.outputs.command == 'review' || needs.parse.outputs.command == 'pi') + needs.plan.outputs.launch_pi == 'true' permissions: contents: read issues: write @@ -138,3 +277,34 @@ jobs: secrets: DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} WINDMILL_EE_PRIVATE_ACCESS: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} + + # Resolve the `review-launch/` head statuses that `plan` set to pending, so a + # fresh launch's outcome is visible on the PR head (not just on main) and never lingers + # as a stale pending check. Targets the exact SHA `plan` launched against, so a push + # that moved the head mid-review does not stamp a status on the new head. + finalize: + needs: [plan, claude, codex, pi] + if: always() && needs.plan.result == 'success' && needs.plan.outputs.head_sha != '' + runs-on: ubuntu-latest + permissions: + statuses: write + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + HEAD_SHA: ${{ needs.plan.outputs.head_sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + steps: + - name: Finalize launch statuses on the PR head + run: | + set -uo pipefail + finalize() { + agent="$1"; launched="$2"; result="$3" + [ "$launched" = true ] || return 0 + state=$([ "$result" = success ] && echo success || echo failure) + gh api -X POST "repos/$REPO/statuses/$HEAD_SHA" \ + -f "state=$state" -f "context=review-launch/$agent" -f "target_url=$RUN_URL" \ + -f "description=Review $result" >/dev/null 2>&1 || true + } + finalize codex "${{ needs.plan.outputs.launch_codex }}" "${{ needs.codex.result }}" + finalize pi "${{ needs.plan.outputs.launch_pi }}" "${{ needs.pi.result }}" + finalize claude "${{ needs.plan.outputs.launch_claude }}" "${{ needs.claude.result }}" diff --git a/.github/workflows/publish_windows_worker.yml b/.github/workflows/publish_windows_worker.yml index 019de201e6..d159619fba 100644 --- a/.github/workflows/publish_windows_worker.yml +++ b/.github/workflows/publish_windows_worker.yml @@ -56,8 +56,12 @@ jobs: vcpkg.exe integrate install $env:VCPKGRS_DYNAMIC=1 $env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static" - mkdir frontend/build && cd backend + cd backend + # Stub the openapi specs to empty: they are compiled in via an ungated + # include_str! but a worker binary never serves them, so this avoids + # embedding ~2.5MB of spec. New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force + New-Item -Path . -Name "windmill-api/openapi-deref.json" -ItemType "File" -Force cargo build --release --features=ee_windows - name: Rename binary with corresponding architecture run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 382a04c843..e800f74201 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,128 @@ # Changelog +## [1.769.0](https://github.com/windmill-labs/windmill/compare/v1.768.0...v1.769.0) (2026-07-24) + + +### Features + +* **hub:** surface data pipelines in deploy-to-hub drawer ([#10299](https://github.com/windmill-labs/windmill/issues/10299)) ([010059a](https://github.com/windmill-labs/windmill/commit/010059a449f4d0cde0656086f1b3f03a02a1c78c)) +* **pipeline:** collapse secondary top-bar controls into an overflow menu ([#10300](https://github.com/windmill-labs/windmill/issues/10300)) ([1d25d75](https://github.com/windmill-labs/windmill/commit/1d25d7539ed7fb17c47a732d6b1cd7fc19202a04)) + + +### Bug Fixes + +* pin table actions column so it stays visible on narrow screens ([#10301](https://github.com/windmill-labs/windmill/issues/10301)) ([75acf72](https://github.com/windmill-labs/windmill/commit/75acf7207b151f0227600be555bfa5e14dfa8dd1)) +* pin validated DNS address to close SSRF DNS-rebinding TOCTOU ([#10303](https://github.com/windmill-labs/windmill/issues/10303)) ([3cf7a39](https://github.com/windmill-labs/windmill/commit/3cf7a390a37c07a818248c95defa4ecf5bf262e5)) + + +### Performance Improvements + +* optimize get_datatable_full_schema to avoid timeout on large catalogs ([#10304](https://github.com/windmill-labs/windmill/issues/10304)) ([1478d12](https://github.com/windmill-labs/windmill/commit/1478d12eb352b1b7906ccfe3ab5eaafe60ffbe4f)) + +## [1.768.0](https://github.com/windmill-labs/windmill/compare/v1.767.0...v1.768.0) (2026-07-24) + + +### Features + +* Add section to deploy projects to hub ([#9332](https://github.com/windmill-labs/windmill/issues/9332)) ([30eedf9](https://github.com/windmill-labs/windmill/commit/30eedf9ee1754cb2bbf46e82d2b3eac766bd3e6d)) +* **ai-chat:** let the session/global chat create email triggers ([#10282](https://github.com/windmill-labs/windmill/issues/10282)) ([248c875](https://github.com/windmill-labs/windmill/commit/248c8751b1fd5d6f7506a7c3fc8f500dd98af26e)) +* **ai:** enable data pipelines in AI sessions with alpha notice ([#10273](https://github.com/windmill-labs/windmill/issues/10273)) ([1f912f4](https://github.com/windmill-labs/windmill/commit/1f912f410401ec68789d43973d6e0a1f8e595f86)) +* alert on expired online license key ([#10295](https://github.com/windmill-labs/windmill/issues/10295)) ([30d8104](https://github.com/windmill-labs/windmill/commit/30d8104edcbdb7f0c63f186d22ac03000e0d4f07)) +* data-pipeline recorder, interactive player, and deploy-to-hub recording (WIN-2156) ([#10055](https://github.com/windmill-labs/windmill/issues/10055)) ([65e5041](https://github.com/windmill-labs/windmill/commit/65e504146d832b105603ceec89c6de0ad1d66e32)) +* let a workspace fall back to the instance critical alert channels ([#10292](https://github.com/windmill-labs/windmill/issues/10292)) ([717e38a](https://github.com/windmill-labs/windmill/commit/717e38a0c6b5bb2340e236a9d49645a4cebf4849)) +* **monitor:** make between-steps zombie flows hand-recoverable ([#10287](https://github.com/windmill-labs/windmill/issues/10287)) ([f02df7f](https://github.com/windmill-labs/windmill/commit/f02df7fc454b0c2a2afa9ae9848e26af0e379246)) +* surface workspace-script advanced settings in flow editor ([#10289](https://github.com/windmill-labs/windmill/issues/10289)) ([bf16e7d](https://github.com/windmill-labs/windmill/commit/bf16e7d49a7486d37cf9eb1907e80abae47a78a7)) +* **windows:** enable ruby and rlang on the windows worker ([#10279](https://github.com/windmill-labs/windmill/issues/10279)) ([0f1b864](https://github.com/windmill-labs/windmill/commit/0f1b8641f26f0eda8a8517db1a37efafdb173464)) + + +### Bug Fixes + +* **ci:** make /review idempotent per head commit, re-run cancelled reviews in place ([#10283](https://github.com/windmill-labs/windmill/issues/10283)) ([3aaceb7](https://github.com/windmill-labs/windmill/commit/3aaceb7efb3bd232da73b79fcdcf4c712caad7d3)) +* **cli:** surface shared UI changes in sync push dry-run preview ([#10278](https://github.com/windmill-labs/windmill/issues/10278)) ([14c29b7](https://github.com/windmill-labs/windmill/commit/14c29b77e90842b8c3d4395ce50cb44666cd998c)) +* **jobs:** sanitize NUL in completed job result before jsonb insert ([#10274](https://github.com/windmill-labs/windmill/issues/10274)) ([c50a2ab](https://github.com/windmill-labs/windmill/commit/c50a2abad0c222bf8f76ff10b864450120ee9ee7)) +* **monitor:** diagnose zombie-flow OOM on the transition worker, not q.worker ([#10286](https://github.com/windmill-labs/windmill/issues/10286)) ([fa36442](https://github.com/windmill-labs/windmill/commit/fa3644281f30d90aa0c4ca86520a38db5d8b91f0)) +* resolve svelte/style export conditions in raw-app CLI bundler ([#10294](https://github.com/windmill-labs/windmill/issues/10294)) ([9713e60](https://github.com/windmill-labs/windmill/commit/9713e6074d2df55db1331cf31ee1357606b1dbe9)) +* **resources:** apply resource_type changes on update (git-sync pull) — Fixes GIT-932 ([#10277](https://github.com/windmill-labs/windmill/issues/10277)) ([d7a0078](https://github.com/windmill-labs/windmill/commit/d7a0078b58f74f1ffecc03b348e8935a70f02e55)) +* surface workspace ids on duplicate names and explain fork promotion ([#10291](https://github.com/windmill-labs/windmill/issues/10291)) ([9b182aa](https://github.com/windmill-labs/windmill/commit/9b182aaf3879d4d81b6a785222dca75f085e6fb7)) +* treat concurrent_limit/timeout <= 0 as unset instead of a zero cap ([#10288](https://github.com/windmill-labs/windmill/issues/10288)) ([8eb36ce](https://github.com/windmill-labs/windmill/commit/8eb36ce008b4efe2be9a9bfebc91af68070f7a6c)) + +## [1.767.0](https://github.com/windmill-labs/windmill/compare/v1.766.2...v1.767.0) (2026-07-22) + + +### Features + +* **ai-chat:** email triggers in flow/script chat + trigger-intent eval guards (WIN-2228) ([#10267](https://github.com/windmill-labs/windmill/issues/10267)) ([e41440b](https://github.com/windmill-labs/windmill/commit/e41440b344d94ebe5746a89237c3d1be522aa59d)) +* **ai:** improve data-pipeline building in AI sessions (prompt + evals + e2e) ([#10270](https://github.com/windmill-labs/windmill/issues/10270)) ([0819641](https://github.com/windmill-labs/windmill/commit/0819641f3a89abdf114cc5e341a9f585e5d71296)) + + +### Bug Fixes + +* **ai-chat:** improve resource-type search tool description and scoring ([#10272](https://github.com/windmill-labs/windmill/issues/10272)) ([ad53673](https://github.com/windmill-labs/windmill/commit/ad53673a2855e787af9fe10dcc8998ffc7f1b960)) +* **embeddings:** retry on failed init instead of disabling for a day ([#10266](https://github.com/windmill-labs/windmill/issues/10266)) ([2318481](https://github.com/windmill-labs/windmill/commit/2318481f4f2932dc578d5cb64c4c705cf5c324a0)) +* manual resource type sync fetches from hub first, cache as fallback ([#10269](https://github.com/windmill-labs/windmill/issues/10269)) ([07d4b67](https://github.com/windmill-labs/windmill/commit/07d4b674f1dbd24c89f7d1e77094a40e528a3740)) + +## [1.766.2](https://github.com/windmill-labs/windmill/compare/v1.766.1...v1.766.2) (2026-07-22) + + +### Bug Fixes + +* **python,windows:** cross-platform cross-process wheel-install lock ([#10264](https://github.com/windmill-labs/windmill/issues/10264)) ([7e2f1af](https://github.com/windmill-labs/windmill/commit/7e2f1afffb4982099870e7ce32592cf3a447e513)) + +## [1.766.1](https://github.com/windmill-labs/windmill/compare/v1.766.0...v1.766.1) (2026-07-22) + + +### Bug Fixes + +* **jobs:** enforce self_approval_disabled on the UI resume path ([#10262](https://github.com/windmill-labs/windmill/issues/10262)) ([2d24b3a](https://github.com/windmill-labs/windmill/commit/2d24b3ac49b65469072bca929730d2c9ecd54b8f)) + +## [1.766.0](https://github.com/windmill-labs/windmill/compare/v1.765.0...v1.766.0) (2026-07-22) + + +### Features + +* **ai-sessions:** show item preview cards for tools ([#10254](https://github.com/windmill-labs/windmill/issues/10254)) ([703744f](https://github.com/windmill-labs/windmill/commit/703744fb8b76f70384cbc3341d5e1b3dbf30455e)) +* make content search a full CE feature ([#10252](https://github.com/windmill-labs/windmill/issues/10252)) ([d2c5d6f](https://github.com/windmill-labs/windmill/commit/d2c5d6f4b4ede9449407ed3831713d3e3d1d6972)) +* **sessions:** ship AI sessions as beta with legacy-chat opt-out ([#10242](https://github.com/windmill-labs/windmill/issues/10242)) ([0508cdd](https://github.com/windmill-labs/windmill/commit/0508cddf0a86b6a6a043cb31978ca68b76a95d02)) + + +### Bug Fixes + +* accept ssh/scheme-less git repo urls and $var: refs in app repo resolution ([#10246](https://github.com/windmill-labs/windmill/issues/10246)) ([b948efd](https://github.com/windmill-labs/windmill/commit/b948efd3c81aada103bd141eae03fbe626bf6ee5)) +* **copilot:** stop write_flow forcing rawscript code into nested JSON ([#10260](https://github.com/windmill-labs/windmill/issues/10260)) ([ecb1a92](https://github.com/windmill-labs/windmill/commit/ecb1a92070fb18048a6a1467270469e8462a3cbf)) +* prevent u16 underflow in suspend count causing permanent flow deadlock ([#10256](https://github.com/windmill-labs/windmill/issues/10256)) ([68b1fcc](https://github.com/windmill-labs/windmill/commit/68b1fcc5cdd3d217ebb1b9e09c0d52334faf0a63)) +* **prompts:** prefer Bun over Deno for TypeScript runtime selection ([#10253](https://github.com/windmill-labs/windmill/issues/10253)) ([380cf75](https://github.com/windmill-labs/windmill/commit/380cf752ca8fc57eb70f7c8efd57290f7902a030)) +* **tutorials:** repair broken frontend tutorials after UI redesigns ([#10255](https://github.com/windmill-labs/windmill/issues/10255)) ([5685981](https://github.com/windmill-labs/windmill/commit/5685981c9902ff37c3bd271dc5c5ede948a0787a)) + +## [1.765.0](https://github.com/windmill-labs/windmill/compare/v1.764.0...v1.765.0) (2026-07-21) + + +### Features + +* **ai:** open the Compare & Deploy page from chat with item preselection ([#10232](https://github.com/windmill-labs/windmill/issues/10232)) ([572d69e](https://github.com/windmill-labs/windmill/commit/572d69e5ae8ae12207dca175fdb5ea378d852a45)) +* **ai:** session chat nits — empty sends, command picker polish, session-state prompt ([#10233](https://github.com/windmill-labs/windmill/issues/10233)) ([9bc1f62](https://github.com/windmill-labs/windmill/commit/9bc1f6212837155c964a7db7103aabbcf14cbce2)) +* attach text files to chat messages, read on demand via file tools ([#10215](https://github.com/windmill-labs/windmill/issues/10215)) ([d6cf1ef](https://github.com/windmill-labs/windmill/commit/d6cf1ef9872cd0db0f51e4881237bac383e45f61)) +* **git-sync:** enable per-item promotion mode on dev workspaces ([#10205](https://github.com/windmill-labs/windmill/issues/10205)) ([2ce21c9](https://github.com/windmill-labs/windmill/commit/2ce21c9ef86c608d73e12a7e278d2f328e06b6fa)) +* **pipelines:** catalog declared measures and dimensions ([#10190](https://github.com/windmill-labs/windmill/issues/10190)) ([fd51d40](https://github.com/windmill-labs/windmill/commit/fd51d40f1254a206de1f38e59cc3dadce8e13aa5)) +* **triggers:** add AMQP (RabbitMQ) trigger via lapin ([#10230](https://github.com/windmill-labs/windmill/issues/10230)) ([68debab](https://github.com/windmill-labs/windmill/commit/68debab877c6dc8ee3732e0c23d467db85fd4584)) +* unified read-only `diff` chat tool (drafts, fork vs parent, search) ([#10211](https://github.com/windmill-labs/windmill/issues/10211)) ([9739d5a](https://github.com/windmill-labs/windmill/commit/9739d5a2c2a21beded8848ea9049b2816196a898)) + + +### Bug Fixes + +* **apps:** let entitled viewers read pre-existing S3 files from deployed apps ([#10245](https://github.com/windmill-labs/windmill/issues/10245)) ([4a89824](https://github.com/windmill-labs/windmill/commit/4a898247a21ae918fa952a993fbe407ebc49e404)) +* **db:** repair s3 asset paths missing default-storage leading slash ([#10243](https://github.com/windmill-labs/windmill/issues/10243)) ([555c751](https://github.com/windmill-labs/windmill/commit/555c751016fea4087be09fa2520af331cc872bfc)) +* **frontend:** curl fallback for +Variable/+Resource in bash sandbox mode ([#10235](https://github.com/windmill-labs/windmill/issues/10235)) ([28966bd](https://github.com/windmill-labs/windmill/commit/28966bdbf190ac461aa5259e04d6c15cf699fd94)) +* **frontend:** limit compare & deploy rows to the active direction ([#10234](https://github.com/windmill-labs/windmill/issues/10234)) ([7ac27c1](https://github.com/windmill-labs/windmill/commit/7ac27c1ef240729fc38d5fa506b96bf60be74e25)) +* **frontend:** prevent browser back-swipe navigation over monaco editors ([#10229](https://github.com/windmill-labs/windmill/issues/10229)) ([b0bf256](https://github.com/windmill-labs/windmill/commit/b0bf25683ba3aac3e94bbe9d6bbd6e22545e734e)) +* **parsers:** keep s3 asset path suffix verbatim to preserve storage distinction ([#10241](https://github.com/windmill-labs/windmill/issues/10241)) ([7fb8a2e](https://github.com/windmill-labs/windmill/commit/7fb8a2e390cef3cd01a33c89ee55d3f53cb1e20f)) +* **parser:** spurious pg arg inferred from placeholders in comments ([#10226](https://github.com/windmill-labs/windmill/issues/10226)) ([d24e176](https://github.com/windmill-labs/windmill/commit/d24e1768163c10fbbc144bd1126457e42d282e37)) +* **pg:** actionable error when s3object input exceeds jsonb 256MB cap ([#10228](https://github.com/windmill-labs/windmill/issues/10228)) ([6e42633](https://github.com/windmill-labs/windmill/commit/6e4263364325e4d4616697c6da601bbc887ff43c)) +* **postgres-triggers:** enforce resource-path scopes on ancillary routes ([#10222](https://github.com/windmill-labs/windmill/issues/10222)) ([39058c0](https://github.com/windmill-labs/windmill/commit/39058c0a01c35cf553daae3af37c6d326a7c8763)) +* return to parent workspace when a fork is deleted remotely ([#9898](https://github.com/windmill-labs/windmill/issues/9898)) ([32994df](https://github.com/windmill-labs/windmill/commit/32994df427e6df6747caf1ba6371512d5a5be5c2)) +* **s3:** support instance-policy credentials in object storage tests ([#10238](https://github.com/windmill-labs/windmill/issues/10238)) ([ec63244](https://github.com/windmill-labs/windmill/commit/ec6324409d6c0e9ddcda5d110ac570ca79fe9e38)) +* **triggers:** apply scope-path filtering to list and fix update scope check ([#10220](https://github.com/windmill-labs/windmill/issues/10220)) ([adc555d](https://github.com/windmill-labs/windmill/commit/adc555d1722cb59aa5b88503b3105357d76f5b71)) +* **worker:** mount /dev/shm as tmpfs in the Docker v2 nsjail sandbox ([#10240](https://github.com/windmill-labs/windmill/issues/10240)) ([2caee41](https://github.com/windmill-labs/windmill/commit/2caee41fdfe1e69010a1f4544d45aaa5db49f590)) + ## [1.764.0](https://github.com/windmill-labs/windmill/compare/v1.763.0...v1.764.0) (2026-07-20) diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index 2ab8f47f37..64cacabd58 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -304,13 +304,20 @@ export function getBenchmarkJobLogs(workspace: string, jobId: string): string { */ const benchmarkDrafts = new Map< string, - { workspace: string; kind: UserDraftItemKind; path: string; value: unknown } + { workspace: string; kind: UserDraftItemKind; path: string; value: unknown; createdAt: string } >() -// Fixed timestamp so artifacts stay deterministic. No eval simulates a -// concurrent writer, so every save is accepted and the conflict branch is -// never taken — the syncer just records this as its `last_sync` baseline. -const BENCHMARK_DRAFT_TIMESTAMP = '1970-01-01T00:00:00.000Z' +// Counter-based timestamps: deterministic run-to-run (same event order → same +// values) but MONOTONIC per update, because production bumps a draft row's +// created_at on every upsert and the diff snapshot cache keys patch reuse on +// it — a fixed timestamp would serve stale patches after an edit. No eval +// simulates a concurrent writer, so every save is accepted and the conflict +// branch is never taken. +let benchmarkDraftClock = 0 +function nextBenchmarkDraftTimestamp(): string { + benchmarkDraftClock += 1 + return new Date(benchmarkDraftClock * 1000).toISOString() +} function benchmarkDraftKey(workspace: string, kind: string, path: string): string { return `${workspace}::${kind}::${path}` @@ -341,7 +348,8 @@ export function seedBenchmarkDraft( workspace, kind, path, - value + value, + createdAt: nextBenchmarkDraftTimestamp() }) } @@ -354,6 +362,7 @@ export function updateBenchmarkDraft(input: { }): UpdateDraftResponse { const key = benchmarkDraftKey(input.workspace, input.kind, input.path) const value = input.requestBody?.value + const createdAt = nextBenchmarkDraftTimestamp() if (value == null) { benchmarkDrafts.delete(key) } else { @@ -361,10 +370,11 @@ export function updateBenchmarkDraft(input: { workspace: input.workspace, kind: input.kind, path: input.path, - value + value, + createdAt }) } - return { status: 'saved', current_timestamp: BENCHMARK_DRAFT_TIMESTAMP } + return { status: 'saved', current_timestamp: createdAt } } /** Mirror `DraftService.getDraftForUser`: 404-shaped throw when absent so the @@ -378,7 +388,7 @@ export function getBenchmarkDraftForUser(input: { if (!entry) { throw Object.assign(new Error(`no draft for "${input.path}"`), { status: 404 }) } - return { value: entry.value, created_at: BENCHMARK_DRAFT_TIMESTAMP } + return { value: entry.value, created_at: entry.createdAt } } /** Mirror `DraftService.getOwnDraft`: `null` (200) when absent — unlike @@ -392,7 +402,18 @@ export function getBenchmarkOwnDraft(input: { if (!entry) { return null } - return { value: entry.value, created_at: BENCHMARK_DRAFT_TIMESTAMP } + return { value: entry.value, created_at: entry.createdAt } +} + +/** Whether a deployed benchmark item exists for a draft row's kind+path — + * drives `draft_only`, which production computes against the deployed tables. */ +function benchmarkDeployedExists(workspace: string, kind: UserDraftItemKind, path: string): boolean { + if (kind === 'script') return Boolean(getBenchmarkScriptByPath(workspace, path)) + if (kind === 'flow') return Boolean(getBenchmarkFlowByPath(workspace, path)) + if (kind === 'app' || kind === 'raw_app') return Boolean(getBenchmarkAppByPath(workspace, path)) + // Drawer kinds (variables/resources/schedules/triggers) have no deployed + // benchmark stores today. + return false } /** Mirror `DraftService.listDrafts`: metadata rows (no value) for a workspace. */ @@ -403,9 +424,9 @@ export function listBenchmarkDrafts(workspace: string): ListDraftsResponse { kind: entry.kind, path: entry.path, summary: (entry.value as { summary?: string } | null)?.summary, - draft_only: true, + draft_only: !benchmarkDeployedExists(workspace, entry.kind, entry.path), legacy_draft: false, - created_at: BENCHMARK_DRAFT_TIMESTAMP + created_at: entry.createdAt })) } diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index 1532928bff..c07d104f47 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -126,13 +126,32 @@ vi.mock('$lib/gen', async () => { hasBenchmarkWorkspace(data.workspace) ? Boolean(getBenchmarkScriptByPath(data.workspace, data.path)) : actual.ScriptService.existsScriptByPath(data), - getScriptByPath: async (data: { workspace: string; path: string }) => { + getScriptByPath: async (data: { workspace: string; path: string; getDraft?: boolean }) => { if (hasBenchmarkWorkspace(data.workspace)) { const script = getBenchmarkScriptByPath(data.workspace, data.path) + // `getDraft` mirrors production's overlay: the row plus the caller's + // draft and a `no_deployed` marker (draft-only item). The diff tool + // reads through this shape — without it every draft looks absent. + const draft = data.getDraft + ? getBenchmarkOwnDraft({ workspace: data.workspace, kind: 'script', path: data.path }) + : null if (!script) { - throw new Error(`Script "${data.path}" not found in benchmark workspace`) + if (data.getDraft && draft) { + return { + ...(draft.value as Record), + path: data.path, + draft: draft.value, + no_deployed: true + } + } + throw Object.assign( + new Error(`Script "${data.path}" not found in benchmark workspace`), + { status: 404 } + ) } - return script + return data.getDraft + ? { ...script, draft: draft?.value ?? undefined, no_deployed: false } + : script } return actual.ScriptService.getScriptByPath(data) }, @@ -166,13 +185,30 @@ vi.mock('$lib/gen', async () => { hasBenchmarkWorkspace(data.workspace) ? Boolean(getBenchmarkFlowByPath(data.workspace, data.path)) : actual.FlowService.existsFlowByPath(data), - getFlowByPath: async (data: { workspace: string; path: string }) => { + getFlowByPath: async (data: { workspace: string; path: string; getDraft?: boolean }) => { if (hasBenchmarkWorkspace(data.workspace)) { const flow = getBenchmarkFlowByPath(data.workspace, data.path) + // Mirror production's `getDraft` overlay (see getScriptByPath above). + const draft = data.getDraft + ? getBenchmarkOwnDraft({ workspace: data.workspace, kind: 'flow', path: data.path }) + : null if (!flow) { - throw new Error(`Flow "${data.path}" not found in benchmark workspace`) + if (data.getDraft && draft) { + return { + ...(draft.value as Record), + path: data.path, + draft: draft.value, + no_deployed: true + } + } + throw Object.assign( + new Error(`Flow "${data.path}" not found in benchmark workspace`), + { status: 404 } + ) } - return flow + return data.getDraft + ? { ...flow, draft: draft?.value ?? undefined, no_deployed: false } + : flow } return actual.FlowService.getFlowByPath(data) }, @@ -341,13 +377,37 @@ vi.mock('$lib/gen', async () => { hasBenchmarkWorkspace(data.workspace) ? (listBenchmarkApps(data.workspace) ?? []) : actual.AppService.listApps(data), - getAppByPath: async (data: { workspace: string; path: string }) => { + getAppByPath: async (data: { + workspace: string + path: string + getDraft?: boolean + rawApp?: boolean + }) => { if (hasBenchmarkWorkspace(data.workspace)) { const app = getBenchmarkAppByPath(data.workspace, data.path) + // Mirror production's `getDraft` overlay (see getScriptByPath above). + // Benchmark app drafts live under the raw_app kind. + const draft = data.getDraft + ? getBenchmarkOwnDraft({ workspace: data.workspace, kind: 'raw_app', path: data.path }) + : null if (!app) { - throw new Error(`App "${data.path}" not found in benchmark workspace`) + if (data.getDraft && draft) { + return { + ...(draft.value as Record), + path: data.path, + raw_app: true, + draft: draft.value, + no_deployed: true + } + } + throw Object.assign( + new Error(`App "${data.path}" not found in benchmark workspace`), + { status: 404 } + ) } - return app + return data.getDraft + ? { ...app, draft: draft?.value ?? undefined, no_deployed: false } + : app } return actual.AppService.getAppByPath(data) } diff --git a/ai_evals/cases/flow.yaml b/ai_evals/cases/flow.yaml index a21ae81f17..fa2ca3bee7 100644 --- a/ai_evals/cases/flow.yaml +++ b/ai_evals/cases/flow.yaml @@ -480,3 +480,85 @@ judgeChecklist: - "the flow includes a final top-level step named `webhook_response`" - "`webhook_response` returns `ok: true` and the order summary" + +- id: flow-test17-implicit-schedule-intent + prompt: |- + I want this order processing flow to run on its own every morning at 07:30 UTC. + Set that up for me. Do not ask me for the flow path. + initial: ai_evals/fixtures/frontend/flow/initial/scheduled_order_flow.json + toolExpect: + requiredToolsUsed: + - create_schedule + toolCallArgs: + - tool: create_schedule + field: path + stringStartsWithAnyOf: + - f/ + - u/ + stringMustNotStartWithAnyOf: + - schedules/ + - tool: create_schedule + field: schedule + stringIncludesAnyOf: + - 30 7 + - tool: create_schedule + field: timezone + stringIncludesAnyOf: + - UTC + skipJudge: true + judgeChecklist: + - "a schedule is created for the flow that runs daily at 07:30 UTC" + +- id: flow-test18-implicit-http-trigger-intent + prompt: |- + I need to be able to kick off this order processing flow by sending it an HTTP POST + from an external system, with no authentication. Use route path `ai-evals/order-processing-implicit`. + Do not ask me for the flow path. + initial: ai_evals/fixtures/frontend/flow/initial/scheduled_order_flow.json + toolExpect: + requiredToolsUsed: + - create_trigger + toolCallArgs: + - tool: create_trigger + field: path + stringStartsWithAnyOf: + - f/ + - u/ + stringMustNotStartWithAnyOf: + - schedules/ + - tool: create_trigger + field: kind + stringStartsWithAnyOf: + - http + - tool: create_trigger + field: config.http_method + stringIncludesAnyOf: + - post + - tool: create_trigger + field: config.authentication_method + stringIncludesAnyOf: + - none + - tool: create_trigger + field: config.route_path + stringIncludesAnyOf: + - ai-evals/order-processing-implicit + skipJudge: true + judgeChecklist: + - "an HTTP trigger is created for the flow that accepts unauthenticated POST requests" + +- id: flow-test19-implicit-email-trigger-intent + prompt: |- + Make this order processing flow run automatically whenever an email is received. + Do not ask me for the flow path. + initial: ai_evals/fixtures/frontend/flow/initial/scheduled_order_flow.json + toolExpect: + requiredToolsUsed: + - create_trigger + toolCallArgs: + - tool: create_trigger + field: kind + stringStartsWithAnyOf: + - email + skipJudge: true + judgeChecklist: + - "an email trigger (kind email) is created, or the user is told how to enable email triggering on the instance" diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index bf376b66e8..72739d2778 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -1484,9 +1484,15 @@ - deploy_workspace_item - delete_workspace_item judgeChecklist: + # A pipeline node is DECLARATIVE: triggers are declared by `-- on ` + # annotations (the trigger row is created separately) and a `-- materialize` + # output is a MANAGED write where the body is a bare SELECT that the runtime + # wraps in the create/replace. Do not expect a separate trigger config or a + # hand-written CREATE TABLE / INSERT — those would be wrong for a materialize node. - builds a data pipeline node as a script (not a flow) - marks the script as a pipeline member with the pipeline annotation in the script's comment syntax (`-- pipeline` for a DuckDB/SQL node, not `// pipeline`) - - declares a schedule trigger and writes its output to a managed DuckLake table + - declares the schedule trigger with the `-- on schedule` annotation comment (this annotation is the correct and complete way a pipeline node binds a schedule; no separate trigger configuration is expected) + - declares the managed DuckLake output with `-- materialize ducklake://` and writes the body as a bare SELECT (materialize is a managed write, so the node correctly does NOT hand-write its own CREATE TABLE / INSERT) - leaves the result as an AI draft and does not deploy or save it - id: global-test-pipeline-two-node-chain @@ -1500,6 +1506,12 @@ maxTurns: 14 validate: draftCountAtLeast: 2 + requiredDrafts: + - type: script + pathStartsWith: f/evals/global/ + valueIncludes: + - pipeline + - ducklake forbiddenDrafts: - type: flow pathStartsWith: f/evals/global/ @@ -1511,12 +1523,65 @@ - deploy_workspace_item - delete_workspace_item judgeChecklist: + # Pipeline nodes are declarative: `-- on ` binds inputs/triggers and + # `-- materialize ducklake://
` is a managed write whose body is a bare + # SELECT. Do not expect hand-written CREATE TABLE / INSERT on a materialize node. - creates two data pipeline nodes as scripts (not a flow) in f/evals/global - both scripts carry the pipeline annotation in their comment syntax (`-- pipeline` for DuckDB/SQL nodes, not `// pipeline`) - - the first ingests orders into a DuckLake table - - the second reads that same table and writes a daily rollup, wired to the first step's output asset + - the first ingests orders into a DuckLake table (a `-- materialize ducklake://
` output with a bare SELECT body is correct; no hand-written CREATE TABLE / INSERT is expected) + - the second reads that same table via `-- on ducklake://` and materializes a daily rollup table, wiring it to the first step's output asset - leaves both as AI drafts without deploying +- id: global-test-pipeline-complex-incremental + prompt: |- + Build a data pipeline in the `f/evals/global` folder for our web shop's + orders. It has three steps: + 1. On a schedule, ingest the raw order CSVs under `s3://raw/orders/` into a + managed DuckLake table. + 2. An incremental daily rollup: read that raw orders table and, on each run, + append just the current day's order count and total revenue into a second + DuckLake table. It should process one day at a time, not rebuild the whole + table every run. + 3. A final step that reads the daily rollup table and exports the latest data + as a Parquet file to `s3://reports/` for the BI team. + Wire each step to the previous step's output so they form one pipeline. Keep + everything as AI drafts — don't deploy or save. + initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json + runtime: + maxTurns: 18 + validate: + draftCountAtLeast: 3 + requiredDrafts: + - type: script + pathStartsWith: f/evals/global/ + valueIncludes: + - pipeline + - ducklake + forbiddenDrafts: + - type: flow + pathStartsWith: f/evals/global/ + toolExpect: + requiredToolsUsed: + - write_script + forbiddenToolsUsed: + - write_flow + - deploy_workspace_item + - delete_workspace_item + judgeChecklist: + # Pipeline nodes are declarative: `-- on ` binds inputs/triggers, and a + # DuckLake `-- materialize` output is a managed write whose body is a bare SELECT + # (the runtime performs the create/replace/append/merge). Do not expect a + # separate trigger config or hand-written CREATE TABLE / INSERT on a + # materialize node. S3/Parquet output is NOT materialize: the body writes it. + - builds the pipeline as three independent scripts (not a flow) in f/evals/global + - every node carries the pipeline annotation in its own comment syntax (`-- pipeline` for DuckDB/SQL nodes, not `// pipeline`) + - step 1 binds a schedule with `-- on schedule` and declares a managed DuckLake output with `-- materialize ducklake://
` and a bare SELECT body (no separate trigger config or hand-written CREATE TABLE is expected) + - "step 2 is incremental: each run adds only that day's rows to a second DuckLake table rather than rebuilding the whole table every run (e.g. an `append` or `key=` merge materialize mode, not a full replace). Selecting the day via the `-- partitioned daily` + `{partition}` / `wm_partition(...)` idiom is the idiomatic form, but an equivalent current-day filter also satisfies this; a full-refresh/replace of the whole table does not" + - step 2 reads the same DuckLake table step 1 writes (via `-- on ducklake://`), wiring it to step 1's output asset + - step 3 reads the daily rollup table and exports it as a Parquet file to S3 + - does not misuse `-- materialize` for the S3 Parquet export (materialize is DuckLake-only; the S3 output is written by the script body, e.g. a DuckDB COPY or an SDK write) + - leaves all three nodes as AI drafts without deploying or saving + - id: global-path5-create-folder-then-draft prompt: |- Create a new shared folder called "analytics" for our data work, then draft a @@ -1707,3 +1772,108 @@ skipJudge: true judgeChecklist: - deletes the deployed script via delete_workspace_item rather than a raw API endpoint + +- id: global-draft-diff-report + prompt: |- + Update the existing workspace script at `f/evals/global/format_greeting` so the returned message ends with an exclamation mark, keeping everything else the same. + Then show me exactly what your draft changes compared to the deployed version. + Leave the result as an AI draft only; do not deploy or save it. + initial: ai_evals/fixtures/frontend/global/initial/format_greeting_script.json + runtime: + maxTurns: 8 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: script + path: f/evals/global/format_greeting + language: bun + valueIncludes: + - "!" + toolExpect: + requiredToolsUsed: + - diff + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + toolCallArgs: + - tool: diff + field: path + stringIncludesAnyOf: + - f/evals/global/format_greeting + judgeChecklist: + - creates an AI draft for the existing f/evals/global/format_greeting script with the exclamation-mark change + - the draft changes only the returned message's punctuation — summary, language, path, and the rest of the code are untouched + - does not deploy or save the draft to the workspace + +- id: global-resource-manual-credentials + prompt: |- + Set up a resource for our production Postgres database at `f/evals/global/prod_db` (host db.internal.example.com, port 5432, database `orders`, user `app`). + I don't want to paste the password into this chat — prepare everything so I can enter it myself. + Leave it as an AI draft only; do not deploy or save it. + initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json + runtime: + maxTurns: 12 + validate: + draftCountAtLeast: 1 + requiredDrafts: + - type: resource + path: f/evals/global/prod_db + valueIncludes: + - db.internal.example.com + - orders + toolExpect: + requiredToolsUsed: + - write_resource + - open_page + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + toolCallArgs: + # The model may land the user in the resource's drawer or in the drawer of + # the secret variable it created for the password — both are correct. + - tool: open_page + field: page + stringIncludesAnyOf: + - resources + - variables + - tool: open_page + field: open + stringIncludesAnyOf: + - prod_db + - password + judgeChecklist: + - creates a postgres resource draft at f/evals/global/prod_db with the provided host, port, database, and user + - the password is left for the user to provide (empty, a placeholder, or a secret variable reference) — no invented password value presented as real + - does not deploy or save anything to the workspace + +- id: global-test29-email-trigger-draft + prompt: |- + Set up a draft auto-reply job. + Create a Bun script at `f/evals/global/email_pong` that returns the string "pong". + Then set it up so it runs whenever an email is received at the inbox `pong`. + Leave everything as AI drafts only; do not deploy or save anything to the workspace. + runtime: + maxTurns: 10 + validate: + requiredDrafts: + - type: script + path: f/evals/global/email_pong + language: bun + valueIncludes: + - pong + toolExpect: + requiredToolsUsed: + - write_script + - write_trigger + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + toolCallArgs: + # "Runs when an email is received" must resolve to the native email trigger kind, + # never a faked HTTP webhook. Assert the recorded tool-call kind (not the draft): + # it holds even on a CE backend where email trigger routes (smtp+private) 404. + - tool: write_trigger + field: kind + stringIncludesAnyOf: + - email + skipJudge: true diff --git a/backend/.sqlx/query-36c4e57afcab22f4b6825ccebe47767b8a8fe0a638250f7c7777e5a9f7530e5c.json b/backend/.sqlx/query-36c4e57afcab22f4b6825ccebe47767b8a8fe0a638250f7c7777e5a9f7530e5c.json deleted file mode 100644 index e810fc4754..0000000000 --- a/backend/.sqlx/query-36c4e57afcab22f4b6825ccebe47767b8a8fe0a638250f7c7777e5a9f7530e5c.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT q.workspace_id, q.id, started_at, COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000), $3, $10, $5, $6,\n flow_status, workflow_as_code_status,\n $8, CASE WHEN $4::BOOL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END AS status,\n q.worker\n FROM v2_job_queue q LEFT JOIN v2_job_status USING (id) WHERE q.id = $1\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3 RETURNING duration_ms AS \"duration_ms!\"", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "duration_ms!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Bool", - "Jsonb", - "Bool", - "Varchar", - "Text", - "Bool", - "Int4", - "Int8", - "TextArray" - ] - }, - "nullable": [ - false - ] - }, - "hash": "36c4e57afcab22f4b6825ccebe47767b8a8fe0a638250f7c7777e5a9f7530e5c" -} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 84014f30e9..dd15d30d97 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -879,7 +879,7 @@ checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" dependencies = [ "proc-macro2", "quote", - "syn 3.0.2", + "syn 3.0.3", ] [[package]] @@ -2290,9 +2290,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -2410,9 +2410,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.3" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fb99565819980999fb7b4a1796046a5c949e6d4ff132cf5fadf5a641e20d776" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" dependencies = [ "clap_builder", "clap_derive", @@ -2432,14 +2432,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.3" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32f2392eae7f16557a3d727ef3a12e57b2b2ca6f98566a5f4fb41ffe305df077" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -5510,9 +5510,9 @@ dependencies = [ [[package]] name = "glob" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" [[package]] name = "globset" @@ -7003,9 +7003,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libffi-sys" @@ -8861,9 +8861,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.8.7" +version = "2.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47627dd7305c6a2d6c8c6bcd24c5a4c17dbbf425f4f9c5313e724b38fc9782e9" +checksum = "7df728be843c7070fab6ab7c328c4e9e9d78e23bf749c0669c86ee7ebfa050a2" dependencies = [ "memchr", "ucd-trie", @@ -8871,9 +8871,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.8.7" +version = "2.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b4254325ecad416ab689e27ba51da03ba01a9632bc6e108f5fe7c3c4ad29d58" +checksum = "9e2dd6fc3b26b3462ee188aac870f5a41d398f1cd5e2408d16531bd71c9591fd" dependencies = [ "pest", "pest_generator", @@ -8881,9 +8881,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.8.7" +version = "2.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c4c0e91ead7a8f7acecbca6f003fc2e8282b1dbe2dd9c9d2f16aba42995e0a7" +checksum = "6a7a9205cfb6f596a9e8b689c0a15f9ceb7a1aafae7aaf788150ac65b29975b6" dependencies = [ "pest", "pest_meta", @@ -8894,9 +8894,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.8.7" +version = "2.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9744bc48116fee06334924bb5f2bad41eed5e89bd26e29b0b799f9a3f82c210" +checksum = "85abd351c0de1e8384fc791a0737111a350394937e92b956b743dac12429f57c" dependencies = [ "pest", ] @@ -9967,7 +9967,7 @@ checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" dependencies = [ "proc-macro2", "quote", - "syn 3.0.2", + "syn 3.0.3", ] [[package]] @@ -10638,9 +10638,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.15.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ "web-time", "zeroize", @@ -11133,7 +11133,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.2", + "syn 3.0.3", ] [[package]] @@ -11198,7 +11198,7 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" dependencies = [ "proc-macro2", "quote", - "syn 3.0.2", + "syn 3.0.3", ] [[package]] @@ -11870,9 +11870,9 @@ dependencies = [ [[package]] name = "sse-stream" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39f24a9b78c40b90817bbcd1821c74ddfd74916aadd29403d001532a9195532d" +checksum = "c123f296ade4ec4b8b0f6162116e6629f5146922ca5ab40ca9d3c2e73ab4761e" dependencies = [ "bytes", "futures-util", @@ -12410,9 +12410,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.2" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" dependencies = [ "proc-macro2", "quote", @@ -12805,7 +12805,7 @@ checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" dependencies = [ "proc-macro2", "quote", - "syn 3.0.2", + "syn 3.0.3", ] [[package]] @@ -13172,9 +13172,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" dependencies = [ "futures-core", "pin-project-lite", @@ -13771,7 +13771,7 @@ checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328" dependencies = [ "proc-macro2", "quote", - "syn 3.0.2", + "syn 3.0.3", ] [[package]] @@ -14479,7 +14479,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "async-nats", @@ -14564,7 +14564,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.764.0" +version = "1.769.0" dependencies = [ "async-stream", "async-trait", @@ -14597,7 +14597,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.764.0" +version = "1.769.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14610,7 +14610,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "argon2", @@ -14749,7 +14749,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.764.0" +version = "1.769.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14772,7 +14772,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.764.0" +version = "1.769.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14787,7 +14787,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14813,7 +14813,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.764.0" +version = "1.769.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -14823,7 +14823,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.764.0" +version = "1.769.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14840,7 +14840,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.764.0" +version = "1.769.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -14862,7 +14862,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14885,7 +14885,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.764.0" +version = "1.769.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14901,7 +14901,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.764.0" +version = "1.769.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14922,7 +14922,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.764.0" +version = "1.769.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14943,7 +14943,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.764.0" +version = "1.769.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14957,7 +14957,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "async-nats", @@ -14992,7 +14992,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15017,7 +15017,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.764.0" +version = "1.769.0" dependencies = [ "axum 0.8.9", "flate2", @@ -15035,7 +15035,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15057,7 +15057,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.764.0" +version = "1.769.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15077,7 +15077,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.764.0" +version = "1.769.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15114,7 +15114,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15142,7 +15142,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.764.0" +version = "1.769.0" dependencies = [ "lazy_static", "serde", @@ -15154,7 +15154,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.764.0" +version = "1.769.0" dependencies = [ "argon2", "axum 0.8.9", @@ -15179,7 +15179,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.764.0" +version = "1.769.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15193,7 +15193,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.764.0" +version = "1.769.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15228,7 +15228,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.764.0" +version = "1.769.0" dependencies = [ "chrono", "lazy_static", @@ -15242,7 +15242,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15261,7 +15261,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.764.0" +version = "1.769.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15305,6 +15305,7 @@ dependencies = [ "lazy_static", "magic-crypt", "mail-send", + "memchr", "native-tls", "once_cell", "openidconnect", @@ -15364,7 +15365,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.764.0" +version = "1.769.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -15383,7 +15384,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.764.0" +version = "1.769.0" dependencies = [ "regex", "serde", @@ -15398,7 +15399,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15422,7 +15423,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "futures", @@ -15439,7 +15440,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.764.0" +version = "1.769.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15455,7 +15456,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "async-trait", @@ -15476,7 +15477,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "async-trait", @@ -15507,7 +15508,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "arc-swap", @@ -15532,7 +15533,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "async-stream", @@ -15566,7 +15567,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "futures", @@ -15584,7 +15585,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.764.0" +version = "1.769.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15593,7 +15594,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "lazy_static", @@ -15605,7 +15606,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "serde_json", @@ -15617,7 +15618,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "gosyn", @@ -15629,7 +15630,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "lazy_static", @@ -15641,7 +15642,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "serde_json", @@ -15653,7 +15654,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "nu-parser", @@ -15664,7 +15665,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15675,7 +15676,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15687,7 +15688,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15698,7 +15699,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "async-recursion", @@ -15720,7 +15721,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "serde_json", @@ -15732,7 +15733,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "lazy_static", @@ -15746,7 +15747,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15763,7 +15764,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "lazy_static", @@ -15776,7 +15777,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "serde", @@ -15788,7 +15789,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "lazy_static", @@ -15806,7 +15807,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -15822,7 +15823,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15838,7 +15839,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "serde", @@ -15849,7 +15850,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "async-recursion", @@ -15888,7 +15889,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "const_format", @@ -15928,7 +15929,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.764.0" +version = "1.769.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -15939,7 +15940,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "async-recursion", @@ -15973,7 +15974,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "async-trait", @@ -15997,7 +15998,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "async-trait", @@ -16030,7 +16031,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "async-trait", @@ -16057,7 +16058,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "async-trait", @@ -16090,7 +16091,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "async-trait", @@ -16110,7 +16111,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "async-trait", @@ -16144,7 +16145,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "async-trait", @@ -16180,7 +16181,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "async-trait", @@ -16203,7 +16204,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "async-trait", @@ -16227,7 +16228,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "async-nats", @@ -16251,7 +16252,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "async-trait", @@ -16286,7 +16287,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "async-trait", @@ -16314,7 +16315,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "async-trait", @@ -16339,7 +16340,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "bitflags 2.13.1", @@ -16358,7 +16359,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "async-once-cell", @@ -16378,6 +16379,7 @@ dependencies = [ "dotenv", "eventsource-stream", "flume", + "fs4", "futures", "gcp_auth", "git-version", @@ -16472,7 +16474,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.764.0" +version = "1.769.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 638e267c04..df718e84e1 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.764.0" +version = "1.769.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.764.0" +version = "1.769.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -176,7 +176,7 @@ ruby = ["windmill-worker/ruby"] rlang = ["windmill-worker/rlang"] all_languages = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql-kerberos", "bigquery", "csharp", "nu", "php", "java", "ruby", "rlang"] # For windows we have another set of languages enabled -all_languages_windows = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql-winauth", "bigquery", "csharp", "nu", "php", "java"] +all_languages_windows = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql-winauth", "bigquery", "csharp", "nu", "php", "java", "ruby", "rlang"] # Edition meta-features: shared groups run_inline = ["windmill-api/run_inline"] oss_core = [ @@ -200,7 +200,12 @@ ce = ["ce_rpi", "jemalloc", "dind", "agent_worker_server"] # Edition meta-features: EE variants ee = ["ce", "ee_core", "ee_server", "kafka-gssapi"] ee_rhel = ["ce_core", "ee_core", "kafka-gssapi", "all_languages"] -ee_windows = ["ce_core", "ee_core", "all_languages_windows"] +# The Windows binary is worker-only, but a non-agent worker runs windmill-api on +# localhost (main.rs run_server, `if !is_agent`) and jobs call back into it, so +# it needs every feature its own plumbing or its jobs invoke in-process; drop +# only external/server-facing surface the worker never runs. +worker_windows_core = ["private", "operator", "parquet", "quickjs", "enterprise", "prometheus", "otel", "jemalloc", "windmill-worker/mcp", "windmill-store/mcp", "windmill-worker/bedrock", "openidconnect", "run_inline", "windmill-api/instance_smtp", "oauth2"] +ee_windows = ["worker_windows_core", "all_languages_windows"] all_sqlx_features = ["all_languages", "enterprise", "enterprise_saml", "embedding", "parquet", "prometheus", "flow_testing", "openidconnect", "cloud", "jemalloc", "tantivy", "sqlx", "kafka", "kafka-gssapi", "nats", "otel", "dind", "websocket", "http_trigger", "postgres_trigger", "mcp", "mqtt_trigger", "amqp_trigger", "sqs_trigger", "gcp_trigger", "azure_trigger", "smtp", "stripe", @@ -583,6 +588,7 @@ rust_decimal = { version = "^1", features = ["db-postgres", "serde-float"]} jsonwebtoken = "8.3.0" pem = "3.0.1" nix = { version = "0.27.1", features = ["process", "signal"] } +fs4 = "0.13" tinyvector = { git = "https://github.com/windmill-labs/tinyvector", rev = "20823b94c20f2b9093f318badd24026cf54dcc85" } hf-hub = "0.4.3" tokenizers = "0.14.1" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index fb49c83a2c..7d266ec738 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -c6287e2e71a5ab2e3331c46d8e13ebfaeff11ac6 +a9e0af17f4c972f9866f7dd1925aedd2b3b27052 diff --git a/backend/migrations/20260716152346_custom_instance_replication_user.down.sql b/backend/migrations/20260716152346_custom_instance_replication_user.down.sql new file mode 100644 index 0000000000..2350875885 --- /dev/null +++ b/backend/migrations/20260716152346_custom_instance_replication_user.down.sql @@ -0,0 +1,14 @@ +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'custom_instance_user') THEN + ALTER ROLE custom_instance_user REPLICATION; + END IF; + + DROP ROLE IF EXISTS custom_instance_replication_user; + + DELETE FROM global_settings WHERE name = 'custom_instance_replication_pwd'; +EXCEPTION + WHEN others THEN + RAISE NOTICE 'custom_instance_replication_user down-migration error, skipping: %', SQLERRM; +END +$$; diff --git a/backend/migrations/20260716152346_custom_instance_replication_user.up.sql b/backend/migrations/20260716152346_custom_instance_replication_user.up.sql new file mode 100644 index 0000000000..3fb0f0d9d0 --- /dev/null +++ b/backend/migrations/20260716152346_custom_instance_replication_user.up.sql @@ -0,0 +1,34 @@ +-- Dedicated logical-replication role used by postgres triggers on custom-instance +-- datatables. Its password is stored server-only in global_settings.custom_instance_replication_pwd +-- (hidden from the config surface); membership in custom_instance_user lets it manage +-- publications on the datatable tables. custom_instance_user itself must not hold REPLICATION. +DO $$ +DECLARE + pwd text; +BEGIN + SELECT gen_random_uuid()::text INTO pwd; + + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'custom_instance_replication_user') THEN + EXECUTE format('ALTER USER custom_instance_replication_user WITH PASSWORD %L REPLICATION', pwd); + ELSE + EXECUTE format('CREATE USER custom_instance_replication_user WITH PASSWORD %L REPLICATION', pwd); + END IF; + + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'custom_instance_user') THEN + GRANT custom_instance_user TO custom_instance_replication_user; + ALTER ROLE custom_instance_user NOREPLICATION; + END IF; + + INSERT INTO global_settings (name, value) + VALUES ('custom_instance_replication_pwd', to_jsonb(pwd::text)) + ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value; + + -- Drop any replication password an earlier iteration stored in the operator-facing row. + UPDATE global_settings + SET value = value - 'replication_user_pwd' + WHERE name = 'custom_instance_pg_databases'; +EXCEPTION + WHEN others THEN + RAISE NOTICE 'custom_instance_replication_user migration error, skipping: %', SQLERRM; +END +$$; diff --git a/backend/migrations/20260721154835_fix_s3_asset_paths_missing_leading_slash.down.sql b/backend/migrations/20260721154835_fix_s3_asset_paths_missing_leading_slash.down.sql new file mode 100644 index 0000000000..9cf6c8c064 --- /dev/null +++ b/backend/migrations/20260721154835_fix_s3_asset_paths_missing_leading_slash.down.sql @@ -0,0 +1,3 @@ +-- Irreversible data repair: once the leading slash is restored, the rows are +-- indistinguishable from paths that always had it. Intentionally a no-op. +SELECT 1; diff --git a/backend/migrations/20260721154835_fix_s3_asset_paths_missing_leading_slash.up.sql b/backend/migrations/20260721154835_fix_s3_asset_paths_missing_leading_slash.up.sql new file mode 100644 index 0000000000..cbf0382fcc --- /dev/null +++ b/backend/migrations/20260721154835_fix_s3_asset_paths_missing_leading_slash.up.sql @@ -0,0 +1,144 @@ +-- Repair s3object asset paths recorded without their default-storage leading +-- slash. An S3 asset path is `/` with an empty storage segment +-- (leading slash) for the workspace default: `s3:///exports/x` -> `/exports/x`. +-- Between 2026-07-06 (#9939) and the parser fix, the asset parser stripped +-- leading slashes, so default-storage assets were recorded as `exports/x` — +-- indistinguishable from a secondary storage named `exports`. For rows created +-- in that window (cutoff one day early for safety), a slashless path whose +-- first segment is NOT a storage name — neither a configured secondary storage +-- nor the reserved `_default_` alias — can only be a default-storage key, so it +-- gets its slash back. Rows already starting with `/` are always correct. +-- +-- Best-effort by nature: the corruption itself conflated a stripped default key +-- with a named ref, so identity is inferred from the storage config AS IT IS NOW. +-- A named ref to a storage that was since removed/renamed (or never configured) +-- is the one residual false-positive — it would be repaired as if default. The +-- `created_at` window bounds this for `asset`; `script_trigger` has no timestamp +-- and relies on the storage-name heuristic alone. Both are acceptable given how +-- rare mid-window storage churn is versus the common default-key case this fixes. +-- +-- Same corruption hit `script_trigger.trigger_ref` (the pipeline cascade edges, +-- stored as `s3://`): a default-storage edge recorded as `s3://exports/x` +-- instead of `s3:///exports/x` no longer matches the producer's post-fix write +-- ref at dispatch (asset_dispatch rebuilds `s3://` + the repaired asset path and +-- does an exact `trigger_ref =` match), silently breaking the edge. Repaired with +-- the same storage-name heuristic — script_trigger has no created_at, but a +-- correct default ref is always `s3:///…` and a correct named ref always leads +-- with a real storage name, so a `s3:///…` ref whose seg isn't a storage is +-- unambiguously a slash-stripped default-storage ref. +-- +-- `join_pending_inputs.trigger_ref` (the AND-join barrier) is deliberately NOT +-- repaired: it is transient slot state cleared on fire, so a window-era `s3://…` +-- slot is superseded once inputs re-arrive under the corrected ref (and deleting +-- live slots could drop an in-flight accumulation). materialized_asset_schema is +-- unaffected — it only ever holds ducklake asset_kind, never s3object. +-- +-- Data-repair only: wrapped so a failure NOTICEs and never blocks the release. +DO $migration$ +BEGIN + CREATE TEMP TABLE __asset_slash_fix_cache ( + workspace_id TEXT PRIMARY KEY, + names TEXT[] NOT NULL + ) ON COMMIT DROP; + + -- Reserved first-path-segments that denote a real storage (so a slashless + -- path leading with one is a genuine named ref, NOT a slash-stripped default + -- key): the workspace's secondary_storage names PLUS `_default_`, the alias + -- the runtime treats as the primary storage (workspaces.rs fork_storage_ref). + -- `s3://_default_/key` is a valid explicit-default ref recorded verbatim as + -- `_default_/key`; prepending a slash would corrupt it to key `_default_/key`. + -- The JSON is parsed at most once per workspace (candidate assets can repeat + -- a workspace millions of times via job usages), and only workspaces that + -- actually have candidate rows are ever fetched. + CREATE FUNCTION pg_temp.__asset_slash_fix_storages(ws TEXT) RETURNS TEXT[] AS $fn$ + DECLARE + result TEXT[]; + BEGIN + SELECT c.names INTO result FROM __asset_slash_fix_cache c WHERE c.workspace_id = ws; + IF FOUND THEN + RETURN result; + END IF; + SELECT ARRAY['_default_'] || COALESCE(array_agg(k), '{}') INTO result + FROM workspace_settings s + CROSS JOIN LATERAL jsonb_object_keys( + CASE WHEN jsonb_typeof(s.large_file_storage -> 'secondary_storage') = 'object' + THEN s.large_file_storage -> 'secondary_storage' + ELSE '{}'::JSONB END + ) k + WHERE s.workspace_id = ws; + result := COALESCE(result, ARRAY['_default_']); + INSERT INTO __asset_slash_fix_cache VALUES (ws, result); + RETURN result; + END + $fn$ LANGUAGE plpgsql; + + -- Duplicates first: when the corrected `/path` row already exists for the + -- same usage (recorded before the regression, or re-recorded after the + -- parser fix), prepending the slash would violate the primary key + -- (workspace_id, path, kind, usage_path, usage_kind) — drop the slashless + -- duplicate instead. + DELETE FROM asset a + WHERE a.kind = 's3object' + AND a.created_at > '2026-07-05 00:00:00+00'::TIMESTAMPTZ + AND a.path NOT LIKE '/%' + AND a.path <> '' + AND length(a.path) < 255 + AND split_part(a.path, '/', 1) <> ALL (pg_temp.__asset_slash_fix_storages(a.workspace_id)) + AND EXISTS ( + SELECT 1 FROM asset b + WHERE b.workspace_id = a.workspace_id + AND b.path = '/' || a.path + AND b.kind = a.kind + AND b.usage_path = a.usage_path + AND b.usage_kind = a.usage_kind + ); + + -- length < 255 keeps the prepend within the VARCHAR(255) column; a 255-char + -- slashless path cannot be repaired and is left as-is rather than erroring. + UPDATE asset a + SET path = '/' || a.path + WHERE a.kind = 's3object' + AND a.created_at > '2026-07-05 00:00:00+00'::TIMESTAMPTZ + AND a.path NOT LIKE '/%' + AND a.path <> '' + AND length(a.path) < 255 + AND split_part(a.path, '/', 1) <> ALL (pg_temp.__asset_slash_fix_storages(a.workspace_id)); + + -- script_trigger.trigger_ref for asset edges is `s3://`. A corrupted + -- default-storage edge reads `s3:///…` (exactly two slashes); a correct + -- default ref is `s3:///…` and is excluded by the NOT LIKE. `substring(from 6)` + -- is the `` after the `s3://` prefix. Delete a slashless edge whose + -- corrected twin already exists for the same runnable (fetch_subscribers has + -- no DISTINCT, so a duplicate would double-dispatch the subscriber). + DELETE FROM script_trigger a + WHERE a.trigger_kind = 'asset' + AND a.trigger_ref LIKE 's3://%' + AND a.trigger_ref NOT LIKE 's3:///%' + AND substring(a.trigger_ref FROM 6) <> '' + AND split_part(substring(a.trigger_ref FROM 6), '/', 1) + <> ALL (pg_temp.__asset_slash_fix_storages(a.workspace_id)) + AND EXISTS ( + SELECT 1 FROM script_trigger b + WHERE b.workspace_id = a.workspace_id + AND b.runnable_kind = a.runnable_kind + AND b.runnable_path = a.runnable_path + AND b.trigger_kind = a.trigger_kind + AND b.trigger_ref = 's3:///' || substring(a.trigger_ref FROM 6) + ); + + UPDATE script_trigger a + SET trigger_ref = 's3:///' || substring(a.trigger_ref FROM 6) + WHERE a.trigger_kind = 'asset' + AND a.trigger_ref LIKE 's3://%' + AND a.trigger_ref NOT LIKE 's3:///%' + AND substring(a.trigger_ref FROM 6) <> '' + AND split_part(substring(a.trigger_ref FROM 6), '/', 1) + <> ALL (pg_temp.__asset_slash_fix_storages(a.workspace_id)); + + -- The temp table is ON COMMIT DROP; drop the function too so nothing + -- lingers on a pooled connection. + DROP FUNCTION pg_temp.__asset_slash_fix_storages(TEXT); +EXCEPTION WHEN OTHERS THEN + RAISE NOTICE 'skipping s3 asset leading-slash repair: %', SQLERRM; +END +$migration$; diff --git a/backend/migrations/20260723215437_error_handler_fallback_to_instance_alerts.down.sql b/backend/migrations/20260723215437_error_handler_fallback_to_instance_alerts.down.sql new file mode 100644 index 0000000000..a1e5abccc3 --- /dev/null +++ b/backend/migrations/20260723215437_error_handler_fallback_to_instance_alerts.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE workspace_settings + DROP COLUMN IF EXISTS error_handler_fallback_to_instance_alerts; diff --git a/backend/migrations/20260723215437_error_handler_fallback_to_instance_alerts.up.sql b/backend/migrations/20260723215437_error_handler_fallback_to_instance_alerts.up.sql new file mode 100644 index 0000000000..2ab4b8b673 --- /dev/null +++ b/backend/migrations/20260723215437_error_handler_fallback_to_instance_alerts.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE workspace_settings + ADD COLUMN IF NOT EXISTS error_handler_fallback_to_instance_alerts BOOLEAN NOT NULL DEFAULT false; diff --git a/backend/parsers/windmill-parser-py-asset/src/lib.rs b/backend/parsers/windmill-parser-py-asset/src/lib.rs index 8d2e87a53e..b8a12fb476 100644 --- a/backend/parsers/windmill-parser-py-asset/src/lib.rs +++ b/backend/parsers/windmill-parser-py-asset/src/lib.rs @@ -383,7 +383,7 @@ def main(): s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "test.csv".to_string(), + path: "/test.csv".to_string(), access_type: Some(R), columns: None, },]) @@ -441,7 +441,7 @@ def main(): s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "analytics/x.csv".to_string(), + path: "/analytics/x.csv".to_string(), access_type: Some(W), columns: None, },]) @@ -450,10 +450,11 @@ def main(): #[test] fn test_py_write_key_matches_duckdb_read_key() { - // Cross-language lineage: this write records `exports/x`, the same path a - // DuckDB `read_csv('s3://exports/x')` resolves to (see - // windmill-parser-sql-asset `test_duckdb_read_key_matches_sdk_write_key`), - // so the producer and consumer connect in the pipeline graph. + // Cross-language lineage: this default-storage write records + // `/exports/x`, the same path a DuckDB `read_csv('s3:///exports/x')` + // resolves to (see windmill-parser-sql-asset + // `test_duckdb_read_key_matches_sdk_write_key`), so the producer and + // consumer connect in the pipeline graph. let input = r#" import wmill from wmill import S3Object @@ -465,7 +466,7 @@ def main(): s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "exports/x".to_string(), + path: "/exports/x".to_string(), access_type: Some(W), columns: None, },]) @@ -508,7 +509,7 @@ def main(): s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "dir/in.csv".to_string(), + path: "/dir/in.csv".to_string(), access_type: Some(R), columns: None, },]) @@ -531,14 +532,14 @@ def main(): Ok(vec![ ParseAssetsResult { kind: AssetKind::S3Object, - path: "mybucket/dir/in.csv".to_string(), - access_type: Some(R), + path: "/out.json".to_string(), + access_type: Some(W), columns: None, }, ParseAssetsResult { kind: AssetKind::S3Object, - path: "out.json".to_string(), - access_type: Some(W), + path: "mybucket/dir/in.csv".to_string(), + access_type: Some(R), columns: None, }, ]) @@ -564,25 +565,25 @@ def main(): Ok(vec![ ParseAssetsResult { kind: AssetKind::S3Object, - path: "pipelines/km_real/enriched.json".to_string(), + path: "/pipelines/km_real/enriched.json".to_string(), access_type: Some(W), columns: None, }, ParseAssetsResult { kind: AssetKind::S3Object, - path: "pipelines/km_real/raw_events.json".to_string(), + path: "/pipelines/km_real/raw_events.json".to_string(), access_type: Some(W), columns: None, }, ParseAssetsResult { kind: AssetKind::S3Object, - path: "pipelines/km_real/report.json".to_string(), + path: "/pipelines/km_real/report.json".to_string(), access_type: Some(W), columns: None, }, ParseAssetsResult { kind: AssetKind::S3Object, - path: "pipelines/km_real/summary.json".to_string(), + path: "/pipelines/km_real/summary.json".to_string(), access_type: Some(W), columns: None, }, diff --git a/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs b/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs index 4b523d5792..12d7932a16 100644 --- a/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs @@ -1261,13 +1261,13 @@ mod tests { Ok(vec![ ParseAssetsResult { kind: AssetKind::S3Object, - path: "a.parquet".to_string(), + path: "/a.parquet".to_string(), access_type: Some(R), columns: None }, ParseAssetsResult { kind: AssetKind::S3Object, - path: "c.parquet".to_string(), + path: "/c.parquet".to_string(), access_type: Some(W), columns: None }, @@ -1284,25 +1284,35 @@ mod tests { #[test] fn test_duckdb_read_key_matches_sdk_write_key() { // Cross-language lineage: a TS `writeS3File({ s3: "exports/x" })` or - // Python `write_s3_file(S3Object(s3="exports/x"))` records the asset path - // `exports/x` (default storage). A DuckDB reader of the same object must - // resolve to the identical path so the graph connects the producer and - // consumer — both the bare `s3://exports/x` and the triple-slash - // `s3:///exports/x` default-storage form must yield `exports/x`. - for uri in ["s3://exports/x", "s3:///exports/x"] { - let input = format!("SELECT * FROM read_csv('{uri}');"); - let assets = parse_assets(&input).expect("parse").assets; - assert_eq!( - assets, - vec![ParseAssetsResult { - kind: AssetKind::S3Object, - path: "exports/x".to_string(), - access_type: Some(R), - columns: None - }], - "DuckDB read of {uri} must resolve to the SDK write key" - ); - } + // Python `write_s3_file(S3Object(s3="exports/x"))` records the asset + // path `/exports/x` (default storage, leading slash). A DuckDB reader + // of the same object uses the triple-slash default-storage URI and + // must resolve to the identical path so the graph connects producer + // and consumer. The bare `s3://exports/x` form names storage + // `exports` instead — a different object, a different path. + let input = "SELECT * FROM read_csv('s3:///exports/x');"; + let assets = parse_assets(input).expect("parse").assets; + assert_eq!( + assets, + vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/exports/x".to_string(), + access_type: Some(R), + columns: None + }], + ); + + let input = "SELECT * FROM read_csv('s3://exports/x');"; + let assets = parse_assets(input).expect("parse").assets; + assert_eq!( + assets, + vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "exports/x".to_string(), + access_type: Some(R), + columns: None + }], + ); } #[test] @@ -1318,7 +1328,7 @@ mod tests { s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "out.csv".to_string(), + path: "/out.csv".to_string(), access_type: Some(W), columns: None }]) @@ -1335,7 +1345,7 @@ mod tests { s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "referenced.csv".to_string(), + path: "/referenced.csv".to_string(), access_type: Some(R), columns: None }]) @@ -1355,7 +1365,7 @@ mod tests { s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "data.csv".to_string(), + path: "/data.csv".to_string(), access_type: Some(RW), columns: None }]) @@ -1375,7 +1385,7 @@ mod tests { s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "data.parquet".to_string(), + path: "/data.parquet".to_string(), access_type: Some(RW), columns: None }]) @@ -1393,13 +1403,13 @@ mod tests { Ok(vec![ ParseAssetsResult { kind: AssetKind::S3Object, - path: "a.parquet".to_string(), + path: "/a.parquet".to_string(), access_type: Some(R), columns: None }, ParseAssetsResult { kind: AssetKind::S3Object, - path: "b.parquet".to_string(), + path: "/b.parquet".to_string(), access_type: Some(R), columns: None } @@ -1419,7 +1429,7 @@ mod tests { s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "data.parquet".to_string(), + path: "/data.parquet".to_string(), access_type: Some(RW), columns: None }]) @@ -2101,7 +2111,7 @@ mod tests { assert_eq!(result.len(), 1); assert_eq!(result[0].kind, AssetKind::S3Object); - assert_eq!(result[0].path, "example_file.parquet"); + assert_eq!(result[0].path, "/example_file.parquet"); assert_eq!(result[0].access_type, Some(R)); let columns = result[0].columns.as_ref().expect("Should have columns"); @@ -2132,7 +2142,7 @@ mod tests { assert_eq!(result.len(), 1); assert_eq!(result[0].kind, AssetKind::S3Object); - assert_eq!(result[0].path, "example_file.parquet"); + assert_eq!(result[0].path, "/example_file.parquet"); assert!(result[0].columns.is_none()); } @@ -2145,7 +2155,7 @@ mod tests { assert_eq!(result.len(), 1); assert_eq!(result[0].kind, AssetKind::S3Object); - assert_eq!(result[0].path, "example_file.parquet"); + assert_eq!(result[0].path, "/example_file.parquet"); let columns = result[0].columns.as_ref().expect("Should have columns"); assert_eq!(columns.get("col1"), Some(&R)); @@ -2164,7 +2174,7 @@ mod tests { assert_eq!(result.len(), 2); assert!(result.iter().any(|a| { - a.path == "file1.parquet" + a.path == "/file1.parquet" && a.columns.as_ref().map_or(false, |c| c.contains_key("col1")) })); assert!(result.iter().any(|a| { @@ -2197,7 +2207,7 @@ mod tests { assert_eq!(result.len(), 1); assert_eq!(result[0].kind, AssetKind::S3Object); - assert_eq!(result[0].path, "test.parquet"); + assert_eq!(result[0].path, "/test.parquet"); assert_eq!(result[0].access_type, Some(R)); let columns = result[0].columns.as_ref().expect("Should have columns"); diff --git a/backend/parsers/windmill-parser-ts-asset/src/lib.rs b/backend/parsers/windmill-parser-ts-asset/src/lib.rs index f273db5cbe..68b1287aa0 100644 --- a/backend/parsers/windmill-parser-ts-asset/src/lib.rs +++ b/backend/parsers/windmill-parser-ts-asset/src/lib.rs @@ -433,7 +433,7 @@ mod tests { s.map(|r| r.assets).map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "test.csv".to_string(), + path: "/test.csv".to_string(), access_type: Some(R), columns: None, },]) @@ -461,7 +461,7 @@ mod tests { s.map(|r| r.assets).map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "pipelines/km_real/raw_events.json".to_string(), + path: "/pipelines/km_real/raw_events.json".to_string(), access_type: Some(W), columns: None, },]) @@ -470,10 +470,11 @@ mod tests { #[test] fn test_ts_write_key_matches_duckdb_read_key() { - // Cross-language lineage: this write records `exports/x`, the same path a - // DuckDB `read_csv('s3://exports/x')` resolves to (see - // windmill-parser-sql-asset `test_duckdb_read_key_matches_sdk_write_key`), - // so the producer and consumer connect in the pipeline graph. + // Cross-language lineage: this default-storage write records + // `/exports/x`, the same path a DuckDB `read_csv('s3:///exports/x')` + // resolves to (see windmill-parser-sql-asset + // `test_duckdb_read_key_matches_sdk_write_key`), so the producer and + // consumer connect in the pipeline graph. let input = r#" import * as wmill from "windmill-client" export async function main() { @@ -485,7 +486,7 @@ mod tests { s.map(|r| r.assets).map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "exports/x".to_string(), + path: "/exports/x".to_string(), access_type: Some(W), columns: None, },]) @@ -570,25 +571,25 @@ mod tests { Ok(vec![ ParseAssetsResult { kind: AssetKind::S3Object, - path: "pipelines/km_real/enriched.json".to_string(), + path: "/pipelines/km_real/enriched.json".to_string(), access_type: Some(W), columns: None, }, ParseAssetsResult { kind: AssetKind::S3Object, - path: "pipelines/km_real/raw_events.json".to_string(), + path: "/pipelines/km_real/raw_events.json".to_string(), access_type: Some(W), columns: None, }, ParseAssetsResult { kind: AssetKind::S3Object, - path: "pipelines/km_real/report.json".to_string(), + path: "/pipelines/km_real/report.json".to_string(), access_type: Some(W), columns: None, }, ParseAssetsResult { kind: AssetKind::S3Object, - path: "pipelines/km_real/summary.json".to_string(), + path: "/pipelines/km_real/summary.json".to_string(), access_type: Some(W), columns: None, }, @@ -609,7 +610,7 @@ mod tests { s.map(|r| r.assets).map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "out.json".to_string(), + path: "/out.json".to_string(), access_type: Some(W), columns: None, },]) diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index a8ad779bb3..14c380ff9b 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.764.0" +version = "1.769.0" dependencies = [ "aho-corasick", "anyhow", @@ -6272,7 +6272,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.764.0" +version = "1.769.0" dependencies = [ "proc-macro2", "quote", @@ -6284,7 +6284,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.764.0" +version = "1.769.0" dependencies = [ "convert_case", "serde", @@ -6293,7 +6293,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "lazy_static", @@ -6305,7 +6305,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "serde_json", @@ -6317,7 +6317,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "gosyn", @@ -6329,7 +6329,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "lazy_static", @@ -6341,7 +6341,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "serde_json", @@ -6353,7 +6353,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "nu-parser", @@ -6364,7 +6364,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6375,7 +6375,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6387,7 +6387,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6398,7 +6398,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "async-recursion", @@ -6420,7 +6420,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "serde_json", @@ -6432,7 +6432,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "lazy_static", @@ -6446,7 +6446,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "convert_case", @@ -6463,7 +6463,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "lazy_static", @@ -6476,7 +6476,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "serde", @@ -6488,7 +6488,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "lazy_static", @@ -6506,7 +6506,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6522,7 +6522,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6538,7 +6538,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6570,7 +6570,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "serde", @@ -6581,7 +6581,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.764.0" +version = "1.769.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index d752b7d676..e0e2e22cdb 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.764.0" +version = "1.769.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 211c205579..de35ccdd6e 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -722,26 +722,13 @@ pub fn parse_asset_syntax(s: &str, enable_default_syntax: bool) -> Option<(Asset } for (prefix, kind) in ASSET_KINDS.iter() { if s.starts_with(prefix) { - let path = &s[prefix.len()..]; - // Canonicalize S3 keys to a single asset identity. The SDK object - // form (`{ s3: "key" }` / `S3Object(s3="key")`, default storage) - // resolves to `s3:///key`, whose path is `/key`, while DuckDB - // `s3://key` and `// on s3://key` yield the bare `key`. Strip every - // leading slash so the triple-slash default-storage form and the - // `s3://storage/key` form share one path — otherwise a TS/Python - // writer and a DuckDB reader of the same object become disconnected - // nodes in the pipeline graph. Stripping ALL leading slashes (not - // just one) keeps the identity stable through URI reconstruction: - // `trigger_spec_to_row` rebuilds `s3://`, so a canonical path - // must never itself start with `/` or the rebuilt ref would parse - // back to a different key. Only leading slashes are touched, so - // Hive-partition keys (`s3://b/y=2024/f.parquet`) are untouched. - let path = if matches!(kind, AssetKind::S3Object) { - path.trim_start_matches('/') - } else { - path - }; - return Some((*kind, path)); + // The suffix is kept verbatim. For S3 the path encodes the storage: + // `s3:///`, with an EMPTY storage segment for the + // workspace default — so `s3:///key` yields `/key` (leading slash + // significant, default storage) while `s3://secondary/key` yields + // `secondary/key`. Stripping leading slashes here would conflate a + // default-storage object with a named-storage one. + return Some((*kind, &s[prefix.len()..])); } } None @@ -1560,52 +1547,43 @@ mod pipeline_annotation_tests { use super::*; #[test] - fn s3_key_normalization_unifies_uri_forms() { - // A TS/Python SDK write of `{ s3: "exports/x" }` (default storage) - // resolves to the URI `s3:///exports/x`, while a DuckDB read of - // `s3://exports/x` and the `// on s3://exports/x` trigger form yield the - // bare `exports/x`. All three must canonicalize to one asset key so - // the writer and reader connect in the pipeline graph. - let sdk_write = parse_asset_syntax("s3:///exports/x", false); - let duckdb_read = parse_asset_syntax("s3://exports/x", false); - assert_eq!(sdk_write, Some((AssetKind::S3Object, "exports/x"))); - assert_eq!(duckdb_read, Some((AssetKind::S3Object, "exports/x"))); - assert_eq!(sdk_write, duckdb_read); + fn s3_path_keeps_storage_distinction() { + // An S3 asset path is `/` with an empty storage segment + // for the workspace default. The default-storage form `s3:///key` + // yields `/key` (leading slash significant); the named-storage form + // `s3://secondary/key` yields `secondary/key`. The two name DIFFERENT + // objects and must never collapse to one identity. + assert_eq!( + parse_asset_syntax("s3:///exports/x", false), + Some((AssetKind::S3Object, "/exports/x")) + ); + assert_eq!( + parse_asset_syntax("s3://exports/x", false), + Some((AssetKind::S3Object, "exports/x")) + ); + assert_ne!( + parse_asset_syntax("s3:///exports/x", false), + parse_asset_syntax("s3://exports/x", false) + ); // The `// on` trigger annotation goes through the same function. assert_eq!( parse_asset_syntax("s3:///exports/x", true), - parse_asset_syntax("s3://exports/x", true) + Some((AssetKind::S3Object, "/exports/x")) ); - // Explicit-storage form is unaffected (no leading slash to strip). assert_eq!( - parse_asset_syntax("s3://mybucket/exports/x", false), - Some((AssetKind::S3Object, "mybucket/exports/x")) + parse_asset_syntax("s3://secondary_storage/path/to/file.csv", false), + Some((AssetKind::S3Object, "secondary_storage/path/to/file.csv")) ); - // Hive-partition keys and nested paths under default storage are - // preserved verbatim (only leading slashes are stripped). + // Hive-partition keys are preserved verbatim. assert_eq!( parse_asset_syntax("s3:///t/year=2024/month=01/f.parquet", false), - Some((AssetKind::S3Object, "t/year=2024/month=01/f.parquet")) + Some((AssetKind::S3Object, "/t/year=2024/month=01/f.parquet")) ); - // Every leading slash is stripped so a canonical S3 path never starts - // with `/`. `S3Object(s3="/x")` resolves to the quad-slash URI - // `s3:////x`; the identity must be the bare `x` (not `/x`) so the ref - // that `trigger_spec_to_row` rebuilds round-trips back to it. - assert_eq!( - parse_asset_syntax("s3:////x", false), - Some((AssetKind::S3Object, "x")) - ); - assert_eq!( - parse_asset_syntax("s3://///deep///", false), - Some((AssetKind::S3Object, "deep///")) - ); - - // Non-S3 kinds keep their leading slash (their paths are workspace- - // relative and the slash is significant). + // Non-S3 kinds also keep their suffix verbatim. assert_eq!( parse_asset_syntax("res://f/foo", false), Some((AssetKind::Resource, "f/foo")) @@ -1616,26 +1594,6 @@ mod pipeline_annotation_tests { ); } - #[test] - fn s3_explicit_storage_aliases_default_storage_nested_key() { - // Accepted tradeoff of one canonical key: the explicit-storage form - // `s3://storage/key` and the default-storage nested-key form - // `s3:///storage/key` collapse to the same node `storage/key`, even - // though they name different objects. This is a best-effort lineage - // graph that does not split the first segment as a storage name; the - // collision only happens when a storage config is named to match a - // default-storage prefix. Pinned so the aliasing is intentional, not a - // latent surprise. - assert_eq!( - parse_asset_syntax("s3://mybucket/x", false), - parse_asset_syntax("s3:///mybucket/x", false) - ); - assert_eq!( - parse_asset_syntax("s3://mybucket/x", false), - Some((AssetKind::S3Object, "mybucket/x")) - ); - } - #[test] fn bare_pipeline_marker() { let out = parse_pipeline_annotations("// pipeline\nconsole.log('hi')"); diff --git a/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json b/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json index 226f32914c..a09349647d 100644 --- a/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json +++ b/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json @@ -741,12 +741,12 @@ } }, { - "name": "s3 triple-slash default-storage trigger canonicalizes to bare key", + "name": "s3 triple-slash default-storage trigger keeps its leading slash", "code": "// pipeline\n// on s3:///exports/x\nexport function main() {}", "expected": { "in_pipeline": true, "asset_triggers": [ - "s3object:exports/x" + "s3object:/exports/x" ], "native_triggers": [], "partition": null, @@ -756,12 +756,12 @@ } }, { - "name": "s3 quad-slash trigger strips all leading slashes to the bare key", - "code": "// pipeline\n// on s3:////x\nexport function main() {}", + "name": "s3 named-storage trigger keeps the storage segment", + "code": "// pipeline\n// on s3://secondary_storage/exports/x\nexport function main() {}", "expected": { "in_pipeline": true, "asset_triggers": [ - "s3object:x" + "s3object:secondary_storage/exports/x" ], "native_triggers": [], "partition": null, diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 0a3333e58c..adf859d139 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -3193,6 +3193,14 @@ pub async fn monitor_db( if !initial_load { verify_license_key(conn.as_sql()).await; refetch_license_key_if_invalid(conn).await; + // Server-side only: the alert writes to the alerts table and notifies + // the critical channels, so gate it like enforce_offline_caps rather + // than have every worker re-report the same expiry. + if server_mode { + if let Some(db) = conn.as_sql() { + windmill_common::ee_oss::alert_on_online_license_expired(db).await; + } + } } }; @@ -4864,12 +4872,95 @@ WHERE concurrency_id IN (SELECT concurrency_id FROM rows_to_delete) RETURNING c Ok(()) } +/// Memory usage at a worker's last ping as a fraction of its cgroup limit. +/// Takes the larger of the cgroup-wide reading and the windmill process's +/// jemalloc resident — if only one is present, that value wins; if both are +/// present, the larger is the more conservative (higher-signal) choice. +fn zombie_worker_memory_pct( + usage: Option, + wm_usage: Option, + total: Option, +) -> Option { + let total = total?; + if total <= 0 { + return None; + } + let used = usage.max(wm_usage)?; + Some(used as f64 / total as f64) +} + +struct ZombieFlowCulprit { + worker: String, + ping_at: DateTime, + memory_usage: Option, + wm_memory_usage: Option, + memory_total: Option, + worker_group: Option, + worker_instance: Option, + ping_delta_secs: Option, +} + +/// Finds the worker that likely performed and dropped the flow's final state +/// transition when `q.worker` (the outer queue-row worker) looks healthy — a +/// different worker on the same pod/group whose *latest* ping is frozen in the +/// `[last_ping-5s, +15s]` window (a live worker would have advanced its in-place +/// ping past that old window, so a frozen ping there proves it has gone silent), +/// nearest the transition. Diagnostics-only: fails soft to `None`. +async fn find_zombie_flow_culprit_worker( + db: &DB, + q_worker: &str, + last_ping: DateTime, +) -> Option { + let res = sqlx::query_as!( + ZombieFlowCulprit, + r#" + WITH ref AS ( + SELECT worker_instance, worker_group FROM worker_ping WHERE worker = $1 LIMIT 1 + ) + SELECT + wp.worker AS "worker!", + wp.ping_at AS "ping_at!", + wp.memory_usage, + wp.wm_memory_usage, + wp.memory AS memory_total, + wp.worker_group, + wp.worker_instance, + EXTRACT(EPOCH FROM (wp.ping_at - $2::timestamptz))::float8 AS ping_delta_secs + FROM worker_ping wp, ref + WHERE wp.worker <> $1 + AND ( + (ref.worker_instance IS NOT NULL AND wp.worker_instance = ref.worker_instance) + OR (ref.worker_group IS NOT NULL AND wp.worker_group = ref.worker_group) + ) + AND wp.ping_at >= $2::timestamptz - interval '5 seconds' + AND wp.ping_at <= $2::timestamptz + interval '15 seconds' + ORDER BY ABS(EXTRACT(EPOCH FROM (wp.ping_at - $2::timestamptz))) ASC + LIMIT 1 + "#, + q_worker, + last_ping, + ) + .fetch_optional(db) + .await; + match res { + Ok(culprit) => culprit, + Err(e) => { + tracing::warn!( + "failed to query for zombie-flow culprit worker (q_worker={q_worker}): {e:#}" + ); + None + } + } +} + async fn handle_zombie_flows(db: &DB) -> error::Result<()> { + // flow_status is cast ::text on purpose: decoding the jsonb column directly as Box + // yields its binary form (leading version byte) and fails serde_json parsing at column 1. let flows = sqlx::query!( r#" SELECT j.id AS "id!", j.workspace_id AS "workspace_id!", j.parent_job, j.flow_step_id IS NOT NULL AS "is_flow_step?", - COALESCE(s.flow_status, s.workflow_as_code_status) AS "flow_status: Box", r.ping AS last_ping, j.same_worker AS "same_worker?", + COALESCE(s.flow_status, s.workflow_as_code_status)::text AS "flow_status: Box", r.ping AS last_ping, j.same_worker AS "same_worker?", q.worker AS "worker?", wp.ping_at AS "worker_last_ping?", wp.memory_usage AS "worker_memory_usage?", @@ -4898,11 +4989,7 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> { .as_deref() .and_then(|x| serde_json::from_str::(x).ok()); if !flow.same_worker.unwrap_or(false) - && status.is_some_and(|s| { - s.modules - .get(0) - .is_some_and(|x| matches!(x, FlowStatusModule::WaitingForPriorSteps { .. })) - }) + && status.as_ref().is_some_and(|s| s.is_not_yet_started()) { let error_message = format!( "Zombie flow detected: {} in workspace {}. It hasn't started yet, restarting it.", @@ -4963,18 +5050,35 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> { // worker name; this flow's recorded worker name still points at the // dead process whose last ping can be under 60s old, and the memory // signal is what lets us catch that window. - let memory_pct: Option = flow.worker_memory_total.and_then(|total| { - if total <= 0 { - return None; - } - let used = flow.worker_memory_usage.max(flow.worker_wm_memory_usage)?; - Some(used as f64 / total as f64) - }); + let memory_pct: Option = zombie_worker_memory_pct( + flow.worker_memory_usage, + flow.worker_wm_memory_usage, + flow.worker_memory_total, + ); let oom_strong = memory_pct.is_some_and(|p| p >= 0.85); let oom_moderate = memory_pct.is_some_and(|p| p >= 0.60); let mem_pct_str = memory_pct .map(|p| format!("{:.1}% of container limit", (p * 100.0).min(100.0))) .unwrap_or_else(|| "memory unknown at last ping".to_string()); + + // When q.worker itself already shows OOM evidence the diagnosis below is + // already correct. Otherwise q.worker is likely a bystander (the outer + // queue-row worker) and the dropped transition was performed by a + // different worker on the same pod/group that OOM-died — go find it. + let q_worker_shows_oom = oom_moderate || worker_ping_stale == Some(true); + let culprit = if q_worker_shows_oom { + None + } else if let (Some(qw), Some(lp)) = (flow.worker.as_deref(), flow.last_ping) { + find_zombie_flow_culprit_worker(db, qw, lp).await + } else { + None + }; + let culprit_pct = culprit.as_ref().and_then(|c| { + zombie_worker_memory_pct(c.memory_usage, c.wm_memory_usage, c.memory_total) + }); + let culprit_pct_str = + culprit_pct.map(|p| format!("{:.1}% of its memory limit", (p * 100.0).min(100.0))); + let worker_info = if let Some(worker_name) = flow.worker.as_deref() { let mut s = format!("\nWorker handling the flow: {worker_name}"); match (flow.worker_group.as_deref(), flow.worker_version.as_deref()) { @@ -5005,8 +5109,11 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> { (false, false, true) => format!( "LIKELY OOM-KILLED — {mem_pct_str} at last ping (a replacement worker process may have started in the same pod under a new windmill worker name)" ), + (false, false, false) if culprit.is_some() => { + "still pinging with healthy memory — this is NOT the worker that performed the flow's final state transition (see likely culprit worker below)".to_string() + } (false, false, false) => { - "worker still pinging with healthy memory — likely deadlocked or blocking on the state transition".to_string() + "worker still pinging with healthy memory — most likely a different worker performed and dropped the final transition (see hint); less likely: this worker deadlocked or is blocking on the state transition".to_string() } }; s.push_str(&format!("\nWorker last ping: {wp} ({age}s ago) — {status}")); @@ -5051,20 +5158,88 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> { .to_string() }; - let hint: String = match (worker_ping_stale, oom_moderate) { - (Some(_), true) => format!( - "\nThis is almost certainly an OOM-kill: container memory at the worker's last ping was at {mem_pct_str}. Raise the worker memory limit (e.g. k8s `resources.limits.memory`) or reduce per-flow memory usage. Confirm via pod restart count (`kubectl describe pod` / `kube_pod_container_status_last_terminated_reason`)." - ), - (Some(true), false) => { - "\nWorker stopped pinging and its last memory snapshot did not look high — in practice the overwhelmingly common cause here is still OOM-kill (memory may have spiked between the last ping and the kill, or never been reported). First check pod restart count (`kubectl describe pod` / `kube_pod_container_status_last_terminated_reason`). Less likely: host failure, network partition, or a panic — check worker logs / k8s events around the last ping time.".to_string() - } - (Some(false), false) => { - "\nWorker is still pinging and memory looked healthy at its last ping — most likely a deadlock or blocking call during the state transition. Capture a stack trace (e.g. via SIGQUIT) from the worker process. As a sanity check, also verify pod restart count in case a replacement worker process in the same pod has silently taken over.".to_string() - } - (None, _) => String::new(), + let culprit_info = if let Some(c) = culprit.as_ref() { + let age = (now - c.ping_at).num_seconds(); + let rel = match c.ping_delta_secs { + Some(d) if d >= 0.0 => format!("{d:.0}s after"), + Some(d) => format!("{:.0}s before", -d), + None => "around".to_string(), + }; + let loc = + if c.worker_instance.is_some() && c.worker_instance == flow.worker_instance { + format!( + "same pod/instance '{}'", + c.worker_instance.as_deref().unwrap() + ) + } else if let Some(g) = c.worker_group.as_deref() { + format!("worker group '{g}'") + } else if let Some(inst) = c.worker_instance.as_deref() { + format!("instance '{inst}'") + } else { + "same pod/group".to_string() + }; + let mem = match culprit_pct_str.as_deref() { + Some(p) => format!("was at {p}"), + None => "did not report memory".to_string(), + }; + let mem_detail = match (c.memory_usage, c.wm_memory_usage, c.memory_total) { + (host, wm, Some(total)) => { + let used = host.max(wm); + match used { + Some(u) => format!( + " (memory at last ping: {} of {})", + fmt_mb(u), + fmt_mb(total) + ), + None => format!(" (total available: {})", fmt_mb(total)), + } + } + _ => String::new(), + }; + let q_name = flow.worker.as_deref().unwrap_or("the recorded worker"); + format!( + "\nLikely culprit worker (on {loc}): {} — last pinged {} ({age}s ago, {rel} this flow's last ping) and then stopped pinging; it {mem} at that last ping{mem_detail}. This flow's dropped state transition was most likely performed by this worker and lost to its death (most likely OOM-kill), not a deadlock on {q_name}.", + c.worker, c.ping_at, + ) + } else { + String::new() }; - let service_logs_info = match (flow.worker_instance.as_deref(), flow.worker_last_ping) { + let hint: String = if let Some(c) = culprit.as_ref() { + let q_name = flow.worker.as_deref().unwrap_or("the recorded worker"); + match culprit_pct { + Some(p) if p >= 0.60 => format!( + "\nThis is almost certainly an OOM-kill on a *different* worker: {} (on the same pod/worker group) was at {} at its last ping right around this flow's transition, then stopped pinging. The flow's recorded worker ({q_name}) looks healthy because it is not the worker that performed the dropped transition. Raise the worker memory limit (e.g. k8s `resources.limits.memory`) or reduce per-flow memory usage. Confirm via that pod's restart count (`kubectl describe pod` / `kube_pod_container_status_last_terminated_reason`).", + c.worker, + culprit_pct_str.as_deref().unwrap_or("a high fraction of its limit"), + ), + _ => format!( + "\nMost likely an OOM-kill on a *different* worker: {} (on the same pod/worker group) stopped pinging right around this flow's transition ({last_ping:?}); its memory may have spiked after its last ping or not been reported. The flow's recorded worker ({q_name}) looks healthy because it did not perform the dropped transition. First check that pod's restart count (`kubectl describe pod` / `kube_pod_container_status_last_terminated_reason`). Only if that worker was not OOM-killed, consider a deadlock on {q_name} and capture a stack trace (e.g. via SIGQUIT).", + c.worker, + ), + } + } else { + match (worker_ping_stale, oom_moderate) { + (Some(_), true) => format!( + "\nThis is almost certainly an OOM-kill: container memory at the worker's last ping was at {mem_pct_str}. Raise the worker memory limit (e.g. k8s `resources.limits.memory`) or reduce per-flow memory usage. Confirm via pod restart count (`kubectl describe pod` / `kube_pod_container_status_last_terminated_reason`)." + ), + (Some(true), false) => { + "\nWorker stopped pinging and its last memory snapshot did not look high — in practice the overwhelmingly common cause here is still OOM-kill (memory may have spiked between the last ping and the kill, or never been reported). First check pod restart count (`kubectl describe pod` / `kube_pod_container_status_last_terminated_reason`). Less likely: host failure, network partition, or a panic — check worker logs / k8s events around the last ping time.".to_string() + } + (Some(false), false) => { + format!("\nThe flow's recorded worker is still pinging with healthy memory, but that worker is often NOT the one that performed the final state transition (in nested/subflow/forloop cases the last iteration runs on another worker). First check whether a different worker on the same pod / worker group was OOM-killed around {last_ping:?} (`kubectl describe pod` / `kube_pod_container_status_last_terminated_reason`, and that pod's worker memory metrics). Only if no such worker died, treat this as a deadlock or blocking call on the recorded worker during the state transition and capture a stack trace (e.g. via SIGQUIT).") + } + (None, _) => String::new(), + } + }; + + // Pull logs for the worker (and around the time) we actually blame: the + // culprit's instance/last-ping when one was found, else q.worker's. + let (log_instance, log_ping) = match culprit.as_ref() { + Some(c) => (c.worker_instance.as_deref(), Some(c.ping_at)), + None => (flow.worker_instance.as_deref(), flow.worker_last_ping), + }; + let service_logs_info = match (log_instance, log_ping) { (Some(host), Some(wlp)) => { let after = (wlp - chrono::Duration::seconds(90)) .to_rfc3339_opts(chrono::SecondsFormat::Millis, true); @@ -5118,13 +5293,25 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> { }; let reason = format!( - "{} was hanging in between 2 steps. Last ping: {last_ping:?} (now: {now}){worker_info}{hint}{service_logs_info}", + "{} was hanging in between 2 steps. Last ping: {last_ping:?} (now: {now}){worker_info}{culprit_info}{hint}{service_logs_info}", if flow.is_flow_step.unwrap_or(false) && flow.parent_job.is_some() { format!("Flow was cancelled because subflow {id} ({base_url}/run/{id}?workspace={workspace_id})") } else { format!("Flow {id} ({base_url}/run/{id}?workspace={workspace_id}) was cancelled because it") } ); + let reason = match between_steps_recovery_guidance( + db, + status.as_ref(), + id, + &workspace_id, + &base_url, + ) + .await + { + Some(guidance) => format!("{reason}\n\n{guidance}"), + None => reason, + }; report_critical_error(reason.clone(), db.clone(), Some(&flow.workspace_id), None).await; cancel_zombie_flow_job(db, flow.id, &flow.workspace_id, format!(r#"{reason} @@ -5171,6 +5358,103 @@ Please check your worker logs for more details and feel free to report it to the Ok(()) } +/// When a between-steps zombie's stuck step has every child recorded as a +/// `success` completion, the flow's state is fully derivable: only the final +/// state transition was lost to the worker failure, not any real work. In that +/// case return concrete restart-from-step recovery guidance to append to the +/// cancellation reason / critical alert. Returns `None` when the state isn't +/// derivable (some child missing or not successful), so the existing wording is +/// left untouched. Auto-recovery is deliberately not attempted (a re-driven +/// transition can OOM again on the same aggregated state; a human raises the +/// memory limit first, then restarts). +async fn between_steps_recovery_guidance( + db: &DB, + status: Option<&FlowStatus>, + flow_id: Uuid, + workspace_id: &str, + base_url: &str, +) -> Option { + // The stuck module is the current step, left InProgress because the + // transition that would have marked it Success was dropped. It is only + // derivable when its own cursor reached the end (a serial fan-out reaped + // mid-iteration has unrun work left; while-loops are never derivable). Whether + // restart reuses the children or re-runs the step (final step, or one carrying a + // stop/skip/approval/sleep) is decided by the restart path against the flow + // definition, which the reaper doesn't load; the guidance states both outcomes + // rather than promising reuse the restart might decline. + let status = status?; + let idx = usize::try_from(status.step).ok()?; + let module = status.modules.get(idx)?; + if !module.is_between_steps_complete() { + return None; + } + let step_id = module.id(); + + // Only a top-level deployed flow exposes a working restart-from-step: the run page's + // "Re-start from" button is rendered only for job_kind == 'flow' (a flowpreview, even a + // pathful editor preview, or a singlestepflow does not qualify), and a subflow child + // restarts via its root. Match that surface exactly so the guidance never points at a + // button / endpoint that isn't there; leave the existing wording otherwise. + let restartable = sqlx::query_scalar!( + r#"SELECT (kind = 'flow' AND parent_job IS NULL) AS "restartable!" + FROM v2_job WHERE id = $1"#, + flow_id, + ) + .fetch_one(db) + .await + .ok()?; + if !restartable { + return None; + } + + // Children whose completion the lost transition would have aggregated: the + // loop/branchall iterations, or the single leaf/subflow child. + let child_ids: Vec = module + .flow_jobs() + .filter(|v| !v.is_empty()) + .or_else(|| module.job().map(|j| vec![j]))?; + + // Derivable only when every child is recorded as a success completion. + let success_children = sqlx::query_scalar!( + "SELECT count(*) FROM v2_job_completed + WHERE workspace_id = $1 AND id = ANY($2) AND status = 'success'", + workspace_id, + &child_ids, + ) + .fetch_one(db) + .await + .ok()? + .unwrap_or(0); + if success_children != child_ids.len() as i64 { + return None; + } + let n = child_ids.len(); + + // For loop/branchall, name the completed iteration/branch count so the + // operator can confirm the whole fan-out is intact. + let iteration_hint = match module { + FlowStatusModule::InProgress { iterator: Some(_), .. } => { + format!(" (loop step, all {n} iterations completed)") + } + FlowStatusModule::InProgress { branchall: Some(_), .. } => { + format!(" (branchall step, all {n} branches completed)") + } + _ => String::new(), + }; + + Some(format!( + "RECOVERY: all {n} child job(s) of step `{step_id}`{iteration_hint} completed successfully; \ +only the flow's final state transition was lost to the worker failure above (not any genuine failure), so \ +the completed work is intact. To recover: first change the failure condition (raise the worker memory limit, \ +e.g. k8s `resources.limits.memory`, or move the flow to a larger worker group), then restart from step \ +`{step_id}`. Restart replays only the dropped transition and reuses the completed children where the step's \ +result is fully derivable; a step that is the flow's last, or carries a stop/skip condition, an approval, or a \ +sleep, is re-run instead (re-evaluating those on the larger worker).\n\ + UI: open {base_url}/run/{flow_id}?workspace={workspace_id} and use \"Re-start from {step_id}\".\n\ + API: POST {base_url}/api/w/{workspace_id}/jobs/restart/f/{flow_id} with body {{\"step_id\":\"{step_id}\"}}." + )) +} + async fn cancel_zombie_flow_job( db: &Pool, id: Uuid, @@ -5788,3 +6072,39 @@ mod strike_unarmed_tests { assert_eq!(strike_unarmed(&mut seen, set(&["b"])), vec![key("b")]); } } + +#[cfg(test)] +mod zombie_worker_memory_pct_tests { + use super::zombie_worker_memory_pct; + + #[test] + fn takes_the_larger_of_the_two_readings() { + // Both present: the larger (higher-signal) reading wins, not either + // one unconditionally — a "simplify to `usage.or(wm_usage)`" refactor + // would silently under-report and miss OOMs. + let p = zombie_worker_memory_pct(Some(600), Some(900), Some(1000)).unwrap(); + assert!((p - 0.9).abs() < f64::EPSILON); + } + + #[test] + fn falls_back_to_whichever_reading_is_present() { + assert_eq!( + zombie_worker_memory_pct(Some(700), None, Some(1000)), + Some(0.7) + ); + assert_eq!( + zombie_worker_memory_pct(None, Some(800), Some(1000)), + Some(0.8) + ); + } + + #[test] + fn none_when_no_usage_or_no_valid_total() { + assert_eq!(zombie_worker_memory_pct(None, None, Some(1000)), None); + assert_eq!(zombie_worker_memory_pct(Some(500), Some(500), None), None); + assert_eq!( + zombie_worker_memory_pct(Some(500), Some(500), Some(0)), + None + ); + } +} diff --git a/backend/tests/app_s3_onbehalf.rs b/backend/tests/app_s3_onbehalf.rs index 9dd96e30f1..7b233b505d 100644 --- a/backend/tests/app_s3_onbehalf.rs +++ b/backend/tests/app_s3_onbehalf.rs @@ -1,7 +1,9 @@ -//! Deployed-app S3 reads authorize on-behalf of the app author and are confined -//! to app provenance (declared keys or recent job outputs): a viewer cannot read -//! an arbitrary `file_key` as the author. Requires the `parquet` feature — the -//! real `apps_u/*` S3 handlers are gated on it. +//! Deployed-app S3 reads authorize on-behalf of the app author and are confined to +//! app provenance (declared keys or recent job outputs): an anonymous viewer cannot +//! read an arbitrary `file_key` as the author. A viewer on a full (unscoped) session +//! instead falls back to reading as THEMSELVES (bounded by their own S3 perms), so the +//! gate is exercised here through the anonymous identity it still fully protects. +//! Requires the `parquet` feature — the real `apps_u/*` S3 handlers are gated on it. //! //! `base` fixture: test-user (admin, SECRET_TOKEN); test-user-2 (non-admin, //! SECRET_TOKEN_2, no S3 folder permission). @@ -21,6 +23,19 @@ fn client() -> reqwest::Client { reqwest::Client::new() } +/// Mint an API token for test-user (admin) restricted to `scopes`. +async fn mint_scoped_token(port: u16, scopes: Vec<&str>) -> anyhow::Result { + let resp = authed( + client().post(format!("http://localhost:{port}/api/users/tokens/create")), + ADMIN_TOKEN, + ) + .json(&json!({ "label": "scoped", "scopes": scopes, "workspace_id": "test-workspace" })) + .send() + .await?; + assert_eq!(resp.status(), 201, "mint scoped token"); + Ok(resp.text().await?) +} + fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { builder.header("Authorization", format!("Bearer {}", token)) } @@ -50,62 +65,54 @@ async fn test_deployed_app_s3_onbehalf_provenance(db: Pool) -> anyhow: .await?; assert_eq!(resp.status(), 201, "app create: {}", resp.text().await?); - // GET an app-scoped S3 route as `token`. No workspace storage is configured, - // so a request that clears the provenance gate fails later at the storage - // lookup (or the CE OSS stub), never with "File restricted" — which is what - // lets these assertions distinguish "gate passed" from "gate denied". - let get = |route: &str, token: &'static str| { + // GET an app-scoped S3 route ANONYMOUSLY. Anonymous callers have no viewer + // identity to fall back to, so the provenance gate still fully applies to them + // (unlike logged-in viewers, who now read as themselves — see the union test). + // No workspace storage is configured, so a request that clears the gate fails + // later at the storage lookup (or the CE OSS stub), never with the denial + // message — which is what lets these assertions distinguish pass from deny. + let get = |route: &str| { let url = format!("{ws}/apps_u/{route}"); - authed(client().get(url), token).send() + client().get(url).send() }; - let denied = |body: &str| body.contains("File restricted"); + let denied = |body: &str| body.contains("is not accessible from this app"); - // download_s3_file: author-on-behalf allowed for the declared key, denied for - // a key the app never declared (the confused-deputy guard). - let body = get(&format!("download_s3_file/{APP}?s3={DECLARED}"), USER_TOKEN) + // download_s3_file: allowed for the declared key, denied for a key the app never + // declared (the confused-deputy guard). + let body = get(&format!("download_s3_file/{APP}?s3={DECLARED}")) .await? .text() .await?; assert!(!denied(&body), "declared key must clear the gate: {body}"); - let body = get( - &format!("download_s3_file/{APP}?s3={NON_PROVENANCE}"), - USER_TOKEN, - ) - .await? - .text() - .await?; + let body = get(&format!("download_s3_file/{APP}?s3={NON_PROVENANCE}")) + .await? + .text() + .await?; assert!(denied(&body), "non-provenance key must be denied: {body}"); // load_table_count and load_csv_preview enforce the same gate. The preview's // numeric `limit`/`offset` must deserialize (regression: a flattened query // struct 400s on them under serde_urlencoded). - let body = get( - &format!("load_table_count/{APP}?file_key={DECLARED}"), - USER_TOKEN, - ) - .await? - .text() - .await?; + let body = get(&format!("load_table_count/{APP}?file_key={DECLARED}")) + .await? + .text() + .await?; assert!( !denied(&body), "table_count declared key must clear the gate: {body}" ); - let body = get( - &format!("load_table_count/{APP}?file_key={NON_PROVENANCE}"), - USER_TOKEN, - ) - .await? - .text() - .await?; + let body = get(&format!("load_table_count/{APP}?file_key={NON_PROVENANCE}")) + .await? + .text() + .await?; assert!( denied(&body), "table_count non-provenance key must be denied: {body}" ); - let resp = get( - &format!("load_csv_preview/{APP}?file_key={DECLARED}&limit=5&offset=0"), - USER_TOKEN, - ) + let resp = get(&format!( + "load_csv_preview/{APP}?file_key={DECLARED}&limit=5&offset=0" + )) .await?; let status = resp.status(); let body = resp.text().await?; @@ -116,23 +123,16 @@ async fn test_deployed_app_s3_onbehalf_provenance(db: Pool) -> anyhow: ); // load_file_preview: `read_bytes_from` / `read_bytes_length` are required. - let resp = get( - &format!("load_file_preview/{APP}?file_key={DECLARED}"), - USER_TOKEN, - ) - .await?; + let resp = get(&format!("load_file_preview/{APP}?file_key={DECLARED}")).await?; assert_eq!( resp.status(), 400, "file_preview without byte range must 400: {}", resp.text().await? ); - let body = get( - &format!( - "load_file_preview/{APP}?file_key={DECLARED}&read_bytes_from=0&read_bytes_length=4096" - ), - USER_TOKEN, - ) + let body = get(&format!( + "load_file_preview/{APP}?file_key={DECLARED}&read_bytes_from=0&read_bytes_length=4096" + )) .await? .text() .await?; @@ -144,6 +144,94 @@ async fn test_deployed_app_s3_onbehalf_provenance(db: Pool) -> anyhow: Ok(()) } +/// The viewer-perm union: a viewer on a full (unscoped) session is no longer hard-denied +/// by the provenance gate for a pre-existing file. It falls back to reading as ITSELF +/// (bounded by its own S3 perms downstream), while an anonymous caller (no identity) and +/// a scope-restricted token (can hit `apps_u/*` but not `job_helpers/*`, so the fallback +/// would be a new capability) both stay fully gated with the actionable denial. +#[sqlx::test(fixtures("base"))] +async fn test_deployed_app_s3_viewer_union(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&json!({ + "path": APP, + "summary": "s3 viewer union test", + "value": {}, + "policy": { + "execution_mode": "anonymous", + "triggerables": {}, + "allowed_s3_keys": [{ "s3_path": DECLARED }] + } + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "app create: {}", resp.text().await?); + + let url = format!("{ws}/apps_u/download_s3_file/{APP}?s3={NON_PROVENANCE}"); + + // Anonymous: still gated. The denial is the actionable message and echoes the key. + let body = client().get(&url).send().await?.text().await?; + assert!( + body.contains("is not accessible from this app"), + "anonymous viewer must stay gated with the actionable denial: {body}" + ); + assert!( + body.contains(NON_PROVENANCE), + "denial must echo the requested key: {body}" + ); + + // Logged-in viewer: no longer hard-denied — the gate delegates to reading as the + // viewer, so the request falls through to the storage read (no gate denial in + // EITHER the old or new form). No workspace storage is configured here, so it + // surfaces a downstream storage/OSS error, not a gate denial. + let body = authed(client().get(&url), USER_TOKEN) + .send() + .await? + .text() + .await?; + assert!( + !body.contains("is not accessible from this app") && !body.contains("File restricted"), + "logged-in viewer must delegate to its own read, not be gate-denied: {body}" + ); + + // Scope-restricted token: an `apps:read:` token reaches this route but is + // REJECTED by the route-scope middleware on `job_helpers/*`, so it must NOT get the + // viewer fallback (that would be a capability it cannot obtain directly). It stays + // gated with the denial, unlike the unscoped session above. + let apps_read_scope = format!("apps:read:{APP}"); + let scoped = mint_scoped_token(port, vec![apps_read_scope.as_str()]).await?; + let body = authed(client().get(&url), &scoped) + .send() + .await? + .text() + .await?; + assert!( + body.contains("is not accessible from this app"), + "scope-restricted token must stay gated, not get the viewer fallback: {body}" + ); + + // A filter-tags-only token carries no real scope restriction (the route-scope + // middleware treats it as unscoped), so it can read via job_helpers directly and + // MUST get the viewer fallback here — not be gated like a genuinely scoped token. + let tag_only = mint_scoped_token(port, vec!["if_jobs:filter_tags:default"]).await?; + let body = authed(client().get(&url), &tag_only) + .send() + .await? + .text() + .await?; + assert!( + !body.contains("is not accessible from this app") && !body.contains("File restricted"), + "filter-tags-only token is effectively unscoped and must delegate, not be gated: {body}" + ); + + Ok(()) +} + /// Mint a presigned bearer (`exp=..&sig=..`) exactly as `sign_s3_objects` does: /// `HMAC-SHA256(workspace_key, "file_key={s3}&exp={exp}")` (no storage param, since /// these routes send none). `validate_s3_signature` is `private`-gated, so this test @@ -198,16 +286,19 @@ async fn test_deployed_app_s3_presigned_bypasses_gate(db: Pool) -> any let url = format!("{ws}/apps_u/{route}"); authed(client().get(url), token).send() }; - let denied = |body: &str| body.contains("File restricted"); + let denied = |body: &str| body.contains("is not accessible from this app"); - // Control: NON_PROVENANCE without a signature is denied by the gate. - let body = get( - format!("download_s3_file/{APP}?s3={NON_PROVENANCE}"), - USER_TOKEN, - ) - .await? - .text() - .await?; + // Control: NON_PROVENANCE without a signature is denied by the gate. Sent + // anonymously — a logged-in viewer would instead fall back to reading as + // themselves, so anonymous is the identity that isolates the presigned bypass. + let body = client() + .get(format!( + "{ws}/apps_u/download_s3_file/{APP}?s3={NON_PROVENANCE}" + )) + .send() + .await? + .text() + .await?; assert!( denied(&body), "unsigned non-provenance key must be denied: {body}" @@ -304,12 +395,14 @@ async fn seed_completed_job( } /// A deployed app that renders S3 files it produced (e.g. a SQL query persisted to -/// S3 by a component) must clear the provenance gate for the viewer whose own app -/// run produced them, while (a) a viewer cannot forge provenance by running a -/// runnable directly (no app marker), (b) another app's outputs stay denied, and -/// (c) another viewer's outputs stay denied (cross-viewer isolation). Provenance is -/// keyed on the app-origination marker (`trigger_kind='app'` + `trigger=`) -/// that `execute_component` stamps, plus `created_by = ` for isolation. +/// S3 by a component) must clear the provenance gate for the caller whose own app run +/// produced them, while (a) provenance cannot be forged by running a runnable directly +/// (no app marker), (b) another app's outputs stay denied, and (c) another caller's +/// outputs stay denied (per-caller isolation). Provenance is keyed on the +/// app-origination marker (`trigger_kind='app'` + `trigger=`) that +/// `execute_component` stamps, plus `created_by = ` for isolation. +/// Exercised anonymously: the gate still fully governs anonymous callers, whereas a +/// logged-in viewer would instead fall back to reading as themselves. #[sqlx::test(fixtures("base"))] async fn test_deployed_app_s3_onbehalf_flow_script_provenance( db: Pool, @@ -322,10 +415,10 @@ async fn test_deployed_app_s3_onbehalf_flow_script_provenance( const FS_APP: &str = "u/test-user/s3flowscript"; const OTHER_APP: &str = "u/test-user/other_app"; - // Produced by test-user-2's own app run of THIS app. - const USER_KEY: &str = "results/user2_output.parquet"; - // Produced by test-user's own app run of THIS app. - const ADMIN_KEY: &str = "results/admin_output.parquet"; + // Produced by the anonymous caller's own app run of THIS app. + const OWN_KEY: &str = "results/own_output.parquet"; + // Produced by a DIFFERENT caller's app run of THIS app → isolation, must stay denied. + const OTHER_CALLER_KEY: &str = "results/user2_output.parquet"; // Produced by an app run of a DIFFERENT app → must stay denied. const OTHER_APP_KEY: &str = "results/other_app_output.parquet"; // Produced by a DIRECT run (no app marker) → the forgery attempt, must stay denied. @@ -342,64 +435,49 @@ async fn test_deployed_app_s3_onbehalf_flow_script_provenance( .await?; assert_eq!(resp.status(), 201, "app create: {}", resp.text().await?); - // Seed the produced-file jobs (all within the 3h window). - seed_completed_job(&db, "test-user-2", Some(FS_APP), USER_KEY).await?; - seed_completed_job(&db, "test-user", Some(FS_APP), ADMIN_KEY).await?; - seed_completed_job(&db, "test-user-2", Some(OTHER_APP), OTHER_APP_KEY).await?; - seed_completed_job(&db, "test-user-2", None, FORGED_KEY).await?; + // Seed the produced-file jobs (all within the 3h window). The gate's `created_by` + // filter uses "anonymous" for an unauthenticated caller. + seed_completed_job(&db, "anonymous", Some(FS_APP), OWN_KEY).await?; + seed_completed_job(&db, "test-user-2", Some(FS_APP), OTHER_CALLER_KEY).await?; + seed_completed_job(&db, "anonymous", Some(OTHER_APP), OTHER_APP_KEY).await?; + seed_completed_job(&db, "anonymous", None, FORGED_KEY).await?; - let get = |route: &str, token: &'static str| { + let denied = |body: &str| body.contains("is not accessible from this app"); + // Anonymous GET (borrows `ws`, reusable across calls: the URL is built before the + // `async move` so only the owned `url` is moved into the future, not `ws`). + let anon_body = |route: String| { let url = format!("{ws}/apps_u/{route}"); - authed(client().get(url), token).send() - }; - let denied = |body: &str| body.contains("File restricted"); - let body_of = |route: String, token: &'static str| async move { - get(&route, token).await.unwrap().text().await.unwrap() + async move { + client() + .get(url) + .send() + .await + .unwrap() + .text() + .await + .unwrap() + } }; - // The viewer's own app run's output clears the gate (the case that regressed to - // "File restricted"). - let body = body_of( - format!("download_s3_file/{FS_APP}?s3={USER_KEY}"), - USER_TOKEN, - ) - .await; + // The caller's own app run's output clears the gate (the case that regressed to + // a hard denial). + let body = anon_body(format!("download_s3_file/{FS_APP}?s3={OWN_KEY}")).await; assert!( !denied(&body), - "viewer's own app-produced key must clear the gate: {body}" + "caller's own app-produced key must clear the gate: {body}" ); - // The admin viewer's own app run's output clears — the gate has no admin bypass, - // it just matches the caller's own runs. - let body = body_of( - format!("download_s3_file/{FS_APP}?s3={ADMIN_KEY}"), - ADMIN_TOKEN, - ) - .await; - assert!( - !denied(&body), - "admin's own app-produced key must clear the gate: {body}" - ); - - // Cross-viewer isolation: the admin cannot pull test-user-2's result even though - // it is a genuine app-marked job of the same app (no admin bypass either). - let body = body_of( - format!("download_s3_file/{FS_APP}?s3={USER_KEY}"), - ADMIN_TOKEN, - ) - .await; + // Per-caller isolation: another caller's result stays denied even though it is a + // genuine app-marked job of the same app. + let body = anon_body(format!("download_s3_file/{FS_APP}?s3={OTHER_CALLER_KEY}")).await; assert!( denied(&body), - "another viewer's app-produced key must stay denied (isolation): {body}" + "another caller's app-produced key must stay denied (isolation): {body}" ); // A key produced by a direct run (no app marker) stays denied — the forgery the // app-origination marker closes. - let body = body_of( - format!("download_s3_file/{FS_APP}?s3={FORGED_KEY}"), - USER_TOKEN, - ) - .await; + let body = anon_body(format!("download_s3_file/{FS_APP}?s3={FORGED_KEY}")).await; assert!( denied(&body), "key from a direct run (no app marker) must stay denied: {body}" @@ -407,11 +485,7 @@ async fn test_deployed_app_s3_onbehalf_flow_script_provenance( // A key produced by a DIFFERENT app stays denied — provenance is scoped to THIS // app's path. - let body = body_of( - format!("download_s3_file/{FS_APP}?s3={OTHER_APP_KEY}"), - USER_TOKEN, - ) - .await; + let body = anon_body(format!("download_s3_file/{FS_APP}?s3={OTHER_APP_KEY}")).await; assert!( denied(&body), "key produced by a different app must stay denied: {body}" diff --git a/backend/tests/nativets_jobs.rs b/backend/tests/nativets_jobs.rs index 610d0c5ea8..ff871e2da6 100644 --- a/backend/tests/nativets_jobs.rs +++ b/backend/tests/nativets_jobs.rs @@ -263,6 +263,43 @@ export function main(): number[] { ); } + // -- result + wm_labels carrying a NUL: must complete, jsonb-safe & text[]-safe -- + { + // `\u0000` here is 6 literal chars in the raw string; the JS runtime emits + // a real U+0000. It aborts the jsonb result insert (22P05) and, via + // wm_labels, the `text[]` labels update - both in the completion tx. + // `nul`/label come back stripped; `literal` (escaped backslash + text + // "u0000", no real NUL) survives untouched. + let result = push_and_wait( + &db, + RunJob::from(nativets_code( + r#"//native + +export function main(): {nul: string, literal: string, wm_labels: string[]} { + return { nul: "a\u0000b", literal: "a\\u0000b", wm_labels: ["x\u0000y"] }; +} +"#, + )), + &mut listener, + ) + .await; + assert!(result.success, "nul_result failed: {:?}", result.result); + let val = result.json_result().unwrap(); + assert_eq!(val["nul"], serde_json::json!("ab")); + assert_eq!(val["literal"], serde_json::json!("a\\u0000b")); + + // The wm_labels entry is persisted to the `text[]` column NUL-free. + let labels: Option> = + sqlx::query_scalar("SELECT labels FROM v2_job WHERE id = $1") + .bind(result.id) + .fetch_one(&db) + .await + .unwrap(); + let labels = labels.unwrap_or_default(); + assert!(labels.iter().any(|l| l == "xy"), "expected stripped label, got {labels:?}"); + assert!(!labels.iter().any(|l| l.contains('\0')), "labels must be NUL-free: {labels:?}"); + } + killpill.send(); Ok(()) } diff --git a/backend/tests/suspend_resume.rs b/backend/tests/suspend_resume.rs index 704fa0d256..29fe38d2c3 100644 --- a/backend/tests/suspend_resume.rs +++ b/backend/tests/suspend_resume.rs @@ -352,6 +352,208 @@ mod suspend_resume { Ok(()) } + /// The UI "Resume" button (POST /jobs_u/flow/resume_suspended/:job_id) must reject the + /// triggerer approving their own self_approval_disabled step even when they own the flow + /// path: only admins are exempt from the self-approval restriction, so owning the runnable + /// does not grant the right to self-approve. + #[cfg(feature = "enterprise")] + #[cfg(feature = "deno_core")] + #[sqlx::test(fixtures("base"))] + async fn test_self_approval_disabled_blocks_ui_resume_for_owner( + db: Pool, + ) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let flow_with_self_approval_disabled: FlowValue = serde_json::from_value(json!({ + "modules": [{ + "id": "a", + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main() { return 'step1'; }" + }, + "suspend": { + "required_events": 1, + "user_auth_required": true, + "self_approval_disabled": true + } + }, { + "id": "b", + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main() { return 'step2 - after approval'; }" + } + }] + })) + .unwrap(); + + // Push as a non-admin who owns the flow path, so the owner shortcut is exercised. + let flow = RunJob::from(JobPayload::RawFlow { + value: flow_with_self_approval_disabled, + path: Some("u/test-user-2/test_ui_resume".to_string()), + restarted_from: None, + }) + .push_as(&db, "test-user-2", "test2@windmill.dev") + .await; + + let queue = listen_for_queue(&db).await; + let db_ = db.clone(); + + in_test_worker( + &db, + async move { + let db = db_; + + wait_until_flow_suspends(flow, queue, &db).await; + + let token = windmill_common::auth::create_token_for_owner( + &db, + "test-workspace", + "u/test-user-2", + "test-token", + 100, + "test2@windmill.dev", + &Uuid::nil(), + None, + None, + ) + .await + .unwrap(); + + // Resume via the UI endpoint as the owner who triggered the flow. + let response = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/jobs_u/flow/resume_suspended/{flow}" + )) + .header("Authorization", format!("Bearer {token}")) + .header("Content-Type", "application/json") + .body("{}") + .send() + .await + .unwrap(); + + let status = response.status(); + assert!( + status == reqwest::StatusCode::FORBIDDEN, + "Self-approval via the UI resume endpoint should be blocked for the owner when \ + self_approval_disabled=true. Expected 403 Forbidden, got {}. Response: {}", + status, + response.text().await.unwrap_or_default() + ); + }, + port, + ) + .await; + + server.close().await.unwrap(); + Ok(()) + } + + /// self_approval_disabled must hold even when user_auth_required is not set: the worker must + /// persist the condition and the resume boundary must enforce it for the authenticated + /// triggerer. The triggerer here is a non-owner (folder path they don't own), so the owner + /// shortcut is not involved and this exercises the persistence + authenticated-check path. + #[cfg(feature = "enterprise")] + #[cfg(feature = "deno_core")] + #[sqlx::test(fixtures("base"))] + async fn test_self_approval_disabled_without_user_auth_required( + db: Pool, + ) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // self_approval_disabled without user_auth_required (as a raw-flow/CLI author could set). + let flow_value: FlowValue = serde_json::from_value(json!({ + "modules": [{ + "id": "a", + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main() { return 'step1'; }" + }, + "suspend": { + "required_events": 1, + "self_approval_disabled": true + } + }, { + "id": "b", + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main() { return 'step2 - after approval'; }" + } + }] + })) + .unwrap(); + + // Folder path test-user-2 does not own -> non-owner triggerer (no folders in the base + // fixture), so the owner shortcut is bypassed and only persistence matters here. + let flow = RunJob::from(JobPayload::RawFlow { + value: flow_value, + path: Some("f/system/test_persist".to_string()), + restarted_from: None, + }) + .push_as(&db, "test-user-2", "test2@windmill.dev") + .await; + + let queue = listen_for_queue(&db).await; + let db_ = db.clone(); + + in_test_worker( + &db, + async move { + let db = db_; + + wait_until_flow_suspends(flow, queue, &db).await; + + let token = windmill_common::auth::create_token_for_owner( + &db, + "test-workspace", + "u/test-user-2", + "test-token", + 100, + "test2@windmill.dev", + &Uuid::nil(), + None, + None, + ) + .await + .unwrap(); + + let response = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/jobs_u/flow/resume_suspended/{flow}" + )) + .header("Authorization", format!("Bearer {token}")) + .header("Content-Type", "application/json") + .body("{}") + .send() + .await + .unwrap(); + + let status = response.status(); + assert!( + status == reqwest::StatusCode::FORBIDDEN, + "Self-approval should be blocked when self_approval_disabled=true even without \ + user_auth_required. Expected 403 Forbidden, got {}. Response: {}", + status, + response.text().await.unwrap_or_default() + ); + }, + port, + ) + .await; + + server.close().await.unwrap(); + Ok(()) + } + /// Test that self-approval WORKS when self_approval_disabled is false (default behavior). /// /// This is the complementary test to test_self_approval_disabled_blocks_owner_resume. diff --git a/backend/tests/zombie_flow_recovery.rs b/backend/tests/zombie_flow_recovery.rs new file mode 100644 index 0000000000..fb658dfb03 --- /dev/null +++ b/backend/tests/zombie_flow_recovery.rs @@ -0,0 +1,661 @@ +//! Regression test for hand-recovery of between-steps zombie flows. +//! +//! When a worker is OOM-killed mid state-transition, the zombie monitor +//! (`handle_zombie_flows` → `cancel_job` with force) reaps the flow: it lands in +//! `v2_job_completed` as `canceled`, with its `flow_status` preserved: the step +//! whose transition was lost stays `InProgress` even though all its children +//! completed successfully. This test reproduces that exact terminal state and +//! asserts that a hand-restart from the stuck step reuses every completed child +//! (no re-run) and the flow reaches success. +//! +//! The reaper itself lives in the `windmill` binary crate and is unreachable +//! from an integration test, so we reproduce the state `cancel_job(force)` +//! leaves behind directly; the fix under test is the restart-resolution path, +//! not the detection query. + +#![cfg(feature = "deno_core")] + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_common::flow_status::{BranchChosen, FlowStatus, RestartedFrom}; +use windmill_common::flows::FlowValue; +use windmill_common::jobs::JobPayload; +use windmill_test_utils::*; + +/// Child job UUID for a top-level step in a completed flow's `flow_status` +/// (optionally the iteration index for a ForLoop / BranchAll container). +async fn child_job_id_for_step( + db: &Pool, + flow_job_id: uuid::Uuid, + step_id: &str, + iter: Option, +) -> uuid::Uuid { + let raw: serde_json::Value = sqlx::query_scalar!( + "SELECT flow_status FROM v2_job_completed WHERE id = $1", + flow_job_id + ) + .fetch_one(db) + .await + .unwrap() + .expect("flow_status missing"); + let status: FlowStatus = serde_json::from_value(raw).expect("parse flow_status"); + let module = status + .modules + .iter() + .find(|m| m.id() == step_id) + .expect("step in flow_status"); + match iter { + Some(i) => module.flow_jobs().expect("flow_jobs")[i], + None => module.job().expect("job"), + } +} + +/// A between-steps zombie whose fan-out completed but whose final transition was +/// lost can be hand-restarted from the stuck step, reusing every completed child +/// (including the last iteration) and reaching success. +#[sqlx::test(fixtures("base", "hello"))] +async fn test_between_steps_zombie_restart_reuses_all_children( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // A fan-out ForLoop `fanout` (2 iterations) followed by `after`, which + // consumes the loop's aggregated result. In the zombie scenario `fanout` + // finished all iterations but its final transition was lost, so `after` + // never ran. + let flow_value: FlowValue = serde_json::from_value(json!({ + "modules": [ + { + "id": "fanout", + "value": { + "type": "forloopflow", + "iterator": { "type": "javascript", "expr": "['a', 'b']" }, + "skip_failures": false, + "parallel": false, + "modules": [{ + "id": "inner", + "value": { + "type": "rawscript", + "language": "deno", + "input_transforms": { + "v": { "type": "javascript", "expr": "flow_input.iter.value" } + }, + "content": "export function main(v: string) { return v }" + } + }] + } + }, + { + "id": "after", + "value": { + "type": "rawscript", + "language": "deno", + "input_transforms": { + "loop_res": { "type": "javascript", "expr": "results.fanout" } + }, + "content": "export function main(loop_res: string[]) { return loop_res.join(',') }" + } + } + ] + })) + .unwrap(); + + // Run to completion to obtain real, successful child jobs. + let full_run = RunJob::from(JobPayload::RawFlow { + value: flow_value.clone(), + path: None, + restarted_from: None, + }) + .run_until_complete(&db, false, port) + .await; + assert!(full_run.success, "baseline run should succeed"); + assert_eq!(full_run.json_result().unwrap(), json!("a,b")); + + let orig_iter0 = child_job_id_for_step(&db, full_run.id, "fanout", Some(0)).await; + let orig_iter1 = child_job_id_for_step(&db, full_run.id, "fanout", Some(1)).await; + let orig_after = child_job_id_for_step(&db, full_run.id, "after", None).await; + + // Reproduce the zombie-reaper's terminal state: cancelled by `monitor` with + // `flow_status` frozen mid-transition: `fanout` still `InProgress` (all + // iterations done), `after` never reached. + let mut flow_status: serde_json::Value = sqlx::query_scalar!( + "SELECT flow_status FROM v2_job_completed WHERE id = $1", + full_run.id + ) + .fetch_one(&db) + .await? + .expect("flow_status"); + for m in flow_status["modules"].as_array_mut().unwrap() { + match m["id"].as_str() { + Some("fanout") => { + m["type"] = json!("InProgress"); + // A reaped loop keeps its cursor at the last iteration. + m["iterator"] = json!({ "index": 1, "itered_len": 2 }); + } + Some("after") => *m = json!({ "type": "WaitingForPriorSteps", "id": "after" }), + _ => {} + } + } + flow_status["step"] = json!(0); + sqlx::query!( + "UPDATE v2_job_completed + SET status = 'canceled', canceled_by = 'monitor', canceled_reason = 'zombie flow', + flow_status = $2 + WHERE id = $1", + full_run.id, + flow_status, + ) + .execute(&db) + .await?; + + // Hand-restart from the stuck step. `fanout` is recognised as a derivable + // between-steps zombie (all children succeeded), so it is reused verbatim and + // only the dropped transition onward is replayed. `Some(0)` is the exact value the + // run page's "Re-start from" button sends (a whole-step restart), not `None`. + let restarted = RunJob::from(JobPayload::RestartedFlow { + completed_job_id: full_run.id, + step_id: "fanout".into(), + branch_or_iteration_n: Some(0), + flow_version: None, + branch_chosen: None, + nested: None, + }) + .run_until_complete(&db, false, port) + .await; + + // Flow reaches success, reusing the loop's aggregated result. + assert!( + restarted.success, + "restarted zombie flow should succeed: {:?}", + restarted.json_result() + ); + assert_eq!(restarted.json_result().unwrap(), json!("a,b")); + + // Every completed loop iteration reuses its original child job (no re-run); + // only `after`, which never ran, executes fresh. + let new_iter0 = child_job_id_for_step(&db, restarted.id, "fanout", Some(0)).await; + let new_iter1 = child_job_id_for_step(&db, restarted.id, "fanout", Some(1)).await; + let new_after = child_job_id_for_step(&db, restarted.id, "after", None).await; + assert_eq!(new_iter0, orig_iter0, "loop iteration 0 must be reused"); + assert_eq!(new_iter1, orig_iter1, "loop iteration 1 must be reused"); + assert_ne!(new_after, orig_after, "`after` should run fresh"); + + Ok(()) +} + +/// A serial for-loop reaped *between* iterations (an all-success prefix, but the +/// cursor not yet at the last iteration) must NOT be treated as complete: reuse +/// would silently drop the remaining iterations. A downstream `after` step makes +/// the loop non-final, so the ONLY thing that can prevent reuse here is the +/// cursor-completeness guard; if it regresses, `after` would consume a truncated +/// loop result and this test fails. Restart must re-run the whole loop instead. +#[sqlx::test(fixtures("base", "hello"))] +async fn test_mid_iteration_zombie_not_reused(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let flow_value: FlowValue = serde_json::from_value(json!({ + "modules": [ + { + "id": "fanout", + "value": { + "type": "forloopflow", + "iterator": { "type": "javascript", "expr": "['a', 'b', 'c']" }, + "skip_failures": false, + "parallel": false, + "modules": [{ + "id": "inner", + "value": { + "type": "rawscript", + "language": "deno", + "input_transforms": { + "v": { "type": "javascript", "expr": "flow_input.iter.value" } + }, + "content": "export function main(v: string) { return v }" + } + }] + } + }, + { + "id": "after", + "value": { + "type": "rawscript", + "language": "deno", + "input_transforms": { + "loop_res": { "type": "javascript", "expr": "results.fanout" } + }, + "content": "export function main(loop_res: string[]) { return loop_res.join(',') }" + } + } + ] + })) + .unwrap(); + + let full_run = RunJob::from(JobPayload::RawFlow { + value: flow_value.clone(), + path: None, + restarted_from: None, + }) + .run_until_complete(&db, false, port) + .await; + assert!(full_run.success); + let orig_iter0 = child_job_id_for_step(&db, full_run.id, "fanout", Some(0)).await; + + // Reap after iteration 0: the loop is InProgress with the cursor still on + // iteration 0 (of 3), only iteration 0 recorded; `after` never reached. + let mut flow_status: serde_json::Value = sqlx::query_scalar!( + "SELECT flow_status FROM v2_job_completed WHERE id = $1", + full_run.id + ) + .fetch_one(&db) + .await? + .expect("flow_status"); + for m in flow_status["modules"].as_array_mut().unwrap() { + match m["id"].as_str() { + Some("fanout") => { + m["type"] = json!("InProgress"); + m["iterator"] = json!({ "index": 0, "itered_len": 3 }); + m["flow_jobs"] = json!([m["flow_jobs"][0]]); + m["flow_jobs_success"] = json!([true]); + } + Some("after") => *m = json!({ "type": "WaitingForPriorSteps", "id": "after" }), + _ => {} + } + } + flow_status["step"] = json!(0); + sqlx::query!( + "UPDATE v2_job_completed SET status = 'canceled', canceled_by = 'monitor', + flow_status = $2 WHERE id = $1", + full_run.id, + flow_status, + ) + .execute(&db) + .await?; + + let restarted = RunJob::from(JobPayload::RestartedFlow { + completed_job_id: full_run.id, + step_id: "fanout".into(), + branch_or_iteration_n: None, + flow_version: None, + branch_chosen: None, + nested: None, + }) + .run_until_complete(&db, false, port) + .await; + + // The loop re-runs from scratch: all three iterations execute (so `after` sees + // "a,b,c", not a truncated "a"), and iteration 0 is a fresh job. + assert!( + restarted.success, + "restart should re-run the loop and succeed: {:?}", + restarted.json_result() + ); + assert_eq!(restarted.json_result().unwrap(), json!("a,b,c")); + let new_iter0 = child_job_id_for_step(&db, restarted.id, "fanout", Some(0)).await; + assert_ne!( + new_iter0, orig_iter0, + "iteration 0 must re-run, not be reused" + ); + + Ok(()) +} + +/// A nested restart request targets an inner step of the restart-step container. +/// Even when that container is an eligible between-steps zombie, reuse must NOT +/// fire (it would skip the whole container and ignore the explicit nested target). +/// The inner step must re-run. +#[sqlx::test(fixtures("base", "hello"))] +async fn test_nested_restart_not_swallowed_by_zombie_reuse( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // `branch` is a BranchOne (single child, so branch_or_iteration_n is None on + // restart: the exact shape that would trip zombie reuse) with two inner steps, + // followed by a downstream `after`. + let flow_value: FlowValue = serde_json::from_value(json!({ + "modules": [ + { + "id": "branch", + "value": { + "type": "branchone", + "default": [], + "branches": [{ + "expr": "true", + "modules": [ + { + "id": "inner_first", + "value": { + "type": "rawscript", "language": "deno", + "input_transforms": {}, + "content": "export function main() { return 'first' }" + } + }, + { + "id": "inner_second", + "value": { + "type": "rawscript", "language": "deno", + "input_transforms": { + "first": { "type": "javascript", "expr": "results.inner_first" } + }, + "content": "export function main(first: string) { return `${first}|second` }" + } + } + ] + }] + } + }, + { + "id": "after", + "value": { + "type": "rawscript", "language": "deno", + "input_transforms": { + "b": { "type": "javascript", "expr": "results.branch" } + }, + "content": "export function main(b: string) { return `after:${b}` }" + } + } + ] + })) + .unwrap(); + + let full_run = RunJob::from(JobPayload::RawFlow { + value: flow_value.clone(), + path: None, + restarted_from: None, + }) + .run_until_complete(&db, false, port) + .await; + assert!(full_run.success); + assert_eq!(full_run.json_result().unwrap(), json!("after:first|second")); + let branch_child = child_job_id_for_step(&db, full_run.id, "branch", None).await; + let orig_inner_second = child_job_id_for_step(&db, branch_child, "inner_second", None).await; + + // Reap `branch` as a between-steps zombie (its child completed, transition lost); + // `after` never reached. + let mut flow_status: serde_json::Value = sqlx::query_scalar!( + "SELECT flow_status FROM v2_job_completed WHERE id = $1", + full_run.id + ) + .fetch_one(&db) + .await? + .expect("flow_status"); + for m in flow_status["modules"].as_array_mut().unwrap() { + match m["id"].as_str() { + Some("branch") => m["type"] = json!("InProgress"), + Some("after") => *m = json!({ "type": "WaitingForPriorSteps", "id": "after" }), + _ => {} + } + } + flow_status["step"] = json!(0); + sqlx::query!( + "UPDATE v2_job_completed SET status = 'canceled', canceled_by = 'monitor', + flow_status = $2 WHERE id = $1", + full_run.id, + flow_status, + ) + .execute(&db) + .await?; + + // Nested restart: re-run `inner_second` inside `branch`. Zombie reuse must step + // aside so the nested chain is honored. + let restarted = RunJob::from(JobPayload::RestartedFlow { + completed_job_id: full_run.id, + step_id: "branch".into(), + branch_or_iteration_n: None, + flow_version: None, + branch_chosen: Some(BranchChosen::Branch { branch: 0 }), + nested: Some(Box::new(RestartedFrom { + flow_job_id: branch_child, + step_id: "inner_second".into(), + branch_or_iteration_n: None, + flow_version: None, + branch_chosen: None, + nested: None, + })), + }) + .run_until_complete(&db, false, port) + .await; + + assert!( + restarted.success, + "nested restart of a zombie container should succeed: {:?}", + restarted.json_result() + ); + assert_eq!( + restarted.json_result().unwrap(), + json!("after:first|second") + ); + let new_branch_child = child_job_id_for_step(&db, restarted.id, "branch", None).await; + let new_inner_second = child_job_id_for_step(&db, new_branch_child, "inner_second", None).await; + assert_ne!( + new_inner_second, orig_inner_second, + "the nested target inner_second must re-run, not be skipped by zombie reuse" + ); + + Ok(()) +} + +/// A raw-flow (editor preview) restart queues the request's CURRENT definition, which the editor +/// allows to differ from the completed run. Zombie reuse must not fire there: it would validate the +/// stored step and synthesize Success from the old children, skipping the user's edit. The edited +/// step must re-run and downstream must observe its new result. +#[sqlx::test(fixtures("base", "hello"))] +async fn test_raw_flow_restart_does_not_reuse_edited_step( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let flow_of = |suffix: &str| -> FlowValue { + serde_json::from_value(json!({ + "modules": [ + { + "id": "fanout", + "value": { + "type": "forloopflow", + "iterator": { "type": "javascript", "expr": "['a', 'b']" }, + "skip_failures": false, + "parallel": false, + "modules": [{ + "id": "inner", + "value": { + "type": "rawscript", "language": "deno", + "input_transforms": { + "v": { "type": "javascript", "expr": "flow_input.iter.value" } + }, + "content": format!("export function main(v: string) {{ return v + '{suffix}' }}") + } + }] + } + }, + { + "id": "after", + "value": { + "type": "rawscript", "language": "deno", + "input_transforms": { + "loop_res": { "type": "javascript", "expr": "results.fanout" } + }, + "content": "export function main(loop_res: string[]) { return loop_res.join(',') }" + } + } + ] + })) + .unwrap() + }; + + let full_run = + RunJob::from(JobPayload::RawFlow { value: flow_of(""), path: None, restarted_from: None }) + .run_until_complete(&db, false, port) + .await; + assert!(full_run.success); + assert_eq!(full_run.json_result().unwrap(), json!("a,b")); + let orig_iter0 = child_job_id_for_step(&db, full_run.id, "fanout", Some(0)).await; + + let mut flow_status: serde_json::Value = sqlx::query_scalar!( + "SELECT flow_status FROM v2_job_completed WHERE id = $1", + full_run.id + ) + .fetch_one(&db) + .await? + .expect("flow_status"); + for m in flow_status["modules"].as_array_mut().unwrap() { + match m["id"].as_str() { + Some("fanout") => { + m["type"] = json!("InProgress"); + m["iterator"] = json!({ "index": 1, "itered_len": 2 }); + } + Some("after") => *m = json!({ "type": "WaitingForPriorSteps", "id": "after" }), + _ => {} + } + } + flow_status["step"] = json!(0); + sqlx::query!( + "UPDATE v2_job_completed SET status = 'canceled', canceled_by = 'monitor', + flow_status = $2 WHERE id = $1", + full_run.id, + flow_status, + ) + .execute(&db) + .await?; + + let restarted = RunJob::from(JobPayload::RawFlow { + value: flow_of("X"), + path: None, + restarted_from: Some(RestartedFrom { + flow_job_id: full_run.id, + step_id: "fanout".into(), + branch_or_iteration_n: None, + flow_version: None, + branch_chosen: None, + nested: None, + }), + }) + .run_until_complete(&db, false, port) + .await; + + // The edited step must run: results reflect the new definition, not the reused old children. + assert!( + restarted.success, + "edited raw-flow restart should succeed: {:?}", + restarted.json_result() + ); + assert_eq!(restarted.json_result().unwrap(), json!("aX,bX")); + let new_iter0 = child_job_id_for_step(&db, restarted.id, "fanout", Some(0)).await; + assert_ne!( + new_iter0, orig_iter0, + "the edited fanout step must re-run, not be reused" + ); + + Ok(()) +} + +/// Only a flow reaped by the zombie monitor (canceled_by = 'monitor') is eligible for reuse. A +/// plain force-cancel at the same boundary (a child succeeded, its parent transition not yet +/// landed) yields the identical InProgress/all-success shape but must keep restart-from-step +/// semantics: the selected step re-runs. +#[sqlx::test(fixtures("base", "hello"))] +async fn test_non_monitor_cancel_is_not_reused(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let flow_value: FlowValue = serde_json::from_value(json!({ + "modules": [ + { + "id": "fanout", + "value": { + "type": "forloopflow", + "iterator": { "type": "javascript", "expr": "['a', 'b']" }, + "skip_failures": false, + "parallel": false, + "modules": [{ + "id": "inner", + "value": { + "type": "rawscript", "language": "deno", + "input_transforms": { + "v": { "type": "javascript", "expr": "flow_input.iter.value" } + }, + "content": "export function main(v: string) { return v }" + } + }] + } + }, + { + "id": "after", + "value": { + "type": "rawscript", "language": "deno", + "input_transforms": { + "loop_res": { "type": "javascript", "expr": "results.fanout" } + }, + "content": "export function main(loop_res: string[]) { return loop_res.join(',') }" + } + } + ] + })) + .unwrap(); + + let full_run = RunJob::from(JobPayload::RawFlow { + value: flow_value.clone(), + path: None, + restarted_from: None, + }) + .run_until_complete(&db, false, port) + .await; + assert!(full_run.success); + let orig_iter0 = child_job_id_for_step(&db, full_run.id, "fanout", Some(0)).await; + + // Same frozen-transition shape as a zombie, but canceled by a USER, not the monitor. + let mut flow_status: serde_json::Value = sqlx::query_scalar!( + "SELECT flow_status FROM v2_job_completed WHERE id = $1", + full_run.id + ) + .fetch_one(&db) + .await? + .expect("flow_status"); + for m in flow_status["modules"].as_array_mut().unwrap() { + match m["id"].as_str() { + Some("fanout") => { + m["type"] = json!("InProgress"); + m["iterator"] = json!({ "index": 1, "itered_len": 2 }); + } + Some("after") => *m = json!({ "type": "WaitingForPriorSteps", "id": "after" }), + _ => {} + } + } + flow_status["step"] = json!(0); + sqlx::query!( + "UPDATE v2_job_completed SET status = 'canceled', canceled_by = 'admin', + flow_status = $2 WHERE id = $1", + full_run.id, + flow_status, + ) + .execute(&db) + .await?; + + let restarted = RunJob::from(JobPayload::RestartedFlow { + completed_job_id: full_run.id, + step_id: "fanout".into(), + branch_or_iteration_n: None, + flow_version: None, + branch_chosen: None, + nested: None, + }) + .run_until_complete(&db, false, port) + .await; + + assert!(restarted.success); + let new_iter0 = child_job_id_for_step(&db, restarted.id, "fanout", Some(0)).await; + assert_ne!( + new_iter0, orig_iter0, + "a non-monitor cancel must re-run the step, not reuse the child" + ); + + Ok(()) +} diff --git a/backend/windmill-ai/src/utils.rs b/backend/windmill-ai/src/utils.rs index 66b68e9b6f..328d2e0458 100644 --- a/backend/windmill-ai/src/utils.rs +++ b/backend/windmill-ai/src/utils.rs @@ -18,20 +18,19 @@ lazy_static::lazy_static! { /// use it instead of the shared `HTTP_CLIENT`. Redirects are governed by /// `ALLOW_AI_BASE_URL_REDIRECTS` (disabled by default). Mirrors the API proxy /// client (windmill-api/src/ai.rs). + /// + /// This pooled client does no DNS pinning: callers reaching a user-controlled + /// base_url must go through [`pinned_ai_client_for`] so the connect targets + /// the SSRF-validated address (DNS-rebinding TOCTOU). It is the safe default + /// only for trusted/fixed hosts. pub static ref AI_HTTP_CLIENT: reqwest::Client = { - let redirect = if *ALLOW_AI_BASE_URL_REDIRECTS { + if *ALLOW_AI_BASE_URL_REDIRECTS { tracing::warn!( "ALLOW_AI_BASE_URL_REDIRECTS is enabled - the AI HTTP client will follow \ redirects, weakening SSRF protection on provider base URLs" ); - reqwest::redirect::Policy::default() - } else { - reqwest::redirect::Policy::none() - }; - configure_client(reqwest::ClientBuilder::new() - .user_agent("windmill/beta") - .connect_timeout(std::time::Duration::from_secs(10)) - .redirect(redirect)) + } + ai_http_client_builder() .build() .expect("Failed to build AI HTTP client - check system TLS configuration") }; @@ -64,6 +63,67 @@ lazy_static::lazy_static! { }; } +/// Shared configuration for every client that targets a user-configured AI +/// provider `base_url` (the pooled [`AI_HTTP_CLIENT`] and per-request DNS-pinned +/// clients from [`pinned_ai_client_for`]). Redirects are disabled by default +/// because the SSRF check on base_url is single-shot; DNS pinning likewise only +/// covers the original host, so a redirect could bounce a validated public host +/// into a private/internal one (see `ALLOW_AI_BASE_URL_REDIRECTS`). +pub fn ai_http_client_builder() -> reqwest::ClientBuilder { + let redirect = if *ALLOW_AI_BASE_URL_REDIRECTS { + reqwest::redirect::Policy::default() + } else { + reqwest::redirect::Policy::none() + }; + configure_client( + reqwest::ClientBuilder::new() + .user_agent("windmill/beta") + .connect_timeout(std::time::Duration::from_secs(10)) + .redirect(redirect), + ) +} + +/// Build the client for a single outbound AI request to `url`, pinning DNS to +/// the SSRF-validated address so the connect cannot rebind to an internal IP +/// after the check (DNS-rebinding TOCTOU). +/// +/// Returns the shared pooled [`AI_HTTP_CLIENT`] unchanged when there is nothing +/// to pin — an IP-literal host, or a deployment that opted into private AI +/// endpoints via `ALLOW_PRIVATE_AI_BASE_URLS`. The same opt-out and error hint +/// as `AIProvider::get_base_url` apply, so this is consistent with the +/// credential-time validation while additionally closing the connect-time window. +pub async fn pinned_ai_client_for( + url: &str, +) -> windmill_common::error::Result> { + use std::borrow::Cow; + use windmill_common::error::{to_anyhow, Error}; + use windmill_common::ssrf::SsrfValidationError; + + if *crate::ai_providers::ALLOW_PRIVATE_AI_BASE_URLS { + return Ok(Cow::Borrowed(&AI_HTTP_CLIENT)); + } + + let target = windmill_common::ssrf::validate_url_for_ssrf(url) + .await + .map_err(|e| match e { + e @ SsrfValidationError::Private { .. } => Error::BadRequest(format!( + "{e}. If you need to use private/internal AI endpoints, \ + set the ALLOW_PRIVATE_AI_BASE_URLS=true environment variable" + )), + e => Error::from(e), + })?; + + if target.pinned_addrs().is_empty() { + return Ok(Cow::Borrowed(&AI_HTTP_CLIENT)); + } + + let client = target + .apply_dns_pinning(ai_http_client_builder()) + .build() + .map_err(to_anyhow)?; + Ok(Cow::Owned(client)) +} + /// AWS Bedrock do not handle structured output query param, so we use a tool for structured output. Same for every Claude models. pub fn should_use_structured_output_tool(provider: &AIProvider, model: &str) -> bool { model.contains("claude") || provider == &AIProvider::AWSBedrock diff --git a/backend/windmill-api-auth/src/lib.rs b/backend/windmill-api-auth/src/lib.rs index 961f88d547..8120c16605 100644 --- a/backend/windmill-api-auth/src/lib.rs +++ b/backend/windmill-api-auth/src/lib.rs @@ -275,6 +275,15 @@ fn scope_restrictions(scopes: Option<&[String]>) -> Option> { (!restrictions.is_empty()).then_some(restrictions) } +/// True when the token carries no real scope restriction — unscoped, an empty scope +/// list, or only `if_jobs:filter_tags:` filters — so it holds the full privileges of +/// its user and can reach any non-job route they are authorized for (mirrors +/// `check_scopes` / `check_route_access`). A `false` result means the token is +/// genuinely scope-restricted. +pub fn is_effectively_unscoped(scopes: Option<&[String]>) -> bool { + scope_restrictions(scopes).is_none() +} + /// Enforce monotonic privilege when a token lifecycle endpoint mints or rescopes /// a credential on behalf of `authed`: the resulting credential must never be /// more privileged than the caller's own token. diff --git a/backend/windmill-api-embeddings/src/lib.rs b/backend/windmill-api-embeddings/src/lib.rs index 016e72be91..35816dfaa8 100644 --- a/backend/windmill-api-embeddings/src/lib.rs +++ b/backend/windmill-api-embeddings/src/lib.rs @@ -56,6 +56,10 @@ lazy_static::lazy_static! { pub static ref EMBEDDINGS_DB: Arc>> = Arc::new(RwLock::new(None)); pub static ref MODEL_INSTANCE: Arc>>> = Arc::new(RwLock::new(None)); pub static ref HUB_EMBEDDINGS_PULLING_INTERVAL_SECS: u64 = std::env::var("HUB_EMBEDDINGS_PULLING_INTERVAL_SECS").ok().map(|x| x.parse::().ok()).flatten().unwrap_or(3600 * 24); + // On a failed init/refresh we retry after this short interval instead of the + // full pulling interval, so a transient startup error doesn't leave the + // embeddings DB uninitialized for a whole day. + pub static ref HUB_EMBEDDINGS_RETRY_INTERVAL_SECS: u64 = std::env::var("HUB_EMBEDDINGS_RETRY_INTERVAL_SECS").ok().map(|x| x.parse::().ok()).flatten().unwrap_or(60); } #[cfg(feature = "embedding")] @@ -112,6 +116,26 @@ pub struct ResourceTypeResult { score: f32, schema: Option, } + +/// Drop results whose score falls more than `max_relative_drop` below the best +/// match, so a strong hit isn't diluted by weakly-related entries that merely +/// clear the similarity floor. Expects `results` sorted by descending score and +/// a positive top score (guaranteed by the caller's similarity threshold). +#[cfg(feature = "embedding")] +fn trim_to_top_score( + results: Vec, + max_relative_drop: f32, + score: impl Fn(&T) -> f32, +) -> Vec { + if results.len() <= 1 { + return results; + } + let top_score = score(&results[0]); + results + .into_iter() + .take_while(|r| (top_score - score(r)) / top_score <= max_relative_drop) + .collect() +} #[cfg(feature = "embedding")] async fn query_resource_types( Query(query): Query, @@ -511,15 +535,7 @@ impl EmbeddingsDb { }) .collect(); - let mut results = results?; - - if results.len() > 1 { - let top_score = results[0].score; - results = results - .into_iter() - .take_while(|r| (top_score - r.score) / top_score <= 0.05) - .collect(); - } + let results = trim_to_top_score(results?, 0.05, |r| r.score); Ok(results) } @@ -560,7 +576,7 @@ impl EmbeddingsDb { Some(0.75), ); - let results: Result<_> = results + let results: Result> = results .iter() .map(|r| { let metadata = r @@ -582,7 +598,9 @@ impl EmbeddingsDb { }) .collect(); - results + let results = trim_to_top_score(results?, 0.05, |r| r.score); + + Ok(results) } } @@ -601,42 +619,67 @@ pub fn load_embeddings_db(db: &Pool) -> () { if !disable_embedding { let db_clone = db.clone(); tokio::spawn(async move { - let model_instance = ModelInstance::new().await; - if let Ok(model_instance) = model_instance { - let mut model_instance_lock = MODEL_INSTANCE.write().await; - *model_instance_lock = Some(Arc::new(model_instance)); - drop(model_instance_lock); - loop { - update_embeddings_db(&db_clone).await; - tokio::time::sleep(std::time::Duration::from_secs( - *HUB_EMBEDDINGS_PULLING_INTERVAL_SECS, - )) - .await; + // Keep retrying model init: a transient failure here must not + // permanently disable embeddings until the next process restart. + // Backoff decays to the pulling interval so an environment where it + // can never succeed (e.g. air-gapped, embeddings left enabled) + // settles into ~1 attempt/interval rather than a tight error loop. + let mut backoff_secs = *HUB_EMBEDDINGS_RETRY_INTERVAL_SECS; + loop { + match ModelInstance::new().await { + Ok(model_instance) => { + let mut model_instance_lock = MODEL_INSTANCE.write().await; + *model_instance_lock = Some(Arc::new(model_instance)); + break; + } + Err(e) => { + tracing::error!( + "Failed to initialize model instance: {}. Retrying in {}s...", + e, + backoff_secs + ); + tokio::time::sleep(std::time::Duration::from_secs(backoff_secs)).await; + backoff_secs = backoff_secs + .saturating_mul(2) + .min(*HUB_EMBEDDINGS_PULLING_INTERVAL_SECS); + } } - } else { - tracing::error!( - "Failed to initialize model instance: {}", - model_instance.err().unwrap() - ); + } + let mut backoff_secs = *HUB_EMBEDDINGS_RETRY_INTERVAL_SECS; + loop { + let sleep_secs = if update_embeddings_db(&db_clone).await { + backoff_secs = *HUB_EMBEDDINGS_RETRY_INTERVAL_SECS; + *HUB_EMBEDDINGS_PULLING_INTERVAL_SECS + } else { + let secs = backoff_secs; + backoff_secs = backoff_secs + .saturating_mul(2) + .min(*HUB_EMBEDDINGS_PULLING_INTERVAL_SECS); + secs + }; + tokio::time::sleep(std::time::Duration::from_secs(sleep_secs)).await; } }); } } #[cfg(feature = "embedding")] -pub async fn update_embeddings_db(db: &Pool) -> () { +pub async fn update_embeddings_db(db: &Pool) -> bool { if let Some(model_instance) = MODEL_INSTANCE.read().await.as_ref() { tracing::info!("Creating embeddings DB..."); let new_embeddings_db = EmbeddingsDb::new(&db, model_instance.clone()).await; if let Err(e) = new_embeddings_db.as_ref() { tracing::error!("Failed to create embeddings db: {}", e); + false } else { let mut embeddings_db = EMBEDDINGS_DB.write().await; *embeddings_db = new_embeddings_db.ok(); tracing::info!("Created embeddings DB"); + true } } else { tracing::error!("Could not update embeddings DB, model instance not initialized"); + false } } @@ -659,3 +702,31 @@ pub fn workspaced_service() -> Router { pub fn global_service() -> Router { Router::new() } + +#[cfg(all(test, feature = "embedding"))] +mod tests { + use super::trim_to_top_score; + + #[test] + fn trims_scores_more_than_5pct_below_top() { + // top=1.0, cutoff at 0.95: 0.96 stays (0.04 drop), 0.93 is the first + // beyond the cutoff so take_while stops there and drops the tail. + let kept = trim_to_top_score(vec![1.0f32, 0.97, 0.96, 0.93, 0.9], 0.05, |s| *s); + assert_eq!(kept, vec![1.0, 0.97, 0.96]); + } + + #[test] + fn keeps_all_when_tightly_clustered() { + let kept = trim_to_top_score(vec![0.9f32, 0.89, 0.88], 0.05, |s| *s); + assert_eq!(kept, vec![0.9, 0.89, 0.88]); + } + + #[test] + fn passes_through_zero_or_one_result() { + assert_eq!( + trim_to_top_score(Vec::::new(), 0.05, |s| *s), + Vec::::new() + ); + assert_eq!(trim_to_top_score(vec![0.42f32], 0.05, |s| *s), vec![0.42]); + } +} diff --git a/backend/windmill-api-flows/src/flows.rs b/backend/windmill-api-flows/src/flows.rs index ff555113c3..97e6999e23 100644 --- a/backend/windmill-api-flows/src/flows.rs +++ b/backend/windmill-api-flows/src/flows.rs @@ -101,11 +101,7 @@ async fn list_search_flows( Path(w_id): Path, Extension(user_db): Extension, ) -> JsonResult> { - #[cfg(feature = "enterprise")] let n = 1000; - - #[cfg(not(feature = "enterprise"))] - let n = 3; let mut tx = user_db.begin(&authed).await?; let allowed = build_scope_path_predicate(&authed, "flows", "read"); @@ -531,6 +527,11 @@ async fn create_flow( } check_scopes(&authed, || format!("flows:write:{}", nf.path))?; + // A `<= 0` flow timeout is "unset", not a 0-second limit that kills every run instantly. + // (The concurrency settings inside the flow value are normalized on deserialization; see + // ConcurrencySettings.) Runtime guards also protect already-stored rows. + nf.timeout = windmill_common::runnable_settings::none_if_non_positive(nf.timeout); + if let RuleCheckResult::Blocked(msg) = check_deploy_rules( &w_id, AuditAuthorable::username(&authed), @@ -1016,7 +1017,9 @@ async fn update_flow( } let flow_path = flow_path.to_path(); // The URL identifies the flow being updated; the body path is only needed to rename. - let nf = ef.into_new_flow(flow_path); + let mut nf = ef.into_new_flow(flow_path); + // A `<= 0` flow timeout is "unset", not a 0-second limit (see create_flow). + nf.timeout = windmill_common::runnable_settings::none_if_non_positive(nf.timeout); check_scopes(&authed, || format!("flows:write:{}", flow_path))?; if let RuleCheckResult::Blocked(msg) = check_deploy_rules( diff --git a/backend/windmill-api-groups/src/folders.rs b/backend/windmill-api-groups/src/folders.rs index 5e204a6f2c..3551435dc7 100644 --- a/backend/windmill-api-groups/src/folders.rs +++ b/backend/windmill-api-groups/src/folders.rs @@ -243,6 +243,7 @@ async fn create_folder( Path(w_id): Path, Json(mut ng): Json, ) -> Result { + crate::check_demo_workspace_restriction(&authed, &w_id, "Folder creation")?; if let Some(labels) = ng.labels.as_mut() { dedup_labels(labels); } @@ -418,6 +419,12 @@ async fn update_folder( return Err(Error::PermissionDenied(msg)); } + // update_folder can also grant permissions (owners / extra_perms / default_permissioned_as), + // so it is a sharing path and must honor the demo-workspace sharing restriction. + if ng.owners.is_some() || ng.extra_perms.is_some() || ng.default_permissioned_as.is_some() { + crate::check_demo_workspace_restriction(&authed, &w_id, "Sharing")?; + } + let mut sqlb = SqlBuilder::update_table("folder"); sqlb.and_where_eq("name", "?".bind(&name)); sqlb.and_where_eq("workspace_id", "?".bind(&w_id)); @@ -820,6 +827,7 @@ async fn add_owner( Path((w_id, name)): Path<(String, String)>, Json(Owner { owner, .. }): Json, ) -> Result { + crate::check_demo_workspace_restriction(&authed, &w_id, "Sharing")?; let mut tx = user_db.begin(&authed).await?; not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?; @@ -884,6 +892,13 @@ async fn remove_owner( Path((w_id, name)): Path<(String, String)>, Json(Owner { owner, write }): Json, ) -> Result { + // remove_owner with a `write` value is a grant path: it jsonb_set's the owner's + // permission level into extra_perms (only write=None is a pure revoke), so the + // demo-workspace sharing restriction must apply when a level is being set. + if write.is_some() { + crate::check_demo_workspace_restriction(&authed, &w_id, "Sharing")?; + } + let mut tx = user_db.begin(&authed).await?; not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?; diff --git a/backend/windmill-api-groups/src/granular_acls.rs b/backend/windmill-api-groups/src/granular_acls.rs index e413165825..5a7af05cbe 100644 --- a/backend/windmill-api-groups/src/granular_acls.rs +++ b/backend/windmill-api-groups/src/granular_acls.rs @@ -96,6 +96,7 @@ async fn add_granular_acl( Path((w_id, path)): Path<(String, StripPath)>, Json(GranularAcl { owner, write }): Json, ) -> Result { + crate::check_demo_workspace_restriction(&authed, &w_id, "Sharing")?; let path = path.to_path(); let (kind, path) = path diff --git a/backend/windmill-api-groups/src/groups.rs b/backend/windmill-api-groups/src/groups.rs index 263daad624..d2a6c7d16e 100644 --- a/backend/windmill-api-groups/src/groups.rs +++ b/backend/windmill-api-groups/src/groups.rs @@ -234,6 +234,7 @@ async fn create_group( Path(w_id): Path, Json(ng): Json, ) -> Result { + crate::check_demo_workspace_restriction(&authed, &w_id, "Group creation")?; let mut tx = user_db.begin(&authed).await?; check_name_conflict(&mut tx, &w_id, &ng.name).await?; diff --git a/backend/windmill-api-groups/src/lib.rs b/backend/windmill-api-groups/src/lib.rs index 75b9349105..7abfa426ee 100644 --- a/backend/windmill-api-groups/src/lib.rs +++ b/backend/windmill-api-groups/src/lib.rs @@ -2,3 +2,23 @@ pub mod folder_history; pub mod folders; pub mod granular_acls; pub mod groups; + +use windmill_api_auth::ApiAuthed; +use windmill_common::{error::Error, worker::CLOUD_HOSTED}; + +/// The public demo workspace on the managed cloud is kept clean and consistent by +/// restricting folder creation, item sharing, and group creation for non-admins. +/// `action` is a short noun phrase completing "… is disabled …" (e.g. +/// "Folder creation", "Sharing"). Returns `Err(BadRequest)` when the caller is blocked. +pub fn check_demo_workspace_restriction( + authed: &ApiAuthed, + w_id: &str, + action: &str, +) -> Result<(), Error> { + if *CLOUD_HOSTED && w_id == "demo" && !authed.is_admin { + return Err(Error::BadRequest(format!( + "{action} is disabled in the demo workspace. Create your own workspace to keep the demo clean and consistent." + ))); + } + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/resources.rs b/backend/windmill-api-integration-tests/tests/resources.rs index cc78056176..3cc59be473 100644 --- a/backend/windmill-api-integration-tests/tests/resources.rs +++ b/backend/windmill-api-integration-tests/tests/resources.rs @@ -275,6 +275,19 @@ async fn test_resource_endpoints(db: Pool) -> anyhow::Result<()> { let body = resp.json::().await?; assert_eq!(body["description"], "Updated description"); + // --- update (resource_type) --- + // An update that only changes resource_type must persist it. + let resp = authed(client().post(resource_url(port, "update", "u/test-user/new_resource"))) + .json(&json!({"resource_type": "mcp_server"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + let resp = authed_get(port, "get", "u/test-user/new_resource").await; + let body = resp.json::().await?; + assert_eq!(body["resource_type"], "mcp_server"); + // --- update_value --- let resp = authed(client().post(resource_url( port, diff --git a/backend/windmill-api-integration-tests/tests/workspaces.rs b/backend/windmill-api-integration-tests/tests/workspaces.rs index c319a1b9ec..a681fd982e 100644 --- a/backend/windmill-api-integration-tests/tests/workspaces.rs +++ b/backend/windmill-api-integration-tests/tests/workspaces.rs @@ -901,6 +901,64 @@ async fn test_get_copilot_info_ignores_empty_instance_ai_row( Ok(()) } +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_error_handler_instance_alerts_fallback(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + + let stored = || async { + sqlx::query_scalar!( + "SELECT error_handler_fallback_to_instance_alerts FROM workspace_settings WHERE workspace_id = 'test-workspace'" + ) + .fetch_one(&db) + .await + }; + + let resp = authed(client().post(format!("{base}/edit_error_handler"))) + .json(&json!({"path": null, "fallback_to_instance_alerts": true})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "enable: {}", resp.text().await?); + assert!(stored().await?); + + // A client that predates the setting (the CLI pushing settings.yaml) omits the field and + // must not silently turn it back off. + let resp = authed(client().post(format!("{base}/edit_error_handler"))) + .json(&json!({"path": null, "extra_args": null})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "omitted: {}", resp.text().await?); + assert!(stored().await?); + + sqlx::query!( + "UPDATE workspace SET parent_workspace_id = 'test-workspace' WHERE id = 'test-workspace'" + ) + .execute(&db) + .await?; + let resp = authed(client().post(format!("{base}/edit_error_handler"))) + .json(&json!({"path": null, "fallback_to_instance_alerts": true})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400, "fork must be rejected"); + + // The settings page stops offering the option once the workspace is a fork, so its next save + // sends `false`: that must go through rather than lock the whole error handler behind a 400. + let resp = authed(client().post(format!("{base}/edit_error_handler"))) + .json(&json!({"path": null, "fallback_to_instance_alerts": false})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "disable on fork: {}", resp.text().await?); + assert!(!stored().await?); + + Ok(()) +} + #[sqlx::test(migrations = "../migrations", fixtures("base"))] async fn test_get_imports(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index 64f997ff06..c2f023ce5a 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -157,12 +157,8 @@ async fn list_search_scripts( Extension(user_db): Extension, ) -> JsonResult> { let mut tx = user_db.begin(&authed).await?; - #[cfg(feature = "enterprise")] let n = 10000; - #[cfg(not(feature = "enterprise"))] - let n = 10; - let allowed = build_scope_path_predicate(&authed, "scripts", "read"); let rows = sqlx::query_as!( SearchScript, @@ -918,6 +914,13 @@ async fn create_script_internal<'c>( } check_scopes(&authed, || format!("scripts:write:{}", ns.path))?; + // Normalize positive-only settings so a `<= 0` value (e.g. a CLI-pushed `0`) persists as + // disabled rather than as a zero-slot concurrency cap or a 0-second timeout. Deserialization + // already normalizes the concurrency fields; re-applying here also covers `timeout` and any + // NewScript built in-process rather than from a request body. + ns.timeout = windmill_common::runnable_settings::none_if_non_positive(ns.timeout); + ns.concurrency_settings = ns.concurrency_settings.normalized(); + guard_script_from_debounce_data(&ns).await?; let codebase = ns.codebase.as_ref(); diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index 268feedcc6..bfd0848bd4 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -32,11 +32,9 @@ use windmill_common::DB; use ee_oss::validate_license_key; use windmill_common::usernames::generate_instance_username_for_all_users; -#[cfg(feature = "enterprise")] -use axum::extract::Query; use axum::{ body::Body, - extract::{Extension, Path}, + extract::{Extension, Path, Query}, response::Response, routing::{get, post}, Json, Router, @@ -283,15 +281,15 @@ pub async fn test_s3_bucket( let mut list = client.list(Some( &windmill_object_store::object_store_reexports::Path::from("".to_string()), )); - let first_file = list.next().await; - if first_file.is_some() { - if let Err(e) = first_file.as_ref().unwrap() { + match list.next().await { + Some(Err(e)) => { tracing::error!("error listing bucket: {e:#}"); - error::Error::internal_err(format!("Failed to list files in blob storage: {e:#}")); + return Err(error::Error::internal_err(format!( + "Failed to list files in blob storage: {e:#}" + ))); } - tracing::info!("Listed files: {:?}", first_file.unwrap()); - } else { - tracing::info!("No files in blob storage"); + Some(Ok(first_file)) => tracing::info!("Listed files: {:?}", first_file), + None => tracing::info!("No files in blob storage"), } let path = windmill_object_store::object_store_reexports::Path::from(format!( @@ -1580,6 +1578,7 @@ async fn refresh_custom_instance_user_pwd( ) -> JsonResult<()> { require_super_admin(&db, &authed.email).await?; windmill_common::utils::refresh_custom_instance_user_pwd(&db).await?; + windmill_common::utils::refresh_custom_instance_replication_user_pwd(&db).await?; Ok(Json(())) } @@ -1710,11 +1709,23 @@ async fn setup_custom_instance_pg_database_inner( )) })?; + // The replication attribute lives on a dedicated role used by postgres trigger + // connections. The getter creates the role (with its stored password) when the + // migration couldn't. + if let Err(e) = windmill_common::utils::get_custom_pg_instance_replication_password(db).await { + tracing::error!("Failed to ensure custom_instance_replication_user exists: {e:#}"); + } if let Err(e) = client - .batch_execute(&format!("ALTER ROLE custom_instance_user REPLICATION;")) + .batch_execute( + "ALTER ROLE custom_instance_replication_user REPLICATION; + GRANT custom_instance_user TO custom_instance_replication_user; + ALTER ROLE custom_instance_user NOREPLICATION;", + ) .await { - tracing::error!("Failed to grant replication permission to custom_instance_user: {e:#}"); + tracing::error!( + "Failed to grant replication permission to custom_instance_replication_user: {e:#}" + ); } logs.grant_permissions = "OK".to_string(); @@ -1976,25 +1987,53 @@ async fn fetch_resource_types_from_hub() -> error::Result, +} + async fn sync_cached_resource_types( Extension(db): Extension, authed: ApiAuthed, + Query(SyncResourceTypesQuery { name }): Query, ) -> error::Result { require_super_admin(&db, &authed.email).await?; use windmill_common::worker::HUB_RT_CACHE_DIR; let cache_path = format!("{}/resource_types.json", *HUB_RT_CACHE_DIR); - let cached_types = match tokio::fs::read_to_string(&cache_path).await { - Ok(content) => serde_json::from_str::>(&content).map_err(|e| { - error::Error::InternalErr(format!("Failed to parse cached resource types: {}", e)) - })?, - Err(_) => fetch_resource_types_from_hub().await?, + // Manual sync is hub-first so it lands newly-published hub types on demand. The + // on-disk cache is only a fallback for when the hub is unreachable (airgapped + // installs / network error); refreshing it is left to the daily cache-rt cron and + // the startup sync in main.rs, which own the offline path. + let (resource_types, from_hub) = match fetch_resource_types_from_hub().await { + Ok(types) => { + tracing::info!("Fetched {} resource types live from the hub", types.len()); + (types, true) + } + Err(hub_err) => { + tracing::warn!( + "Live hub fetch failed ({hub_err}), falling back to on-disk cache at {cache_path}" + ); + match tokio::fs::read_to_string(&cache_path).await { + Ok(content) => { + let parsed = serde_json::from_str::>(&content) + .map_err(|e| { + error::Error::InternalErr(format!( + "Failed to parse cached resource types: {}", + e + )) + })?; + (parsed, false) + } + Err(_) => return Err(hub_err), + } + } }; let mut synced_count = 0; - for rt in &cached_types { + for rt in &resource_types { let exists: Option = sqlx::query_scalar!( "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3)", &rt.name, @@ -2023,10 +2062,27 @@ async fn sync_cached_resource_types( synced_count += 1; } + // If a specific type was requested and is still absent after syncing, surface an + // explicit not-found instead of a silent "Synced 0". Word it by source so the + // cache-fallback path does not claim it checked the hub. + if let Some(name) = name.as_deref() { + if !resource_types.iter().any(|rt| rt.name == name) { + let source = if from_hub { + "on the hub" + } else { + "in the cached resource types (hub unreachable)" + }; + return Err(error::Error::NotFound(format!( + "resource type '{}' not found {}", + name, source + ))); + } + } + Ok(format!( "Synced {} resource types ({} unchanged)", synced_count, - cached_types.len() - synced_count + resource_types.len() - synced_count )) } diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 0c0929a666..ec6b06c3eb 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -303,6 +303,7 @@ pub struct WorkspaceSettings { pub success_handler: Option, #[serde(skip_serializing_if = "Option::is_none")] pub public_app_execution_limit_per_minute: Option, + pub error_handler_fallback_to_instance_alerts: bool, } /// Subset of `WorkspaceSettings` that is safe to return to any workspace @@ -452,6 +453,8 @@ struct CreateWorkspace { name: String, username: Option, color: Option, + #[serde(default)] + error_handler_fallback_to_instance_alerts: bool, } #[derive(Deserialize)] @@ -559,6 +562,9 @@ pub struct EditErrorHandlerNew { pub muted_on_cancel: bool, #[serde(default)] pub muted_on_user_path: bool, + /// Left as `None` by clients that predate the setting (the CLI among them), which must + /// keep the stored value rather than silently reset it on every settings push. + pub fallback_to_instance_alerts: Option, } // Legacy format for error handler (flat fields from old CLI) @@ -587,6 +593,7 @@ impl EditErrorHandler { extra_args: legacy.error_handler_extra_args, muted_on_cancel: legacy.error_handler_muted_on_cancel, muted_on_user_path: false, // Old format doesn't have this field + fallback_to_instance_alerts: None, }, } } @@ -827,21 +834,37 @@ async fn reject_dev_label_matching_tracked_branch( Ok(()) } -/// Reject parent-only git-sync settings on a fork workspace. Auto-pull, fork -/// PRs, and promotion mode are all configured at the parent: repo → fork sync is -/// routed by the parent's webhook/poller (`sync_forks`), a fork-owned auto-pull -/// would register a second webhook on the same GitHub repo per fork, and a -/// fork's deploys always go to its `wm-fork/**` branch so a promotion repo could -/// never take effect there. +/// Reject parent-only git-sync settings on a fork workspace. Auto-pull and fork +/// PRs are configured at the parent: repo → fork sync is routed by the parent's +/// webhook/poller (`sync_forks`), and a fork-owned auto-pull would register a +/// second webhook on the same GitHub repo per fork. Promotion mode is rejected +/// on throwaway forks (their deploys always go to their `wm-fork/**` branch, so +/// a promotion repo could never take effect) but allowed on a **dev workspace**, +/// which deploys per-item `wm_deploy/**` branches that promote into the parent. async fn reject_parent_only_git_sync_settings_on_fork<'a>( db: &DB, w_id: &str, - mut repos: impl Iterator, + repos: impl Iterator, ) -> Result<()> { - let offending = repos.find_map(|r| { + let row = sqlx::query!( + "SELECT parent_workspace_id, is_dev_workspace FROM workspace WHERE id = $1", + w_id + ) + .fetch_optional(db) + .await?; + let is_fork = row + .as_ref() + .and_then(|r| r.parent_workspace_id.as_ref()) + .is_some() + || w_id.starts_with(windmill_common::workspaces::WM_FORK_PREFIX); + if !is_fork { + return Ok(()); + } + let is_dev = row.map(|r| r.is_dev_workspace).unwrap_or(false); + let offending = repos.into_iter().find_map(|r| { if r.auto_pull.as_ref().is_some_and(|a| a.enabled) { Some("Auto-pull") - } else if r.use_individual_branch.unwrap_or(false) { + } else if r.use_individual_branch.unwrap_or(false) && !is_dev { Some("Promotion mode") } else if r.fork_open_prs { Some("Opening PRs for fork deploys") @@ -849,17 +872,7 @@ async fn reject_parent_only_git_sync_settings_on_fork<'a>( None } }); - let Some(offending) = offending else { - return Ok(()); - }; - let parent = sqlx::query_scalar!( - "SELECT parent_workspace_id FROM workspace WHERE id = $1", - w_id - ) - .fetch_optional(db) - .await? - .flatten(); - if parent.is_some() || w_id.starts_with(windmill_common::workspaces::WM_FORK_PREFIX) { + if let Some(offending) = offending { return Err(Error::BadRequest(format!( "{offending} cannot be configured on a fork workspace: it is managed from the parent workspace's git sync settings" ))); @@ -968,7 +981,8 @@ async fn get_settings( auto_invite, error_handler, success_handler, - public_app_execution_limit_per_minute + public_app_execution_limit_per_minute, + error_handler_fallback_to_instance_alerts FROM workspace_settings WHERE @@ -3314,6 +3328,95 @@ async fn check_open_prs_license<'a>( Ok(()) } +/// Promotion mode (`use_individual_branch`: per-item `wm_deploy/**` deploy +/// branches) is an EE feature; runtime-gate it like auto-pull and PR creation +/// so an enterprise binary without an active plan can't enable it via either +/// git-sync edit endpoint. +#[cfg(feature = "enterprise")] +async fn check_promotion_license<'a>( + mut repos: impl Iterator, +) -> Result<()> { + if repos.any(|r| r.use_individual_branch.unwrap_or(false)) { + check_git_sync_ee_license("Promotion mode").await?; + } + Ok(()) +} + +/// Promotion on a dev workspace needs the dev-aware sync script (hub >= 28796): +/// an older pinned script bundles a CLI that force-disables per-item branches +/// on every fork, so enabling promotion would silently keep deploying to the +/// env-label branch. Reject with an actionable error instead (the dispatcher +/// demotes inherited configs the same way). Roots run promotion on any script +/// version, and auto-managed repositories (no pin) always use the latest. +#[cfg(feature = "enterprise")] +async fn check_dev_promotion_script_version<'a>( + db: &DB, + w_id: &str, + repos: impl Iterator, +) -> Result<()> { + let mut offending: Option = None; + for r in repos { + if !r.use_individual_branch.unwrap_or(false) { + continue; + } + if !r.is_script_meets_min_version(28796)? { + offending = Some(r.effective_script_path().to_string()); + break; + } + } + let Some(offending) = offending else { + return Ok(()); + }; + let is_dev = sqlx::query!( + "SELECT parent_workspace_id, is_dev_workspace FROM workspace WHERE id = $1", + w_id + ) + .fetch_optional(db) + .await? + .map(|r| r.is_dev_workspace) + .unwrap_or(false); + if !is_dev { + return Ok(()); + } + Err(Error::BadRequest(format!( + "Promotion mode on a dev workspace requires git sync script version 28796 or newer, \ + but this repository pins '{offending}'. Update the pinned sync script (or reset it to \ + auto-managed) first." + ))) +} + +/// A dev workspace's promotion must target its parent ("prod") workspace's own +/// git repository (same URL and branch) — that is what "promote to prod" means. +/// A fork-created dev inherits prod's repo; an **attached** dev keeps its own, +/// which may be unrelated. Reject enabling promotion on a repo the parent does +/// not track so the UI can't present an unrelated repo as prod's target. The +/// deploy path re-checks the same invariant (a resource edit could break it +/// after save), via the shared `dev_promotion_target_matches_parent`. +#[cfg(all(feature = "enterprise", feature = "private"))] +async fn check_dev_promotion_targets_parent_repo<'a>( + db: &DB, + w_id: &str, + repos: impl Iterator, +) -> Result<()> { + for r in repos.filter(|r| r.use_individual_branch.unwrap_or(false)) { + if !windmill_common::git_sync_ee::dev_promotion_target_matches_parent( + db, + w_id, + &r.git_repo_resource_path, + ) + .await? + { + return Err(Error::BadRequest( + "Promotion mode on a dev workspace must reuse the parent workspace's git repository \ + (same URL and branch), but this repository is not one the parent tracks — promotion \ + would target a repository the parent does not sync with." + .to_string(), + )); + } + } + Ok(()) +} + #[cfg(feature = "enterprise")] async fn check_git_sync_access(_db: &DB, _w_id: &str) -> Result<()> { Ok(()) @@ -3458,6 +3561,25 @@ async fn edit_git_sync_config( } #[cfg(feature = "enterprise")] check_open_prs_license(git_sync_settings.repositories.iter()).await?; + #[cfg(feature = "enterprise")] + check_promotion_license(git_sync_settings.repositories.iter()).await?; + #[cfg(feature = "enterprise")] + check_dev_promotion_script_version(&db, &w_id, git_sync_settings.repositories.iter()) + .await?; + #[cfg(all(feature = "enterprise", feature = "private"))] + check_dev_promotion_targets_parent_repo(&db, &w_id, git_sync_settings.repositories.iter()) + .await?; + // Promotion mode: EE only (mirrors edit_git_sync_repository). + #[cfg(not(feature = "enterprise"))] + if git_sync_settings + .repositories + .iter() + .any(|r| r.use_individual_branch.unwrap_or(false)) + { + return Err(Error::BadRequest( + "Promotion mode is an Enterprise Edition feature".to_string(), + )); + } // Preserve server-owned auto-pull state (webhook id/secret, synced sha, last // status) that the redacted GET response omits — otherwise a whole-config // save from the UI would drop the webhook secret (breaking delivery) or @@ -3593,7 +3715,8 @@ async fn edit_git_sync_config( tracing::warn!("git auto-pull: webhook field persist error: {}", e); } for (path, hook_id) in removed_webhooks { - if let Ok(url) = windmill_common::git_sync_ee::resolve_repo_url(&db, &w_id, &path).await + if let Ok(url) = + windmill_common::git_sync_ee::resolve_repo_url_interpolated(&db, &w_id, &path).await { let _ = windmill_common::git_sync_ee::delete_repo_webhook(&db, &w_id, &url, hook_id) @@ -3670,6 +3793,13 @@ async fn edit_git_sync_repository( } #[cfg(feature = "enterprise")] check_open_prs_license(std::iter::once(&new_config.repository)).await?; + #[cfg(feature = "enterprise")] + check_promotion_license(std::iter::once(&new_config.repository)).await?; + #[cfg(feature = "enterprise")] + check_dev_promotion_script_version(&db, &w_id, std::iter::once(&new_config.repository)).await?; + #[cfg(all(feature = "enterprise", feature = "private"))] + check_dev_promotion_targets_parent_repo(&db, &w_id, std::iter::once(&new_config.repository)) + .await?; // Promotion mode: EE only #[cfg(not(feature = "enterprise"))] @@ -3945,7 +4075,7 @@ async fn delete_git_sync_repository( // Removal is durable now — best-effort delete the GitHub webhook. #[cfg(all(feature = "enterprise", feature = "private"))] if let Some(hook_id) = webhook_to_delete { - if let Ok(url) = windmill_common::git_sync_ee::resolve_repo_url( + if let Ok(url) = windmill_common::git_sync_ee::resolve_repo_url_interpolated( &db, &w_id, &request.git_repo_resource_path, @@ -4244,6 +4374,19 @@ async fn edit_error_handler( let mut tx = db.begin().await?; + if let Some(fallback_to_instance_alerts) = ee.fallback_to_instance_alerts { + if fallback_to_instance_alerts { + ensure_instance_alert_fallback_allowed(&mut tx, &w_id).await?; + } + sqlx::query!( + "UPDATE workspace_settings SET error_handler_fallback_to_instance_alerts = $1 WHERE workspace_id = $2", + fallback_to_instance_alerts, + &w_id + ) + .execute(&mut *tx) + .await?; + } + sqlx::query_as!( Group, "INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING", @@ -4322,7 +4465,16 @@ async fn edit_error_handler( ActionKind::Update, &w_id, Some(&authed.email), - Some([("error_handler", &format!("{:?}", ee.path)[..])].into()), + Some( + [ + ("error_handler", &format!("{:?}", ee.path)[..]), + ( + "fallback_to_instance_alerts", + &format!("{:?}", ee.fallback_to_instance_alerts)[..], + ), + ] + .into(), + ), ) .await?; tx.commit().await?; @@ -4799,6 +4951,36 @@ async fn session_workspace_status( Ok(Json(statuses)) } +/// The instance critical alert channels belong to the instance operator, who on cloud is +/// not the workspace owner and never opted into a tenant's job failures. Fork workspaces run +/// throwaway copies of their parent's runnables, so instance-wide operational alerting must +/// stay a property of the real workspace. +async fn ensure_instance_alert_fallback_allowed<'c>( + tx: &mut Transaction<'c, Postgres>, + w_id: &str, +) -> Result<()> { + if *CLOUD_HOSTED { + return Err(Error::BadRequest( + "Reporting to the instance critical alert channels is not available on cloud" + .to_string(), + )); + } + let is_fork = sqlx::query_scalar!( + r#"SELECT (parent_workspace_id IS NOT NULL) AS "is_fork!" FROM workspace WHERE id = $1"#, + w_id + ) + .fetch_optional(&mut **tx) + .await? + .unwrap_or(false); + if is_fork { + return Err(Error::BadRequest( + "Reporting to the instance critical alert channels cannot be enabled on a fork workspace" + .to_string(), + )); + } + Ok(()) +} + pub async fn check_w_id_conflict<'c>(tx: &mut Transaction<'c, Postgres>, w_id: &str) -> Result<()> { if w_id == "global" { return Err(windmill_common::error::Error::BadRequest( @@ -4975,12 +5157,16 @@ async fn create_workspace( ) .execute(&mut *tx) .await?; + if nw.error_handler_fallback_to_instance_alerts { + ensure_instance_alert_fallback_allowed(&mut tx, &nw.id).await?; + } sqlx::query!( "INSERT INTO workspace_settings - (workspace_id, color) - VALUES ($1, $2)", + (workspace_id, color, error_handler_fallback_to_instance_alerts) + VALUES ($1, $2, $3)", nw.id, nw.color, + nw.error_handler_fallback_to_instance_alerts, ) .execute(&mut *tx) .await?; @@ -6557,7 +6743,7 @@ async fn enforce_fork_depth( /// True if `raw` (the text form of a `json` value) contains a genuine `\u0000` /// NUL escape: a `u0000` preceded by an ODD run of backslashes. Mirrors the -/// parity rule in `strip_null_chars` (windmill-api `apps.rs`) — an even run +/// parity rule in `windmill_common::utils::strip_json_nul` — an even run /// (`\\u0000`) is an escaped backslash then the literal text "u0000" (common in /// minified JS) and is jsonb-safe. A genuine NUL is exactly what the /// `json`→`jsonb` re-encode in `clone_apps` / `clone_flows` rejects with @@ -7097,8 +7283,12 @@ async fn attach_dev_workspace( ) .execute(&mut *tx) .await?; + // Clearing the instance-alert opt-in here keeps the stored setting truthful for a workspace + // that becomes parent-managed: dispatch enforces the fork boundary on its own, but a lingering + // `true` would survive a later detach and would make the settings page submit a value the API + // rejects on a fork. sqlx::query!( - "UPDATE workspace_settings SET deploy_to = $1 WHERE workspace_id = $2", + "UPDATE workspace_settings SET deploy_to = $1, error_handler_fallback_to_instance_alerts = false WHERE workspace_id = $2", &prod_w_id, &dev_w_id ) @@ -7173,7 +7363,8 @@ async fn attach_dev_workspace( // (their auto_pull is gone), so remove them from GitHub. #[cfg(all(feature = "enterprise", feature = "private"))] for (path, hook_id) in stripped_webhooks { - if let Ok(url) = windmill_common::git_sync_ee::resolve_repo_url(&db, &dev_w_id, &path).await + if let Ok(url) = + windmill_common::git_sync_ee::resolve_repo_url_interpolated(&db, &dev_w_id, &path).await { let _ = windmill_common::git_sync_ee::delete_repo_webhook(&db, &dev_w_id, &url, hook_id) @@ -7210,7 +7401,8 @@ async fn attach_dev_workspace( } /// Reverse [`attach_dev_workspace`] / clear the dev designation: unset the dev flag and remove the -/// prod lock. The workspace keeps its `parent_workspace_id` (it remains an ordinary fork). +/// prod lock. Whether `parent_workspace_id` is kept depends on the workspace's origin (see the +/// UPDATE below): a genuine fork stays a fork, a standalone workspace returns to standalone. async fn detach_dev_workspace( authed: ApiAuthed, Extension(db): Extension, @@ -10109,6 +10301,8 @@ const FEATURE_USAGE_KINDS: &[(&str, &str)] = &[ ("ai_session", "deployed"), ("ai_session", "archived"), ("ai_session", "deleted"), + ("ai_session", "beta_optout"), + ("ai_session", "beta_optin"), ("ai_chat", "message"), ("ai_chat", "model"), ("ai_chat", "tool"), diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index 0f8fd04075..2a120d08f1 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -108,7 +108,7 @@ pub(crate) async fn change_workspace_id( // Duplicate workspace settings (keep copy in old workspace for reference) info!("Duplicating workspace_settings table"); sqlx::query!( - "INSERT INTO workspace_settings SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, deploy_to, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler FROM workspace_settings WHERE workspace_id = $2", + "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, deploy_to, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, deploy_to, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts FROM workspace_settings WHERE workspace_id = $2", &rw.new_id, &old_id ) @@ -805,7 +805,8 @@ pub(crate) async fn change_workspace_id( #[cfg(all(feature = "enterprise", feature = "private"))] for (path, hook_id) in stale_webhooks { if let Ok(url) = - windmill_common::git_sync_ee::resolve_repo_url(&db, &rw.new_id, &path).await + windmill_common::git_sync_ee::resolve_repo_url_interpolated(&db, &rw.new_id, &path) + .await { let _ = windmill_common::git_sync_ee::delete_repo_webhook(&db, &rw.new_id, &url, hook_id) diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index b764e6ff83..ee789945f5 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -26,7 +26,11 @@ kafka = ["dep:windmill-trigger-kafka", "windmill-store/kafka"] kafka-gssapi = ["kafka", "windmill-trigger-kafka/kafka-gssapi"] nats = ["dep:windmill-trigger-nats", "windmill-store/nats"] websocket = ["dep:windmill-trigger-websocket"] -smtp = ["dep:mail-parser", "dep:openssl", "windmill-common/smtp", "dep:windmill-trigger-email"] +smtp = ["instance_smtp", "dep:mail-parser", "dep:openssl", "dep:windmill-trigger-email"] +# Outbound instance-SMTP email (the send_email_with_instance_smtp endpoint and +# critical alerts) without the inbound email trigger's openssl/mail-parser deps. +# `smtp` is the full trigger + endpoint; `instance_smtp` is the endpoint only. +instance_smtp = ["windmill-common/smtp"] license = ["dep:rsa", "windmill-api-settings/license"] zip = ["dep:async_zip"] oauth2 = ["dep:windmill-oauth", "windmill-store/oauth2"] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 69446ef84f..890b85d596 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.764.0 + version: 1.769.0 title: Windmill API contact: @@ -1330,7 +1330,7 @@ paths: /settings/refresh_custom_instance_user_pwd: post: - summary: Refreshes the password for the custom_instance_user + summary: Refreshes the passwords for the custom_instance_user and the custom_instance_replication_user (used by postgres triggers) operationId: refreshCustomInstanceUserPwd tags: - setting @@ -3617,6 +3617,9 @@ paths: public_app_execution_limit_per_minute: type: integer description: Rate limit for public app executions per minute per server. NULL or 0 means disabled. + error_handler_fallback_to_instance_alerts: + type: boolean + description: Report failed jobs to the instance critical alert channels when no workspace error handler is set. /w/{workspace}/workspaces/get_deploy_to: get: @@ -22963,6 +22966,472 @@ paths: schema: type: string + /w/{workspace}/hub/publish_draft: + post: + summary: create or update a hub project draft + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubDraft + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublishDraftBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/scripts: + post: + summary: publish a script to a hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubScript + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublishScriptBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/flows: + post: + summary: publish a flow to a hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubFlow + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublishFlowBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/apps: + post: + summary: publish an app to a hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubApp + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublishAppBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/raw_apps: + post: + summary: publish a raw app to a hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubRawApp + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublishRawAppBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/raw_apps/{id}/embed: + post: + summary: set or clear the embed url of a hub raw app + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubRawAppEmbed + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: id + in: path + required: true + description: hub id of the raw app + schema: + type: integer + format: int64 + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/RawAppEmbedBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/scripts/{ask_id}/recording: + post: + summary: attach a recording to a hub script + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubScriptRecording + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: ask_id + in: path + required: true + description: hub ask id of the script + schema: + type: integer + format: int64 + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/RecordingBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/flows/{flow_id}/recording: + post: + summary: attach a recording to a hub flow + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubFlowRecording + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: flow_id + in: path + required: true + description: hub id of the flow + schema: + type: integer + format: int64 + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/RecordingBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/projects/{slug}/pipeline_recording: + post: + summary: attach a data-pipeline recording to a hub project + description: | + Requires the caller to be a workspace admin. A data-pipeline recording is + scoped to the whole project (a folder cascade), not a single item. Forwards + the request to the configured Hub scoped to the `{workspace}:{folder}` + source and returns the Hub's status code and raw response body. + operationId: publishHubPipelineRecording + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: slug + in: path + required: true + description: hub project slug + schema: + $ref: "#/components/schemas/HubProjectSlug" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PipelineRecordingBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/resource_types: + post: + summary: publish a resource type to a hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubResourceType + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublishResourceTypeBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/resources: + post: + summary: publish resource placeholders to a hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubResources + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublishResourcesBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/triggers: + post: + summary: publish triggers to a hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubTriggers + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublishTriggersBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/migrations: + post: + summary: publish data table migrations to a hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubMigrations + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublishMigrationsBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/projects/{slug}/export: + get: + summary: export a hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub and returns the Hub's status code and raw response body. + The folder scope is only needed to re-export the caller's own draft; + approved projects are public, so it is optional here. + operationId: getHubProjectExport + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: slug + in: path + required: true + description: hub project slug (3-50 chars, lowercase alphanumeric and hyphens, no leading/trailing hyphen) + schema: + type: string + minLength: 3 + maxLength: 50 + pattern: "^[a-z0-9][a-z0-9-]*[a-z0-9]$" + - name: folder + in: query + required: false + description: folder scoping the Hub project source (`{workspace}:{folder}`) + schema: + type: string + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/projects/{slug}/submit: + post: + summary: submit a hub project draft for review + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: submitHubProject + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: slug + in: path + required: true + description: hub project slug (3-50 chars, lowercase alphanumeric and hyphens, no leading/trailing hyphen) + schema: + type: string + minLength: 3 + maxLength: 50 + pattern: "^[a-z0-9][a-z0-9-]*[a-z0-9]$" + - $ref: "#/components/parameters/HubPublishFolder" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/project: + get: + summary: get the hub project linked to a workspace folder + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: getHubProjectBySource + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + components: securitySchemes: bearerAuth: @@ -23011,6 +23480,16 @@ components: required: true schema: type: string + HubPublishFolder: + name: folder + in: query + required: true + description: | + workspace folder scoping the Hub publication: a workspace can publish + one Hub project per folder and the Hub-side source key is + `{workspace}:{folder}` + schema: + type: string PublicationName: name: publication in: path @@ -23824,6 +24303,9 @@ components: muted_on_user_path: type: boolean default: false + fallback_to_instance_alerts: + type: boolean + description: Report failed jobs to the instance critical alert channels when no workspace error handler is set. Omit to leave the stored value untouched. EditErrorHandlerLegacy: type: object @@ -26475,6 +26957,7 @@ components: - slack - teams - email + - instance_alerts NewSchedule: type: object @@ -29403,6 +29886,10 @@ components: type: string color: type: string + error_handler_fallback_to_instance_alerts: + type: boolean + default: false + description: Report failed jobs to the instance critical alert channels when no workspace error handler is set. Not available on cloud or on fork workspaces. required: - id - name @@ -31943,3 +32430,282 @@ components: - name - owner - private + + HubProjectSlug: + type: string + description: hub project slug (3-50 chars, lowercase alphanumeric and hyphens, no leading/trailing hyphen) + minLength: 3 + maxLength: 50 + pattern: "^[a-z0-9][a-z0-9-]*[a-z0-9]$" + + PublishDraftBody: + type: object + properties: + slug: + $ref: "#/components/schemas/HubProjectSlug" + name: + type: string + summary: + type: string + readme: + type: string + required: + - slug + - name + - summary + + PublishScriptBody: + type: object + properties: + summary: + type: string + app: + type: string + description: + type: string + kind: + type: string + content: + type: string + language: + type: string + schema: + type: object + lockfile: + type: string + path: + type: string + source_path: + type: string + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - summary + - app + - content + - language + - project_slug + + PublishFlowInner: + type: object + properties: + summary: + type: string + description: + type: string + value: + type: object + schema: + type: object + required: + - summary + - value + + PublishFlowBody: + type: object + properties: + flow: + $ref: "#/components/schemas/PublishFlowInner" + apps: + type: array + items: + type: string + path: + type: string + source_path: + type: string + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - flow + - apps + - project_slug + + PublishAppBody: + type: object + properties: + app: + type: object + apps: + type: array + items: + type: string + description: + type: string + summary: + type: string + path: + type: string + source_path: + type: string + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - app + - apps + - summary + - project_slug + + PublishRawAppBody: + type: object + properties: + raw: + type: string + apps: + type: array + items: + type: string + description: + type: string + summary: + type: string + path: + type: string + source_path: + type: string + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - raw + - apps + - summary + - project_slug + + RawAppEmbedBody: + type: object + properties: + external_embed_url: + type: string + nullable: true + description: explicit `null` clears the embed (unpublish) + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - project_slug + + RecordingBody: + type: object + properties: + recording: + type: object + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - project_slug + + PipelineRecordingBody: + type: object + properties: + recording: + type: object + + PublishResourceTypeBody: + type: object + properties: + name: + type: string + schema: + type: object + description: + type: string + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - name + - project_slug + + PublishResourceBody: + type: object + properties: + path: + type: string + resource_type: + type: string + required: + - path + - resource_type + + PublishResourcesBody: + type: object + properties: + resources: + type: array + items: + $ref: "#/components/schemas/PublishResourceBody" + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - resources + - project_slug + + PublishTriggerBody: + type: object + properties: + path: + type: string + kind: + type: string + summary: + type: string + nullable: true + description: + type: string + nullable: true + config: + type: object + script_ask_id: + type: integer + format: int64 + nullable: true + flow_id: + type: integer + format: int64 + nullable: true + required: + - path + - kind + - config + + PublishTriggersBody: + type: object + properties: + triggers: + type: array + items: + $ref: "#/components/schemas/PublishTriggerBody" + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - triggers + - project_slug + + PublishMigrationBody: + type: object + description: one best-effort data table migration attached to a project (per data table) + properties: + datatable_name: + type: string + sql: + type: string + sql_down: + type: string + description: defaults to an empty string when omitted + enabled: + type: boolean + required: + - datatable_name + - sql + - enabled + + PublishMigrationsBody: + type: object + properties: + migrations: + type: array + items: + $ref: "#/components/schemas/PublishMigrationBody" + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - migrations + - project_slug diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 89dff4cfd6..efc2ba7e60 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -104,18 +104,7 @@ lazy_static::lazy_static! { } }; - pub(crate) static ref HTTP_CLIENT: Client = configure_client(reqwest::ClientBuilder::new() - .timeout(std::time::Duration::from_secs(*AI_TIMEOUT_SECS)) - .pool_max_idle_per_host(HTTP_POOL_MAX_IDLE_PER_HOST) - .pool_idle_timeout(Some(std::time::Duration::from_secs(HTTP_POOL_IDLE_TIMEOUT_SECS))) - // The SSRF check in `get_base_url` only validates the configured `base_url`. - // reqwest follows up to 10 redirects by default and does not revalidate the - // hops, so a public base_url could 3xx the server into a private/internal - // address. Disable redirect following so the validated host is the only one - // we ever connect to. AI APIs respond directly and do not rely on redirects, - // so this holds even for ALLOW_PRIVATE_AI_BASE_URLS deployments. - .redirect(reqwest::redirect::Policy::none()) - .user_agent("windmill/beta")) + pub(crate) static ref HTTP_CLIENT: Client = ai_http_client_builder() .build() .expect("Failed to build AI HTTP client - check system TLS configuration"); @@ -129,6 +118,65 @@ pub(crate) fn invalidate_ai_request_cache_for_workspace(workspace_id: &str) { AI_REQUEST_CACHE.retain(|(cached_workspace_id, _), _| cached_workspace_id != workspace_id); } +/// Shared configuration for every outbound AI HTTP client (the pooled +/// [`HTTP_CLIENT`] and the per-request DNS-pinned clients). +/// +/// Redirects are disabled: the SSRF check only validates the configured host, so +/// a public host that 3xx-es could otherwise bounce us to a private/internal +/// address. AI APIs respond directly and do not rely on redirects, so this holds +/// even for ALLOW_PRIVATE_AI_BASE_URLS deployments. DNS pinning likewise only +/// covers the original host, so following a redirect would reopen the hole. +fn ai_http_client_builder() -> reqwest::ClientBuilder { + configure_client( + reqwest::ClientBuilder::new() + .timeout(std::time::Duration::from_secs(*AI_TIMEOUT_SECS)) + .pool_max_idle_per_host(HTTP_POOL_MAX_IDLE_PER_HOST) + .pool_idle_timeout(Some(std::time::Duration::from_secs( + HTTP_POOL_IDLE_TIMEOUT_SECS, + ))) + .redirect(reqwest::redirect::Policy::none()) + .user_agent("windmill/beta"), + ) +} + +/// Build the client for a single outbound AI request to `url`, pinning DNS to +/// the SSRF-validated address so the connect cannot rebind to an internal IP +/// after the check (DNS-rebinding TOCTOU). +/// +/// Returns the shared pooled [`HTTP_CLIENT`] unchanged when there is nothing to +/// pin — an IP-literal host, or a deployment that opted into private AI +/// endpoints via `ALLOW_PRIVATE_AI_BASE_URLS`. The same opt-out and error hint +/// as `get_base_url` apply, so the guard here is consistent with save-time +/// validation while additionally closing the connect-time window. +async fn pinned_ai_client_for(url: &str) -> Result> { + use std::borrow::Cow; + use windmill_common::ssrf::SsrfValidationError; + + if *windmill_ai::ai_providers::ALLOW_PRIVATE_AI_BASE_URLS { + return Ok(Cow::Borrowed(&HTTP_CLIENT)); + } + + let target = windmill_common::ssrf::validate_url_for_ssrf(url) + .await + .map_err(|e| match e { + e @ SsrfValidationError::Private { .. } => Error::BadRequest(format!( + "{e}. If you need to use private/internal AI endpoints, \ + set the ALLOW_PRIVATE_AI_BASE_URLS=true environment variable" + )), + e => Error::from(e), + })?; + + if target.pinned_addrs().is_empty() { + return Ok(Cow::Borrowed(&HTTP_CLIENT)); + } + + let client = target + .apply_dns_pinning(ai_http_client_builder()) + .build() + .map_err(to_anyhow)?; + Ok(Cow::Owned(client)) +} + #[derive(Deserialize, Debug)] struct AIOAuthResource { client_id: String, @@ -303,23 +351,13 @@ async fn get_token_using_oauth( // Validate the resolved token_url against SSRF rules before issuing the request, // mirroring the protection applied to base_url in `get_base_url` (same // ALLOW_PRIVATE_AI_BASE_URLS opt-in). Without this a workspace member could - // point token_url at an internal/metadata address. - if !*windmill_ai::ai_providers::ALLOW_PRIVATE_AI_BASE_URLS { - use windmill_common::ssrf::SsrfValidationError; - windmill_common::ssrf::validate_url_for_ssrf(&resource.token_url) - .await - .map_err(|e| match e { - e @ SsrfValidationError::Private { .. } => Error::BadRequest(format!( - "{e}. If you need to use private/internal AI endpoints, \ - set the ALLOW_PRIVATE_AI_BASE_URLS=true environment variable" - )), - e => Error::from(e), - })?; - } + // point token_url at an internal/metadata address. The returned client pins + // DNS to the validated address so the connect cannot rebind after the check. + let client = pinned_ai_client_for(&resource.token_url).await?; let mut params = HashMap::new(); params.insert("grant_type", "client_credentials"); params.insert("scope", "https://cognitiveservices.azure.com/.default"); - let response = HTTP_CLIENT + let response = client .post(resource.token_url) .form(¶ms) .basic_auth(resource.client_id, Some(resource.client_secret)) @@ -420,8 +458,11 @@ fn is_sse_response(headers: &HeaderMap) -> bool { .unwrap_or(false) } -fn proxy_request_to_request_builder(proxy_request: ProxyRequest) -> RequestBuilder { - let mut request = HTTP_CLIENT.request(proxy_request.method.clone(), &proxy_request.url); +fn proxy_request_to_request_builder( + client: &Client, + proxy_request: ProxyRequest, +) -> RequestBuilder { + let mut request = client.request(proxy_request.method.clone(), &proxy_request.url); for (header_name, header_value) in &proxy_request.headers { request = request.header(header_name.as_str(), header_value.as_str()); } @@ -546,6 +587,8 @@ async fn global_proxy( custom_headers: HashMap::new(), }; + let client = pinned_ai_client_for(&credentials.base_url).await?; + if matches!(proxy_mode, ProxyExecutionMode::NativeGoogleAi) { let proxy_args = ProxyBuildArgs { method: &method, @@ -558,8 +601,8 @@ async fn global_proxy( audit_global_ai_request(&db, &authed).await?; let response = match ai_path.as_str() { - "chat/completions" => handle_google_ai_chat_proxy(&HTTP_CLIENT, &proxy_args).await, - "models" => handle_google_ai_models_proxy(&HTTP_CLIENT, &proxy_args).await, + "chat/completions" => handle_google_ai_chat_proxy(&client, &proxy_args).await, + "models" => handle_google_ai_models_proxy(&client, &proxy_args).await, _ => Err(Error::BadRequest(format!( "Unsupported Google AI path: {}", ai_path @@ -582,7 +625,7 @@ async fn global_proxy( body: &body, credentials: &credentials, })?; - proxy_request_to_request_builder(proxy_request) + proxy_request_to_request_builder(&client, proxy_request) } ProxyExecutionMode::NativeGoogleAi | ProxyExecutionMode::NativeAwsBedrock => { return Err(Error::BadRequest(format!( @@ -826,9 +869,11 @@ async fn proxy( credentials: &credentials, }; + let client = pinned_ai_client_for(&credentials.base_url).await?; + let response = match ai_path.as_str() { - "chat/completions" => handle_google_ai_chat_proxy(&HTTP_CLIENT, &proxy_args).await, - "models" => handle_google_ai_models_proxy(&HTTP_CLIENT, &proxy_args).await, + "chat/completions" => handle_google_ai_chat_proxy(&client, &proxy_args).await, + "models" => handle_google_ai_models_proxy(&client, &proxy_args).await, _ => Err(Error::BadRequest(format!( "Unsupported Google AI path: {}", ai_path @@ -887,7 +932,8 @@ async fn proxy( body: &body, credentials: &credentials, })?; - proxy_request_to_request_builder(proxy_request) + let client = pinned_ai_client_for(&credentials.base_url).await?; + proxy_request_to_request_builder(&client, proxy_request) } ProxyExecutionMode::NativeGoogleAi => { return Err(Error::internal_err( diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index fe10229af3..e8ac1340ca 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -69,7 +69,7 @@ use windmill_common::{ users::username_to_permissioned_as, utils::{ http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, require_admin, - Pagination, RunnableKind, StripPath, + strip_json_nul, Pagination, RunnableKind, StripPath, }, variables::{build_crypt, build_crypt_with_key_suffix, encrypt}, worker::{to_raw_value, CLOUD_HOSTED}, @@ -383,11 +383,7 @@ async fn list_search_apps( // apps' definitions. `check_scopes` uses ScopeDefinition::includes, where run // does NOT include read, so it correctly denies such tokens. check_scopes(&authed, || "apps:read".to_string())?; - #[cfg(feature = "enterprise")] let n = 1000; - - #[cfg(not(feature = "enterprise"))] - let n = 3; let mut tx = user_db.begin(&authed).await?; let allowed = build_scope_path_predicate(&authed, "apps", "read"); @@ -1832,50 +1828,6 @@ fn custom_path_conflict_error( } } -/// App values live in a `json` column, which — unlike `jsonb` — accepts the -/// `\u0000` escape. Any later `json`→`jsonb` conversion (a workspace fork's -/// `clone_apps`, search indexing, …) then aborts with "unsupported Unicode -/// escape sequence". Strip genuine NULs so the value is jsonb-safe before it -/// lands in the DB; the usual source is a binary file such as `.DS_Store` -/// accidentally bundled into a raw app's file map. A real NUL is unstorable -/// either way, and frontend code that needs the character writes it as the -/// source escape `\u0000`, which JSON-encodes to `\\u0000` (an escaped -/// backslash — the even-parity case below) and is left untouched. -/// -/// Returns `Cow::Borrowed` (no allocation) when the value is already clean. -fn strip_null_chars(raw: &str) -> Cow<'_, str> { - let bytes = raw.as_bytes(); - let mut out: Option = None; - let mut copied_to = 0; - let mut search_from = 0; - // A genuine NUL is `\u0000`: a `u0000` introduced by an *odd* run of - // backslashes. An even run (`\\u0000`) is an escaped backslash then the - // literal text "u0000" (common in minified JS regexes) and is preserved. - while let Some(rel) = raw[search_from..].find("u0000") { - let at = search_from + rel; - let mut backslashes = 0; - let mut j = at; - while j > 0 && bytes[j - 1] == b'\\' { - backslashes += 1; - j -= 1; - } - if backslashes % 2 == 1 { - // Drop the escaping backslash + `u0000` — the 6 chars in [at-1, at+5). - let out = out.get_or_insert_with(String::new); - out.push_str(&raw[copied_to..at - 1]); - copied_to = at + 5; - } - search_from = at + 5; - } - match out { - Some(mut out) => { - out.push_str(&raw[copied_to..]); - Cow::Owned(out) - } - None => Cow::Borrowed(raw), - } -} - async fn create_app_internal<'a>( authed: ApiAuthed, db: sqlx::Pool, @@ -2021,7 +1973,7 @@ async fn create_app_internal<'a>( .await?; // `.get()` keeps the raw text (and thus key order); strip any NUL so the // `json`→`jsonb` conversion downstream (fork, indexing) can't choke on it. - let value = strip_null_chars(app.value.0.get()); + let value = strip_json_nul(app.value.0.get()); if matches!(value, Cow::Owned(_)) { tracing::warn!(path = %app.path, "stripped NUL character(s) from app value on create"); } @@ -2606,7 +2558,7 @@ async fn update_app_internal<'a>( // `.get()` keeps the raw text (and thus key order); strip any NUL so the // `json`→`jsonb` conversion downstream (fork, indexing) can't choke on it. - let value = strip_null_chars(nvalue.0.get()); + let value = strip_json_nul(nvalue.0.get()); if matches!(value, Cow::Owned(_)) { tracing::warn!(path = %npath, "stripped NUL character(s) from app value on update"); } @@ -3883,6 +3835,18 @@ async fn get_on_behalf_authed_from_app( Ok((on_behalf_authed, policy)) } +/// Which identity a deployed `apps_u/*` S3 read runs as. +#[cfg(feature = "parquet")] +enum AppS3ReadIdentity { + /// The gate passed: read with the policy's on-behalf identity (the app author in + /// author-mode, the viewer in viewer-mode). + OnBehalf, + /// The gate did not pass but a logged-in, non-embed viewer is present: read with + /// the viewer's OWN identity so the downstream S3 permission check self-enforces + /// their entitlement (never the author's). + AsViewer(ApiAuthed), +} + #[cfg(feature = "parquet")] async fn check_if_allowed_to_access_s3_file_from_app( db: &DB, @@ -3891,7 +3855,7 @@ async fn check_if_allowed_to_access_s3_file_from_app( w_id: &str, path: &str, policy: &Policy, -) -> Result<()> { +) -> Result { let is_app_embed = opt_authed.as_ref().is_some_and(|authed| { windmill_api_auth::scopes::has_app_embed_sentinel(authed.scopes.as_deref()) }); @@ -3912,7 +3876,7 @@ async fn check_if_allowed_to_access_s3_file_from_app( &db, ) .await?; - return Ok(()); + return Ok(AppS3ReadIdentity::OnBehalf); } } @@ -3921,24 +3885,25 @@ async fn check_if_allowed_to_access_s3_file_from_app( // get_workspace_s3_resource_and_check_paths already bounds the read by // their own perms — no provenance gate (it would over-restrict). Embed // tokens are excluded (untrusted app JS stays confined below). - Ok(()) - } else { - // Author-mode/embed: confine reads to the app's declared keys or files THIS - // app produced, else a viewer could launder the author's S3 perms via an - // arbitrary file_key (confused deputy). Provenance is the un-forgeable - // app-origination marker (`trigger_kind='app'` + `trigger=`); - // `created_by=` is ANDed only as a per-viewer isolation filter (it - // can narrow — one viewer can't read another's result — never forge). - let creator = opt_authed - .as_ref() - .map(|authed| authed.username.clone()) - .unwrap_or_else(|| "anonymous".to_string()); - let allowed = policy.allowed_s3_keys.as_ref().is_some_and(|keys| { - keys.iter() - .any(|key| key.s3_path == file_query.s3 && key.storage == file_query.storage) - }) || { - sqlx::query_scalar!( - r#"SELECT EXISTS ( + return Ok(AppS3ReadIdentity::OnBehalf); + } + + // Author-mode/embed: confine reads to the app's declared keys or files THIS + // app produced, else a viewer could launder the author's S3 perms via an + // arbitrary file_key (confused deputy). Provenance is the un-forgeable + // app-origination marker (`trigger_kind='app'` + `trigger=`); + // `created_by=` is ANDed only as a per-viewer isolation filter (it + // can narrow — one viewer can't read another's result — never forge). + let creator = opt_authed + .as_ref() + .map(|authed| authed.username.clone()) + .unwrap_or_else(|| "anonymous".to_string()); + let allowed = policy.allowed_s3_keys.as_ref().is_some_and(|keys| { + keys.iter() + .any(|key| key.s3_path == file_query.s3 && key.storage == file_query.storage) + }) || { + sqlx::query_scalar!( + r#"SELECT EXISTS ( SELECT 1 FROM v2_job_completed c JOIN v2_job j USING (id) WHERE j.workspace_id = $2 AND c.started_at > now() - interval '3 hours' @@ -3947,21 +3912,45 @@ async fn check_if_allowed_to_access_s3_file_from_app( AND j.trigger = $3 AND j.created_by = $4 )"#, - file_query.s3, - w_id, - path, - creator, - ) - .fetch_one(db) - .await? - .unwrap_or(false) - }; + file_query.s3, + w_id, + path, + creator, + ) + .fetch_one(db) + .await? + .unwrap_or(false) + }; - if !allowed { - Err(Error::BadRequest("File restricted".to_string())) - } else { - Ok(()) + if allowed { + return Ok(AppS3ReadIdentity::OnBehalf); + } + + // Gate denied. A viewer whose token is effectively unscoped falls back to reading + // as THEMSELVES: the file is still bounded by their own S3 perms downstream, and + // such a token can already fetch it via `job_helpers/download_s3_file`, so the + // fallback adds zero capability. `is_effectively_unscoped` (the same predicate the + // route-scope middleware uses) is what makes that true: a genuinely scope-restricted + // token (e.g. `apps:read:`) is allowed on `apps_u/*` but REJECTED on + // `job_helpers/*`, so serving it the file here WOULD be a new capability — it stays + // gated. `!is_app_embed` keeps that confinement explicit (embed tokens carry the + // `app_embed` scope, so they are already scope-restricted). Anonymous callers (no + // identity) also have no viewer to fall back to. Only the confused-deputy denial + // reaches the message below. + match opt_authed.as_ref() { + Some(viewer) + if !is_app_embed + && windmill_api_auth::is_effectively_unscoped(viewer.scopes.as_deref()) => + { + Ok(AppS3ReadIdentity::AsViewer(viewer.clone())) } + _ => Err(Error::BadRequest(format!( + "S3 file \"{}\" is not accessible from this app. A deployed app running on \ + behalf of its author only serves files it generated, files in its declared \ + allowlist, or presigned files. To expose a pre-existing file, sign it \ + (signS3Object / sign_s3_object) or set the app's execution mode to \"viewer\".", + file_query.s3 + ))), } } @@ -4029,7 +4018,7 @@ async fn download_s3_file_from_app( get_on_behalf_authed_from_app(&db, &path, &w_id, &opt_authed, force_viewer_allowed_s3_keys) .await?; - check_if_allowed_to_access_s3_file_from_app( + let read_authed = match check_if_allowed_to_access_s3_file_from_app( &db, &opt_authed, &query.file_query, @@ -4037,10 +4026,14 @@ async fn download_s3_file_from_app( &path, &policy, ) - .await?; + .await? + { + AppS3ReadIdentity::OnBehalf => on_behalf_authed, + AppS3ReadIdentity::AsViewer(viewer) => viewer, + }; download_s3_file_internal( - OptJobAuthed { authed: on_behalf_authed, job_id: None }, + OptJobAuthed { authed: read_authed, job_id: None }, &db, None, &w_id, @@ -4092,9 +4085,15 @@ async fn app_s3_on_behalf_and_provenance( } let (on_behalf_authed, policy) = get_on_behalf_authed_from_app(db, path, w_id, opt_authed, None).await?; - check_if_allowed_to_access_s3_file_from_app(db, opt_authed, file_query, w_id, path, &policy) - .await?; - Ok(crate::db::OptJobAuthed { authed: on_behalf_authed, job_id: None }) + let read_authed = match check_if_allowed_to_access_s3_file_from_app( + db, opt_authed, file_query, w_id, path, &policy, + ) + .await? + { + AppS3ReadIdentity::OnBehalf => on_behalf_authed, + AppS3ReadIdentity::AsViewer(viewer) => viewer, + }; + Ok(crate::db::OptJobAuthed { authed: read_authed, job_id: None }) } // The app-scoped display ops carry the app path in the URL and everything else @@ -4798,62 +4797,3 @@ mod embed_token_tests { assert!(parse_embed_policy("not json").is_err()); } } - -#[cfg(test)] -mod strip_null_chars_tests { - use super::strip_null_chars; - use std::borrow::Cow; - - // Build `{"k":"u0000"}` without writing the escape literally - // (a real NUL can't live in Rust source). Odd n => the trailing `u0000` is a - // genuine NUL escape; even n => an escaped backslash then the text "u0000". - fn doc(backslashes: usize) -> String { - format!(r#"{{"k":"{}u0000"}}"#, "\\".repeat(backslashes)) - } - - #[test] - fn strips_genuine_null_escape() { - // 1 backslash: the NUL escape is dropped, the string value becomes "". - assert_eq!(strip_null_chars(&doc(1)).as_ref(), r#"{"k":""}"#); - // 3 backslashes: escaped backslash + NUL -> keep the escaped backslash. - let three = doc(3); - let out = strip_null_chars(&three); - assert_eq!(out.as_ref(), r#"{"k":"\\"}"#); - // Result is now valid, NUL-free JSON (i.e. jsonb-safe). - let v: serde_json::Value = serde_json::from_str(out.as_ref()).unwrap(); - assert!(!v["k"].as_str().unwrap().as_bytes().contains(&0u8)); - } - - #[test] - fn preserves_escaped_backslash_then_literal_u0000() { - // Even runs are the literal text "u0000" (e.g. a minified JS regex char - // class) and must be returned untouched, with no allocation. - for n in [2usize, 4] { - let s = doc(n); - let out = strip_null_chars(&s); - assert_eq!(out.as_ref(), s.as_str()); - assert!(matches!(out, Cow::Borrowed(_)), "n={n} should be borrowed"); - } - } - - #[test] - fn preserves_clean_values() { - // Plain value, and the bare token "u0000" with no preceding backslash. - for s in [r#"{"files":{"/index.tsx":"hello"}}"#, r#"{"k":"u0000"}"#] { - let out = strip_null_chars(s); - assert_eq!(out.as_ref(), s); - assert!(matches!(out, Cow::Borrowed(_))); - } - // The escape for a literal backslash char (`u005c`) then text "u0000": - // the only "u0000" match is preceded by `c` (0 backslashes) -> no NUL. - let s = format!(r#"{{"k":"{}u005cu0000"}}"#, "\\"); - assert!(matches!(strip_null_chars(&s), Cow::Borrowed(_))); - } - - #[test] - fn strips_multiple_and_preserves_surrounding() { - // Mirrors the .DS_Store case: several NULs interleaved with real text. - let s = format!(r#"{{"a":"x{b}u0000{b}u0000y","b":"ok"}}"#, b = "\\"); - assert_eq!(strip_null_chars(&s).as_ref(), r#"{"a":"xy","b":"ok"}"#); - } -} diff --git a/backend/windmill-api/src/drafts.rs b/backend/windmill-api/src/drafts.rs index a4ebf74ec0..b44b0e64be 100644 --- a/backend/windmill-api/src/drafts.rs +++ b/backend/windmill-api/src/drafts.rs @@ -19,6 +19,7 @@ use windmill_common::{ error::{Error, Result}, user_drafts::{DraftUserRef, UserDraftItemKind, ENCRYPTED_DRAFT_PREFIX}, users::resolve_username_to_email, + utils::strip_json_nul, variables::{build_crypt, encrypt}, }; @@ -353,7 +354,7 @@ async fn update_draft( // `draft.value` is a `json` column, so a U+0000 (NUL) would persist as an // escape and later make any `->>`/`to_jsonb` extraction raise `22P05`. // Strip it here so a NUL never reaches the column. - let serialized = strip_json_nul(serialized); + let serialized = strip_json_nul(&serialized); // Upsert. The conflict check rides on the DO UPDATE WHERE clause — // when the row is newer than `last_sync`, RETURNING yields nothing. // `created_at` defaults to `now()` but the migration overrides it ($8) @@ -371,7 +372,7 @@ async fn update_draft( email, path, kind as UserDraftItemKind, - serialized, + serialized.as_ref(), req.last_sync, req.force, req.created_at, @@ -521,53 +522,6 @@ async fn migrate_legacy_draft( } } -/// Remove every U+0000 (NUL) from a serialized JSON document so it is safe to -/// store in the `json`-typed `draft.value` (a NUL there would later make any -/// `->>`/`to_jsonb` extraction raise `22P05`). -/// -/// A NUL can only appear in JSON text as a backslash-u0000 escape, and a -/// backslash only ever occurs inside a string, so one backslash-parity-aware -/// pass removes every real NUL escape — covering values and keys alike — while -/// leaving a legitimate `\\u0000` (an escaped backslash followed by the literal -/// text `u0000`) intact. O(n) over the bytes with no `serde_json::Value` tree to -/// allocate, and the fast path (no such substring at all) returns the input -/// untouched. The slow path is reached not only by genuinely poisoned values but -/// by any value that legitimately contains `u0000` after a backslash (e.g. script -/// source), so it must stay allocation-light for potentially large drafts. -fn strip_json_nul(serialized: String) -> String { - if !serialized.contains("\\u0000") { - return serialized; - } - let bytes = serialized.as_bytes(); - let mut out: Vec = Vec::with_capacity(bytes.len()); - let mut i = 0; - while i < bytes.len() { - if bytes[i] != b'\\' { - out.push(bytes[i]); - i += 1; - continue; - } - // Consume the whole run of backslashes. An even run is N/2 escaped - // backslashes and leaves the next char unescaped; an odd run ends in an - // escaping backslash, so a following `u0000` is a real NUL escape. - let run_start = i; - while i < bytes.len() && bytes[i] == b'\\' { - i += 1; - } - let run = i - run_start; - if run % 2 == 1 && bytes[i..].starts_with(b"u0000") { - // Drop the escaping backslash + `u0000`; keep the leading literal pairs. - out.extend(std::iter::repeat(b'\\').take(run - 1)); - i += 5; - } else { - out.extend(std::iter::repeat(b'\\').take(run)); - } - } - // Only whole ASCII backslash-u0000 escapes were removed, so the bytes remain - // valid UTF-8 (and valid JSON). - String::from_utf8(out).expect("removing a NUL escape preserves valid UTF-8") -} - /// For variable-kind drafts with `variable.is_secret == true`, encrypt /// `variable.value` with the workspace crypt key and mark it /// `$encrypted:` so the secret never persists in plaintext at rest. @@ -859,65 +813,3 @@ async fn require_can_read_path( } Err(Error::NotFound(format!("no draft visible at {path}"))) } - -#[cfg(test)] -mod tests { - use super::strip_json_nul; - - // Parse the (NUL-free) result so assertions read clearly. - fn parsed(s: String) -> serde_json::Value { - serde_json::from_str(&s).expect("strip_json_nul must return valid JSON") - } - - #[test] - fn clean_value_is_returned_byte_for_byte() { - let s = r#"{"summary":"all good","n":1}"#.to_string(); - assert_eq!(strip_json_nul(s.clone()), s); - } - - #[test] - fn real_nul_in_value_is_stripped() { - let out = strip_json_nul(r#"{"summary":"hi\u0000there"}"#.to_string()); - assert!(!out.contains(r"\u0000")); - assert_eq!(parsed(out)["summary"], "hithere"); - } - - #[test] - fn legit_escaped_backslash_is_a_noop() { - // JSON "a\\u0000b" decodes to the 8-char string a,backslash,u,0,0,0,0,b - // — not a NUL — so the value is already clean and round-trips byte-for-byte. - let s = r#"{"summary":"a\\u0000b"}"#.to_string(); - assert_eq!(strip_json_nul(s.clone()), s); - } - - #[test] - fn collision_real_and_literal_both_handled() { - // "a" carries a real NUL; "b" carries the literal text backslash-u0000. - // The value walk strips the former and leaves the latter intact — the - // pathological case that needed a fallback in SQL is trivial in Rust. - let v = parsed(strip_json_nul( - r#"{"a":"x\u0000y","b":"p\\u0000q"}"#.to_string(), - )); - assert_eq!(v["a"], "xy"); - assert_eq!(v["b"], "p\\u0000q"); - } - - #[test] - fn nested_values_and_keys_are_cleaned() { - let out = strip_json_nul( - r#"{"o":{"k\u0000":["a\u0000b",{"deep\u0000":"v\u0000"}]}}"#.to_string(), - ); - assert!(!out.contains(r"\u0000")); - let v = parsed(out); - assert_eq!(v["o"]["k"][0], "ab"); - assert_eq!(v["o"]["k"][1]["deep"], "v"); - } - - #[test] - fn odd_backslash_run_keeps_literal_drops_nul() { - // JSON "a\\\u0000b" is an escaped backslash (kept) immediately followed by - // a real NUL escape (dropped) -> decodes to a,backslash,b. - let v = parsed(strip_json_nul(r#"{"x":"a\\\u0000b"}"#.to_string())); - assert_eq!(v["x"], "a\\b"); - } -} diff --git a/backend/windmill-api/src/hub_publish.rs b/backend/windmill-api/src/hub_publish.rs new file mode 100644 index 0000000000..fab473ea41 --- /dev/null +++ b/backend/windmill-api/src/hub_publish.rs @@ -0,0 +1,543 @@ +use crate::auth::Tokened; +use crate::db::ApiAuthed; +use crate::HTTP_CLIENT; +use axum::{ + extract::{FromRequestParts, Json, Path, Query, RawPathParams}, + http::{request::Parts, StatusCode}, + response::{IntoResponse, Response}, + routing::{get, post}, + Router, +}; +use serde::{Deserialize, Deserializer, Serialize}; +use windmill_common::{ + error::{to_anyhow, Error}, + utils::require_admin, + HUB_BASE_URL, +}; + +pub fn workspaced_service() -> Router { + Router::new() + .route("/publish_draft", post(publish_draft)) + .route("/scripts", post(publish_script)) + .route("/flows", post(publish_flow)) + .route("/apps", post(publish_app)) + .route("/raw_apps", post(publish_raw_app)) + .route("/raw_apps/{id}/embed", post(publish_raw_app_embed)) + .route( + "/scripts/{ask_id}/recording", + post(publish_script_recording), + ) + .route("/flows/{flow_id}/recording", post(publish_flow_recording)) + .route( + "/projects/{slug}/pipeline_recording", + post(publish_pipeline_recording), + ) + .route("/resource_types", post(publish_resource_type)) + .route("/resources", post(publish_resources)) + .route("/triggers", post(publish_triggers)) + .route("/migrations", post(publish_migrations)) + .route("/projects/{slug}/export", get(get_project_export)) + .route("/projects/{slug}/submit", post(submit_project)) + .route("/project", get(get_project_by_source)) +} + +#[derive(Deserialize)] +struct HubScope { + folder: Option, +} + +fn validate_folder(folder: &str) -> Result<(), Error> { + let ok = !folder.is_empty() + && folder.len() <= 255 + && folder + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-'); + if ok { + Ok(()) + } else { + Err(Error::BadRequest(format!("invalid folder: {folder}"))) + } +} + +fn source_key(workspace: &str, folder: &str) -> Result { + validate_folder(folder)?; + Ok(format!("{workspace}:{folder}")) +} + +fn validate_project_slug(slug: &str) -> Result<(), Error> { + let ok = slug.len() >= 3 + && slug.len() <= 50 + && !slug.starts_with('-') + && !slug.ends_with('-') + && slug + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-'); + if ok { + Ok(()) + } else { + Err(Error::BadRequest(format!("invalid project slug: {slug}"))) + } +} + +/// A Hub project slug that is valid by construction: deserialization (from a +/// request body or a path segment) is the only way to obtain one and it runs +/// `validate_project_slug`, so no handler can forward or interpolate an +/// unvalidated slug into a Hub URL. +#[derive(Serialize)] +#[serde(transparent)] +struct ProjectSlug(String); + +impl<'de> Deserialize<'de> for ProjectSlug { + fn deserialize>(d: D) -> Result { + let s = String::deserialize(d)?; + validate_project_slug(&s).map_err(serde::de::Error::custom)?; + Ok(ProjectSlug(s)) + } +} + +impl std::fmt::Display for ProjectSlug { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +/// The single gate every Hub endpoint goes through: an admin caller, their +/// token (the Hub authenticates it back against this instance's whoami), and +/// the validated `workspace_id:folder` source key scoping ownership Hub-side. +/// Handlers can only reach the Hub via this extractor's methods, so a new +/// endpoint cannot forget the admin check or folder validation. +/// +/// A workspace can publish one Hub project per folder. The stable, never-mutated +/// link key is `workspace_id:folder_name` (folder name is the path segment and is +/// never renamed — only display_name changes). `:` is safe: neither workspace ids +/// nor folder names (alphanumeric, underscore, hyphen) contain it. +struct HubPublishCtx { + source_id: Option, + token: String, +} + +impl FromRequestParts for HubPublishCtx +where + S: Send + Sync, +{ + type Rejection = Response; + + async fn from_request_parts( + parts: &mut Parts, + state: &S, + ) -> std::result::Result { + let authed = ApiAuthed::from_request_parts(parts, state) + .await + .map_err(IntoResponse::into_response)?; + let tokened = Tokened::from_request_parts(parts, state) + .await + .map_err(IntoResponse::into_response)?; + let params = RawPathParams::from_request_parts(parts, state) + .await + .map_err(IntoResponse::into_response)?; + let workspace = params + .iter() + .find(|(k, _)| *k == "workspace_id") + .map(|(_, v)| v.to_owned()); + let Query(scope) = Query::::from_request_parts(parts, state) + .await + .map_err(IntoResponse::into_response)?; + let build = || -> Result { + require_admin(authed.is_admin, &authed.username)?; + let workspace = workspace.ok_or_else(|| { + Error::internal_err( + "hub publish route must be nested under /w/{workspace_id}".to_string(), + ) + })?; + let source_id = scope + .folder + .as_deref() + .map(|f| source_key(&workspace, f)) + .transpose()?; + Ok(HubPublishCtx { source_id, token: tokened.token }) + }; + build().map_err(IntoResponse::into_response) + } +} + +impl HubPublishCtx { + fn require_source(&self) -> Result<&str, Error> { + self.source_id + .as_deref() + .ok_or_else(|| Error::BadRequest("missing folder query param".to_string())) + } + + async fn post( + &self, + path: &str, + body: &T, + ) -> Result<(StatusCode, String), Error> { + forward_to_hub(path, self.require_source()?, &self.token, body).await + } + + async fn get(&self, path: &str) -> Result<(StatusCode, String), Error> { + get_from_hub(path, self.require_source()?, &self.token).await + } + + /// GET without requiring a folder scope. Only for reads the Hub allows + /// publicly (e.g. exporting an approved project); the empty source id makes + /// the Hub skip the ownership match. + async fn get_maybe_unscoped(&self, path: &str) -> Result<(StatusCode, String), Error> { + get_from_hub(path, self.source_id.as_deref().unwrap_or(""), &self.token).await + } +} + +#[derive(Deserialize, Serialize)] +struct PublishDraftBody { + slug: ProjectSlug, + name: String, + summary: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + readme: Option, +} + +async fn publish_draft( + ctx: HubPublishCtx, + Json(body): Json, +) -> Result { + ctx.post("/projects", &body).await +} + +#[derive(Deserialize, Serialize)] +struct PublishScriptBody { + summary: String, + app: String, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + #[serde(skip_serializing_if = "Option::is_none")] + kind: Option, + content: String, + language: String, + #[serde(skip_serializing_if = "Option::is_none")] + schema: Option, + #[serde(skip_serializing_if = "Option::is_none")] + lockfile: Option, + #[serde(skip_serializing_if = "Option::is_none")] + path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + source_path: Option, + project_slug: ProjectSlug, +} + +async fn publish_script( + ctx: HubPublishCtx, + Json(body): Json, +) -> Result { + ctx.post("/scripts/add", &body).await +} + +#[derive(Deserialize, Serialize)] +struct PublishFlowInner { + summary: String, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + value: serde_json::Value, + #[serde(skip_serializing_if = "Option::is_none")] + schema: Option, +} + +#[derive(Deserialize, Serialize)] +struct PublishFlowBody { + flow: PublishFlowInner, + apps: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + source_path: Option, + project_slug: ProjectSlug, +} + +async fn publish_flow( + ctx: HubPublishCtx, + Json(body): Json, +) -> Result { + ctx.post("/flows", &body).await +} + +#[derive(Deserialize, Serialize)] +struct PublishAppBody { + app: serde_json::Value, + apps: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + summary: String, + #[serde(skip_serializing_if = "Option::is_none")] + path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + source_path: Option, + project_slug: ProjectSlug, +} + +async fn publish_app( + ctx: HubPublishCtx, + Json(body): Json, +) -> Result { + ctx.post("/apps", &body).await +} + +#[derive(Deserialize, Serialize)] +struct PublishRawAppBody { + raw: String, + apps: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + summary: String, + #[serde(skip_serializing_if = "Option::is_none")] + path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + source_path: Option, + project_slug: ProjectSlug, +} + +async fn publish_raw_app( + ctx: HubPublishCtx, + Json(body): Json, +) -> Result { + ctx.post("/raw_apps", &body).await +} + +#[derive(Deserialize, Serialize)] +struct RawAppEmbedBody { + // No skip_serializing_if: `null` must reach the Hub to clear the embed (unpublish). + external_embed_url: Option, + project_slug: ProjectSlug, +} + +async fn publish_raw_app_embed( + ctx: HubPublishCtx, + Path((_workspace, id)): Path<(String, i64)>, + Json(body): Json, +) -> Result { + ctx.post(&format!("/raw_apps/{}/embed", id), &body).await +} + +#[derive(Deserialize, Serialize)] +struct RecordingBody { + #[serde(skip_serializing_if = "Option::is_none")] + recording: Option, + project_slug: ProjectSlug, +} + +async fn publish_script_recording( + ctx: HubPublishCtx, + Path((_workspace, ask_id)): Path<(String, i64)>, + Json(body): Json, +) -> Result { + ctx.post(&format!("/scripts/{}/recording", ask_id), &body) + .await +} + +async fn publish_flow_recording( + ctx: HubPublishCtx, + Path((_workspace, flow_id)): Path<(String, i64)>, + Json(body): Json, +) -> Result { + ctx.post(&format!("/flows/{}/recording", flow_id), &body) + .await +} + +// A data-pipeline recording is scoped to the whole project (a folder cascade), +// not a single Hub item, so the slug comes from the path (validated by +// construction) and only the opaque recording is forwarded. +#[derive(Deserialize, Serialize)] +struct PipelineRecordingBody { + #[serde(skip_serializing_if = "Option::is_none")] + recording: Option, +} + +async fn publish_pipeline_recording( + ctx: HubPublishCtx, + Path((_workspace, slug)): Path<(String, ProjectSlug)>, + Json(body): Json, +) -> Result { + ctx.post(&format!("/projects/{}/pipeline_recording", slug), &body) + .await +} + +#[derive(Deserialize, Serialize)] +struct PublishResourceTypeBody { + name: String, + #[serde(skip_serializing_if = "Option::is_none")] + schema: Option, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + project_slug: ProjectSlug, +} + +async fn publish_resource_type( + ctx: HubPublishCtx, + Json(body): Json, +) -> Result { + ctx.post( + &format!("/projects/{}/resource_types", body.project_slug), + &body, + ) + .await +} + +#[derive(Deserialize, Serialize)] +struct PublishResourceBody { + path: String, + resource_type: String, +} + +#[derive(Deserialize, Serialize)] +struct PublishResourcesBody { + resources: Vec, + project_slug: ProjectSlug, +} + +async fn publish_resources( + ctx: HubPublishCtx, + Json(body): Json, +) -> Result { + ctx.post(&format!("/projects/{}/resources", body.project_slug), &body) + .await +} + +#[derive(Deserialize, Serialize)] +struct PublishTriggerBody { + path: String, + kind: String, + #[serde(skip_serializing_if = "Option::is_none")] + summary: Option, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + config: serde_json::Value, + #[serde(skip_serializing_if = "Option::is_none")] + script_ask_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + flow_id: Option, +} + +#[derive(Deserialize, Serialize)] +struct PublishTriggersBody { + triggers: Vec, + project_slug: ProjectSlug, +} + +async fn publish_triggers( + ctx: HubPublishCtx, + Json(body): Json, +) -> Result { + ctx.post(&format!("/projects/{}/triggers", body.project_slug), &body) + .await +} + +// One best-effort data table migration attached to a project (per data table). +#[derive(Deserialize, Serialize)] +struct PublishMigrationBody { + datatable_name: String, + sql: String, + #[serde(default)] + sql_down: String, + enabled: bool, +} + +#[derive(Deserialize, Serialize)] +struct PublishMigrationsBody { + migrations: Vec, + project_slug: ProjectSlug, +} + +async fn publish_migrations( + ctx: HubPublishCtx, + Json(body): Json, +) -> Result { + ctx.post( + &format!("/projects/{}/migrations", body.project_slug), + &body, + ) + .await +} + +// Export is owner-scoped only when re-exporting your own draft; approved +// projects are public, so the folder scope is optional here. +async fn get_project_export( + ctx: HubPublishCtx, + Path((_workspace, slug)): Path<(String, ProjectSlug)>, +) -> Result { + ctx.get_maybe_unscoped(&format!("/projects/{}/export", slug)) + .await +} + +async fn get_project_by_source(ctx: HubPublishCtx) -> Result { + ctx.get("/projects/by_source").await +} + +async fn submit_project( + ctx: HubPublishCtx, + Path((_workspace, slug)): Path<(String, ProjectSlug)>, +) -> Result { + ctx.post( + &format!("/projects/{}/submit", slug), + &serde_json::json!({}), + ) + .await +} + +// The Hub has no auth of its own: it validates bearer tokens by calling this +// instance's /api/users/whoami. Forwarding the caller's own token logs them in +// on the Hub as themselves (account auto-created on first use). +async fn get_from_hub( + path: &str, + source_id: &str, + token: &str, +) -> Result<(StatusCode, String), Error> { + let url = format!("{}{}", **HUB_BASE_URL.load(), path); + + let res = HTTP_CLIENT + .get(&url) + .query(&[("source_id", source_id)]) + .bearer_auth(token) + .send() + .await + .map_err(|e| Error::InternalErr(format!("hub request failed: {e}")))?; + + let status = StatusCode::from_u16(res.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY); + let text = res + .text() + .await + .map_err(|e| Error::InternalErr(format!("hub response read failed: {e}")))?; + + Ok((status, text)) +} + +async fn forward_to_hub( + path: &str, + source_id: &str, + token: &str, + body: &T, +) -> Result<(StatusCode, String), Error> { + let url = format!("{}{}", **HUB_BASE_URL.load(), path); + + let mut payload = serde_json::to_value(body).map_err(to_anyhow)?; + let obj = payload + .as_object_mut() + .ok_or_else(|| Error::internal_err("hub publish body must be a JSON object".to_string()))?; + obj.insert( + "source_id".to_string(), + serde_json::Value::String(source_id.to_string()), + ); + + let res = HTTP_CLIENT + .post(&url) + .bearer_auth(token) + .json(&payload) + .send() + .await + .map_err(|e| Error::InternalErr(format!("hub request failed: {e}")))?; + + let status = StatusCode::from_u16(res.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY); + let text = res + .text() + .await + .map_err(|e| Error::InternalErr(format!("hub response read failed: {e}")))?; + + Ok((status, text)) +} diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 14cbb76c91..fe1f01f4e6 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -25,7 +25,7 @@ use std::time::Instant; use tokio::io::AsyncReadExt; use tower::ServiceBuilder; use url::Url; -#[cfg(all(feature = "enterprise", feature = "smtp"))] +#[cfg(all(feature = "enterprise", feature = "instance_smtp"))] use windmill_common::auth::is_super_admin_email; use windmill_common::auth::TOKEN_PREFIX_LEN; #[cfg(feature = "run_inline")] @@ -54,7 +54,7 @@ use windmill_common::workspace_dependencies::{ RawWorkspaceDependencies, MIN_VERSION_WORKSPACE_DEPENDENCIES, }; use windmill_common::DYNAMIC_INPUT_CACHE; -#[cfg(all(feature = "enterprise", feature = "smtp"))] +#[cfg(all(feature = "enterprise", feature = "instance_smtp"))] use windmill_common::{email_oss::send_email_html, server::load_smtp_config}; use windmill_object_store::upload_artifact_to_store; #[cfg(feature = "run_inline")] @@ -1692,7 +1692,7 @@ impl<'a> GetQuery<'a> { } } -#[cfg(all(feature = "smtp", feature = "enterprise"))] +#[cfg(all(feature = "instance_smtp", feature = "enterprise"))] async fn send_workspace_trigger_failure_email_notification( db: &DB, w_id: &str, @@ -1855,7 +1855,7 @@ struct SendEmail { error: Value, } -#[cfg(all(feature = "enterprise", feature = "smtp"))] +#[cfg(all(feature = "enterprise", feature = "instance_smtp"))] async fn send_email_with_instance_smtp( authed: ApiAuthed, Extension(db): Extension, @@ -1913,7 +1913,7 @@ async fn send_email_with_instance_smtp( Ok(Json(resp)) } -#[cfg(not(all(feature = "enterprise", feature = "smtp")))] +#[cfg(not(all(feature = "enterprise", feature = "instance_smtp")))] async fn send_email_with_instance_smtp( _authed: ApiAuthed, Extension(_db): Extension, @@ -3192,17 +3192,9 @@ async fn resume_suspended( } // Check approval conditions - let approval_conditions = if is_wac { - flow.flow_status - .as_ref() - .and_then(|v| v.get("approval_conditions")) - .and_then(|v| serde_json::from_value::(v.clone()).ok()) - } else { - flow.flow_status - .as_ref() - .and_then(|v| serde_json::from_value::(v.clone()).ok()) - .and_then(|fs| fs.approval_conditions) - }; + let approval_conditions = extract_approval_conditions(flow.flow_status.as_ref(), is_wac); + + let trigger_email = flow.email.as_deref().unwrap_or(""); if let Some(ref ac) = approval_conditions { if ac.user_auth_required && opt_authed.is_none() { @@ -3214,6 +3206,13 @@ async fn resume_suspended( // If logged in, check authorization rules if let Some(ref authed) = opt_authed { + // self_approval_disabled applies to owners too (only admins are exempt), so it is + // enforced before the owner shortcut below. A token-only (anonymous) resume is treated as + // capability-based and intentionally not gated here; see resume_suspended_job. + if let Some(ref ac) = approval_conditions { + require_not_self_approval(authed, ac, trigger_email)?; + } + let is_admin = authed.is_admin; let is_owner = flow .script_path @@ -3222,7 +3221,6 @@ async fn resume_suspended( .unwrap_or(false); if !is_admin && !is_owner { - let trigger_email = flow.email.as_deref().unwrap_or(""); conditionally_require_authed_user( Some(authed.clone()), approval_conditions.clone(), @@ -3347,10 +3345,11 @@ struct ApprovalInfo { } /// Whether `opt_authed` is allowed to approve — and therefore view — this approval step. -/// Mirrors the authorization performed at the resume boundary: workspace admins and owners -/// of the runnable always qualify; otherwise the approval conditions (user_auth_required / -/// user_groups_required / self_approval_disabled) decide. When the step does not require auth, -/// an anonymous (token-only) caller qualifies. +/// Mirrors the authorization performed at the resume boundary: workspace admins always qualify; +/// self_approval_disabled then bars the triggerer even when they own the runnable; otherwise +/// owners qualify and the remaining approval conditions (user_auth_required / +/// user_groups_required) decide. When the step does not require auth, an anonymous (token-only) +/// caller qualifies. fn can_approve_step( opt_authed: &Option, approval_conditions: &Option, @@ -3362,6 +3361,12 @@ fn can_approve_step( if authed.is_admin { return true; } + // self_approval_disabled applies to owners too, so it gates the owner shortcut. + if let Some(ref ac) = approval_conditions { + if require_not_self_approval(authed, ac, trigger_email).is_err() { + return false; + } + } let is_owner = script_path .map(|p| require_owner_of_path(authed, p).is_ok()) .unwrap_or(false); @@ -3648,8 +3653,10 @@ async fn resume_suspended_job_internal( // Get flow info - works for step-level, flow-level, and WAC approval let (flow_info, is_flow_level, is_wac) = get_flow_info_for_resume(job_id, &db).await?; - // HMAC secret = full capability. Skip approval_conditions checks. - // Authorization rules are enforced by the new resume_suspended endpoint instead. + // HMAC secret = full capability. Skip approval_conditions checks: possession of the full + // resume URL is the authorization (it is only disclosed to intended approvers, e.g. when a + // step returns it). Identity-based rules, including self_approval_disabled, are enforced by + // the resume_suspended endpoint instead. let exists = sqlx::query_scalar!( r#" @@ -4095,6 +4102,45 @@ pub async fn get_suspended_job_flow( Ok(Json(SuspendedJobFlow { job: flow, approvers, view_token }).into_response()) } +/// Read the step's approval_conditions from the suspended flow status. For classic flows they +/// live inside the deserialized `FlowStatus`; for workflow-as-code they are a top-level +/// `approval_conditions` key in the status JSON. +fn extract_approval_conditions( + flow_status: Option<&serde_json::Value>, + is_wac: bool, +) -> Option { + if is_wac { + flow_status + .and_then(|v| v.get("approval_conditions")) + .and_then(|v| serde_json::from_value::(v.clone()).ok()) + } else { + flow_status + .and_then(|v| serde_json::from_value::(v.clone()).ok()) + .and_then(|fs| fs.approval_conditions) + } +} + +/// The flow's triggerer may not approve their own suspended step when the step sets +/// `self_approval_disabled`. Only admins are exempt: owning the runnable does not grant +/// the right to approve your own run, so this must be enforced at every resume boundary +/// independently of the owner shortcut (which only waives user_auth_required / +/// user_groups_required). +fn require_not_self_approval( + authed: &ApiAuthed, + approval_conditions: &ApprovalConditions, + trigger_email: &str, +) -> error::Result<()> { + if approval_conditions.self_approval_disabled + && !authed.is_admin + && authed.email.eq(trigger_email) + { + return Err(Error::PermissionDenied( + "Self-approval is disabled for this flow step".to_string(), + )); + } + Ok(()) +} + fn conditionally_require_authed_user( _authed: Option, approval_conditions_opt: Option, @@ -4106,14 +4152,8 @@ fn conditionally_require_authed_user( let approval_conditions = approval_conditions_opt.unwrap(); // Check self-approval independently of user_auth_required - if approval_conditions.self_approval_disabled { - if let Some(ref authed) = _authed { - if !authed.is_admin && authed.email.eq(_trigger_email) { - return Err(Error::PermissionDenied( - "Self-approval is disabled for this flow step".to_string(), - )); - } - } + if let Some(ref authed) = _authed { + require_not_self_approval(authed, &approval_conditions, _trigger_email)?; } if approval_conditions.user_auth_required { diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index d56928a712..ce66ec22da 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -94,6 +94,7 @@ mod granular_acls; mod group_history; mod groups; mod health; +mod hub_publish; #[cfg(feature = "private")] pub mod indexer_ee; mod indexer_oss; @@ -656,6 +657,7 @@ pub async fn run_server( .nest("/volumes", volumes_oss::workspaced_service()) .nest("/workers", windmill_api_workers::workspaced_service()) .nest("/workspaces", workspaces::workspaced_service()) + .nest("/hub", hub_publish::workspaced_service()) .nest( "/data_metrics", windmill_api_workspaces::data_metrics::workspaced_service(), diff --git a/backend/windmill-api/src/live_migrations.rs b/backend/windmill-api/src/live_migrations.rs index d5c759fc18..7c359300c3 100644 --- a/backend/windmill-api/src/live_migrations.rs +++ b/backend/windmill-api/src/live_migrations.rs @@ -19,6 +19,30 @@ pub async fn custom_migrations(migrator: &mut CustomMigrator) -> Result<(), Erro tracing::error!("Could not apply flow versioning fix migration: {err:#}"); } + if let Err(err) = normalize_custom_instance_user_attributes(migrator).await { + tracing::error!("Could not normalize custom_instance_user attributes: {err:#}"); + } + + Ok(()) +} + +// Converged on every boot, not once: the one-shot migration swallows errors (it must not +// abort startup without superuser), and an older instance sharing the cluster can re-add +// the attribute. REPLICATION belongs only on custom_instance_replication_user. +async fn normalize_custom_instance_user_attributes( + migrator: &mut CustomMigrator, +) -> Result<(), Error> { + let has_replication = sqlx::query_scalar::<_, bool>( + "SELECT rolreplication FROM pg_roles WHERE rolname = 'custom_instance_user'", + ) + .fetch_optional(migrator.connection()) + .await?; + if has_replication == Some(true) { + sqlx::query("ALTER ROLE custom_instance_user NOREPLICATION") + .execute(migrator.connection()) + .await?; + tracing::info!("Normalized custom_instance_user attributes"); + } Ok(()) } diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index d91fb209db..e45bba05c3 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -38,6 +38,7 @@ anyhow.workspace = true serde.workspace = true serde_json.workspace = true serde_yml.workspace = true +memchr.workspace = true erased-serde = "0.4" chrono.workspace = true chrono-tz.workspace = true diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index d1bbed1a18..d6a4156f38 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -567,11 +567,9 @@ mod trigger_ref_roundtrip_tests { // `trigger_spec_to_row` rebuilds a stored ref as `s3://`, and // `parse_asset_trigger_ref` parses it back. The two must be inverse for // every S3 URI form, or a consumer's `// on` trigger lands on a different - // graph node than the producer's inferred write. Because `parse_asset_syntax` - // strips ALL leading slashes, a canonical path never starts with `/`, so the - // naive `prefix + path` rebuild round-trips — including the `S3Object(s3="/x")` - // quad-slash case that previously desynced (path `/x` rebuilt to `s3:///x`, - // which re-parsed to `x`). + // graph node than the producer's inferred write. `parse_asset_syntax` + // keeps the URI suffix verbatim (a default-storage path starts with `/`), + // so the naive `prefix + path` rebuild round-trips for every form. fn roundtrip(uri: &str) -> String { let (pkind, path) = parse_asset_syntax(uri, false).expect("parse uri"); assert_eq!(pkind, PAssetKind::S3Object); @@ -590,11 +588,10 @@ mod trigger_ref_roundtrip_tests { #[test] fn s3_trigger_ref_roundtrips_for_every_uri_form() { - assert_eq!(roundtrip("s3:///exports/x"), "exports/x"); // SDK default storage - assert_eq!(roundtrip("s3://exports/x"), "exports/x"); // DuckDB / bare + assert_eq!(roundtrip("s3:///exports/x"), "/exports/x"); // SDK default storage + assert_eq!(roundtrip("s3://exports/x"), "exports/x"); // named storage `exports` assert_eq!(roundtrip("s3://mybucket/exports/x"), "mybucket/exports/x"); // explicit - assert_eq!(roundtrip("s3:////x"), "x"); // S3Object(s3="/x") quad-slash - assert_eq!(roundtrip("s3:///y=2024/f.parquet"), "y=2024/f.parquet"); // Hive + assert_eq!(roundtrip("s3:///y=2024/f.parquet"), "/y=2024/f.parquet"); // Hive } } diff --git a/backend/windmill-common/src/ee_oss.rs b/backend/windmill-common/src/ee_oss.rs index 26b9c87b7e..2510df0960 100644 --- a/backend/windmill-common/src/ee_oss.rs +++ b/backend/windmill-common/src/ee_oss.rs @@ -83,6 +83,11 @@ pub async fn enforce_offline_caps(_db: &DB) -> anyhow::Result
{owner_name} - {#if can_write} + {#if can_write && !restricted}
import { FolderService } from '$lib/gen' import { workspaceStore, userStore } from '$lib/stores' + import { isDemoWorkspaceRestricted } from '$lib/cloud' import { ChevronDown, Pen, PlusIcon } from 'lucide-svelte' import { Button, Drawer, DrawerContent } from './common' import FolderEditor from './FolderEditor.svelte' @@ -13,6 +14,10 @@ const VALID_FOLDER_NAME = /^[a-zA-Z_0-9-]+$/ + const restricted = $derived( + isDemoWorkspaceRestricted($workspaceStore, $userStore?.is_admin, $userStore?.is_super_admin) + ) + let folders: { name: string; write: boolean }[] = $state([]) let filterText: string = $state('') let selectOpen: boolean = $state(false) @@ -137,7 +142,7 @@ ) function handleSelectKeydown(e: KeyboardEvent) { - if (e.key === 'Enter' && selectOpen && noMatchingItems) { + if (e.key === 'Enter' && selectOpen && noMatchingItems && !restricted) { e.preventDefault() selectOpen = false openCreateFolder() @@ -233,18 +238,20 @@ /> {/snippet} {#snippet bottomSnippet({ close })} - + {#if !restricted} + + {/if} {/snippet}
diff --git a/frontend/src/lib/components/ForkWorkspaceBanner.svelte b/frontend/src/lib/components/ForkWorkspaceBanner.svelte index e2bfae895e..b97740b135 100644 --- a/frontend/src/lib/components/ForkWorkspaceBanner.svelte +++ b/frontend/src/lib/components/ForkWorkspaceBanner.svelte @@ -1,7 +1,8 @@ + +
+
+ {#snippet header()} + + The script will be executed on a worker configured to listen to this worker group tag + (queue). For instance, you could setup an "highmem", or "gpu" tag. + + {/snippet} + +
+ +
+ {#snippet header()} + + Allowed concurrency within a given timeframe + + {/snippet} + { + if (script.concurrent_limit && script.concurrent_limit != undefined) { + script.concurrent_limit = undefined + script.concurrency_time_window_s = undefined + script.concurrency_key = undefined + } else { + script.concurrent_limit = 1 + } + }} + options={{ right: 'Concurrency limits' }} + /> + {#if Boolean(script.concurrent_limit)} +
+ + + +
+ {/if} +
+ +
+ {#snippet header()} + + Cache the results for each possible inputs + + {/snippet} +
+ !!script.cache_ttl, (v) => (script.cache_ttl = v ? 300 : undefined)} + options={{ right: 'Cache the results for each possible inputs' }} + /> + {#if script.cache_ttl} +
How long to keep the cache valid
+ + script.cache_ignore_s3_path, (v) => (script.cache_ignore_s3_path = v || undefined) + } + options={{ + right: 'Ignore S3 Object paths for caching purposes', + rightTooltip: + 'If two S3 objects passed as input have the same content, they will hit the same cache entry, regardless of their path.' + }} + /> + {/if} +
+
+ +
+ {#snippet header()} + + Add a custom timeout for this script + + {/snippet} +
+ { + if (script.timeout && script.timeout != undefined) { + script.timeout = undefined + } else { + script.timeout = 300 + } + }} + options={{ right: 'Add a custom timeout for this script' }} + /> + {#if Boolean(script.timeout)} + Timeout duration + + {/if} +
+
+ +
+ {#snippet header()} + + Debounce Jobs + + {/snippet} + +
+ +
+ {#snippet header()} + + Restart the script upon ending unless cancelled + + {/snippet} + { + script.restart_unless_cancelled = script.restart_unless_cancelled ? undefined : true + }} + options={{ right: 'Restart upon ending unless cancelled' }} + /> +
+ +
+ {#snippet header()} + + In this mode, the script is meant to be run on dedicated workers that run the script at + native speed. Can reach >1500rps per dedicated worker. Only available on enterprise + edition and for Python3, Deno, Bun and Bunnative. + + {/snippet} + { + script.dedicated_worker = script.dedicated_worker ? undefined : true + }} + options={{ right: 'Script is run on dedicated workers' }} + /> + {#if script.dedicated_worker} +
+ + A worker group needs to be configured to listen to this script. Select it in the dedicated + workers section of the worker group configuration. + +
+ {/if} +
+ +
+ {#snippet header()} + + The logs, arguments and results of the job will be completely deleted from Windmill after + the specified delay once it is complete. Set to 0 for immediate deletion. The deletion is + irreversible. This settings ONLY applies when the script is used within a flow or triggered + synchronously. + {#if !$enterpriseLicense} + This option is only available on Windmill Enterprise Edition. + {/if} + + {/snippet} +
+ { + script.delete_after_secs = script.delete_after_secs != null ? undefined : 0 + }} + options={{ right: 'Delete logs, arguments and results after completion' }} + /> + {#if script.delete_after_secs != null} + + {/if} +
+
+ + {#if !isCloudHosted()} +
+ {#snippet header()} + + Jobs from script labeled as high priority take precedence over the other jobs when in the + jobs queue. + {#if !$enterpriseLicense}This is a feature only available on enterprise edition.{/if} + + {/snippet} + 0} + on:change={() => { + script.priority = script.priority ? undefined : 100 + }} + options={{ right: 'Label as high priority' }} + > + {#snippet right()} + { + if (script.priority && script.priority > 100) { + script.priority = 100 + } else if (script.priority && script.priority < 0) { + script.priority = 0 + } + }} + /> + {/snippet} + +
+ {/if} +
diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index d7214e9a24..4007f3bf0f 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -87,6 +87,7 @@ import DefaultScripts from './DefaultScripts.svelte' import { getContext, onMount, setContext, tick, untrack } from 'svelte' import EditorHeader from './EditorHeader.svelte' + import ScriptSettingsBadges from './ScriptSettingsBadges.svelte' import AutosaveIndicator from './AutosaveIndicator.svelte' import LabelsInput from './LabelsInput.svelte' @@ -1971,12 +1972,26 @@ {onOpenOthersDrafts} /> {/if} + {#if !condensedHeader} + {@const canOpenRuntime = + customUi?.topBar?.settings != false && + customUi?.settingsPanel?.disableRuntime !== true} + { + selectedTab = 'runtime' + metadataOpen = true + } + : undefined} + /> + {/if}
- {#if $enterpriseLicense && initialPath != ''} + {#if $enterpriseLicense && initialPath != '' && !inSessionPane} {/if} diff --git a/frontend/src/lib/components/ScriptSettingsBadges.svelte b/frontend/src/lib/components/ScriptSettingsBadges.svelte new file mode 100644 index 0000000000..8d73609891 --- /dev/null +++ b/frontend/src/lib/components/ScriptSettingsBadges.svelte @@ -0,0 +1,42 @@ + + +{#if badges.length > 0} +
+ {#each badges as badge (badge.key)} + + + onclick?.(badge.key) : undefined} + aria-label={`${badge.label}: ${badge.detail}`} + /> + {#snippet text()} + {badge.label} — {badge.detail} + {/snippet} + + {/each} +
+{/if} diff --git a/frontend/src/lib/components/ShareModal.svelte b/frontend/src/lib/components/ShareModal.svelte index 0996e6ca7b..29da96cf98 100644 --- a/frontend/src/lib/components/ShareModal.svelte +++ b/frontend/src/lib/components/ShareModal.svelte @@ -21,9 +21,14 @@ import { safeSelectItems } from './select/utils.svelte' import Toggle from './Toggle.svelte' import { Trash } from 'lucide-svelte' + import { DEMO_RESTRICTION_HINT, isDemoWorkspaceRestricted } from '$lib/cloud' const dispatch = createEventDispatcher() + let restricted = $derived( + isDemoWorkspaceRestricted($workspaceStore, $userStore?.is_admin, $userStore?.is_super_admin) + ) + type Kind = | 'script' | 'group_' @@ -42,6 +47,7 @@ | 'postgres_trigger' | 'gcp_trigger' | 'azure_trigger' + | 'amqp_trigger' | 'email_trigger' | 'volume' let kind: Kind @@ -268,7 +274,9 @@ > {/if}
- {#if own} + {#if own && restricted} + {DEMO_RESTRICTION_HINT} + {:else if own}
(owner = '')}> @@ -310,7 +318,7 @@
{owner} {#if own} + >{#if own && !restricted}
- {:else}{write}{/if}
{#if own} diff --git a/frontend/src/lib/components/SyncResourceTypes.svelte b/frontend/src/lib/components/SyncResourceTypes.svelte index dd343e43f6..acda8c0567 100644 --- a/frontend/src/lib/components/SyncResourceTypes.svelte +++ b/frontend/src/lib/components/SyncResourceTypes.svelte @@ -6,13 +6,19 @@ interface Props { onSynced?: () => void + // When set, the endpoint returns an explicit not-found error if the hub does + // not know this type (the sync itself still refreshes the whole list). + resourceType?: string } - let { onSynced = undefined }: Props = $props() + let { onSynced = undefined, resourceType = undefined }: Props = $props() let hubRtSync = usePromise( async () => { - const res = await fetch('/api/settings/sync_cached_resource_types', { method: 'POST' }) + const url = resourceType + ? `/api/settings/sync_cached_resource_types?name=${encodeURIComponent(resourceType)}` + : '/api/settings/sync_cached_resource_types' + const res = await fetch(url, { method: 'POST' }) if (!res.ok) { const body = await res.text() throw new Error(body || res.statusText) diff --git a/frontend/src/lib/components/TestConnection.svelte b/frontend/src/lib/components/TestConnection.svelte index 7c59a2a324..37e3e152db 100644 --- a/frontend/src/lib/components/TestConnection.svelte +++ b/frontend/src/lib/components/TestConnection.svelte @@ -3,15 +3,16 @@ import { Database, Loader2 } from 'lucide-svelte' import Button from './common/button/Button.svelte' + import Tooltip from './meltComponents/Tooltip.svelte' import { sendUserToast } from '$lib/toast' import { workspaceStore } from '$lib/stores' import { tryEvery } from '$lib/utils' interface Props { - workspaceOverride?: string | undefined; - resourceType: string | undefined; - args?: Record | any; - buttonTextOverride?: string | undefined; + workspaceOverride?: string | undefined + resourceType: string | undefined + args?: Record | any + buttonTextOverride?: string | undefined } let { @@ -19,13 +20,15 @@ resourceType, args = {}, buttonTextOverride = undefined - }: Props = $props(); + }: Props = $props() const scripts: { [key: string]: { code: string lang: string argName: string + // Shown as an info tooltip next to the button, e.g. to clarify where the test executes + tooltip?: string additionalCheck?: (testResult: CompletedJob) => CompletedJob } } = { @@ -97,7 +100,9 @@ export async function main(s3: S3) { } `, lang: 'bun', - argName: 's3' + argName: 's3', + tooltip: + 'The storage operations of this test run on the Windmill server (the API process), not on the worker. If no access key/secret key is set, the ambient AWS credentials of the server (environment variables, instance role) are used — scripts using this resource directly through an S3 SDK resolve credentials on the worker instead, so results may differ.' }, azure_blob: { code: ` @@ -125,7 +130,9 @@ export async function main(s3: S3) { } `, lang: 'bun', - argName: 's3' + argName: 's3', + tooltip: + 'The storage operations of this test run on the Windmill server (the API process), not on the worker.' }, graphql: { code: '{ __typename }', @@ -171,7 +178,9 @@ export async function main(bucket: any) { } `, lang: 'bun', - argName: 'bucket' + argName: 'bucket', + tooltip: + "The storage operations of this test run on the Windmill server (the API process). If no credentials are configured, the server's ambient credentials for the configured provider (environment variables, instance role) are used." } } @@ -236,13 +245,20 @@ export async function main(bucket: any) { } -{#if Object.keys(scripts).includes(resourceType || '')} - + {#if scripts[resourceType].tooltip} + + {#snippet text()}{scripts[resourceType].tooltip}{/snippet} + {/if} - {buttonTextOverride ?? 'Test connection'} - + {/if} diff --git a/frontend/src/lib/components/WorkspaceDeployLayout.svelte b/frontend/src/lib/components/WorkspaceDeployLayout.svelte index b7d87d6321..491d649a44 100644 --- a/frontend/src/lib/components/WorkspaceDeployLayout.svelte +++ b/frontend/src/lib/components/WorkspaceDeployLayout.svelte @@ -34,6 +34,7 @@ deploymentStatus: Record allSelected?: boolean emptyMessage?: string + hideSelection?: boolean children?: Snippet // Snippets for customization @@ -64,6 +65,7 @@ deploymentStatus, allSelected = false, emptyMessage = 'No items to deploy', + hideSelection = false, header, alerts, selectAllActions, @@ -150,12 +152,13 @@ {@render alerts()} {/if} - + {#if items.length > 0 || selectAllActions}
- {#if items.length > 0} + {#if items.length > 0 && !hideSelection}
{/if} - {#if $enterpriseLicense && $appPath != ''} + {#if $enterpriseLicense && $appPath != '' && !inSessionPane}
diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte index 64bd2790cb..3e5e9d8cef 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte @@ -171,6 +171,19 @@ * initial viewport fit re-arms when it changes, so switching folders * in-place gets a fresh fit. */ viewportFitKey?: string + /** Tint the currently-running runnable nodes amber (the compute is + * happening now). Opt-in — the recorder's replay player turns it on so + * the active transform stands out; the live editor stays calm. */ + highlightActiveRun?: boolean + /** Asset node id (`asset:${kind}:${path}`) → a monotonic "recompute" nonce. + * When a node's nonce changes, it flashes a fading green background — its + * producer just recomputed it. Driven by the replay player frame-by-frame. */ + recomputedAssetIds?: ReadonlyMap + /** Let the wheel zoom the canvas (and swallow the page scroll while doing + * so). Default true for the full-height editor/player. Set false when the + * canvas is embedded inline inside a scrollable container, so a wheel + * gesture over it scrolls the container instead of being captured. */ + scrollZoom?: boolean } let { graph, @@ -199,7 +212,10 @@ boundPick, onPickEnd, showMinimap = true, - viewportFitKey = '' + viewportFitKey = '', + highlightActiveRun = false, + recomputedAssetIds, + scrollZoom = true }: Props = $props() // `${kind}:${path}` ids for the hovered / pinned runs (both script and flow @@ -385,7 +401,10 @@ producers: producersByAsset.get(`${a.kind}:${a.path}`) ?? [], onRunProducer, dataTestGuarded, - producerFailed + producerFailed, + // Bumped by the replay player when this asset's producer just + // recomputed it — the node flashes green and fades. + recomputePulse: recomputedAssetIds?.get(assetId) } }) } @@ -474,6 +493,9 @@ downstreamCount: downstreamByScript.get(r.path) ?? 0, downstreamUnsavedCount: downstreamUnsavedByScript.get(r.path) ?? 0, runState, + // Amber-tint this node while it's the transform actively running + // (replay player only — the live editor keeps its calm styling). + highlightRunning: highlightActiveRun, // Bounded-cascade entrypoint: only valid starts (schedule / // manual roots) with downstream get the "Run downstream up // to…" menu item. @@ -1172,6 +1194,8 @@ nodesDraggable={false} nodesConnectable={false} elementsSelectable + zoomOnScroll={scrollZoom} + preventScrolling={scrollZoom} zoomOnDoubleClick={false} connectionLineType={ConnectionLineType.SmoothStep} defaultEdgeOptions={{ type: 'asset' }} diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte index 425565353a..d48ecbe09e 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte @@ -493,6 +493,10 @@ // restores its input. Guarded on the path so a staging round-trip // (emit → page → runFormInitialArgs) doesn't re-seed and loop. The read-only // branch uses PipelineScriptView's own onArgsChange instead. + // Declared before the pre-effect that seeds it: a `$state` referenced by an + // earlier-registered `$effect.pre` hits a TDZ ("Cannot access 'args' before + // initialization") when the pane remounts and the pre-effect runs before this + // line executes. let args = $state>({}) let argsSeedPath: string | undefined = undefined $effect.pre(() => { diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte index 762bdbc0fc..999559a0c7 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte @@ -74,6 +74,10 @@ // asset failed. Escalates the guard badge from "protected" to a // failed-run outcome (rolled-back on EE, published-anyway on CE). producerFailed?: boolean + // Monotonic nonce bumped by the replay player when this asset's + // producer just recomputed it. A change triggers a one-shot green + // fade so a freshly-written table stands out as the run progresses. + recomputePulse?: number } // SvelteFlow injects this on the node component when the user clicks // the node. Combined with our own `hovered` state to drive the @@ -170,6 +174,13 @@ onmouseleave={() => (hovered = false)} role="presentation" > + {#if data.recomputePulse !== undefined} + {#key data.recomputePulse} + +
+ {/key} + {/if}
+ + diff --git a/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte b/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte index 4cb65ee8f3..d8a977539b 100644 --- a/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte @@ -48,6 +48,9 @@ // Last-run status + run count observed this session (from the // folder queue poll). Undefined until the first observed run. runState?: RunnableRunState + // Opt-in (replay player): when this node is the transform actively + // running, tint its whole surface amber so the compute is unmissable. + highlightRunning?: boolean // True for nodes synthesized from local drafts (script not yet // persisted). Same convention as `unsaved` on triggers/edges. unsaved?: boolean @@ -103,6 +106,9 @@ let hover = $state(false) let menuOpen = $state(false) let running = $state(false) + + // Amber "computing now" surface, gated so only the replay player lights it up. + let computingNow = $derived(data.highlightRunning === true && data.runState?.status === 'running') // Popover state for the on-node Run-button caret. Sticky while open so // `showRun` (which gates the whole pill) stays true even after the // pointer leaves the node — otherwise picking an option would unmount @@ -195,7 +201,9 @@ 'flex items-center rounded-md drop-shadow-sm overflow-hidden border transition-colors', 'bg-surface border-gray-400 dark:border-gray-600 hover:border-gray-500 dark:hover:border-gray-500', selected && 'bg-surface-accent-selected border-border-selected', - data.unsaved && 'border-2 border-dashed border-gray-400 dark:border-gray-500' + data.unsaved && 'border-2 border-dashed border-gray-400 dark:border-gray-500', + computingNow && + 'bg-amber-50 dark:bg-amber-900/30 border-amber-400 dark:border-amber-600 animate-pulse' )} style="width: {NODE.width}px; min-height: {NODE.height}px;" title={nodeTooltip} diff --git a/frontend/src/lib/components/assets/AssetGraph/boundedCascade.test.ts b/frontend/src/lib/components/assets/AssetGraph/boundedCascade.test.ts index e4692de6b8..89c9f24708 100644 --- a/frontend/src/lib/components/assets/AssetGraph/boundedCascade.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/boundedCascade.test.ts @@ -509,15 +509,13 @@ describe('assetUriToNodeId', () => { expect(assetUriToNodeId('ducklake://lake/t')).toBe('ducklake:lake/t') expect(assetUriToNodeId('not-a-uri')).toBeUndefined() }) - it('strips leading slashes from S3 keys so s3:/// and s3:// share a node', () => { - // Mirror of Rust `parse_asset_syntax`: `--to s3:///exports/x` must resolve - // to the same canonical node as the graph's `s3object:exports/x`. - expect(assetUriToNodeId('s3:///exports/x')).toBe('s3object:exports/x') - expect(assetUriToNodeId('s3:///exports/x')).toBe(assetUriToNodeId('s3://exports/x')) - // All leading slashes are stripped so a canonical key never starts with - // `/` (the quad-slash `S3Object(s3="/x")` form collapses to `x`). - expect(assetUriToNodeId('s3:////x')).toBe('s3object:x') + it('keeps the S3 storage distinction (verbatim suffix)', () => { + // Mirror of Rust `parse_asset_syntax`: the suffix is kept verbatim, so a + // default-storage `s3:///exports/x` resolves to `s3object:/exports/x` + // while `s3://exports/x` names storage `exports` — a different node. + expect(assetUriToNodeId('s3:///exports/x')).toBe('s3object:/exports/x') + expect(assetUriToNodeId('s3://exports/x')).toBe('s3object:exports/x') // Hive-partition keys and non-S3 kinds are untouched. - expect(assetUriToNodeId('s3:///t/y=2024/f.parquet')).toBe('s3object:t/y=2024/f.parquet') + expect(assetUriToNodeId('s3:///t/y=2024/f.parquet')).toBe('s3object:/t/y=2024/f.parquet') }) }) diff --git a/frontend/src/lib/components/assets/AssetGraph/boundedCascade.ts b/frontend/src/lib/components/assets/AssetGraph/boundedCascade.ts index 810ab36cc1..f3ed9a4f66 100644 --- a/frontend/src/lib/components/assets/AssetGraph/boundedCascade.ts +++ b/frontend/src/lib/components/assets/AssetGraph/boundedCascade.ts @@ -38,11 +38,10 @@ export function assetUriToNodeId(uri: string): string | undefined { // `s3` is the URI prefix for the `s3object` asset kind (mirrors the CLI // `assetUri` and the canvas). All other kinds use their name verbatim. const kind = prefix === 's3' ? 's3object' : prefix - // Mirror Rust `parse_asset_syntax`: strip all leading slashes from S3 keys so - // `s3:///key` (default storage) and `s3://key` resolve to the same node id - // and a canonical key never starts with `/`. - const path = kind === 's3object' ? m[2].replace(/^\/+/, '') : m[2] - return `${kind}:${path}` + // The suffix is kept verbatim (mirrors Rust `parse_asset_syntax`): an S3 + // path encodes the storage, with a leading `/` for the workspace default + // (`s3:///key` → `/key`) vs `s3://secondary/key` → `secondary/key`. + return `${kind}:${m[2]}` } // Native trigger kinds that fan out *per event*: a single event always flows diff --git a/frontend/src/lib/components/assets/AssetGraph/cascadeRun.ts b/frontend/src/lib/components/assets/AssetGraph/cascadeRun.ts index ede66ebe4b..407dc886f9 100644 --- a/frontend/src/lib/components/assets/AssetGraph/cascadeRun.ts +++ b/frontend/src/lib/components/assets/AssetGraph/cascadeRun.ts @@ -20,6 +20,11 @@ import type { AssetGraphResponse } from './types' export const CASCADE_POLL_INTERVAL_MS = 1000 export const CASCADE_JOB_TIMEOUT_MS = 30 * 60 * 1000 +// Data-asset kinds a pipeline graph resolves — the `asset_kinds` filter for the +// `/assets/graph` fetch. Shared so the pipeline editor and deploy-to-hub request +// the same nodes/edges (and can't silently diverge when a kind is added). +export const DATA_ASSET_KINDS = ['s3object', 'ducklake', 'datatable', 'volume'] + export type LocalScriptContent = { content: string language: Preview['language'] diff --git a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts index 34b9ac2eab..5fdbc4c748 100644 --- a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts +++ b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts @@ -277,18 +277,11 @@ function stripTrailingKvOpts(s: string): string { function parseAssetSyntax(s: string): PipelineTriggerAsset | undefined { for (const [prefix, kind] of ASSET_PREFIXES) { if (s.startsWith(prefix)) { - let path = s.slice(prefix.length) - // Mirror the Rust `parse_asset_syntax` S3 canonicalization: strip all - // leading slashes so the SDK object form (`s3:///key`, default - // storage) and DuckDB / `// on s3://key` share one asset path, and a - // canonical key never starts with `/` (so ref reconstruction - // round-trips). Without this the live graph preview would show - // disconnected `/key` and `key` nodes. S3-only; leading slashes only, - // so Hive-partition keys are untouched. - if (kind === 's3object') { - path = path.replace(/^\/+/, '') - } - return { kind, path } + // The suffix is kept verbatim, mirroring the Rust `parse_asset_syntax`. + // For S3 the path encodes the storage: `s3:///key` yields `/key` + // (default storage, leading slash significant) while + // `s3://secondary/key` yields `secondary/key` — two different objects. + return { kind, path: s.slice(prefix.length) } } } return undefined diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.test.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.test.ts index a1a82870c8..e5f3b74e3d 100644 --- a/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.test.ts @@ -114,6 +114,37 @@ describe('pipeline AI direct-draft helpers', () => { expect(drafts().size).toBe(0) }) + // The output_kind seed is a random placeholder that live inference overwrites + // and deploy re-derives, so it must not read as detected lineage — else a + // dynamic/unwritten output looks wired when the deployed script has no edge. + it('does not report the output_kind seed as detected lineage', async () => { + vi.spyOn(ScriptService, 'getScriptByPath').mockRejectedValue(new Error('404')) + const { handle, drafts } = makeHandle() + const res = await handle.proposeNode({ + path: 'f/x/seeded', + language: 'duckdb' as any, + content: '-- pipeline\n-- on schedule\nSELECT 1', + outputKind: 'ducklake' as any + }) + expect(drafts().get('f/x/seeded')?.outputAssets?.length).toBeGreaterThan(0) + expect(res.detectedWrites).toEqual([]) + }) + + // `inferAssets` returns the `// materialize` target separately from body + // writes, so it must be folded into detectedWrites or a canonical materialize + // node would falsely read as having no output. + it('reports the `-- materialize` target as a detected write', async () => { + vi.spyOn(ScriptService, 'getScriptByPath').mockRejectedValue(new Error('404')) + const { handle } = makeHandle() + const res = await handle.proposeNode({ + path: 'f/x/mat', + language: 'duckdb' as any, + content: '-- pipeline\n-- on schedule\n-- materialize ducklake://main/out\nSELECT 1', + outputKind: 'ducklake' as any + }) + expect(res.detectedWrites).toEqual(['ducklake://main/out']) + }) + it('editNode rejects a path outside the open folder', async () => { const { handle, drafts } = makeHandle() await expect(handle.editNode('f/other/foo', '-- pipeline')).rejects.toThrow(/open folder/) diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.ts index 3658bc5a32..a6874d9b64 100644 --- a/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.ts +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.ts @@ -7,7 +7,7 @@ import { type AssetWithAltAccessType } from '$lib/components/assets/lib' import { assetUri, autoOutputAsset, type PipelineOutputKind } from './pipelineTemplates' -import { parsePipelineAnnotations } from './parsePipelineAnnotations' +import { parsePipelineAnnotations, scd2CurrentTargetPath } from './parsePipelineAnnotations' import type { AssetGraphResponse } from './types' import type { PipelineAIChatHelpers, @@ -107,6 +107,31 @@ async function inferDraftAssets( } } +// `inferAssets` returns body reads/writes but NOT the `// materialize` target, +// which the parser surfaces separately (resolveGraph adds it the same way). +// A managed materialize node's output is real and deployable, so fold its +// target(s) into the detected writes. +function materializeWrites(content: string): Array<{ kind: AssetKind; path: string }> { + const m = parsePipelineAnnotations(content).materialize + if (!m) return [] + const out = [{ kind: m.targetKind, path: m.targetPath }] + const current = scd2CurrentTargetPath(m) + if (current) out.push({ kind: m.targetKind, path: current }) + return out +} + +function dedupeAssets( + assets: Array<{ kind: AssetKind; path: string }> +): Array<{ kind: AssetKind; path: string }> { + const seen = new Set() + return assets.filter((a) => { + const key = `${a.kind}:${a.path}` + if (seen.has(key)) return false + seen.add(key) + return true + }) +} + export function createPipelineAiHelpers(deps: PipelineAiHelperDeps): PipelineAIChatHelpers { // A staged draft is always persisted into the OPEN folder's data_pipeline // bundle, so a path outside the folder would silently land an unrelated script @@ -240,17 +265,30 @@ export function createPipelineAiHelpers(deps: PipelineAiHelperDeps): PipelineAIC (outputKind ? autoOutputAsset(outputKind as PipelineOutputKind, deps.getFolder(), language) : undefined) + // Effective outputs = what actually becomes an output edge on the canvas: + // body/annotation-inferred writes, or the output_kind seed as a fallback. + const outputAssets = + inferred.writes.length > 0 ? inferred.writes : seeded ? [seeded] : undefined const next = new Map(drafts) next.set(path, { localId: deps.newDraftLocalId(), script: makePipelineScript(language, path, content, new Date().toISOString()), - outputAssets: inferred.writes.length > 0 ? inferred.writes : seeded ? [seeded] : undefined, + outputAssets, inputAssets: inferred.reads }) deps.setDrafts(next) deps.onShowDrafts?.() deps.onProposeNode?.(path) - return { path } + // Report deployable lineage only (body writes + `// materialize` target), + // never the random `seeded` placeholder — else a dynamic/unwritten output + // reads as wired when the deployed script has no such edge. + return { + path, + detectedReads: inferred.reads.map(assetUri), + detectedWrites: dedupeAssets([...inferred.writes, ...materializeWrites(content)]).map( + assetUri + ) + } }, editNode: async (path, content) => { deps.ensureEditable?.() @@ -273,16 +311,24 @@ export function createPipelineAiHelpers(deps: PipelineAiHelperDeps): PipelineAIC baseScript = await ScriptService.getScriptByPath({ workspace, path }) } const inferred = await inferDraftAssets(baseScript.language, content) + const outputAssets = inferred.writes.length > 0 ? inferred.writes : existing?.outputAssets const next = new Map(drafts) next.set(path, { localId: existing?.localId ?? deps.newDraftLocalId(), script: { ...baseScript, content }, - outputAssets: inferred.writes.length > 0 ? inferred.writes : existing?.outputAssets, + outputAssets, inputAssets: inferred.reads }) deps.setDrafts(next) deps.onShowDrafts?.() deps.onProposeNode?.(path) + // Deployable lineage only (see proposeNode): body writes + materialize target. + return { + detectedReads: inferred.reads.map(assetUri), + detectedWrites: dedupeAssets([...inferred.writes, ...materializeWrites(content)]).map( + assetUri + ) + } }, removeProposedNode: async (path) => { if (!deps.getDrafts().has(path)) { diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.test.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.test.ts index 00fdd085ed..08ec089491 100644 --- a/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.test.ts @@ -9,16 +9,15 @@ import { // The seeded draft asset (`autoOutputAsset`, stored as `outputAssets` and used // by resolveGraph for inactive-draft node identity) must match the asset // identity the deploy-time / wasm parser infers from the generated body. The -// parser canonicalizes any S3 URI by stripping the `s3://` prefix and all -// leading slashes (see backend `parse_asset_syntax`); if the seed carried a -// leading slash while the body wrote `s3:///key`, the preview would render a -// duplicate `/key` node and a phantom post-deploy drift. This pins the two in -// lockstep so that class of drift can't regress. +// parser keeps the suffix after `s3://` verbatim (see backend +// `parse_asset_syntax`), so a default-storage object's path carries a leading +// slash (`s3:///key` → `/key`). If the seed and the body's write URI disagree, +// the preview renders a duplicate node and a phantom post-deploy drift. This +// pins the two in lockstep so that class of drift can't regress. -// Mirror of the parser's S3 canonicalization for a raw `s3://…` URI. +// Mirror of the parser's S3 path extraction for a raw `s3://…` URI. function canonicalS3Key(uri: string): string { - const rest = uri.replace(/^s3:\/\//, '') - return rest.replace(/^\/+/, '') + return uri.replace(/^s3:\/\//, '') } const S3_KINDS: PipelineOutputKind[] = ['s3_parquet', 's3_object'] @@ -32,10 +31,10 @@ describe('pipelineTemplates S3 seed/body parity', () => { expect(output).toBeDefined() const asset = output! - // The seed must be a canonical slashless key so it matches the - // identity the parser infers from the generated body. + // The seed must carry the default-storage leading slash so it + // matches the identity the parser infers from the generated body. expect(asset.kind).toBe('s3object') - expect(asset.path.startsWith('/')).toBe(false) + expect(asset.path.startsWith('/')).toBe(true) const body = generatePipelineDraft({ language, diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts index bf0c14d16e..1e26f2bb31 100644 --- a/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts @@ -175,16 +175,16 @@ export function autoOutputAsset( case 'ducklake': case 'materialize': return { kind: 'ducklake', path: `main/${adj}_${pick(TABLE_NOUNS)}_${slug}` } - // s3 outputs use the canonical slashless key. `parse_asset_syntax` - // normalizes `s3:///` (default storage) and `s3://` to the - // bare ``, so the seeded draft asset must be slashless to match the - // deploy-time inferred identity — otherwise the post-deploy drift check - // would flag the output as a phantom `/`-prefixed node. The generated - // bodies still emit the `s3:///` default-storage URI for runtime I/O. + // s3 paths carry the canonical leading slash of a default-storage + // object (`s3:///` parses to path `/`). The deploy-time + // parser stores writes in that form — a slashless seeded path would + // never match it, and the post-deploy drift check would report the + // output as lost (it isn't; the key differs by one '/'). Bodies emit + // the path verbatim after `s3://`, so the slash round-trips. case 's3_parquet': return { kind: 's3object', - path: `pipelines/${folder}/${adj}_${pick(DATASET_NOUNS)}_${slug}.parquet` + path: `/pipelines/${folder}/${adj}_${pick(DATASET_NOUNS)}_${slug}.parquet` } case 's3_object': { // duckdb's natural output for a generic blob is CSV (one COPY TO @@ -194,7 +194,7 @@ export function autoOutputAsset( const ext = language === 'duckdb' ? 'csv' : 'json' return { kind: 's3object', - path: `pipelines/${folder}/${adj}_${pick(FILE_NOUNS)}_${slug}.${ext}` + path: `/pipelines/${folder}/${adj}_${pick(FILE_NOUNS)}_${slug}.${ext}` } } // A macro library produces no asset — its "output" is the registry @@ -222,13 +222,6 @@ export function assetUri(asset: { kind: AssetKind; path: string }): string { return `${ASSET_URI_PREFIX[asset.kind]}${asset.path}` } -// Bare object key for the SDK's `{ s3: }` / `s3:///` forms. Asset -// paths are already canonical slashless keys; strip stray leading slashes -// defensively so the emitted key never starts with '/'. -function s3Key(path: string): string { - return path.replace(/^\/+/, '') -} - // Splits a datatable asset path (`/` or `/.
`) // into its constituent parts. The `.
` grammar is owned by // `parseDbInputFromAssetSyntax` in $lib/utils.ts (which consumes a full @@ -438,11 +431,12 @@ function bodyTs(ctx: TemplateContext): string { if (!input) return '' switch (input.kind) { case 's3object': - // `s3:///` URI — one spelling shared with the `// on - // s3:///…` annotation form (the object literal `{ s3: }` - // is equivalent). + // `input.path` encodes storage as `/` (an empty + // storage segment — leading slash — is the workspace default). + // Emit it verbatim after `s3://` so a named-storage input keeps + // its storage; stripping the slash reads the default-storage key. return [ - ` const buf = await wmill.loadS3File(${JSON.stringify(`s3:///${s3Key(input.path)}`)})`, + ` const buf = await wmill.loadS3File(${JSON.stringify(`s3://${input.path}`)})`, ` const rows = JSON.parse(new TextDecoder().decode(buf))`, `` ].join('\n') @@ -468,10 +462,10 @@ function bodyTs(ctx: TemplateContext): string { switch (outputKind) { case 's3_parquet': case 's3_object': - // `s3:///` URI — see the loadS3File note above. + // `s3:///` URI — see the loadS3File note above. return [ ` const payload = new TextEncoder().encode(JSON.stringify(rows))`, - ` await wmill.writeS3File(${JSON.stringify(`s3:///${s3Key(output.path)}`)}, payload)` + ` await wmill.writeS3File(${JSON.stringify(`s3://${output.path}`)}, payload)` ].join('\n') case 'datatable': { const dbName = output.path.split('/')[0] ?? 'main' @@ -525,11 +519,12 @@ function bodyPython(ctx: TemplateContext): string { if (!input) return '' switch (input.kind) { case 's3object': - // `s3:///` URI — SDK string params must be s3:// URIs - // (bare keys are rejected), and this form matches the - // `# on s3:///…` annotation spelling. + // SDK string params must be s3:// URIs (bare keys are rejected). + // `input.path` encodes storage as `/` (empty storage + // segment — leading slash — is the workspace default), so emit it + // verbatim after `s3://` to preserve a named-storage input. return [ - ` buf = wmill.load_s3_file(${JSON.stringify(`s3:///${s3Key(input.path)}`)})`, + ` buf = wmill.load_s3_file(${JSON.stringify(`s3://${input.path}`)})`, ` import json; rows = json.loads(buf.decode("utf-8"))` ].join('\n') case 'datatable': @@ -552,10 +547,10 @@ function bodyPython(ctx: TemplateContext): string { switch (outputKind) { case 's3_parquet': case 's3_object': - // `s3:///` URI — see the load_s3_file note above. + // `s3:///` URI — see the load_s3_file note above. return [ ` import json`, - ` wmill.write_s3_file(${JSON.stringify(`s3:///${s3Key(output.path)}`)}, json.dumps(rows).encode("utf-8"))` + ` wmill.write_s3_file(${JSON.stringify(`s3://${output.path}`)}, json.dumps(rows).encode("utf-8"))` ].join('\n') case 'datatable': { const dbName = output.path.split('/')[0] ?? 'main' @@ -647,7 +642,7 @@ function bodyDuckdb(ctx: TemplateContext): string { if (!input) return null switch (input.kind) { case 's3object': - return `read_parquet('s3:///${input.path}')` + return `read_parquet('s3://${input.path}')` case 'datatable': // `pg` is the attached Postgres catalog (see ATTACH above). // Use a 2-part `pg.
` ref so the asset parser maps it @@ -670,7 +665,7 @@ function bodyDuckdb(ctx: TemplateContext): string { `COPY (`, ` SELECT *`, ` FROM ${fromExpr}`, - `) TO 's3:///${output.path}' (FORMAT 'parquet');` + `) TO 's3://${output.path}' (FORMAT 'parquet');` ) } break @@ -681,7 +676,7 @@ function bodyDuckdb(ctx: TemplateContext): string { `COPY (`, ` SELECT *`, ` FROM ${fromExpr}`, - `) TO 's3:///${output.path}' (FORMAT 'csv', HEADER);` + `) TO 's3://${output.path}' (FORMAT 'csv', HEADER);` ) } break diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index aefa123713..a267e71e8c 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -39,6 +39,7 @@ import { getAiChatManager } from './aiChatManagerContext' import ChatTypingIndicator from './ChatTypingIndicator.svelte' import AIChatInput from './AIChatInput.svelte' + import AttachedFilesBar from './files/AttachedFilesBar.svelte' import QueuedMessageChip from './QueuedMessageChip.svelte' import JobsSegment from './JobsSegment.svelte' import { getModifierKey } from '$lib/utils' @@ -272,8 +273,8 @@ // File attachment is GLOBAL-mode only. const canAttachFiles = $derived(aiChatManager.mode === AIMode.GLOBAL && !disabled) - // Steers the OS file picker toward text + image formats (soft hint; images attach to - // the message, other files link as text context after a content sniff). + // Steers the OS file picker toward text + image formats (soft hint; both attach + // to the message — text files after a content sniff). const TEXT_FILE_ACCEPT = 'image/*,text/*,.txt,.csv,.tsv,.json,.jsonl,.ndjson,.md,.markdown,.log,.yaml,.yml,.toml,.ini,.cfg,.conf,.env,.xml,.html,.htm,.css,.js,.mjs,.cjs,.ts,.tsx,.jsx,.py,.rb,.rs,.go,.java,.kt,.c,.h,.cpp,.cc,.cs,.php,.sh,.bash,.zsh,.sql,.svelte,.vue,.dockerfile' let fileInputEl = $state(null) @@ -361,16 +362,30 @@ e.preventDefault() const dt = e.dataTransfer if (!dt) return - // Images attach to the message; other files link as text context. Images are - // reserved from dt.files BEFORE any await (a send mid-ingestion would land - // them on the next message), and dt.files is the only place a disk-less drag - // exists — a cross-tab image resolves every getAsFileSystemHandle() to null. + // Images and loose text files attach to the message; folders link as session + // assets. Images are reserved from dt.files BEFORE any await (a send + // mid-ingestion would land them on the next message), and dt.files is the + // only place a disk-less drag exists — a cross-tab image resolves every + // getAsFileSystemHandle() to null. const flatFiles = Array.from(dt.files ?? []) const topLevelImages = flatFiles.filter(isImageFile) const imageWork: Promise[] = [] if (topLevelImages.length > 0) { imageWork.push(aiChatInput?.addImages(topLevelImages) ?? Promise.resolve()) } + // Text-file routing must await handle/entry resolution before it can call + // addTextFiles — hold sending across that window (taken BEFORE the first + // await) or a send mid-resolution would land the drop on the next message. + const releaseSendHold = aiChatInput?.holdSendForIngestion() + try { + await routeDroppedTextAndFolders(dt, flatFiles) + } finally { + releaseSendHold?.() + } + await Promise.all(imageWork) + } + + async function routeDroppedTextAndFolders(dt: DataTransfer, flatFiles: File[]) { if (canUseFsAccess) { // getAsFileSystemHandle calls are kicked off synchronously inside this call. const handles = await handlesFromDataTransfer(dt) @@ -381,9 +396,9 @@ handles.length === 0 ? flatFiles : await Promise.all(handles.filter(isFileHandle).map((h) => h.getFile())) - // Files are always snapshotted (handle discarded). + // Loose text files attach to the message, like images. const textFiles = looseFiles.filter((f) => !isImageFile(f)) - if (textFiles.length > 0) await handleAddFiles(textFiles) + if (textFiles.length > 0) await aiChatInput?.addTextFiles(textFiles) // Folders link as a live handle. for (const h of handles.filter(isDirectoryHandle)) { await addDirHandle(h) @@ -395,19 +410,25 @@ // (no entry API), fall back to the flat dt.files. const entries = await readDroppedEntries(Array.from(dt.items ?? [])) const source: FileToAttach[] = entries.length > 0 ? entries : flatFiles - // Only top-level images attach to the message, and those were already - // reserved from dt.files before the walk — drop them here so they aren't - // re-reported as skipped non-text. Folder-nested images are deliberately - // NOT attached (the FSA path never extracts folder contents either); they - // ride the text ingestion and are summarized as skipped. - const textEntries = source.filter((entry) => { + // Top-level files attach to the message (images were already reserved + // from dt.files before the walk). Folder children keep riding the + // session store as a snapshot — including nested images, which are + // deliberately NOT attached (the FSA path never extracts folder + // contents either); they are summarized as skipped there. + const topLevelText: File[] = [] + const folderEntries: FileToAttach[] = [] + for (const entry of source) { const file = entry instanceof File ? entry : entry.file - const nested = !(entry instanceof File) && entry.path?.includes('/') - return !isImageFile(file) || !!nested - }) - if (textEntries.length > 0) await handleAddFiles(textEntries) + const nested = !(entry instanceof File) && !!entry.path?.includes('/') + if (nested) { + folderEntries.push(entry) + } else if (!isImageFile(file)) { + topLevelText.push(file) + } + } + if (folderEntries.length > 0) await handleAddFiles(folderEntries) + if (topLevelText.length > 0) await aiChatInput?.addTextFiles(topLevelText) } - await Promise.all(imageWork) } async function onFileInputChange(e: Event) { @@ -418,7 +439,7 @@ const textFiles = picked.filter((f) => !isImageFile(f)) // Reserved before the text work is awaited — see onPanelDrop. const imageWork = imageFiles.length > 0 ? aiChatInput?.addImages(imageFiles) : undefined - if (textFiles.length > 0) await handleAddFiles(textFiles) + if (textFiles.length > 0) await aiChatInput?.addTextFiles(textFiles) await imageWork } input.value = '' // allow re-selecting the same file @@ -754,10 +775,11 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> {/if} - + mention deselects). Hence showContext={false} below. Session-scoped + assets (attached files/folders) render in the footer row instead. --> {#if inputPreface} {@render inputPreface()} {/if} @@ -863,12 +885,12 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. -->

Attach files or link a folder

- Text files stay in your browser, and a folder is linked live from disk. - The assistant lists, searches, and reads them on demand, so their contents - are sent only when it reads one. + Files and images attach to your next message. Images are seen directly; + file contents stay in your browser and are read on demand.

- Images are sent with your next message, so the assistant can see them. + A linked folder is a session-wide resource: the assistant lists, searches, + and reads its files whenever it needs them.

{/snippet} @@ -876,7 +898,7 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> {/snippet} + `accept` only steers the picker; the content sniff at attach is authoritative. --> {:else}
+ {#if aiChatManager.mode === AIMode.GLOBAL} + + {/if} {#if !hideModeSelector} {/if} diff --git a/frontend/src/lib/components/copilot/chat/AIChatInput.svelte b/frontend/src/lib/components/copilot/chat/AIChatInput.svelte index 427856d291..336246b368 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatInput.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatInput.svelte @@ -2,10 +2,10 @@ import AppAvailableContextList from './AppAvailableContextList.svelte' import ContextElementBadge from './ContextElementBadge.svelte' import ContextTextarea from './ContextTextarea.svelte' - import AttachedFilesBar from './files/AttachedFilesBar.svelte' import autosize from '$lib/autosize' import { contextElementKey, + createAttachedFileContextElement, isSameContextElement, type AppDomSelectorElement, type ContextElement @@ -31,6 +31,16 @@ } from './imageUtils' import { modelSupportsVision } from '../modelConfig' import { tryGetCurrentModel } from '$lib/aiStore' + import { createLongHash } from '$lib/editorLangUtils' + import { + fileToAttachedTextFile, + MAX_ATTACHED_FILES, + MAX_CONVERSATION_FILE_BYTES, + MAX_TEXT_FILE_BYTES, + textByteLength, + type AttachedTextFile + } from './textFileUtils' + import { MessageDraft } from './messageDraft.svelte' import ExpandableImage, { isImageViewerOpen } from '$lib/components/common/image/ExpandableImage.svelte' @@ -46,6 +56,7 @@ initialInstructions?: string initialPastes?: PasteAttachment[] initialImages?: AttachedImage[] + initialFiles?: AttachedTextFile[] editingMessageIndex?: number | null onEditEnd?: () => void className?: string @@ -76,6 +87,7 @@ initialInstructions = '', initialPastes = undefined, initialImages = undefined, + initialFiles = undefined, editingMessageIndex = null, onEditEnd = () => {}, className = '', @@ -142,16 +154,22 @@ let contextTextareaComponent: ContextTextarea | undefined = $state() let instructionsTextareaComponent: HTMLTextAreaElement | undefined = $state() - let instructions = $state(untrack(() => initialInstructions)) + // The four lanes that ship with the next send — text, collapsed big-paste + // blobs, per-message images, per-message text files — owned by one draft so + // every aggregation applies the draft rules. The composer keeps only the + // async in-flight accounting (pending counters, byte reservations). + const draft = new MessageDraft( + untrack(() => ({ + text: initialInstructions, + pastes: initialPastes ?? [], + images: initialImages ?? [], + files: initialFiles ?? [] + })) + ) $effect(() => { - const text = instructions + const text = draft.text untrack(() => onDraftChange?.(text)) }) - // Collapsed big-paste blobs referenced by tokens in `instructions`. - let pastes = $state(untrack(() => initialPastes ?? [])) - // Per-message image attachments (drag/drop/paste), GLOBAL mode only. One-shot: - // they attach to the next send and clear, unlike the persistent attached-files store. - let images = $state(untrack(() => initialImages ?? [])) // Images being decoded right now. Holds off sending so a message can never go // out without an attachment the user already dropped, and reserves cap slots // against a concurrent drop. @@ -171,10 +189,10 @@ sendUserToast(`${model.model} can't read images. Switch to a vision model first.`, true) return } - // Count decodes already in flight: two drops that both read `images.length` + // Count decodes already in flight: two drops that both read the image count // before either resolves would each claim the same free slots and overshoot // the cap. - const remaining = MAX_ATTACHED_IMAGES - images.length - pendingImages + const remaining = MAX_ATTACHED_IMAGES - draft.images.length - pendingImages if (remaining <= 0) { sendUserToast(`You can attach up to ${MAX_ATTACHED_IMAGES} images.`, true) return @@ -210,7 +228,7 @@ failed++ } } - if (added.length > 0) images = [...images, ...added] + if (added.length > 0) draft.addImages(added) if (failed > 0) sendUserToast(`Could not attach ${failed} image(s).`, true) } finally { pendingImages -= batch.length @@ -218,7 +236,140 @@ } function removeImage(index: number) { - images = images.filter((_, i) => i !== index) + draft.images = draft.images.filter((_, i) => i !== index) + } + + // Files being read right now — same send-hold/slot-reservation role as pendingImages. + let pendingFiles = $state(0) + // Drop routing resolves file-system handles/entries asynchronously before it + // can call addTextFiles/addImages; a send during that window would land the + // dropped files on the NEXT message. Holds block sending (no slot or chip + // impact) until the drop handler finishes routing. + let ingestionHolds = $state(0) + export function holdSendForIngestion(): () => void { + ingestionHolds += 1 + let released = false + return () => { + if (!released) { + released = true + ingestionHolds -= 1 + } + } + } + // Bytes those in-flight reads have claimed against the conversation budget: + // two overlapping drops that both read the budget before either lands would + // otherwise each spend the same remaining allowance. + let pendingFileBytes = $state(0) + + // Publish this composer's staged bytes (committed attachments + in-flight + // reads) to the manager so a concurrently-mounted composer — the edit box + // while editing an earlier message — sees them in its own budget check and + // the two can't each spend the whole conversation allowance. + const composerKey = untrack(() => createLongHash()) + let stagedBytes = $derived( + draft.files.reduce((sum, f) => sum + textByteLength(f.content), 0) + pendingFileBytes + ) + $effect(() => { + aiChatManager.setComposerStaged(composerKey, editingMessageIndex, stagedBytes) + }) + $effect(() => () => aiChatManager.clearComposerStaged(composerKey)) + + /** Attach dropped/picked text files (sniffed + bounded). GLOBAL mode only. */ + export async function addTextFiles(candidates: File[]) { + if (aiChatManager.mode !== AIMode.GLOBAL) return + if (candidates.length === 0) return + const remaining = MAX_ATTACHED_FILES - draft.files.length - pendingFiles + if (remaining <= 0) { + sendUserToast(`You can attach up to ${MAX_ATTACHED_FILES} files.`, true) + return + } + const oversized = candidates.filter((f) => f.size > MAX_TEXT_FILE_BYTES) + if (oversized.length > 0) { + const mb = Math.round(MAX_TEXT_FILE_BYTES / 1_000_000) + sendUserToast( + `${oversized.length} file(s) over ${mb}MB were skipped — link their folder to read them on demand.`, + true + ) + } + const usable = candidates.filter((f) => f.size <= MAX_TEXT_FILE_BYTES) + if (usable.length === 0) return + let batch = usable.slice(0, remaining) + if (batch.length < usable.length) { + sendUserToast( + `You can attach up to ${MAX_ATTACHED_FILES} files; ${usable.length - batch.length} were skipped.`, + true + ) + } + // Conversation-level byte budget: transcript + queue + every live + // composer's stage (this one and, mid-edit, the other) + this composer's + // own pending reads. File content is persisted with every history save, so + // an unbounded total would grow the chat record without limit. The + // transcript sum skips any message a composer is editing — that composer's + // stage stands in for it, so counting both would charge those bytes twice. + let budget = + MAX_CONVERSATION_FILE_BYTES - + aiChatManager.attachmentBytesExcluding(composerKey) - + draft.files.reduce((sum, f) => sum + textByteLength(f.content), 0) - + pendingFileBytes + const withinBudget: File[] = [] + for (const f of batch) { + if (f.size <= budget) { + withinBudget.push(f) + budget -= f.size + } + } + if (withinBudget.length < batch.length) { + const mb = Math.round(MAX_CONVERSATION_FILE_BYTES / 1_000_000) + sendUserToast( + `${batch.length - withinBudget.length} file(s) skipped — this conversation reached its ${mb}MB attachment budget. Link a folder to read larger sets on demand.`, + true + ) + } + batch = withinBudget + if (batch.length === 0) return + pendingFiles += batch.length + const reservedBytes = batch.reduce((sum, f) => sum + f.size, 0) + pendingFileBytes += reservedBytes + try { + const reads: { name: string; content: string }[] = [] + let skipped = 0 + for (const file of batch) { + try { + const attached = await fileToAttachedTextFile(file) + if (attached) reads.push(attached) + else skipped++ + } catch { + skipped++ + } + } + // Commit through the draft in one synchronous step — fold (dedupe, + // courtesy rename) and decoded-byte admission both run against the live + // list, so another batch landing between this one's file reads can't be + // missed, and malformed input that inflates on decode can't slip past the + // raw-size admission above. This batch's own raw reservation is excluded + // from the budget — the decoded sizes replace it. + const liveBudget = + MAX_CONVERSATION_FILE_BYTES - + aiChatManager.attachmentBytesExcluding(composerKey) - + draft.files.reduce((sum, f) => sum + textByteLength(f.content), 0) - + (pendingFileBytes - reservedBytes) + const { droppedAtBudget } = draft.addFiles(reads, liveBudget) + if (droppedAtBudget > 0) { + const mb = Math.round(MAX_CONVERSATION_FILE_BYTES / 1_000_000) + sendUserToast( + `${droppedAtBudget} file(s) skipped — this conversation reached its ${mb}MB attachment budget. Link a folder to read larger sets on demand.`, + true + ) + } + if (skipped > 0) sendUserToast(`Skipped ${skipped} file(s) (non-text).`, true) + } finally { + pendingFiles -= batch.length + pendingFileBytes -= reservedBytes + } + } + + function removeFile(index: number) { + draft.files = draft.files.filter((_, i) => i !== index) } // App mode @ mention state @@ -250,9 +401,9 @@ * leave duplicate tokens. */ export function insertMention(title: string) { const target = `@${title}` - if (instructions.split(/\s+/).includes(target)) return - const sep = instructions.length === 0 || /\s$/.test(instructions) ? '' : ' ' - instructions = `${instructions}${sep}${target} ` + if (draft.text.split(/\s+/).includes(target)) return + const sep = draft.text.length === 0 || /\s$/.test(draft.text) ? '' : ' ' + draft.text = `${draft.text}${sep}${target} ` } /** Strip every `@title` token from the textarea — used when the user @@ -268,7 +419,7 @@ contextTextareaComponent?.unsyncMention(title) const escaped = title.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') const re = new RegExp(`(^|\\s)@${escaped}(\\s|$)`, 'g') - instructions = instructions.replace(re, (_m, lead, trail) => { + draft.text = draft.text.replace(re, (_m, lead, trail) => { // Boundary on at least one side → drop the mention entirely. if (!lead || !trail) return '' // Middle of text: keep ONE of the bracketing whitespace chars so @@ -296,12 +447,23 @@ export function restoreInstructions( value: string, restoredPastes: PasteAttachment[] = [], - restoredImages: AttachedImage[] = [] + restoredImages: AttachedImage[] = [], + restoredFiles: AttachedTextFile[] = [] ): boolean { - if (instructions.trim() || images.length > 0 || pendingImages > 0) return false - instructions = value - pastes = restoredPastes - images = restoredImages + // Attachments still decoding/reading (or mid-drop-routing) count as + // occupancy too — they belong to a draft the user started even though + // their lane is still empty. + if (pendingImages > 0 || pendingFiles > 0 || ingestionHolds > 0) return false + if ( + !draft.replaceIfEmpty({ + text: value, + pastes: restoredPastes, + images: restoredImages, + files: restoredFiles + }) + ) { + return false + } focusInput() return true } @@ -311,24 +473,30 @@ * the user typed is lost. Restored images join whatever is already * attached, up to the cap — dropping them would lose the attachment * silently, which is the whole reason the queue carries them. */ - export function prependText(text: string, restoredImages: AttachedImage[] = []): boolean { - // Whether the restored text landed on top of a draft the user was already - // writing: both instructions now share one composer, so the caller must keep - // both their contexts rather than replacing one with the other. - const mergedIntoDraft = !!text && !!instructions.trim() - // An image-only restore has empty text; prepending it would only add blank lines. - if (text) { - instructions = instructions.trim() ? `${text}\n\n${instructions}` : text + export function prependText( + text: string, + restoredImages: AttachedImage[] = [], + restoredFiles: AttachedTextFile[] = [] + ): boolean { + // mergedIntoDraft: the restored text landed on top of a draft the user was + // already writing — both instructions now share one composer, so the caller + // must keep both their contexts rather than replacing one with the other. + const { mergedIntoDraft, droppedImages, droppedFiles } = draft.prepend({ + text, + images: restoredImages, + files: restoredFiles + }) + if (droppedImages > 0) { + sendUserToast( + `You can attach up to ${MAX_ATTACHED_IMAGES} images; ${droppedImages} restored image(s) were dropped.`, + true + ) } - if (restoredImages.length > 0) { - const merged = [...images, ...restoredImages] - if (merged.length > MAX_ATTACHED_IMAGES) { - sendUserToast( - `You can attach up to ${MAX_ATTACHED_IMAGES} images; ${merged.length - MAX_ATTACHED_IMAGES} restored image(s) were dropped.`, - true - ) - } - images = merged.slice(0, MAX_ATTACHED_IMAGES) + if (droppedFiles > 0) { + sendUserToast( + `You can attach up to ${MAX_ATTACHED_FILES} files; ${droppedFiles} restored file(s) were dropped.`, + true + ) } focusInput() return mergedIntoDraft @@ -336,8 +504,8 @@ /** Insert a plain @filename mention for an attached file (used by the @ menu Files category). */ export function insertFileMention(name: string) { - const sep = instructions.length === 0 || instructions.endsWith(' ') ? '' : ' ' - instructions = `${instructions}${sep}${formatMention(name)} ` + const sep = draft.text.length === 0 || draft.text.endsWith(' ') ? '' : ' ' + draft.text = `${draft.text}${sep}${formatMention(name)} ` focusInput() } @@ -429,8 +597,8 @@ function sendRequest() { // The send button is disabled while decoding, but Enter reaches here directly. - // Sending now would drop the in-flight images onto the following message. - if (pendingImages > 0) { + // Sending now would drop the in-flight attachments onto the following message. + if (pendingImages > 0 || pendingFiles > 0 || ingestionHolds > 0) { return } if (aiChatManager.loading) { @@ -444,17 +612,16 @@ // chips picked at press time. if ( editingMessageIndex === null && - (instructions.trim() || - images.length > 0 || - (aiChatManager.mode === AIMode.GLOBAL && selectedContext.length > 0)) + (!draft.isEmpty || (aiChatManager.mode === AIMode.GLOBAL && selectedContext.length > 0)) ) { - aiChatManager.queueMessage(expanded(chatDraft(instructions, pastes)), images, [ - ...selectedContext - ]) + const sent = draft.take() + aiChatManager.queueMessage( + expanded(chatDraft(sent.text, sent.pastes)), + sent.images, + [...selectedContext], + sent.files + ) contextTextareaComponent?.clearForSend() - instructions = '' - pastes = [] - images = [] } return } @@ -462,25 +629,30 @@ // In edit mode selectedContext is the edit box's own copy (seeded from the // message's original chips), so send exactly what's shown — the user may // have added or removed chips. + const sent = draft.take() aiChatManager.restartGeneration( editingMessageIndex, - instructions, - pastes, - images, - selectedContext + sent.text, + sent.pastes, + sent.images, + selectedContext, + sent.files ) onEditEnd() } else { - aiChatManager.sendRequest({ instructions, pastes, images }) + const sent = draft.take() + aiChatManager.sendRequest({ + instructions: sent.text, + pastes: sent.pastes, + images: sent.images, + files: sent.files + }) // clearForSend() pre-zaps the textarea's mention-sync so the wipe // doesn't drop `selectedContext` before `AIChatManager.beforeSend` // snapshots it. Only mounted in SCRIPT/FLOW/GLOBAL — APP and the - // fallback textarea still rely on the plain `instructions = ''` - // reset (no `@`-mention state to coordinate). + // fallback textarea still rely on the draft reset alone (no + // `@`-mention state to coordinate). contextTextareaComponent?.clearForSend() - instructions = '' - pastes = [] - images = [] } } @@ -489,7 +661,7 @@ // for the conversation bubble and expands them for the LLM inside the manager. function submitRequest() { if (onSendRequest) { - onSendRequest(expanded(chatDraft(instructions, pastes))) + onSendRequest(expanded(chatDraft(draft.text, draft.pastes))) } else { sendRequest() } @@ -661,7 +833,7 @@ } function handleAppInput(_e: Event) { - const words = instructions.split(/\s+/) + const words = draft.text.split(/\s+/) const lastWord = words[words.length - 1] if ( @@ -680,9 +852,9 @@ function handleAppContextSelection(contextElement: ContextElement) { void addContextToSelection(contextElement) // Update instructions with the selected context title - const index = instructions.lastIndexOf('@') + const index = draft.text.lastIndexOf('@') if (index !== -1) { - instructions = instructions.substring(0, index) + `@${contextElement.title}` + draft.text = draft.text.substring(0, index) + `@${contextElement.title}` } showAppContextTooltip = false } @@ -696,7 +868,7 @@ {#snippet sendStopButton()} {@const isLoading = loading ?? aiChatManager.loading} - {@const emptyDraft = instructions.trim().length === 0 && images.length === 0} + {@const emptyDraft = draft.isEmpty} +{#snippet badgeRow()} + {@const contextChips = showContext ? selectedContext : domSelectorChips} + {#if contextChips.length > 0 || draft.files.length > 0 || pendingFiles > 0}
- {#each selectedContext as element (contextKey(element))} + {#each contextChips as element (contextKey(element))} { selectedContext = selectedContext?.filter((c) => !isSameContextElement(c, element)) - removeMention(element.title) + if (showContext) removeMention(element.title) }} /> {/each} -
- {/if} -{/snippet} - - -{#snippet domSelectorChipRow()} - {#if domSelectorChips.length > 0} -
- {#each domSelectorChips as element (contextKey(element))} + {#each draft.files as file, i (i)} { - selectedContext = selectedContext?.filter((c) => !isSameContextElement(c, element)) - }} + onDelete={() => removeFile(i)} /> {/each} + {#each { length: pendingFiles } as _, i (i)} +
+ +
+ {/each}
{/if} {/snippet} {#snippet imageChipsRow()} - {#if images.length > 0 || pendingImages > 0} -
- {#each images as image, i (i)} + {#if draft.images.length > 0 || pendingImages > 0} +
+ {#each draft.images as image, i (i)}
@@ -811,10 +987,13 @@
void addImages(files) + ? (pasted) => void addImages(pasted) + : undefined} + onTextFiles={aiChatManager.mode === AIMode.GLOBAL + ? (pasted) => void addTextFiles(pasted) : undefined} {availableContext} {selectedContext} @@ -833,16 +1012,7 @@ {onKeyDown} > {#snippet leading()} - {#if aiChatManager.mode === AIMode.GLOBAL} -
- -
- {/if} - {#if showContext} - {@render contextPickerRow()} - {:else} - {@render domSelectorChipRow()} - {/if} + {@render badgeRow()} {@render imageChipsRow()} {/snippet}
@@ -854,12 +1024,12 @@
{:else if aiChatManager.mode === AIMode.APP} {#if showContext} - {@render contextPickerRow()} + {@render badgeRow()} {/if}
+ + Markdown supported. Editable any time before and after publication. + + +
+
+ + Data table migrations +
+ {#if s.migrationsGenerating} +
+ + Detecting data tables used by this project… +
+ {:else if s.migrationDrafts.length === 0} + + No data table usage detected in this project's scripts, flows, or raw apps. + + {:else} + + We detected these data tables. When included, the migration recreates their tables + on import. Best-effort — review and edit before publishing. + + {#each s.migrationDrafts as m (m.datatable_name)} +
+
+ {m.datatable_name} + +
+ +
+ {/each} + {/if} +
+
+ {#snippet actions()} + + {/snippet} + + + {/key} +{/if} diff --git a/frontend/src/lib/components/workspaceSettings/MigrationSqlEditor.svelte b/frontend/src/lib/components/workspaceSettings/MigrationSqlEditor.svelte new file mode 100644 index 0000000000..76e3da3e07 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/MigrationSqlEditor.svelte @@ -0,0 +1,40 @@ + + +
+
+ {#each [{ id: 'up', label: 'Up' }, { id: 'down', label: 'Down' }] as t (t.id)} + + {/each} +
+ {#key generation} +
+ {#if tab === 'up'} + + {:else} + + {/if} +
+ {/key} +
diff --git a/frontend/src/lib/components/workspaceSettings/deployToHubSession.svelte.ts b/frontend/src/lib/components/workspaceSettings/deployToHubSession.svelte.ts new file mode 100644 index 0000000000..96cb8b117c --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/deployToHubSession.svelte.ts @@ -0,0 +1,1820 @@ +import { untrack } from 'svelte' +import { base } from '$lib/base' +import { + AppService, + FlowService, + JobService, + RawAppService, + ResourceService, + ScriptService, + WorkspaceService, + ScheduleService +} from '$lib/gen' +import { sendUserToast } from '$lib/toast' +import { sleep, emptySchema } from '$lib/utils' +import { computeSecretUrl } from '$lib/components/apps/editor/appDeploy.svelte' +import { + buildProjectBundle, + buildPathMap, + classifyPath, + extractScriptRefs, + extractFlowRefs, + extractAppRefs, + extractTriggerConfigResourceRefs, + extractVarRefsFromValue, + rewriteTriggerConfig, + rewriteVarRefsInValue, + type BundleDeps, + type BundledItem, + type FetchedItem, + type ItemKind, + type ItemRef, + type ProjectBundle +} from './projectBundle' +import { + detectDatatableTables, + generateDatatableMigrations, + type GeneratedMigration +} from './projectMigrations' +import type { Kind } from '$lib/utils_deployable' +import type { AssetGraphResponse } from '$lib/components/assets/AssetGraph/types' +import { + CASCADE_JOB_TIMEOUT_MS, + CASCADE_POLL_INTERVAL_MS, + DATA_ASSET_KINDS +} from '$lib/components/assets/AssetGraph/cascadeRun' +import { capturePipelineRecording } from '$lib/components/recording/pipelineRecording.svelte' +import type { PipelineRecording } from '$lib/components/recording/types' +import { + TRIGGER_KINDS, + listAllWorkspaceTriggers, + triggerResourcePath, + triggerHandlerRefs, + portableTriggerConfig, + type WorkspaceTrigger, + type WorkspaceTriggerKind +} from '../triggers/workspaceTriggersList' + +export type Phase = 'predeploy' | 'draft' | 'under_review' | 'live' +export type RecStatus = 'none' | 'recorded' +export interface DeployItem { + key: string + path: string + kind: Kind + summary?: string + rec: RecStatus + published?: boolean + publicUrl?: string + [k: string]: unknown +} + +export const canRecord = (k: Kind) => k === 'script' || k === 'flow' +// Legacy raw apps live only in the `raw_app` table, but the iframe share flow +// drives AppService (the `app` table), so it can only target apps stored there. +export const canShareAsIframe = (it: DeployItem): boolean => + it.kind === 'app' || (it.kind === 'raw_app' && it.appTable === true) + +// Hub rehydration only carries draft membership, not the live share state of an +// app. Copy the public-execution flag, public URL, and app-table origin from the +// loaded workspace items onto matching draft items so a still-public app keeps its +// Public badge, Unpublish, and iframe controls after its draft is reopened. Returns +// the original array unchanged when nothing needs merging (stable reference). +export function mergeShareState( + draftItems: DeployItem[], + workspaceItems: DeployItem[] +): DeployItem[] { + if (draftItems.length === 0 || workspaceItems.length === 0) return draftItems + const byKey = new Map(workspaceItems.map((w) => [w.key, w])) + let changed = false + const merged = draftItems.map((d) => { + const w = byKey.get(d.key) + if (!w) return d + if (w.published !== d.published || w.publicUrl !== d.publicUrl || w.appTable !== d.appTable) { + changed = true + return { ...d, published: w.published, publicUrl: w.publicUrl, appTable: w.appTable } + } + return d + }) + return changed ? merged : draftItems +} + +export function sanitizeSlug(s: string): string { + return s + .toLowerCase() + .replace(/[_\s]+/g, '-') + .replace(/[^a-z0-9-]/g, '') + .replace(/-+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 50) + .replace(/-+$/g, '') +} +const SLUG_RE = /^[a-z0-9][a-z0-9-]{1,48}[a-z0-9]$/ +export function isValidSlug(s: string): boolean { + return SLUG_RE.test(s) +} + +export type RunState = 'idle' | 'running' | 'success' | 'failed' + +const ITEM_KIND_ROUTE: Record = { + script: 'scripts/get', + flow: 'flows/get', + app: 'apps/get', + raw_app: 'apps_raw/get' +} + +const HIDDEN_RESOURCE_TYPES = new Set(['app_theme', 'state', 'cache']) + +// Prune a folder's asset graph to a set of scripts so a pipeline recording only +// runs, renders and samples the project's included members — a deselected branch +// (its nodes, code, logs/results and table samples) never enters the recording. +// Assets kept are only those an included script touches; edges/triggers only +// those anchored on an included runnable. +function pruneGraphToScripts(graph: AssetGraphResponse, scripts: Set): AssetGraphResponse { + const runnables = graph.runnables.filter((r) => scripts.has(r.path)) + const edges = graph.edges.filter((e) => scripts.has(e.runnable_path)) + const keptAssets = new Set(edges.map((e) => `${e.asset_kind}:${e.asset_path}`)) + const assets = graph.assets.filter((a) => keptAssets.has(`${a.kind}:${a.path}`)) + const triggers = graph.triggers.filter((t) => scripts.has(t.runnable_path)) + const macro_edges = graph.macro_edges?.filter( + (m) => scripts.has(m.consumer_path) && scripts.has(m.lib_path) + ) + const test_edges = graph.test_edges?.filter( + (t) => scripts.has(t.runnable_path) && scripts.has(t.producer_path) + ) + return { assets, runnables, edges, triggers, macro_edges, test_edges } +} + +function typesFromSchema(schema: any): string[] { + const out = new Set() + const props = schema?.properties + if (props && typeof props === 'object') { + for (const key of Object.keys(props)) { + const fmt = props[key]?.format + if (typeof fmt === 'string' && fmt.startsWith('resource-')) { + out.add(fmt.slice('resource-'.length)) + } + } + } + return [...out] +} + +type DependencyUsage = + | { role: 'input'; label: string; kind: ItemKind; itemPath: string } + | { role: 'hardcoded'; label: string; kind: ItemKind; path: string; itemPath: string } + | { role: 'trigger'; label: string; triggerKind: WorkspaceTriggerKind; path: string } +export interface DependencyType { + resource_type: string + hasHardcoded: boolean + usages: DependencyUsage[] +} + +interface SessionDeps { + hasEeLicense: () => boolean +} + +/** + * All state and async operations for one Deploy-to-Hub surface, bound to an + * immutable (workspace, folder) pair. A workspace or folder change never mutates + * a session — `useDeployToHubSession` replaces the instance, so in-flight async + * work keeps writing to the discarded object and cannot leak into the new scope. + * The only invalidation tokens left are intra-session (competing calls on the + * same session), not lifecycle guards. + */ +export class DeployToHubSession { + readonly workspace: string + readonly folder: string + /** `f/`-prefixed folder path the project is scoped to. */ + readonly selectedFolder: string + + #disposed = false + #deps: SessionDeps + + phase = $state('predeploy') + workspaceItems = $state([]) + draftItems = $state([]) + workspaceTriggers = $state([]) + triggersLoading = $state(false) + // True when a trigger kind's discovery failed (not a feature-gated 404): + // the trigger list may be incomplete, so publishing is blocked until a + // retry succeeds. + triggerDiscoveryFailed = $state(false) + schedulePreviews = $state>({}) + manualDeselected = $state>(new Set()) + loading = $state(false) + workspaceRateLimit = $state(undefined) + deploymentStatus = $state< + Record + >({}) + deploying = $state(false) + + recordTarget = $state() + recordArgs = $state>({}) + recordValid = $state(true) + recordSchema = $state>(emptySchema()) + recordSchemaLoading = $state(false) + runState = $state('idle') + runJobId = $state(undefined) + runResult = $state(undefined) + runError = $state(undefined) + recordings = $state>({}) + + // Project-level data-pipeline recording. Unlike script/flow recordings (one + // job per item) a pipeline is the whole folder cascade, so it gets a single + // recording: the resolved asset graph, per-node status timeline, per-node job + // streams and asset samples — replayed by PipelineRecordingReplay. + pipelineGraph = $state(undefined) + pipelineRunState = $state('idle') + pipelineRecordingResult = $state(undefined) + pipelineRunError = $state(undefined) + pipelineRecorded = $state(false) + + publishTarget = $state() + publishing = $state(false) + + hubName = $state('') + hubSummary = $state('') + hubReadme = $state('') + effectiveSlug = $state('') + hubItemIds = $state>({}) + + // Best-effort data table migrations for the bundle, editable in the drawer and + // pushed on deploy. Regenerated when the bundle drawer opens. + migrationDrafts = $state([]) + migrationsGenerating = $state(false) + // Bumped whenever the drafts are (re)generated, to re-key the Monaco editors so + // they pick up the fresh SQL (Monaco doesn't sync external `code` changes). + migrationsGeneration = $state(0) + + bundlePreview = $state(undefined) + detectingResources = $state(false) + // Data tables (→ tables) the current selection reads/writes, detected off the + // same bundle preview. Drives the predeploy "Data table dependencies" summary; + // the editable migration itself is generated in the bundle drawer. + datatableUsage = $state>>(new Map()) + detectingDatatables = $state(false) + + submitting = $state(false) + syncing = $state(false) + + // Intra-session tokens: latest call wins among competing calls on this session. + #triggerLoadTok = 0 + #recordRunTok = 0 + #pipelineRunTok = 0 + #migrationsTok = 0 + #schedulePreviewsInFlight = new Set() + // Preview-only cache: toggling checkboxes re-runs the closure walk, but item + // contents don't change mid-session. deployAll bypasses this and fetches fresh. + #previewItemCache = new Map>() + #previewTypeCache = new Map>() + + constructor(workspace: string, folder: string, deps: SessionDeps) { + this.workspace = workspace + this.folder = folder + this.selectedFolder = `f/${folder}` + this.#deps = deps + } + + dispose() { + this.#disposed = true + // Invalidate any in-flight pipeline cascade poll so it stops on the next + // tick instead of polling to the timeout against a discarded session. + this.#pipelineRunTok++ + } + + load() { + void this.#loadWorkspace() + void this.#loadTriggers() + void this.rehydrateFromHub() + void this.#loadPipelineGraph() + } + + filteredWorkspaceItems = $derived( + this.workspaceItems.filter((i) => i.path.startsWith(this.selectedFolder + '/')) + ) + // Derived (not merged at load time) so it settles regardless of which of the + // racing loads (#loadWorkspace / rehydrateFromHub) finishes last. + draftItemsWithLocalState = $derived(mergeShareState(this.draftItems, this.workspaceItems)) + items = $derived( + this.phase === 'predeploy' ? this.filteredWorkspaceItems : this.draftItemsWithLocalState + ) + selectedItems = $derived( + this.phase === 'predeploy' + ? this.filteredWorkspaceItems.filter((i) => !this.manualDeselected.has(i.key)) + : [] + ) + selectedItemKeys = $derived(this.selectedItems.map((i) => i.key)) + allSelected = $derived( + this.phase === 'predeploy' && + this.selectedItemKeys.length === this.filteredWorkspaceItems.length + ) + recordableItems = $derived(this.items.filter((i) => canRecord(i.kind))) + allRecorded = $derived( + this.recordableItems.length > 0 && this.recordableItems.every((i) => i.rec === 'recorded') + ) + // Pipeline members of this project's folder (`// pipeline` scripts). + pipelineScriptPaths = $derived( + (this.pipelineGraph?.runnables ?? []) + .filter((r) => r.usage_kind === 'script' && r.in_pipeline) + .map((r) => r.path) + ) + // The subset actually in the Hub project — so a member the user deselected from + // the bundle is neither executed nor embedded (with its code/logs/samples) in + // the recording. In the draft phase `items` is the project's membership. + recordablePipelineScriptPaths = $derived( + this.pipelineScriptPaths.filter((p) => + this.items.some((i) => i.kind === 'script' && i.path === p) + ) + ) + pipelineScriptPathSet = $derived(new Set(this.pipelineScriptPaths)) + isPipelineProject = $derived(this.pipelineScriptPaths.length > 0) + hubSlug = $derived(this.effectiveSlug || sanitizeSlug(this.hubName)) + + relevantTriggers = $derived.by(() => { + const selectedScripts = new Set( + this.selectedItems.filter((i) => i.kind === 'script').map((i) => i.path) + ) + const selectedFlows = new Set( + this.selectedItems.filter((i) => i.kind === 'flow').map((i) => i.path) + ) + return this.workspaceTriggers.filter((t) => + t.is_flow ? selectedFlows.has(t.script_path) : selectedScripts.has(t.script_path) + ) + }) + + triggersByKind = $derived.by(() => { + const out = new Map() + for (const t of this.relevantTriggers) { + const arr = out.get(t.kind) ?? [] + arr.push(t) + out.set(t.kind, arr) + } + return Array.from(out.entries()).sort((a, b) => a[0].localeCompare(b[0])) + }) + + runnableSummaryByPath = $derived.by(() => { + const m = new Map() + for (const it of this.workspaceItems) { + if (it.kind === 'script' || it.kind === 'flow') { + m.set(`${it.kind}:${it.path}`, it.summary) + } + } + return m + }) + + // `hasHardcoded` = pinned via $res: path (relocated as a stub); else input-only. + dependencyTypes = $derived.by(() => { + const b = this.bundlePreview + if (!b) return [] as DependencyType[] + const stubByNewPath = new Map(b.resourceStubs.map((s) => [s.newPath, s])) + const byType = new Map() + const ensure = (rt: string) => { + let e = byType.get(rt) + if (!e) { + e = { resource_type: rt, hasHardcoded: false, usages: [] } + byType.set(rt, e) + } + return e + } + for (const it of b.items) { + const label = (it.summary?.trim() || it.path) ?? it.path + const refs = + it.kind === 'flow' + ? extractFlowRefs(it.value).filter((r) => r.kind === 'resource') + : it.kind === 'app' + ? extractAppRefs(it.value) + : extractScriptRefs(it.content ?? '') + for (const r of refs) { + const stub = stubByNewPath.get(r.path) + if (!stub || HIDDEN_RESOURCE_TYPES.has(stub.resource_type)) continue + const e = ensure(stub.resource_type) + e.hasHardcoded = true + e.usages.push({ + role: 'hardcoded', + label, + kind: it.kind, + path: stub.originalPath, + itemPath: it.path + }) + } + for (const t of typesFromSchema(it.schema)) { + if (HIDDEN_RESOURCE_TYPES.has(t)) continue + ensure(t).usages.push({ role: 'input', label, kind: it.kind, itemPath: it.path }) + } + } + // Resources referenced only by a trigger (no item uses them in code) — + // its kind resource field or any `$res:` token in its config. + const stubByOriginal = new Map(b.resourceStubs.map((s) => [s.originalPath, s])) + for (const t of this.relevantTriggers) { + const refs = new Set( + extractTriggerConfigResourceRefs(portableTriggerConfig(t.kind, t.config)) + ) + const rp = triggerResourcePath(t) + if (rp) refs.add(rp) + for (const ref of refs) { + const stub = stubByOriginal.get(ref) + if (!stub || HIDDEN_RESOURCE_TYPES.has(stub.resource_type)) continue + ensure(stub.resource_type).usages.push({ + role: 'trigger', + label: t.summary?.trim() || t.path, + triggerKind: t.kind, + path: stub.originalPath + }) + } + } + return [...byType.values()].sort((a, b) => a.resource_type.localeCompare(b.resource_type)) + }) + + toggleItem = (item: { key: string }) => { + const next = new Set(this.manualDeselected) + if (next.has(item.key)) next.delete(item.key) + else next.add(item.key) + this.manualDeselected = next + } + selectAll = () => { + this.manualDeselected = new Set() + } + deselectAll = () => { + this.manualDeselected = new Set(this.filteredWorkspaceItems.map((i) => i.key)) + } + + #folderQs(): string { + return `?folder=${encodeURIComponent(this.folder)}` + } + + itemUrl(kind: ItemKind, path: string): string | undefined { + if (!path) return undefined + return `${base}/${ITEM_KIND_ROUTE[kind]}/${path}?workspace=${this.workspace}` + } + triggerListUrl(kind: WorkspaceTriggerKind): string { + return `${base}/${TRIGGER_KINDS[kind].route}?workspace=${this.workspace}` + } + + #patchItem(key: string, patch: Partial) { + this.workspaceItems = this.workspaceItems.map((i) => (i.key === key ? { ...i, ...patch } : i)) + this.draftItems = this.draftItems.map((i) => (i.key === key ? { ...i, ...patch } : i)) + } + + async #listAllPages( + fetcher: (params: { perPage: number; page: number }) => Promise + ): Promise { + const perPage = 100 + const out: T[] = [] + for (let page = 1; page <= 1000; page++) { + const batch = await fetcher({ perPage, page }) + out.push(...batch) + if (batch.length < perPage) return out + } + return out + } + + async #loadWorkspace() { + const workspace = this.workspace + this.loading = true + try { + const [apps, rawApps, flows, scripts, settings] = await Promise.all([ + this.#listAllPages((p) => AppService.listApps({ workspace, ...p })), + this.#listAllPages((p) => RawAppService.listRawApps({ workspace, ...p })), + this.#listAllPages((p) => FlowService.listFlows({ workspace, ...p })), + this.#listAllPages((p) => ScriptService.listScripts({ workspace, ...p })), + WorkspaceService.getSettings({ workspace }).catch(() => undefined) + ]) + if (this.#disposed) return + + this.workspaceRateLimit = settings?.public_app_execution_limit_per_minute + + const next: DeployItem[] = [] + const publicApps = apps.filter((a) => a.execution_mode === 'anonymous') + const publicUrls = await Promise.all(publicApps.map((a) => this.#resolvePublicUrl(a.path))) + const publicUrlByPath = new Map(publicApps.map((a, i) => [a.path, publicUrls[i]])) + for (const a of apps) { + const isPublic = a.execution_mode === 'anonymous' + // Raw apps live in the `app` table (value = files/runnables) but must be + // published to the Hub as raw apps, not low-code apps. + const isRaw = (a as any).raw_app === true + next.push({ + key: `${isRaw ? 'raw_app' : 'app'}:${a.path}`, + path: a.path, + kind: isRaw ? 'raw_app' : 'app', + appTable: isRaw || undefined, + summary: a.summary, + rec: 'none', + published: isPublic, + publicUrl: isPublic ? publicUrlByPath.get(a.path) : undefined + }) + } + for (const a of rawApps) { + next.push({ + key: `raw_app:${a.path}`, + path: a.path, + kind: 'raw_app', + summary: a.summary, + rec: 'none' + }) + } + for (const f of flows) { + next.push({ + key: `flow:${f.path}`, + path: f.path, + kind: 'flow', + summary: f.summary, + rec: 'none' + }) + } + for (const s of scripts) { + next.push({ + key: `script:${s.path}`, + path: s.path, + kind: 'script', + summary: s.summary, + rec: 'none' + }) + } + if (this.#disposed) return + this.workspaceItems = next + } catch (e: any) { + if (!this.#disposed) { + sendUserToast(`Failed to load project items: ${e?.message ?? e}`, true) + } + } finally { + if (!this.#disposed) this.loading = false + } + } + + /** Re-fetch triggers, e.g. after the EE license hydrates late. */ + reloadTriggers() { + void this.#loadTriggers() + } + + async #loadTriggers() { + const tok = ++this.#triggerLoadTok + this.triggersLoading = true + try { + const { triggers, failedKinds } = await listAllWorkspaceTriggers(this.workspace, { + includeEeOnly: this.#deps.hasEeLicense(), + onError: (message) => { + if (!this.#disposed) sendUserToast(message, true) + } + }) + if (this.#disposed || tok !== this.#triggerLoadTok) return + this.workspaceTriggers = triggers + this.triggerDiscoveryFailed = failedKinds.length > 0 + } finally { + if (!this.#disposed && tok === this.#triggerLoadTok) this.triggersLoading = false + } + } + + async #resolvePublicUrl(path: string): Promise { + try { + const secret = await AppService.getPublicSecretOfApp({ workspace: this.workspace, path }) + return computeSecretUrl(secret) + } catch { + return undefined + } + } + + async rehydrateFromHub() { + try { + const res = await fetch(`/api/w/${this.workspace}/hub/project${this.#folderQs()}`, { + credentials: 'include', + headers: { accept: 'application/json' } + }) + if (this.#disposed) return + if (!res.ok) return // 404 = no project published for this folder yet + const p = JSON.parse(await res.text()) + if (this.#disposed || !p?.slug) return + this.effectiveSlug = p.slug + this.hubName = p.name ?? '' + this.hubSummary = p.summary ?? '' + this.hubReadme = p.readme ?? '' + this.phase = + p.status === 'live' ? 'live' : p.status === 'under_review' ? 'under_review' : 'draft' + const ids: Record = {} + this.draftItems = (p.items ?? []).map((it: any) => { + const wpath = it.source_path ?? it.path + const key = `${it.kind}:${wpath}` + if (typeof it.hub_id === 'number') ids[key] = it.hub_id + return { + key, + path: wpath, + kind: it.kind as Kind, + summary: it.summary ?? undefined, + rec: it.has_recording ? 'recorded' : 'none' + } satisfies DeployItem + }) + this.hubItemIds = ids + } catch {} + } + + /** Kick off schedule-preview fetches for any relevant schedule trigger missing one. */ + ensureSchedulePreviews() { + for (const t of this.relevantTriggers) { + if (t.kind !== 'schedule') continue + const c = t.config as any + const key = `${c.schedule}|${c.timezone}` + if (this.schedulePreviews[key] || this.#schedulePreviewsInFlight.has(key)) continue + this.#schedulePreviewsInFlight.add(key) + ScheduleService.previewSchedule({ + requestBody: { + schedule: c.schedule, + timezone: c.timezone, + cron_version: c.cron_version ?? 'v2' + } + }) + .then((dates) => { + this.schedulePreviews = { ...this.schedulePreviews, [key]: dates.slice(0, 3) } + }) + .catch(() => {}) + .finally(() => this.#schedulePreviewsInFlight.delete(key)) + } + } + + /** + * Rebuild the predeploy bundle preview (resource + data table dependency + * summaries), debounced so rapid checkbox toggles coalesce into one walk. + * Reads its reactive inputs synchronously and returns a cancel function, so + * it can be driven from an `$effect` with proper cleanup. + */ + queueBundlePreview(): (() => void) | undefined { + if (this.phase !== 'predeploy') { + this.bundlePreview = undefined + this.datatableUsage = new Map() + return undefined + } + this.detectingResources = true + this.detectingDatatables = true + const slug = this.hubSlug + const seed: ItemRef[] = [ + ...this.selectedItems + .filter((i) => i.kind !== 'resource') + .map((i) => ({ kind: i.kind as ItemRef['kind'], path: i.path })), + ...this.#triggerHandlerSeed(this.relevantTriggers, slug) + ] + const triggerResources = this.#triggerResourcePaths(this.relevantTriggers) + const triggerVars = this.#triggerVarPaths(this.relevantTriggers) + let cancelled = false + const timer = setTimeout(() => { + buildProjectBundle(seed, slug, this.#cachedBundleDeps(), triggerResources, triggerVars) + .then((b) => { + if (cancelled) return + this.bundlePreview = b + // Detect data table usage off the same fetched items. + detectDatatableTables(b.items) + .then((usage) => { + if (!cancelled) this.datatableUsage = usage + }) + .finally(() => { + if (!cancelled) this.detectingDatatables = false + }) + }) + .finally(() => { + if (!cancelled) this.detectingResources = false + }) + }, 250) + return () => { + cancelled = true + clearTimeout(timer) + } + } + + #buildBundleDeps(): BundleDeps { + const workspace = this.workspace + return { + fetchItem: async (ref: ItemRef): Promise => { + try { + if (ref.kind === 'script') { + const s = await ScriptService.getScriptByPath({ workspace, path: ref.path }) + return { + kind: 'script', + path: ref.path, + summary: s.summary, + description: s.description ?? undefined, + content: s.content, + language: s.language, + schema: s.schema, + lock: s.lock ?? undefined, + scriptKind: typeof s.kind === 'string' ? s.kind.toLowerCase() : 'script' + } + } else if (ref.kind === 'flow') { + const f = await FlowService.getFlowByPath({ workspace, path: ref.path }) + return { + kind: 'flow', + path: ref.path, + summary: f.summary, + description: f.description ?? undefined, + value: f.value, + schema: f.schema + } + } else if (ref.kind === 'app') { + const a = await AppService.getAppByPath({ workspace, path: ref.path }) + return { kind: 'app', path: ref.path, summary: a.summary, value: a.value } + } else if (ref.kind === 'raw_app') { + // Modern raw apps live in the `app` table: fetch source files + + // runnables + the compiled bundle, and shape them into the `raw` + // payload the Hub's RawAppView expects (JSON is valid YAML). + const isModern = this.workspaceItems.some( + (i) => i.kind === 'raw_app' && i.path === ref.path && i.appTable + ) + if (isModern) { + const a = await AppService.getAppByPath({ workspace, path: ref.path }) + const secret = await AppService.getPublicSecretOfLatestVersionOfApp({ + workspace, + path: ref.path + }) + // The compiled JS bundle is required; a missing one means the app + // was never built/deployed, so fail loudly instead of pushing a blank app. + const [jsRes, cssRes] = await Promise.all([ + fetch(`/api/w/${workspace}/apps/get_data/v/${secret}.js`, { + credentials: 'include' + }), + fetch(`/api/w/${workspace}/apps/get_data/v/${secret}.css`, { + credentials: 'include' + }) + ]) + if (!jsRes.ok) { + throw new Error(`raw app ${ref.path} has no compiled bundle — deploy it first`) + } + const js = await jsRes.text() + const css = cssRes.ok ? await cssRes.text() : '' + const v: any = a.value ?? {} + const content = JSON.stringify({ + files: { ...(v.files ?? {}), '/bundle.js': js, '/bundle.css': css }, + runnables: v.runnables ?? {}, + // Preserve the full-code app's explicit data table declaration so it + // survives publish/import and feeds migration detection. + ...(v.data !== undefined ? { data: v.data } : {}), + ...(v.datatables !== undefined ? { datatables: v.datatables } : {}) + }) + return { kind: 'raw_app', path: ref.path, summary: a.summary, content } + } + const r = await fetch(`/api/w/${workspace}/raw_apps/get_data/0/${ref.path}`, { + credentials: 'include' + }) + if (!r.ok) return undefined + return { kind: 'raw_app', path: ref.path, content: await r.text() } + } + } catch (e: any) { + return undefined + } + return undefined + }, + resolveResourceType: async (path: string): Promise => { + try { + const r = await ResourceService.getResource({ workspace, path }) + return r.resource_type ?? undefined + } catch (e: any) { + return undefined + } + } + } + } + + #cachedBundleDeps(): BundleDeps { + const deps = this.#buildBundleDeps() + // Memoize only successful lookups: a miss (undefined) is likely transient, so + // evict it once it resolves. Otherwise a fixed/retried dependency can never + // clear `bundlePreview.unresolved` until the whole session is recreated. + const memoize = ( + cache: Map>, + key: string, + run: () => Promise + ) => { + let p = cache.get(key) + if (!p) { + p = run() + cache.set(key, p) + void p.then((r) => { + if (r === undefined && cache.get(key) === p) cache.delete(key) + }) + } + return p + } + return { + fetchItem: (ref) => + memoize(this.#previewItemCache, `${ref.kind}:${ref.path}`, () => deps.fetchItem(ref)), + resolveResourceType: (path) => + memoize(this.#previewTypeCache, path, () => deps.resolveResourceType(path)) + } + } + + async #postHub(path: string, body: unknown): Promise | undefined> { + const res = await fetch(`/api/w/${this.workspace}${path}${this.#folderQs()}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + credentials: 'include', + body: JSON.stringify(body) + }) + const text = await res.text() + if (!res.ok) throw new Error(text) + try { + return JSON.parse(text) + } catch { + return undefined + } + } + + async regenerateMigrations() { + const tok = ++this.#migrationsTok + this.migrationsGenerating = true + try { + // Same handler-augmented seed as deployAll: a data table used only by a + // bundled trigger handler must still get its migration. + const seed: ItemRef[] = [ + ...this.selectedItems + .filter((i) => i.kind !== 'resource') + .map((i) => ({ kind: i.kind as ItemRef['kind'], path: i.path })), + ...this.#triggerHandlerSeed(this.relevantTriggers, this.hubSlug || 'project') + ] + // Detection is independent of the final slug (data table refs aren't + // relocated), so any placeholder slug works for this throwaway bundle. + const bundle = await buildProjectBundle( + seed, + this.hubSlug || 'project', + this.#buildBundleDeps(), + [] + ) + const usage = await detectDatatableTables(bundle.items) + const drafts = await generateDatatableMigrations(this.workspace, usage) + if (this.#disposed || tok !== this.#migrationsTok) return + this.migrationDrafts = drafts + this.migrationsGeneration++ + } catch (e: any) { + if (!this.#disposed && tok === this.#migrationsTok) { + this.migrationDrafts = [] + this.migrationsGeneration++ + // Toast so a genuine failure isn't mistaken for "no data table usage". + sendUserToast(`Could not generate data table migrations: ${e?.message ?? e}`, true) + } + } finally { + if (!this.#disposed && tok === this.#migrationsTok) this.migrationsGenerating = false + } + } + + /** Prefill bundle metadata and start migration detection (bundle drawer opening). */ + prepareBundle() { + this.hubName = this.hubName || this.folder + void this.regenerateMigrations() + } + + /** + * Create the Hub draft then push the full bundle. `deploying` is set + * synchronously before the first request so a double-click cannot start a + * second publish, and the whole run is refused while triggers are still + * loading — an incomplete `relevantTriggers` snapshot would permanently + * omit triggers (and their handlers and migrations) from the draft. + * `onDraftCreated` fires once the draft exists (the bundle drawer closes + * there while items continue publishing). + */ + async publishBundle(onDraftCreated?: () => void): Promise { + if (this.deploying || this.triggersLoading || this.triggerDiscoveryFailed) return + this.deploying = true + try { + if (!(await this.#createDraft())) return + onDraftCreated?.() + await this.#deployAll() + } finally { + this.deploying = false + } + } + + /** + * Create the Hub draft project. Returns true when the draft exists and + * publishing can proceed. + */ + async #createDraft(): Promise { + this.hubName = this.hubName.trim() + this.hubSummary = this.hubSummary.trim() + this.hubReadme = this.hubReadme.trim() + try { + const res = await fetch(`/api/w/${this.workspace}/hub/publish_draft${this.#folderQs()}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + credentials: 'include', + body: JSON.stringify({ + slug: this.hubSlug, + name: this.hubName, + summary: this.hubSummary || this.hubName, + readme: this.hubReadme || undefined + }) + }) + const text = await res.text() + if (!res.ok) { + sendUserToast(`Hub draft creation failed: ${text}`, true) + return false + } + // Abort if Hub didn't echo a slug — guessing here lands items under + // a folder the Hub never locked. + let returnedSlug: string | undefined + try { + const parsed = JSON.parse(text) + if (typeof parsed?.slug === 'string') returnedSlug = parsed.slug + } catch {} + if (!returnedSlug) { + sendUserToast(`Hub did not return a slug. Aborting publish to avoid path drift.`, true) + return false + } + // Session replaced mid-request (workspace/folder switch): publishing now + // would push another scope's items into this draft. Abort. + if (this.#disposed) { + sendUserToast(`Workspace changed during publish — aborted to avoid mixing items.`, true) + return false + } + this.effectiveSlug = returnedSlug + return true + } catch (e: any) { + sendUserToast(`Hub draft creation failed: ${e?.message ?? e}`, true) + return false + } + } + + async #pushBundledItem(slug: string, it: BundledItem): Promise { + const key = `${it.kind}:${it.path}` + if (it.kind === 'script') { + const resp = await this.#postHub('/hub/scripts', { + summary: it.summary || it.newPath, + app: slug, + description: it.description ?? '', + kind: it.scriptKind ?? 'script', + content: it.content, + language: it.language, + schema: it.schema ?? undefined, + lockfile: it.lock ?? undefined, + path: it.newPath, + source_path: it.path, + project_slug: slug + }) + if (typeof resp?.id === 'number') this.hubItemIds = { ...this.hubItemIds, [key]: resp.id } + } else if (it.kind === 'flow') { + const resp = await this.#postHub('/hub/flows', { + flow: { + summary: it.summary || it.newPath, + description: it.description ?? undefined, + value: it.value, + schema: it.schema ?? undefined + }, + apps: [], + path: it.newPath, + source_path: it.path, + project_slug: slug + }) + if (typeof resp?.id === 'number') this.hubItemIds = { ...this.hubItemIds, [key]: resp.id } + } else if (it.kind === 'app') { + await this.#postHub('/hub/apps', { + app: it.value, + apps: [], + summary: it.summary || it.newPath, + description: undefined, + path: it.newPath, + source_path: it.path, + project_slug: slug + }) + } else if (it.kind === 'raw_app') { + const resp = await this.#postHub('/hub/raw_apps', { + raw: it.content ?? '', + apps: [], + summary: it.summary || it.newPath, + path: it.newPath, + source_path: it.path, + description: undefined, + project_slug: slug + }) + if (typeof resp?.id === 'number') this.hubItemIds = { ...this.hubItemIds, [key]: resp.id } + } + } + + // Handler runnables (trigger error handlers, schedule on_* handlers) ship + // with the bundle like the primary runnables do; hub refs stay external. + #triggerHandlerSeed(triggers: WorkspaceTrigger[], slug: string): ItemRef[] { + return triggers.flatMap(triggerHandlerRefs).filter((r) => classifyPath(r.path, slug) !== 'hub') + } + + // Every resource a trigger's exported config references: the kind-specific + // broker/auth field plus any `$res:` token nested in it (schedule args, + // handler extra args, …) — all must enter the bundle path map. + #triggerResourcePaths(triggers: WorkspaceTrigger[]): string[] { + const out = new Set() + for (const t of triggers) { + const rp = triggerResourcePath(t) + if (rp) out.add(rp) + for (const p of extractTriggerConfigResourceRefs(portableTriggerConfig(t.kind, t.config))) { + out.add(p) + } + } + return [...out] + } + + // Every whole-string `$var:`/`$jsonvar:` value a trigger's config resolves (SQS + // queue_url, schedule args, …) — relocated through the bundle map like item vars. + #triggerVarPaths(triggers: WorkspaceTrigger[]): string[] { + const out = new Set() + for (const t of triggers) { + for (const p of extractVarRefsFromValue(portableTriggerConfig(t.kind, t.config))) out.add(p) + } + return [...out] + } + + async #pushTriggers( + slug: string, + resourcePathMap: Map, + relevant: WorkspaceTrigger[] + ): Promise { + const pathMap = buildPathMap( + relevant.map((t) => t.path), + slug + ) + const triggers: Array> = [] + const skipped: string[] = [] + for (const t of relevant) { + const itemKind: ItemKind = t.is_flow ? 'flow' : 'script' + const runnableKey = `${itemKind}:${t.script_path}` + const hubId = this.hubItemIds[runnableKey] + if (!hubId) { + skipped.push(t.path) + continue + } + // Full-config remap: resource paths, error-handler paths, schedule on_* + // handler refs and whole-string `$var:` values all relocate through the map. + const config = rewriteVarRefsInValue( + rewriteTriggerConfig(portableTriggerConfig(t.kind, t.config), resourcePathMap), + resourcePathMap + ) + triggers.push({ + path: pathMap.get(t.path) ?? t.path, + kind: t.kind, + summary: t.summary ?? null, + description: (t.config as any)?.description ?? null, + config, + script_ask_id: t.is_flow ? null : hubId, + flow_id: t.is_flow ? hubId : null + }) + } + if (skipped.length > 0) { + sendUserToast( + `Skipped ${skipped.length} trigger(s) whose runnable did not publish: ${skipped.join(', ')}`, + true + ) + } + // Full-set sync: always push (an empty list clears the Hub's triggers on a + // re-deploy), so removing every trigger doesn't leave stale ones on the Hub. + await this.#postHub('/hub/triggers', { triggers, project_slug: slug }) + } + + // Builtin types (git_repository, ...) aren't in resource_type — push with empty schema. + async #pushResourceTypes(slug: string, types: string[]): Promise { + const results = await Promise.all( + types.map(async (name) => { + let schema: unknown = undefined + let description: string | undefined = undefined + try { + const rt = await ResourceService.getResourceType({ + workspace: this.workspace, + path: name + }) + schema = rt.schema ?? undefined + description = rt.description ?? undefined + } catch (e: any) {} + try { + await this.#postHub('/hub/resource_types', { + name, + schema, + description, + project_slug: slug + }) + return 0 + } catch (e: any) { + sendUserToast(`Resource type ${name} push failed: ${e?.message ?? e}`, true) + return 1 + } + }) + ) + return results.reduce((a: number, b) => a + b, 0) + } + + async #deployAll() { + const slug = this.hubSlug + // Snapshot the selection up-front: `selectedItems`/`relevantTriggers` are + // derived from live workspace data and `migrationDrafts` is edited in the + // drawer — the deploy must publish exactly what the user confirmed. + const itemsSnapshot = this.selectedItems.slice() + const triggersSnapshot = this.relevantTriggers.slice() + const migrationsSnapshot = this.migrationDrafts.slice() + this.hubItemIds = {} + this.deploymentStatus = {} + let failures = 0 + try { + const seed: ItemRef[] = [ + ...itemsSnapshot + .filter((i) => i.kind !== 'resource') + .map((i) => ({ kind: i.kind as ItemRef['kind'], path: i.path })), + ...this.#triggerHandlerSeed(triggersSnapshot, slug) + ] + const triggerResources = this.#triggerResourcePaths(triggersSnapshot) + const triggerVars = this.#triggerVarPaths(triggersSnapshot) + const bundle = await buildProjectBundle( + seed, + slug, + this.#buildBundleDeps(), + triggerResources, + triggerVars + ) + // Full path map (incl. unresolved) so a trigger's resource path is always + // relocated — never leaks the publisher's original private path to the Hub. + const resourcePathMap = bundle.pathMap + + // A dangling reference (a selected root or transitive runnable that failed + // to fetch, or a resource whose type can't be resolved) means the bundle + // doesn't close: the root would silently vanish, or a published item would + // still point at the publisher's private source-workspace path. Refuse to + // publish until every reference resolves rather than ship a broken project. + if (bundle.unresolved.length > 0) { + sendUserToast( + `Cannot publish: ${bundle.unresolved.length} unresolved reference(s): ${bundle.unresolved.join(', ')}. Deselect or fix them, then retry.`, + true + ) + return + } + + // Bundle building is slow — bail before the first Hub write if the session + // was replaced (workspace/folder switch) in the meantime. + if (this.#disposed) return + + // Types come from $res: stubs AND schema inputs (resource-). + const inputTypes = bundle.items + .flatMap((i) => typesFromSchema(i.schema)) + .filter((t) => !HIDDEN_RESOURCE_TYPES.has(t)) + const types = [ + ...new Set([...bundle.resourceStubs.map((s) => s.resource_type), ...inputTypes]) + ] + const depFailures = await this.#pushResourceTypes(slug, types) + + // Input-type deps with no path get a conventional f// stub. + const stubsByPath = new Map() + for (const s of bundle.resourceStubs) + stubsByPath.set(s.newPath, { path: s.newPath, resource_type: s.resource_type }) + for (const t of inputTypes) { + const path = `f/${slug}/${t}` + if (!stubsByPath.has(path)) stubsByPath.set(path, { path, resource_type: t }) + } + const stubs = [...stubsByPath.values()] + if (stubs.length > 0) { + try { + await this.#postHub('/hub/resources', { resources: stubs, project_slug: slug }) + } catch (e: any) { + sendUserToast(`Resource sync failed: ${e?.message ?? e}`, true) + failures++ + } + } + failures += depFailures + if (failures > 0) { + sendUserToast( + `Resource dependency sync failed — items not published to avoid broken references.`, + true + ) + return + } + + for (const it of bundle.items) { + // Stop writing item status / Hub IDs once the session is replaced — + // continuing would publish into a project the user has moved away from. + if (this.#disposed) return + const key = `${it.kind}:${it.path}` + this.deploymentStatus = { ...this.deploymentStatus, [key]: { status: 'loading' } } + try { + await this.#pushBundledItem(slug, it) + this.deploymentStatus = { ...this.deploymentStatus, [key]: { status: 'deployed' } } + } catch (e: any) { + failures++ + this.deploymentStatus = { + ...this.deploymentStatus, + [key]: { status: 'failed', error: e?.message ?? String(e) } + } + } + } + // A re-bundle clears the Hub-side embed (idempotent replace), so re-push it + // for any raw app that is already public — keeps the live iframe in sync + // without forcing an unpublish/share round-trip. Updates by hub id, safe in parallel. + const embedResults = await Promise.all( + bundle.items + .filter((it) => it.kind === 'raw_app') + .map(async (it) => { + const hubId = this.hubItemIds[`${it.kind}:${it.path}`] + const src = itemsSnapshot.find((i) => i.kind === 'raw_app' && i.path === it.path) + if (!hubId || !src?.published) return 0 + // The re-bundle cleared the embed; a public raw app with no resolved URL + // can't have its iframe restored, so it's an incomplete publish too — + // count it (like a push failure) so the draft can't become submit-ready. + if (!src.publicUrl) { + sendUserToast(`Cannot restore the iframe for ${it.path}: missing public URL`, true) + return 1 + } + try { + await this.#pushRawAppEmbed(hubId, src.publicUrl) + return 0 + } catch (e: any) { + sendUserToast(`Failed to sync iframe for ${it.path}: ${e?.message ?? e}`, true) + return 1 + } + }) + ) + failures += embedResults.reduce((a: number, b) => a + b, 0) + if (this.#disposed) return + try { + await this.#pushTriggers(slug, resourcePathMap, triggersSnapshot) + } catch (e: any) { + sendUserToast(`Trigger sync failed: ${e?.message ?? e}`, true) + failures++ + } + + // Full-set sync: always push (an empty list clears the Hub's migrations on + // a re-deploy). The Hub drops empty-SQL entries, so disabled placeholders + // don't persist. + try { + await this.#postHub('/hub/migrations', { + migrations: migrationsSnapshot.map((m) => ({ + datatable_name: m.datatable_name, + sql: m.sql, + sql_down: m.sql_down, + enabled: m.enabled + })), + project_slug: slug + }) + } catch (e: any) { + sendUserToast(`Data table migration sync failed: ${e?.message ?? e}`, true) + failures++ + } + + await sleep(150) + if (this.#disposed) return + // An incomplete push must never become submittable: a failed transitive item + // can leave a pushed runnable pointing at content that never landed. Stay in + // predeploy (deploymentStatus keeps the failed items visible) so re-publishing + // retries every write — createDraft and the item pushes are idempotent. + if (failures > 0) { + sendUserToast( + `Publish incomplete: ${failures} write(s) failed. Nothing was submitted — fix them and re-publish.`, + true + ) + return + } + this.deploymentStatus = {} + this.recordings = {} + // Deterministic baseline so a transient Hub read failure can't leave the + // UI stuck in `predeploy`; rehydrate then upgrades to authoritative state. + this.draftItems = itemsSnapshot.map((i) => ({ ...i, rec: 'none' })) + this.phase = 'draft' + await this.rehydrateFromHub() + sendUserToast(`Draft created on the Hub. Add recordings before submitting for review.`) + } finally { + this.deploying = false + } + } + + submitForReview = async () => { + const slug = this.hubSlug + if (!slug) return + this.submitting = true + try { + const res = await fetch( + `/api/w/${this.workspace}/hub/projects/${encodeURIComponent(slug)}/submit${this.#folderQs()}`, + { + method: 'POST', + credentials: 'include', + headers: { 'Content-Type': 'application/json' }, + body: '{}' + } + ) + if (!res.ok) { + sendUserToast(`Submit for review failed: ${await res.text()}`, true) + return + } + this.phase = 'under_review' + sendUserToast('Submitted for review by the Windmill team.') + } finally { + this.submitting = false + } + } + + syncWithHub = async () => { + this.syncing = true + try { + if (this.phase === 'draft') { + await this.#loadWorkspace() + const prev = new Map(this.draftItems.map((i) => [i.key, { rec: i.rec }])) + this.draftItems = this.workspaceItems + .filter((i) => prev.has(i.key)) + .map((i) => ({ ...i, rec: prev.get(i.key)?.rec ?? 'none' })) + } else { + // under_review / live: re-fetch the Hub project to pick up an + // admin status change (under_review -> live). + const before = this.phase + await this.rehydrateFromHub() + sendUserToast( + this.phase === before + ? 'Still waiting for review.' + : this.phase === 'live' + ? 'Approved — your project is now live.' + : `Status updated: ${this.phase}.` + ) + } + } catch (e: any) { + sendUserToast(`Sync failed: ${e?.message ?? e}`, true) + } finally { + this.syncing = false + } + } + + startNewDraft = () => { + this.draftItems = [] + this.recordings = {} + this.phase = 'predeploy' + } + + /** Reset record-drawer state and load the target's schema. */ + async openRecord(it: DeployItem) { + const tok = ++this.#recordRunTok + this.recordTarget = it + this.recordArgs = {} + this.recordValid = true + this.recordSchema = emptySchema() + this.recordSchemaLoading = true + this.runState = 'idle' + this.runJobId = undefined + this.runResult = undefined + this.runError = undefined + try { + if (it.kind === 'script') { + const s = await ScriptService.getScriptByPath({ + workspace: this.workspace, + path: it.path + }) + if (tok !== this.#recordRunTok) return + this.recordSchema = (s.schema as Record) ?? emptySchema() + } else if (it.kind === 'flow') { + const f = await FlowService.getFlowByPath({ workspace: this.workspace, path: it.path }) + if (tok !== this.#recordRunTok) return + this.recordSchema = (f.schema as Record) ?? emptySchema() + } + } catch (e: any) { + if (tok !== this.#recordRunTok) return + sendUserToast(`Failed to load schema: ${e?.message ?? e}`, true) + } finally { + if (tok === this.#recordRunTok) this.recordSchemaLoading = false + } + } + + /** Invalidate any in-flight record run/poll (record drawer closed). */ + cancelRecordRun = () => { + this.#recordRunTok++ + } + + runJob = async () => { + const it = this.recordTarget + if (!it) return + const tok = ++this.#recordRunTok + this.runState = 'running' + this.runJobId = undefined + this.runResult = undefined + this.runError = undefined + try { + let jobId: string + if (it.kind === 'script') { + jobId = await JobService.runScriptByPath({ + workspace: this.workspace, + path: it.path, + requestBody: this.recordArgs + }) + } else if (it.kind === 'flow') { + jobId = await JobService.runFlowByPath({ + workspace: this.workspace, + path: it.path, + requestBody: this.recordArgs + }) + } else { + if (tok === this.#recordRunTok) this.runState = 'idle' + return + } + if (tok !== this.#recordRunTok) return + this.runJobId = jobId + await this.#pollJobUntilComplete(jobId, tok) + } catch (e: any) { + if (tok !== this.#recordRunTok) return + this.runState = 'failed' + this.runError = `Failed to start: ${e?.message ?? e}` + } + } + + async #pollJobUntilComplete(jobId: string, tok: number) { + // First check immediately (fast scripts complete in ms), then back off to 2s. + const deadline = Date.now() + 5 * 60_000 + let interval = 250 + while (Date.now() < deadline) { + if (tok !== this.#recordRunTok) return + try { + const r = await JobService.getCompletedJobResultMaybe({ + workspace: this.workspace, + id: jobId + }) + if (tok !== this.#recordRunTok) return + if (r.completed) { + this.runResult = r.result + if (r.success) { + this.runState = 'success' + } else { + this.runState = 'failed' + this.runError = typeof r.result === 'string' ? r.result : JSON.stringify(r.result) + } + return + } + } catch (e: any) { + if (tok !== this.#recordRunTok) return + this.runState = 'failed' + this.runError = `Polling failed: ${e?.message ?? e}` + return + } + await sleep(interval) + interval = Math.min(interval * 2, 2000) + } + if (tok !== this.#recordRunTok) return + this.runState = 'failed' + this.runError = 'Timed out after 5 minutes' + } + + async #buildScriptRecording(it: DeployItem, jobId: string) { + const workspace = this.workspace + const s = await ScriptService.getScriptByPath({ workspace, path: it.path }) + const job = await JobService.getCompletedJob({ workspace, id: jobId }) + const initial_job = { ...(job as any), type: 'CompletedJob' } + const events = [{ t: 0, data: { completed: true, job: initial_job } }] + const duration = (initial_job.duration_ms as number) ?? 0 + return { + version: 1, + type: 'script' as const, + recorded_at: new Date().toISOString(), + script_path: it.path, + total_duration_ms: duration, + code: s.content, + language: s.language, + args: (job.args ?? {}) as Record, + schema: s.schema, + job: { initial_job, events } + } + } + + async #buildFlowRecording(it: DeployItem, jobId: string) { + const workspace = this.workspace + const f = await FlowService.getFlowByPath({ workspace, path: it.path }) + const root = (await JobService.getCompletedJob({ workspace, id: jobId })) as any + const jobs: Record = {} + const collect = async (j: any) => { + const stamped = { ...j, type: 'CompletedJob' } + jobs[j.id] = { + initial_job: stamped, + events: [{ t: 0, data: { completed: true, job: stamped } }] + } + const modules = (j.flow_status?.modules ?? []).filter( + (m: any) => m.job && typeof m.job === 'string' + ) + // Sub-jobs at the same level are independent reads. + await Promise.all( + modules.map(async (m: any) => { + try { + const sub = (await JobService.getCompletedJob({ workspace, id: m.job })) as any + await collect(sub) + } catch { + /* sub-job missing — skip */ + } + }) + ) + } + await collect(root) + return { + version: 1, + recorded_at: new Date().toISOString(), + flow_path: it.path, + total_duration_ms: (root.duration_ms as number) ?? 0, + flow: { + path: it.path, + value: f.value, + schema: f.schema ?? { type: 'object', properties: {}, required: [] }, + summary: f.summary ?? '', + archived: false, + edited_at: '', + edited_by: '', + extra_perms: {} + }, + jobs + } + } + + /** Save the current successful run as the Hub recording. Returns true on success. */ + async saveRecording(): Promise { + const it = this.recordTarget + if (!it || !this.runJobId || this.runState !== 'success') return false + const hubId = this.hubItemIds[it.key] + if (!hubId) { + sendUserToast(`Push the bundle to the Hub first before saving recordings`, true) + return false + } + if (it.kind !== 'script' && it.kind !== 'flow') { + sendUserToast(`Recordings only supported for script/flow`, true) + return false + } + try { + const recording = + it.kind === 'script' + ? await this.#buildScriptRecording(it, this.runJobId) + : await this.#buildFlowRecording(it, this.runJobId) + const path = it.kind === 'script' ? 'scripts' : 'flows' + await this.#postHub(`/hub/${path}/${hubId}/recording`, { + recording, + project_slug: this.hubSlug + }) + this.recordings = { ...this.recordings, [it.key]: this.runJobId } + this.#patchItem(it.key, { rec: 'recorded' }) + sendUserToast(`Recording saved — job ${this.runJobId}`) + return true + } catch (e: any) { + sendUserToast(`Failed to save recording: ${e?.message ?? e}`, true) + return false + } + } + + /** Resolve the project folder's asset graph so a data-pipeline project can be + * detected and its whole-folder cascade recorded. Best-effort — a project + * with no pipeline just never shows the pipeline record card. */ + async #loadPipelineGraph() { + try { + const params = new URLSearchParams({ + folder: this.folder, + asset_kinds: DATA_ASSET_KINDS.join(',') + }) + const res = await fetch(`/api/w/${this.workspace}/assets/graph?${params}`, { + credentials: 'include' + }) + if (!res.ok) throw new Error(`GET /assets/graph → ${res.status}`) + const graph = (await res.json()) as AssetGraphResponse + if (this.#disposed) return + this.pipelineGraph = graph + } catch { + // No pipeline graph — the pipeline record card simply stays hidden. + } + } + + /** Run the whole-folder cascade and capture it into a single PipelineRecording. + * Deployed-only (no drafts) and arg-less — unlike the editor it seeds no + * per-node input, so a root that needs uploaded data or a schedule's static + * payload records a failure the user can see and fix rather than a green run. */ + runPipelineRecording = async () => { + const fullGraph = this.pipelineGraph + const scripts = this.recordablePipelineScriptPaths + if (!fullGraph || scripts.length === 0) return + const scriptSet = new Set(scripts) + // Scope the graph to the project's members so the run, the recorded graph + // (rendered by the player) and the asset samples all exclude deselected + // branches. + const graph = pruneGraphToScripts(fullGraph, scriptSet) + const tok = ++this.#pipelineRunTok + this.pipelineRunState = 'running' + this.pipelineRecordingResult = undefined + this.pipelineRunError = undefined + // A previous save's badge must not linger over a fresh, unsaved re-run. + this.pipelineRecorded = false + const workspace = this.workspace + try { + const { recording, result } = await capturePipelineRecording({ + workspace, + folder: this.folder, + graph, + scriptPaths: scriptSet, + launch: (path) => + JobService.runScriptByPath({ + workspace, + path, + // Skip the backend asset-trigger dispatcher: the cascade engine owns + // the whole closure (parity with the pipeline editor's bounded run). + requestBody: { _wmill_skip_asset_dispatch: true } + }), + waitTerminal: (jobId) => this.#waitJobTerminal(jobId, tok) + }) + if (tok !== this.#pipelineRunTok) return + this.pipelineRecordingResult = recording + // A dependency cycle drops its members from the schedule, so an all- or + // partially-cyclic run leaves the recording missing steps (and an empty + // schedule reports `ok`). Treat any dropped cyclic member as a failure so + // an incomplete pipeline can't be saved as a successful recording. + if (result.cyclic.length > 0) { + this.pipelineRunState = 'failed' + this.pipelineRunError = `Cannot record — ${result.cyclic.length} script(s) on a dependency cycle: ${result.cyclic.join(', ')}` + } else if (result.ok) { + this.pipelineRunState = 'success' + } else { + this.pipelineRunState = 'failed' + const failed = [...result.statuses.entries()] + .filter(([, s]) => s.status === 'failure') + .map(([p]) => p) + this.pipelineRunError = + failed.length > 0 ? `Failed at ${failed.join(', ')}` : 'Cascade did not complete' + } + } catch (e: any) { + if (tok !== this.#pipelineRunTok) return + this.pipelineRunState = 'failed' + this.pipelineRunError = `Failed to run pipeline: ${e?.message ?? e}` + } + } + + // Poll a launched step to terminal, matching the pipeline editor's cascade + // timeout (DuckLake/DuckDB steps routinely exceed a few minutes). Adds the + // `#pipelineRunTok` cancellation the shared `makeWaitJobTerminal` lacks. + async #waitJobTerminal(jobId: string, tok: number): Promise<'success' | 'failure'> { + const deadline = Date.now() + CASCADE_JOB_TIMEOUT_MS + while (Date.now() < deadline) { + if (tok !== this.#pipelineRunTok) throw new Error('cancelled') + try { + const r = await JobService.getCompletedJobResultMaybe({ + workspace: this.workspace, + id: jobId, + getStarted: false + }) + if (r.completed) return r.success ? 'success' : 'failure' + } catch { + // transient — retry on the next tick + } + await sleep(CASCADE_POLL_INTERVAL_MS) + } + throw new Error(`Timed out waiting for job ${jobId}`) + } + + /** Save the captured pipeline recording to the Hub, scoped to the project + * (a pipeline is the whole folder, not a single Hub item). Returns true on + * success. */ + async savePipelineRecording(): Promise { + const recording = this.pipelineRecordingResult + if (!recording || this.pipelineRunState !== 'success') return false + if (this.phase === 'predeploy') { + sendUserToast(`Push the project to the Hub first before saving its pipeline recording`, true) + return false + } + try { + await this.#postHub(`/hub/projects/${this.hubSlug}/pipeline_recording`, { recording }) + this.pipelineRecorded = true + sendUserToast(`Pipeline recording saved`) + return true + } catch (e: any) { + sendUserToast(`Failed to save pipeline recording: ${e?.message ?? e}`, true) + return false + } + } + + // Set the Hub raw app's live-iframe URL (or clear it with null). The Hub renders + // from external_embed_url; project_slug scopes ownership. + async #pushRawAppEmbed(hubId: number, url: string | null) { + await this.#postHub(`/hub/raw_apps/${hubId}/embed`, { + external_embed_url: url, + project_slug: this.hubSlug + }) + } + + // Flip an app/raw app between public (anonymous) and private (publisher) and keep + // the Hub raw-app iframe in sync. Returns the resolved public URL when shared. + async #setAppShared(it: DeployItem, shared: boolean): Promise { + const workspace = this.workspace + const hubId = it.kind === 'raw_app' ? this.hubItemIds[it.key] : undefined + // Sharing a raw app as an iframe needs its Hub item to wire the embed. Fail + // before flipping the app public so it can't be left anonymous with no embed. + if (shared && it.kind === 'raw_app' && !hubId) { + throw new Error('Push the bundle to the Hub first to share the live iframe') + } + const app = await AppService.getAppByPath({ workspace, path: it.path }) + const prevMode = (app.policy?.execution_mode ?? 'publisher') as 'anonymous' | 'publisher' + const nextMode = (shared ? 'anonymous' : 'publisher') as 'anonymous' | 'publisher' + const setMode = (mode: 'anonymous' | 'publisher', message: string) => + AppService.updateApp({ + workspace, + path: it.path, + requestBody: { + policy: { ...(app.policy ?? {}), execution_mode: mode }, + deployment_message: message + } + }) + // Undo the policy flip so the app's public state stays consistent when a later + // step of the share fails. Best-effort: a revert failure must not mask the cause. + const rollback = () => setMode(prevMode, 'Revert iframe share').catch(() => {}) + await setMode(nextMode, shared ? 'Share as iframe' : 'Unshare iframe') + const url = shared ? ((await this.#resolvePublicUrl(it.path)) ?? null) : null + // A share with no resolvable public URL is incomplete (no embeddable link, no + // Unpublish control); don't leave the app anonymous while reporting success. + if (shared && url === null) { + await rollback() + throw new Error(`Could not resolve the public URL for ${it.path}`) + } + if (hubId && it.kind === 'raw_app' && (!shared || url)) { + try { + await this.#pushRawAppEmbed(hubId, shared ? url : null) + } catch (e) { + await rollback() + throw e + } + } + return url + } + + /** Make the publish target public. Returns true on success. */ + async confirmPublish(): Promise { + const it = this.publishTarget + if (!it || !canShareAsIframe(it)) return false + this.publishing = true + try { + const url = await this.#setAppShared(it, true) + this.#patchItem(it.key, { published: true, publicUrl: url ?? undefined }) + sendUserToast(`${it.path} is now public`) + return true + } catch (e: any) { + sendUserToast(`Failed to publish: ${e?.message ?? e}`, true) + return false + } finally { + this.publishing = false + } + } + + unpublishApp = async (it: DeployItem) => { + if (!canShareAsIframe(it)) return + try { + await this.#setAppShared(it, false) + this.#patchItem(it.key, { published: false, publicUrl: undefined }) + sendUserToast('App unpublished') + } catch (e: any) { + sendUserToast(`Failed to unpublish: ${e?.message ?? e}`, true) + } + } +} + +/** + * Owns the session lifecycle: a new `DeployToHubSession` is created whenever the + * (workspace, folder) identity actually changes — a spurious same-value store + * emit reuses the live session — and the previous one is disposed, which is the + * single mechanism invalidating its in-flight work. Also hosts the reactive + * plumbing the session itself can't (license-hydration reload, schedule + * previews, debounced bundle preview). + */ +export function useDeployToHubSession(args: { + workspace: () => string | undefined + folder: () => string + hasEeLicense: () => boolean +}) { + let session = $state() + + $effect(() => { + const workspace = args.workspace() + const folder = args.folder() + if (!workspace) return + untrack(() => { + if (session && session.workspace === workspace && session.folder === folder) return + session?.dispose() + const next = new DeployToHubSession(workspace, folder, { + hasEeLicense: args.hasEeLicense + }) + session = next + next.load() + }) + }) + + // The EE license hydrates async; if it lands after a license-less trigger load, + // EE kinds stay empty. Re-fetch on false→true (the session reads the license + // getter at call time). + let prevHadLicense: boolean | undefined = undefined + $effect(() => { + const hasLicense = args.hasEeLicense() + untrack(() => { + if (hasLicense && prevHadLicense === false) session?.reloadTriggers() + prevHadLicense = hasLicense + }) + }) + + // Leaving/entering predeploy invalidates manual selection tweaks. + $effect(() => { + const s = session + if (!s) return + s.phase + untrack(() => { + s.manualDeselected = new Set() + }) + }) + + // Schedule previews for relevant schedule triggers (deduped in the session). + $effect(() => { + session?.ensureSchedulePreviews() + }) + + // Debounced predeploy bundle preview; the session reads its reactive inputs + // synchronously and returns the cancel function used as effect cleanup. + $effect(() => { + const s = session + if (!s) return + return s.queueBundlePreview() + }) + + return { + get session() { + return session + } + } +} diff --git a/frontend/src/lib/components/workspaceSettings/deployToHubSession.test.ts b/frontend/src/lib/components/workspaceSettings/deployToHubSession.test.ts new file mode 100644 index 0000000000..7b61251fcc --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/deployToHubSession.test.ts @@ -0,0 +1,54 @@ +import { describe, it, expect } from 'vitest' +import { canShareAsIframe, mergeShareState, type DeployItem } from './deployToHubSession.svelte' + +function item(over: Partial & Pick): DeployItem { + return { rec: 'none', ...over } +} + +describe('canShareAsIframe', () => { + it('allows low-code apps and app-table raw apps', () => { + expect(canShareAsIframe(item({ key: 'app:f/a', path: 'f/a', kind: 'app' }))).toBe(true) + expect( + canShareAsIframe(item({ key: 'raw_app:f/r', path: 'f/r', kind: 'raw_app', appTable: true })) + ).toBe(true) + }) + it('hides the action for legacy raw apps (raw_app table only)', () => { + // Legacy entries from RawAppService carry no appTable flag; AppService can't load them. + expect(canShareAsIframe(item({ key: 'raw_app:f/r', path: 'f/r', kind: 'raw_app' }))).toBe(false) + }) + it('never offers the action for flows or scripts', () => { + expect(canShareAsIframe(item({ key: 'flow:f/f', path: 'f/f', kind: 'flow' }))).toBe(false) + }) +}) + +describe('mergeShareState', () => { + it('carries live public-share state from workspace items onto matching drafts', () => { + const drafts = [item({ key: 'app:f/a', path: 'f/a', kind: 'app' })] + const workspace = [ + item({ + key: 'app:f/a', + path: 'f/a', + kind: 'app', + published: true, + publicUrl: 'https://x/app' + }) + ] + const merged = mergeShareState(drafts, workspace) + expect(merged[0].published).toBe(true) + expect(merged[0].publicUrl).toBe('https://x/app') + }) + it('restores the app-table origin so app-table raw apps stay shareable', () => { + const drafts = [item({ key: 'raw_app:f/r', path: 'f/r', kind: 'raw_app' })] + const workspace = [item({ key: 'raw_app:f/r', path: 'f/r', kind: 'raw_app', appTable: true })] + expect(canShareAsIframe(mergeShareState(drafts, workspace)[0])).toBe(true) + }) + it('returns the same reference when nothing changes', () => { + const drafts = [item({ key: 'flow:f/f', path: 'f/f', kind: 'flow' })] + expect(mergeShareState(drafts, drafts)).toBe(drafts) + }) + it('leaves drafts without a workspace match untouched', () => { + const drafts = [item({ key: 'app:f/gone', path: 'f/gone', kind: 'app' })] + const merged = mergeShareState(drafts, [item({ key: 'app:f/a', path: 'f/a', kind: 'app' })]) + expect(merged).toBe(drafts) + }) +}) diff --git a/frontend/src/lib/components/workspaceSettings/projectBundle.test.ts b/frontend/src/lib/components/workspaceSettings/projectBundle.test.ts new file mode 100644 index 0000000000..669c0a43de --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectBundle.test.ts @@ -0,0 +1,869 @@ +import { describe, it, expect } from 'vitest' +import { + classifyPath, + extractScriptRefs, + extractFlowRefs, + extractAppRefs, + buildPathMap, + rewriteContent, + rewriteTriggerConfig, + rewriteFlowValue, + rewriteAppValue, + extractRawAppRefs, + rewriteRawAppContent, + buildProjectBundle, + retargetProjectExport, + collectExportVarPaths, + extractTriggerConfigResourceRefs, + extractVarRefsFromValue, + type ProjectExport, + type FetchedItem, + type ItemRef +} from './projectBundle' + +describe('classifyPath', () => { + it('internal for paths under the project folder', () => { + expect(classifyPath('f/proj/db', 'proj')).toBe('internal') + expect(classifyPath('f/proj', 'proj')).toBe('internal') + }) + it('hub for hub paths', () => { + expect(classifyPath('hub/16043/discord/send', 'proj')).toBe('hub') + }) + it('external for user and other folders', () => { + expect(classifyPath('u/admin/db', 'proj')).toBe('external') + expect(classifyPath('f/other/db', 'proj')).toBe('external') + }) + it('does not treat a prefix-only match as internal', () => { + expect(classifyPath('f/project2/db', 'proj')).toBe('external') + }) +}) + +describe('extractScriptRefs', () => { + it('finds $res: and res:// resource refs, deduped', () => { + const c = `const a = "$res:u/admin/db"; const b = "res://f/x/api"; const c2 = "$res:u/admin/db"` + expect(extractScriptRefs(c)).toEqual([ + { kind: 'resource', path: 'u/admin/db' }, + { kind: 'resource', path: 'f/x/api' } + ]) + }) + it('returns nothing when no refs', () => { + expect(extractScriptRefs('export async function main() {}')).toEqual([]) + }) +}) + +describe('extractFlowRefs', () => { + it('finds inline-code, static-input, and script-path refs', () => { + const value = { + modules: [ + { + id: 'a', + value: { + type: 'rawscript', + content: 'const db = "$res:u/admin/pg"', + input_transforms: { + other: { type: 'static', value: '$res:f/shared/api' }, + expr1: { type: 'javascript', expr: 'flow_input.x' } + } + } + }, + { + id: 'b', + value: { + type: 'branchone', + branches: [ + { + modules: [ + { id: 'c', value: { type: 'script', path: 'u/admin/my_script' } }, + { id: 'd', value: { type: 'script', path: 'hub/123/x/y' } } + ] + } + ], + default: [{ id: 'e', value: { type: 'rawscript', content: 'no refs' } }] + } + } + ] + } + const refs = extractFlowRefs(value) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/pg' }) + expect(refs).toContainEqual({ kind: 'resource', path: 'f/shared/api' }) + expect(refs).toContainEqual({ kind: 'script', path: 'u/admin/my_script' }) + expect(refs).toContainEqual({ kind: 'script', path: 'hub/123/x/y' }) + // a javascript expr (flow_input) is not a hardcoded ref + expect(refs.filter((r) => r.path === 'flow_input.x')).toEqual([]) + }) + it('finds sub-flow refs from type: flow steps', () => { + const value = { + modules: [ + { id: 'a', value: { type: 'flow', path: 'u/admin/sub_flow' } }, + { id: 'b', value: { type: 'flow', path: 'hub/9/x/y' } } + ] + } + const refs = extractFlowRefs(value) + expect(refs).toContainEqual({ kind: 'flow', path: 'u/admin/sub_flow' }) + expect(refs).toContainEqual({ kind: 'flow', path: 'hub/9/x/y' }) + }) +}) + +describe('buildPathMap', () => { + it('reparents into the project folder keeping the leaf name', () => { + const m = buildPathMap(['u/admin/db', 'f/other/api'], 'proj') + expect(m.get('u/admin/db')).toBe('f/proj/db') + expect(m.get('f/other/api')).toBe('f/proj/api') + }) + it('suffixes collisions deterministically', () => { + const m = buildPathMap(['u/alice/db', 'f/shared/db', 'u/bob/db'], 'proj') + // sorted: f/shared/db, u/alice/db, u/bob/db + expect(m.get('f/shared/db')).toBe('f/proj/db') + expect(m.get('u/alice/db')).toBe('f/proj/db_2') + expect(m.get('u/bob/db')).toBe('f/proj/db_3') + }) + it('maps internal paths to themselves, preserving subfolder depth', () => { + const m = buildPathMap(['f/proj/api', 'f/proj/sub/deep/script'], 'proj') + expect(m.get('f/proj/api')).toBe('f/proj/api') + expect(m.get('f/proj/sub/deep/script')).toBe('f/proj/sub/deep/script') + }) + it('does not flatten two internal items sharing a leaf name', () => { + const m = buildPathMap(['f/proj/a/x', 'f/proj/b/x'], 'proj') + expect(m.get('f/proj/a/x')).toBe('f/proj/a/x') + expect(m.get('f/proj/b/x')).toBe('f/proj/b/x') + }) + it('relocates an external onto a suffix when its leaf collides with an internal path', () => { + const m = buildPathMap(['f/proj/db', 'u/admin/db'], 'proj') + expect(m.get('f/proj/db')).toBe('f/proj/db') + expect(m.get('u/admin/db')).toBe('f/proj/db_2') + }) +}) + +describe('rewriteContent', () => { + it('rewrites mapped refs and leaves unmapped ones', () => { + const map = new Map([['u/admin/db', 'f/proj/db']]) + expect(rewriteContent('x = "$res:u/admin/db"', map)).toBe('x = "$res:f/proj/db"') + expect(rewriteContent('x = "res://u/admin/db"', map)).toBe('x = "$res:f/proj/db"') + expect(rewriteContent('x = "$res:hub/1/a/b"', map)).toBe('x = "$res:hub/1/a/b"') + }) + it('does not partial-match a longer path', () => { + const map = new Map([['u/admin/db', 'f/proj/db']]) + // u/admin/db2 must not be rewritten by the u/admin/db entry + expect(rewriteContent('x = "$res:u/admin/db2"', map)).toBe('x = "$res:u/admin/db2"') + }) +}) + +describe('rewriteTriggerConfig', () => { + const map = new Map([ + ['f/proj/kafka', 'f/target/kafka'], + ['f/proj/script', 'f/target/script'] + ]) + it('remaps plain resource path fields', () => { + expect( + rewriteTriggerConfig({ kafka_resource_path: 'f/proj/kafka', group_id: 'g1' }, map) + ).toEqual({ kafka_resource_path: 'f/target/kafka', group_id: 'g1' }) + }) + it('remaps nested objects, arrays, and $res: tokens', () => { + expect( + rewriteTriggerConfig( + { + nested: { path: 'f/proj/script' }, + list: ['f/proj/kafka', 'unrelated'], + code: 'x = "$res:f/proj/kafka"' + }, + map + ) + ).toEqual({ + nested: { path: 'f/target/script' }, + list: ['f/target/kafka', 'unrelated'], + code: 'x = "$res:f/target/kafka"' + }) + }) + it('leaves non-matching strings and non-string values untouched', () => { + const config = { url: 'wss://example.com', port: 9092, enabled: true, extra: null } + expect(rewriteTriggerConfig(config, map)).toEqual(config) + }) +}) + +describe('rewriteFlowValue', () => { + it('rewrites inline code, static inputs, and script paths; clones input', () => { + const map = new Map([ + ['u/admin/pg', 'f/proj/pg'], + ['f/shared/api', 'f/proj/api'], + ['u/admin/my_script', 'f/proj/my_script'] + ]) + const value = { + modules: [ + { + id: 'a', + value: { + type: 'rawscript', + content: 'const db = "$res:u/admin/pg"', + input_transforms: { other: { type: 'static', value: '$res:f/shared/api' } } + } + }, + { id: 'b', value: { type: 'script', path: 'u/admin/my_script' } }, + { id: 'c', value: { type: 'script', path: 'hub/1/keep/me' } } + ] + } + const out = rewriteFlowValue(value, map) + expect(out.modules[0].value.content).toBe('const db = "$res:f/proj/pg"') + expect(out.modules[0].value.input_transforms.other.value).toBe('$res:f/proj/api') + expect(out.modules[1].value.path).toBe('f/proj/my_script') + expect(out.modules[2].value.path).toBe('hub/1/keep/me') + // original untouched (deep clone) + expect(value.modules[0].value.content).toBe('const db = "$res:u/admin/pg"') + }) +}) + +// A trimmed app value: a runnable-by-path component, a hub runnable, a $res in an +// inline script, and incidental `f/...` text that must NOT be rewritten. +const appValue = () => ({ + grid: [ + { + data: { + componentInput: { + runnable: { type: 'runnableByPath', runType: 'script', path: 'u/admin/charts' } + } + } + }, + { + data: { + componentInput: { + runnable: { type: 'runnableByPath', runType: 'flow', path: 'f/shared/sync' } + } + } + }, + { + data: { + componentInput: { + runnable: { type: 'runnableByPath', runType: 'hubscript', path: 'hub/1/keep' } + } + } + } + ], + hiddenInlineScripts: [ + { name: 'h', inlineScript: { content: 'x = "$res:u/admin/pg"', language: 'deno' } } + ], + someLabel: 'see docs at f/shared/sync for details' +}) + +describe('extractAppRefs', () => { + it('extracts runnable-by-path scripts/flows and $res resources, skips hub', () => { + const refs = extractAppRefs(appValue()) + expect(refs).toContainEqual({ kind: 'script', path: 'u/admin/charts' }) + expect(refs).toContainEqual({ kind: 'flow', path: 'f/shared/sync' }) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/pg' }) + expect(refs.some((r) => r.path === 'hub/1/keep')).toBe(false) + }) +}) + +describe('rewriteAppValue', () => { + it('relocates runnable paths and $res, leaves hub refs and incidental text intact', () => { + const map = new Map([ + ['u/admin/charts', 'f/proj/charts'], + ['f/shared/sync', 'f/proj/sync'], + ['u/admin/pg', 'f/proj/pg'] + ]) + const value = appValue() + const out = rewriteAppValue(value, map) + expect(out.grid[0].data.componentInput.runnable.path).toBe('f/proj/charts') + expect(out.grid[1].data.componentInput.runnable.path).toBe('f/proj/sync') + expect(out.grid[2].data.componentInput.runnable.path).toBe('hub/1/keep') + expect(out.hiddenInlineScripts[0].inlineScript.content).toBe('x = "$res:f/proj/pg"') + // incidental text untouched + expect(out.someLabel).toBe('see docs at f/shared/sync for details') + // original untouched (deep clone) + expect(value.grid[0].data.componentInput.runnable.path).toBe('u/admin/charts') + }) +}) + +describe('raw app (value.raw JSON string)', () => { + const rawContent = () => + JSON.stringify({ + runnables: { + a: { type: 'path', runType: 'flow', path: 'u/admin/sync' }, + b: { type: 'path', runType: 'script', path: 'f/shared/calc' }, + c: { type: 'path', runType: 'hubscript', path: 'hub/1/keep' } + }, + files: { '/bundle.js': 'const conn = "$res:u/admin/pg"' } + }) + + it('extractRawAppRefs sees nested runnables and $res, skips hub', () => { + const refs = extractRawAppRefs(rawContent()) + expect(refs).toContainEqual({ kind: 'flow', path: 'u/admin/sync' }) + expect(refs).toContainEqual({ kind: 'script', path: 'f/shared/calc' }) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/pg' }) + expect(refs.some((r) => r.path === 'hub/1/keep')).toBe(false) + }) + + it('rewriteRawAppContent relocates nested runnable paths and $res', () => { + const map = new Map([ + ['u/admin/sync', 'f/proj/sync'], + ['f/shared/calc', 'f/proj/calc'], + ['u/admin/pg', 'f/proj/pg'] + ]) + const out = JSON.parse(rewriteRawAppContent(rawContent(), map)) + expect(out.runnables.a.path).toBe('f/proj/sync') + expect(out.runnables.b.path).toBe('f/proj/calc') + expect(out.runnables.c.path).toBe('hub/1/keep') + expect(out.files['/bundle.js']).toBe('const conn = "$res:f/proj/pg"') + }) + + it('falls back to $res scan on non-JSON content', () => { + expect(extractRawAppRefs('x = "$res:u/admin/pg"')).toContainEqual({ + kind: 'resource', + path: 'u/admin/pg' + }) + expect( + rewriteRawAppContent('x = "$res:u/admin/pg"', new Map([['u/admin/pg', 'f/proj/pg']])) + ).toBe('x = "$res:f/proj/pg"') + }) +}) + +describe('buildProjectBundle', () => { + // A flow that calls an external script which itself hardcodes a resource. + const flow: FetchedItem = { + kind: 'flow', + path: 'u/admin/my_flow', + summary: 'Flow', + value: { + modules: [ + { id: 'a', value: { type: 'script', path: 'u/admin/helper' } }, + { + id: 'b', + value: { + type: 'rawscript', + content: 'const x = "$res:f/shared/api"', + input_transforms: {} + } + } + ] + } + } + const helper: FetchedItem = { + kind: 'script', + path: 'u/admin/helper', + summary: 'Helper', + language: 'bun', + content: 'const db = "$res:u/admin/pg"; export async function main(){}' + } + + const deps = { + fetchItem: async (ref: ItemRef) => { + if (ref.path === 'u/admin/my_flow') return flow + if (ref.path === 'u/admin/helper') return helper + return undefined + }, + resolveResourceType: async (path: string) => { + if (path === 'u/admin/pg') return 'postgresql' + if (path === 'f/shared/api') return 'http_api' + return undefined + } + } + + it('pulls in referenced scripts + resources and rewrites everything under the folder', async () => { + const bundle = await buildProjectBundle( + [{ kind: 'flow', path: 'u/admin/my_flow' }], + 'proj', + deps + ) + + // flow + transitively-pulled helper script are both bundled + const byPath = Object.fromEntries(bundle.items.map((i) => [i.path, i])) + expect(Object.keys(byPath).sort()).toEqual(['u/admin/helper', 'u/admin/my_flow']) + + // items relocated under f/proj/ + expect(byPath['u/admin/my_flow'].newPath).toBe('f/proj/my_flow') + expect(byPath['u/admin/helper'].newPath).toBe('f/proj/helper') + + // flow's script-path ref rewritten to the helper's new path + expect(byPath['u/admin/my_flow'].value.modules[0].value.path).toBe('f/proj/helper') + // flow inline + helper code resource refs rewritten + expect(byPath['u/admin/my_flow'].value.modules[1].value.content).toBe( + 'const x = "$res:f/proj/api"' + ) + expect(byPath['u/admin/helper'].content).toContain('"$res:f/proj/pg"') + + // resource stubs created at new paths with resolved types + const stubs = Object.fromEntries(bundle.resourceStubs.map((s) => [s.originalPath, s])) + expect(stubs['u/admin/pg'].newPath).toBe('f/proj/pg') + expect(stubs['u/admin/pg'].resource_type).toBe('postgresql') + expect(stubs['f/shared/api'].resource_type).toBe('http_api') + + expect(bundle.unresolved).toEqual([]) + }) + + it('pulls in a sub-flow referenced by a type: flow step and rewrites its path', async () => { + const parent: FetchedItem = { + kind: 'flow', + path: 'u/admin/parent_flow', + value: { modules: [{ id: 'a', value: { type: 'flow', path: 'u/admin/sub_flow' } }] } + } + const sub: FetchedItem = { + kind: 'flow', + path: 'u/admin/sub_flow', + value: { + modules: [{ id: 'a', value: { type: 'script', path: 'hub/1/keep/me' } }] + } + } + const d = { + fetchItem: async (ref: ItemRef) => { + if (ref.path === 'u/admin/parent_flow') return parent + if (ref.path === 'u/admin/sub_flow') return sub + return undefined + }, + resolveResourceType: async () => undefined + } + const bundle = await buildProjectBundle( + [{ kind: 'flow', path: 'u/admin/parent_flow' }], + 'proj', + d + ) + const byPath = Object.fromEntries(bundle.items.map((i) => [i.path, i])) + // both flows bundled + expect(Object.keys(byPath).sort()).toEqual(['u/admin/parent_flow', 'u/admin/sub_flow']) + // parent's type: flow ref rewritten to the sub-flow's new path + expect(byPath['u/admin/parent_flow'].value.modules[0].value.path).toBe('f/proj/sub_flow') + expect(byPath['u/admin/sub_flow'].newPath).toBe('f/proj/sub_flow') + // hub ref inside the sub-flow left untouched + expect(byPath['u/admin/sub_flow'].value.modules[0].value.path).toBe('hub/1/keep/me') + expect(bundle.unresolved).toEqual([]) + }) + + it('leaves hub script references untouched and does not fetch them', async () => { + const hubFlow: FetchedItem = { + kind: 'flow', + path: 'u/admin/hub_flow', + value: { modules: [{ id: 'a', value: { type: 'script', path: 'hub/1/x/y' } }] } + } + const d = { + fetchItem: async (ref: ItemRef) => (ref.path === 'u/admin/hub_flow' ? hubFlow : undefined), + resolveResourceType: async () => undefined + } + const bundle = await buildProjectBundle([{ kind: 'flow', path: 'u/admin/hub_flow' }], 'proj', d) + expect(bundle.items.map((i) => i.path)).toEqual(['u/admin/hub_flow']) + expect(bundle.items[0].value.modules[0].value.path).toBe('hub/1/x/y') + expect(bundle.unresolved).toEqual([]) + }) + + it('reports a missing item and an unresolvable resource as unresolved', async () => { + const root: FetchedItem = { + kind: 'flow', + path: 'u/admin/root', + value: { + modules: [ + { id: 'a', value: { type: 'script', path: 'u/admin/gone' } }, + { + id: 'b', + value: { + type: 'rawscript', + content: 'const x = "$res:u/admin/untyped"', + input_transforms: {} + } + } + ] + } + } + const d = { + fetchItem: async (ref: ItemRef) => (ref.path === 'u/admin/root' ? root : undefined), + resolveResourceType: async () => undefined + } + const bundle = await buildProjectBundle([{ kind: 'flow', path: 'u/admin/root' }], 'proj', d) + expect(bundle.unresolved.sort()).toEqual(['u/admin/gone', 'u/admin/untyped']) + }) + + it('relocates $var:/$jsonvar: refs into the slug when it differs from the source folder', async () => { + const flow: FetchedItem = { + kind: 'flow', + path: 'f/source_folder/main', + value: { + flow_env: { CFG: '$jsonvar:f/source_folder/cfg' }, + modules: [ + { + id: 'a', + value: { + type: 'rawscript', + // Whole-value ref is relocated; the inline literal is not. + content: 'return "$var:f/source_folder/key"', + input_transforms: { k: { type: 'static', value: '$var:f/source_folder/key' } } + } + } + ] + } + } + const d = { + fetchItem: async (ref: ItemRef) => (ref.path === 'f/source_folder/main' ? flow : undefined), + resolveResourceType: async () => undefined + } + const bundle = await buildProjectBundle( + [{ kind: 'flow', path: 'f/source_folder/main' }], + 'kit', + d + ) + const v = bundle.items[0].value + expect(v.modules[0].value.input_transforms.k.value).toBe('$var:f/kit/key') + expect(v.flow_env.CFG).toBe('$jsonvar:f/kit/cfg') + // Inline code literal is untouched. + expect(v.modules[0].value.content).toBe('return "$var:f/source_folder/key"') + }) + + it('dedupes a path missing as both a script and a flow', async () => { + // A missing script + flow sharing a path each push the bare path once; the + // list must stay unique so a keyed UI render of it can't collide. + const root: FetchedItem = { + kind: 'flow', + path: 'u/admin/root', + value: { + modules: [ + { id: 'a', value: { type: 'script', path: 'u/admin/dup' } }, + { id: 'b', value: { type: 'flow', path: 'u/admin/dup' } } + ] + } + } + const d = { + fetchItem: async (ref: ItemRef) => (ref.path === 'u/admin/root' ? root : undefined), + resolveResourceType: async () => undefined + } + const bundle = await buildProjectBundle([{ kind: 'flow', path: 'u/admin/root' }], 'proj', d) + expect(bundle.unresolved).toEqual(['u/admin/dup']) + }) +}) + +describe('extractVarRefsFromValue', () => { + it('collects whole-value `$var:`/`$jsonvar:` refs, deduped, walking nested JSON', () => { + const value = { + flow_env: { API: '$var:u/admin/key' }, + modules: [ + { value: { input_transforms: { a: { type: 'static', value: '$var:f/proj/token' } } } }, + { value: { input_transforms: { b: { type: 'static', value: '$jsonvar:u/admin/cfg' } } } }, + { value: { input_transforms: { c: { type: 'static', value: '$var:u/admin/key' } } } } + ] + } + expect(extractVarRefsFromValue(value).sort()).toEqual([ + 'f/proj/token', + 'u/admin/cfg', + 'u/admin/key' + ]) + }) + it('ignores a `$var:` token embedded in inline code (not a whole value)', () => { + // The worker only substitutes a value that *is* the reference, so an inline + // script literal must not be treated as a variable arg. + const value = { + modules: [{ value: { type: 'rawscript', content: 'return "$var:u/example/template"' } }] + } + expect(extractVarRefsFromValue(value)).toEqual([]) + }) +}) + +describe('retargetProjectExport', () => { + const baseExport = (): ProjectExport => ({ + project: { slug: 'proj', name: 'Proj', summary: '', readme: null }, + scripts: [ + { + path: 'f/proj/hello', + content: 'const r = "$res:f/proj/db"', + summary: 'hello' + } + ], + flows: [ + { + path: 'f/proj/main_flow', + value: { + modules: [ + { id: 'a', value: { type: 'script', path: 'f/proj/hello', input_transforms: {} } } + ] + } + } + ], + apps: [ + { + path: 'f/proj/dashboard', + value: { grid: [{ data: { componentInput: { runnable: {} } } }] } + }, + { + path: 'f/proj/rawapp', + app_type: 'raw', + value: { raw: JSON.stringify({ files: {}, runnables: {} }) } + } + ], + resources: [{ path: 'f/proj/db', resource_type: 'postgresql' }], + triggers: [ + { + path: 'f/proj/every_day', + kind: 'schedule', + runnable_path: 'f/proj/hello', + runnable_kind: 'script', + config: { schedule: '0 0 12 * * *' } + }, + { + path: 'f/proj/kafka_in', + kind: 'kafka', + runnable_path: 'f/proj/hello', + runnable_kind: 'script', + config: { kafka_resource_path: 'f/proj/db' } + } + ] + }) + + it('returns the bundle unchanged when the folder matches the slug', () => { + const bundle = baseExport() + expect(retargetProjectExport(bundle, 'proj', 'proj')).toBe(bundle) + }) + + it('relocates every item path and internal reference into the target folder', () => { + const out = retargetProjectExport(baseExport(), 'proj', 'dest') + expect(out.scripts[0].path).toBe('f/dest/hello') + expect(out.scripts[0].content).toContain('$res:f/dest/db') + expect(out.flows[0].path).toBe('f/dest/main_flow') + expect(out.flows[0].value.modules[0].value.path).toBe('f/dest/hello') + expect(out.apps.map((a) => a.path)).toEqual(['f/dest/dashboard', 'f/dest/rawapp']) + expect(out.resources[0].path).toBe('f/dest/db') + expect(out.triggers[0].path).toBe('f/dest/every_day') + expect(out.triggers[0].runnable_path).toBe('f/dest/hello') + // Plain-string resource path in a trigger config is remapped too. + expect(out.triggers[1].config.kafka_resource_path).toBe('f/dest/db') + }) + + it('leaves external and hub paths untouched', () => { + const bundle = baseExport() + bundle.scripts[0].content = 'const a = "$res:u/admin/db"; const b = "$res:hub/1/x"' + const out = retargetProjectExport(bundle, 'proj', 'dest') + expect(out.scripts[0].content).toContain('$res:u/admin/db') + expect(out.scripts[0].content).toContain('$res:hub/1/x') + }) + + it('retargets internal $var:/$jsonvar: refs but leaves external ones', () => { + const bundle = baseExport() + bundle.flows[0].value.modules[0].value.input_transforms = { + key: { type: 'static', value: '$var:f/proj/api_key' }, + ext: { type: 'static', value: '$var:u/admin/personal' } + } + bundle.flows[0].value.flow_env = { CFG: '$jsonvar:f/proj/cfg' } + bundle.triggers[1].config.queue_url = '$var:f/proj/sqs' + const out = retargetProjectExport(bundle, 'proj', 'dest') + const it = out.flows[0].value.modules[0].value.input_transforms + expect(it.key.value).toBe('$var:f/dest/api_key') + expect(it.ext.value).toBe('$var:u/admin/personal') + expect(out.flows[0].value.flow_env.CFG).toBe('$jsonvar:f/dest/cfg') + expect(out.triggers[1].config.queue_url).toBe('$var:f/dest/sqs') + }) + + it('leaves an inert $var: literal embedded in inline code unchanged', () => { + const bundle = baseExport() + // Same path as a real runtime ref, but here it is a literal inside code: it + // must not be rewritten even once the path enters the retarget map. + bundle.flows[0].value.modules[0].value = { + type: 'rawscript', + content: 'return "$var:f/proj/api_key"', + input_transforms: { real: { type: 'static', value: '$var:f/proj/api_key' } } + } + const out = retargetProjectExport(bundle, 'proj', 'dest') + const mod = out.flows[0].value.modules[0].value + expect(mod.content).toBe('return "$var:f/proj/api_key"') + expect(mod.input_transforms.real.value).toBe('$var:f/dest/api_key') + }) +}) + +describe('collectExportVarPaths', () => { + it('gathers variable refs from flows, apps, and triggers (deduped)', () => { + const bundle: ProjectExport = { + project: { slug: 'proj', name: 'P', summary: '', readme: null }, + scripts: [], + flows: [{ path: 'f/proj/f', value: { flow_env: { A: '$var:f/proj/a' }, modules: [] } }], + apps: [ + { + path: 'f/proj/raw', + app_type: 'raw', + value: { raw: JSON.stringify({ runnables: { r: { fields: { x: '$var:u/admin/b' } } } }) } + } + ], + triggers: [{ path: 'f/proj/t', kind: 'sqs', config: { queue_url: '$jsonvar:f/proj/a' } }], + resources: [] + } + expect(collectExportVarPaths(bundle).sort()).toEqual(['f/proj/a', 'u/admin/b']) + }) +}) + +describe('trigger handler relocation', () => { + it('rewriteTriggerConfig remaps script/- and flow/-prefixed handler refs', () => { + const map = new Map([ + ['u/admin/handler', 'f/proj/handler'], + ['u/admin/recovery_flow', 'f/proj/recovery_flow'] + ]) + const out = rewriteTriggerConfig( + { + error_handler_path: 'u/admin/handler', + on_failure: 'script/u/admin/handler', + on_recovery: 'flow/u/admin/recovery_flow', + on_success: 'script/u/admin/unmapped' + }, + map + ) + expect(out.error_handler_path).toBe('f/proj/handler') + expect(out.on_failure).toBe('script/f/proj/handler') + expect(out.on_recovery).toBe('flow/f/proj/recovery_flow') + expect(out.on_success).toBe('script/u/admin/unmapped') + }) + + it('remaps $script:/$flow: only in the url field, never in literal payloads', () => { + const map = new Map([['u/admin/builder', 'f/proj/builder']]) + const out = rewriteTriggerConfig( + { + url: '$script:u/admin/builder', + initial_messages: [{ raw_message: '$script:u/admin/builder' }] + }, + map + ) + expect(out.url).toBe('$script:f/proj/builder') + expect(out.initial_messages[0].raw_message).toBe('$script:u/admin/builder') + }) + + it('leaves literal handler-shaped strings in args untouched', () => { + const map = new Map([['f/proj/handler', 'f/dest/handler']]) + const out = rewriteTriggerConfig( + { + on_failure: 'script/f/proj/handler', + args: { note: 'script/f/proj/handler' } + }, + map + ) + expect(out.on_failure).toBe('script/f/dest/handler') + expect(out.args.note).toBe('script/f/proj/handler') + }) + + it('leaves nested url keys untouched, rewriting only the top-level websocket url', () => { + const map = new Map([['u/admin/builder', 'f/proj/builder']]) + const out = rewriteTriggerConfig( + { + url: '$script:u/admin/builder', + args: { url: '$script:u/admin/builder' } + }, + map + ) + expect(out.url).toBe('$script:f/proj/builder') + expect(out.args.url).toBe('$script:u/admin/builder') + }) + + it('extracts and relocates $res refs nested in static input transform JSON', () => { + const value = { + modules: [ + { + id: 'a', + value: { + type: 'script', + path: 'f/proj/step', + input_transforms: { + provider: { type: 'static', value: { resource: '$res:u/admin/openai' } }, + note: { type: 'static', value: 'plain text' } + } + } + } + ] + } + const refs = extractFlowRefs(value) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/openai' }) + const out = rewriteFlowValue(value, new Map([['u/admin/openai', 'f/proj/openai']])) + const it0 = out.modules[0].value.input_transforms + expect(it0.provider.value).toEqual({ resource: '$res:f/proj/openai' }) + expect(typeof it0.note.value).toBe('string') + }) + + it('retargetProjectExport remaps trigger error handlers with the bundle', () => { + const bundle: ProjectExport = { + project: { slug: 'proj', name: 'P', summary: '', readme: null }, + scripts: [{ path: 'f/proj/handler', content: '' }], + flows: [], + apps: [], + resources: [], + triggers: [ + { + path: 'f/proj/sched', + kind: 'schedule', + runnable_path: 'f/proj/handler', + runnable_kind: 'script', + config: { schedule: '0 0 * * * *', on_failure: 'script/f/proj/handler' } + }, + { + path: 'f/proj/mq', + kind: 'mqtt', + runnable_path: 'f/proj/handler', + runnable_kind: 'script', + config: { error_handler_path: 'f/proj/handler' } + } + ] + } + const out = retargetProjectExport(bundle, 'proj', 'dest') + expect(out.triggers[0].config.on_failure).toBe('script/f/dest/handler') + expect(out.triggers[1].config.error_handler_path).toBe('f/dest/handler') + }) +}) + +describe('extractTriggerConfigResourceRefs', () => { + it('collects $res: tokens nested anywhere in a trigger config', () => { + expect( + extractTriggerConfigResourceRefs({ + schedule: '0 0 * * * *', + args: { channel: '$res:u/admin/slack' }, + on_failure_extra_args: { db: 'res://f/other/pg' }, + error_handler_args: { nested: { deep: '$res:u/admin/slack' } } + }) + ).toEqual(['u/admin/slack', 'f/other/pg']) + }) +}) + +describe('flow_env and preprocessor_module', () => { + const flowValue = { + modules: [], + preprocessor_module: { + id: 'pre', + value: { type: 'script', path: 'u/admin/preproc', input_transforms: {} } + }, + flow_env: { SLACK: '$res:u/admin/slack', PLAIN: 'not-a-ref' } + } + + it('walks nested children of the failure module', () => { + const refs = extractFlowRefs({ + modules: [], + failure_module: { + id: 'failure', + value: { + type: 'forloopflow', + modules: [ + { id: 'f-a', value: { type: 'script', path: 'u/admin/cleanup', input_transforms: {} } } + ] + } + } + }) + expect(refs).toContainEqual({ kind: 'script', path: 'u/admin/cleanup' }) + }) + + it('extractFlowRefs sees preprocessor scripts and flow_env resources', () => { + const refs = extractFlowRefs(flowValue) + expect(refs).toContainEqual({ kind: 'script', path: 'u/admin/preproc' }) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/slack' }) + }) + + it('sees and relocates $res refs nested inside JSON flow_env values', () => { + const value = { + modules: [], + flow_env: { CFG: { db: '$res:u/admin/pg', opts: ['res://u/admin/s3'] } } + } + const refs = extractFlowRefs(value) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/pg' }) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/s3' }) + const map = new Map([ + ['u/admin/pg', 'f/proj/pg'], + ['u/admin/s3', 'f/proj/s3'] + ]) + const out = rewriteFlowValue(value, map) + expect(out.flow_env.CFG.db).toBe('$res:f/proj/pg') + expect(out.flow_env.CFG.opts[0]).toBe('$res:f/proj/s3') + }) + + it('rewriteFlowValue relocates both', () => { + const map = new Map([ + ['u/admin/preproc', 'f/proj/preproc'], + ['u/admin/slack', 'f/proj/slack'] + ]) + const out = rewriteFlowValue(flowValue, map) + expect(out.preprocessor_module.value.path).toBe('f/proj/preproc') + expect(out.flow_env.SLACK).toBe('$res:f/proj/slack') + expect(out.flow_env.PLAIN).toBe('not-a-ref') + }) +}) diff --git a/frontend/src/lib/components/workspaceSettings/projectBundle.ts b/frontend/src/lib/components/workspaceSettings/projectBundle.ts new file mode 100644 index 0000000000..1fd62498d5 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectBundle.ts @@ -0,0 +1,652 @@ +// Pure logic for the "project = folder" Hub bundle. A project is one folder +// `f//...`. Bundling: collect the transitive closure, relocate external +// refs (`u//`, `f//` -> `f//`, `_2`/`_3`… +// on collision) and rewrite them. Hub refs stay external; runtime string-concat +// paths are out of scope. No API/Svelte deps so it's unit-testable. + +import { getAllModules } from '$lib/components/flows/flowExplorer' +import { isRunnableByPath } from '$lib/components/apps/inputType' + +export type RefKind = 'resource' | 'script' | 'flow' + +export interface Ref { + kind: RefKind + /** Bare path, without the `$res:` / `res://` prefix for resources. */ + path: string +} + +export type PathClass = 'internal' | 'hub' | 'external' + +/** A single `$res:PATH` / `res://PATH` token (path captured in group 1). */ +const RES_TOKEN_RE = /(?:\$res:|res:\/\/)([\w\-./]+)/g + +// A whole-string `$var:PATH` / `$jsonvar:PATH` value. The worker substitutes these +// only when an argument value *is* the reference (walking nested JSON), never a +// token embedded in inline code, so the whole value must match. `_KIND` captures +// the prefix (group 1) and path (group 2) so a rewrite can preserve `var`/`jsonvar`. +const VAR_VALUE_RE = /^\$(?:json)?var:([\w\-./]+)$/ +const VAR_VALUE_RE_KIND = /^\$(var|jsonvar):([\w\-./]+)$/ + +// Variable paths a value will resolve at runtime (flow static inputs, flow_env, +// app runnable inputs, trigger config fields). Walk the parsed structure and match +// whole string values so inline code carrying a literal `$var:` string is ignored. +export function extractVarRefsFromValue(value: any): string[] { + const out = new Set() + const walk = (v: any) => { + if (typeof v === 'string') { + const m = VAR_VALUE_RE.exec(v) + if (m) out.add(m[1]) + } else if (Array.isArray(v)) { + for (const x of v) walk(x) + } else if (v && typeof v === 'object') { + for (const k of Object.keys(v)) walk(v[k]) + } + } + walk(value) + return [...out] +} + +export function classifyPath(path: string, slug: string): PathClass { + if (path.startsWith(`f/${slug}/`) || path === `f/${slug}`) return 'internal' + if (path.startsWith('hub/')) return 'hub' + return 'external' +} + +export function extractScriptRefs(content: string): Ref[] { + const out: Ref[] = [] + const seen = new Set() + let m: RegExpExecArray | null + RES_TOKEN_RE.lastIndex = 0 + while ((m = RES_TOKEN_RE.exec(content)) !== null) { + if (!seen.has(m[1])) { + seen.add(m[1]) + out.push({ kind: 'resource', path: m[1] }) + } + } + return out +} + +/** + * References inside a flow value: + * - inline rawscript code with `$res:` (resource) + * - static step inputs whose value is a `$res:` literal (resource) + * - `type: script` steps that reference a script by path (script) + * - `type: flow` steps that reference a sub-flow by path (flow) + */ +export function extractFlowRefs(value: any): Ref[] { + const out: Ref[] = [] + const seen = new Set() + const add = (kind: RefKind, path: string) => { + const key = `${kind}:${path}` + if (!seen.has(key)) { + seen.add(key) + out.push({ kind, path }) + } + } + // getAllModules flattens the whole tree (loops, branches, aiagent tools, + // failure module) so each module only needs local inspection; the + // preprocessor module sits outside `modules` and is walked the same way. + for (const mod of allFlowModules(value)) { + const v: any = (mod as any)?.value + if (!v || typeof v !== 'object') continue + if (v.type === 'script' && typeof v.path === 'string') add('script', v.path) + if (v.type === 'flow' && typeof v.path === 'string') add('flow', v.path) + if (typeof v.content === 'string') { + for (const r of extractScriptRefs(v.content)) add('resource', r.path) + } + const it = v.input_transforms + if (it && typeof it === 'object') { + for (const key of Object.keys(it)) { + const t = it[key] + // Static values can be a bare `$res:` string or arbitrary JSON with + // refs nested anywhere — the worker resolves both, so scan the full + // serialization. + if (t?.type === 'static' && t.value !== undefined) { + const text = typeof t.value === 'string' ? t.value : JSON.stringify(t.value) + for (const r of extractScriptRefs(text)) add('resource', r.path) + } + } + } + } + // flow_env values support `$res:path` references — as whole string values or + // nested inside JSON values (the worker resolves both), so scan the full + // serialization. + if (value?.flow_env && typeof value.flow_env === 'object') { + for (const r of extractScriptRefs(JSON.stringify(value.flow_env))) add('resource', r.path) + } + return out +} + +// Every module of a flow value: the tree under `modules`, the failure module, +// and the preprocessor module (which lives outside `modules`). Any walk over a +// flow's modules must go through this — a walk that misses a module class +// silently drops its dependencies from bundles or migrations. All three go in +// the root list (not getAllModules' failure_module parameter, which appends +// the module without expanding its descendants) so nested children of a +// failure or preprocessor module are walked too. +export function allFlowModules(value: any) { + return getAllModules([ + ...(value?.modules ?? []), + ...(value?.preprocessor_module ? [value.preprocessor_module] : []), + ...(value?.failure_module ? [value.failure_module] : []) + ]) +} + +// Visit every object node in an app value tree (JSON-safe, no cycles). +function walkAppNodes(value: any, visit: (node: Record) => void): void { + if (value == null || typeof value !== 'object') return + if (Array.isArray(value)) { + for (const v of value) walkAppNodes(v, visit) + return + } + visit(value) + for (const k of Object.keys(value)) walkAppNodes(value[k], visit) +} + +// `runnableByPath`/`path` nodes reference a workspace runnable by path. +function runnableRef(node: Record): Ref | undefined { + if (!isRunnableByPath(node as any) || typeof node.path !== 'string') return undefined + if (node.runType === 'flow') return { kind: 'flow', path: node.path } + if (node.runType === 'script') return { kind: 'script', path: node.path } + return undefined // hubscript -> external hub, ignored +} + +// App refs: `$res:` resources anywhere in the value, plus script/flow runnables +// referenced by path in components. +export function extractAppRefs(value: any): Ref[] { + const out: Ref[] = [] + const seen = new Set() + const add = (kind: RefKind, path: string) => { + const key = `${kind}:${path}` + if (!seen.has(key)) { + seen.add(key) + out.push({ kind, path }) + } + } + walkAppNodes(value, (node) => { + const r = runnableRef(node) + if (r) add(r.kind, r.path) + }) + for (const r of extractScriptRefs(JSON.stringify(value ?? {}))) add('resource', r.path) + return out +} + +/** + * Build the relocation map. Internal paths (`f//...`) map to themselves + * and are reserved first; external paths relocate to `f//` (`_2`/`_3`… + * on collision). Input is sorted so suffix assignment is deterministic. + */ +export function buildPathMap(paths: Iterable, slug: string): Map { + const map = new Map() + const used = new Set() + const sorted = [...new Set(paths)].sort() + for (const p of sorted) { + if (classifyPath(p, slug) === 'internal') { + map.set(p, p) + used.add(p) + } + } + for (const old of sorted) { + if (map.has(old)) continue + const name = old.split('/').filter(Boolean).pop() ?? old + let candidate = `f/${slug}/${name}` + let n = 2 + while (used.has(candidate)) candidate = `f/${slug}/${name}_${n++}` + used.add(candidate) + map.set(old, candidate) + } + return map +} + +// Both ref forms normalize to `$res:` on rewrite. +export function rewriteContent(content: string, map: Map): string { + return content.replace(RES_TOKEN_RE, (whole, path) => { + const next = map.get(path) + return next ? `$res:${next}` : whole + }) +} + +// Structurally relocate whole-string `$var:`/`$jsonvar:` values — the only form the +// worker resolves. Walks the parsed value so an inert token embedded in inline code +// or arbitrary text is left untouched, unlike token replacement over serialized +// strings. Only paths present in the map move (the retarget map carries variables). +export function rewriteVarRefsInValue(value: any, map: Map): any { + if (typeof value === 'string') { + const m = VAR_VALUE_RE_KIND.exec(value) + if (m) { + const next = map.get(m[2]) + if (next) return `$${m[1]}:${next}` + } + return value + } + if (Array.isArray(value)) return value.map((v) => rewriteVarRefsInValue(v, map)) + if (value && typeof value === 'object') { + const out: Record = {} + for (const k of Object.keys(value)) out[k] = rewriteVarRefsInValue(value[k], map) + return out + } + return value +} + +/** + * `$res:`/`res://` tokens anywhere in a trigger config — schedule args, + * on_*_extra_args, error_handler_args, … (e.g. the built-in Slack handler + * stores its channel resource this way). These must enter the bundle path map + * so `rewriteTriggerConfig` relocates them and a stub is exported. + */ +export function extractTriggerConfigResourceRefs(config: any): string[] { + return extractScriptRefs(JSON.stringify(config ?? {})).map((r) => r.path) +} + +/** + * Trigger configs reference resources as plain path strings (e.g. + * `kafka_resource_path: "f/slug/db"`), not `$res:` tokens, so token rewriting + * misses them. Deep-walk the config and remap any string that exact-matches a + * map key (map keys are full bundle paths, so an exact match is a reference), + * or a `script/`/`flow/` handler reference (schedules' on_failure + * et al.), falling back to `$res:` token rewriting for embedded refs. + */ +// Top-level config fields whose string values are prefixed runnable refs. +// Prefixed forms are remapped ONLY in these known positions: deciding meaning +// from string shape alone rewrote literal payloads that merely looked like +// refs. Bare-path exact matches and $res: tokens stay position-independent. +const HANDLER_REF_FIELDS = new Set(['on_failure', 'on_recovery', 'on_success']) + +export function rewriteTriggerConfig(config: any, map: Map, depth = 0): any { + if (typeof config === 'string') { + const direct = map.get(config) + if (direct) return direct + return rewriteContent(config, map) + } + if (Array.isArray(config)) return config.map((v) => rewriteTriggerConfig(v, map, depth + 1)) + if (config && typeof config === 'object') { + return Object.fromEntries( + Object.entries(config).map(([k, v]) => { + if (depth === 0 && typeof v === 'string') { + // Websocket url: $script: / $flow:. + if (k === 'url') { + const m = /^\$(script|flow):(.+)$/.exec(v) + if (m && map.has(m[2])) return [k, `$${m[1]}:${map.get(m[2])}`] + } + // Schedule handlers: script/ / flow/. + if (HANDLER_REF_FIELDS.has(k)) { + const m = /^(script|flow)\/(.+)$/.exec(v) + if (m && map.has(m[2])) return [k, `${m[1]}/${map.get(m[2])}`] + } + } + return [k, rewriteTriggerConfig(v, map, depth + 1)] + }) + ) + } + return config +} + +export function rewriteFlowValue(value: any, map: Map): any { + const cloned = JSON.parse(JSON.stringify(value ?? {})) + for (const mod of allFlowModules(cloned)) { + const v: any = (mod as any)?.value + if (!v || typeof v !== 'object') continue + if ( + (v.type === 'script' || v.type === 'flow') && + typeof v.path === 'string' && + map.has(v.path) + ) { + v.path = map.get(v.path) + } + if (typeof v.content === 'string') v.content = rewriteContent(v.content, map) + const it = v.input_transforms + if (it && typeof it === 'object') { + for (const key of Object.keys(it)) { + const t = it[key] + // Mirror extraction: rewrite refs wherever they sit, preserving the + // value's type (a string stays a string, JSON round-trips). + if (t?.type === 'static' && t.value !== undefined) { + if (typeof t.value === 'string') { + t.value = rewriteContent(t.value, map) + } else { + t.value = JSON.parse(rewriteContent(JSON.stringify(t.value), map)) + } + } + } + } + } + if (cloned?.flow_env && typeof cloned.flow_env === 'object') { + // Tokens can sit inside nested JSON values, not just string values; the + // serialize→rewrite→parse round-trip reaches all of them (paths contain + // no characters that would break JSON string literals). + cloned.flow_env = JSON.parse(rewriteContent(JSON.stringify(cloned.flow_env), map)) + } + return cloned +} + +// Relocate `$res:` tokens (one round-trip, also produces a fresh clone) then +// runnable-by-path refs structurally. Incidental `f//` strings stay intact. +export function rewriteAppValue(value: any, map: Map): any { + if (value == null) return value + const cloned = JSON.parse(rewriteContent(JSON.stringify(value), map)) + walkAppNodes(cloned, (node) => { + if (runnableRef(node) && map.has(node.path)) node.path = map.get(node.path) + }) + return cloned +} + +// Raw/compiled apps store their structure as a JSON string (`{ runnables, files }`). +// Parse it so runnable-by-path refs in the runnables map are seen, reusing the +// same walk; fall back to plain `$res:` scanning if it isn't valid JSON. +export function extractRawAppRefs(content: string): Ref[] { + let parsed: any + try { + parsed = JSON.parse(content) + } catch { + return extractScriptRefs(content) + } + return extractAppRefs(parsed) +} + +export function rewriteRawAppContent(content: string, map: Map): string { + let parsed: any + try { + parsed = JSON.parse(content) + } catch { + return rewriteContent(content, map) + } + return JSON.stringify(rewriteAppValue(parsed, map)) +} + +// --------------------------------------------------------------------------- +// Hub project export format (what /projects/{slug}/export returns) and its +// retargeting into a destination folder. Kept here, next to the rewriters, +// so the bundle format is defined in one module for both publish and install. +// --------------------------------------------------------------------------- + +export type ExportItem = Record +export interface ProjectMigration { + datatable_name: string + sql: string + sql_down?: string + enabled: boolean +} +export interface ProjectExport { + project: { slug: string; name: string; summary: string; readme: string | null } + scripts: ExportItem[] + flows: ExportItem[] + apps: ExportItem[] + resources: ExportItem[] + triggers: ExportItem[] + migrations?: ProjectMigration[] +} + +// Map bundled paths `f//...` -> `f//...`. Only enumerated +// paths go in, so rewriters touch real refs, never incidental text. +export function buildRetargetMap( + bundle: ProjectExport, + fromSlug: string, + folder: string +): Map { + const map = new Map() + const prefix = `f/${fromSlug}/` + const add = (p: unknown) => { + if (typeof p === 'string' && p.startsWith(prefix)) { + map.set(p, `f/${folder}/${p.slice(prefix.length)}`) + } + } + for (const s of bundle.scripts) add(s.path) + for (const f of bundle.flows) add(f.path) + for (const a of bundle.apps) add(a.path) + for (const r of bundle.resources) add(r.path) + for (const t of bundle.triggers) { + add(t.path) + add(t.runnable_path) + } + // Variables aren't enumerated in the export; their `$var:`/`$jsonvar:` refs live + // inside item values. Relocate the internal ones so a renamed-folder import + // rewrites them into the target folder instead of retaining the old prefix. + for (const p of collectExportVarPaths(bundle)) add(p) + return map +} + +// Internal-or-external variable paths referenced by the export's flows, apps and +// triggers. Scripts carry no variable args. Raw apps hold their structure in the +// `value.raw` JSON string. +export function collectExportVarPaths(bundle: ProjectExport): string[] { + const out = new Set() + const collect = (value: any) => { + for (const p of extractVarRefsFromValue(value)) out.add(p) + } + for (const f of bundle.flows) collect(f.value) + for (const a of bundle.apps) collect(a.app_type === 'raw' ? safeParseRaw(a.value?.raw) : a.value) + for (const t of bundle.triggers) collect(t.config) + return [...out] +} + +function safeParseRaw(raw: unknown): any { + if (typeof raw !== 'string') return undefined + try { + return JSON.parse(raw) + } catch { + return undefined + } +} + +// Structural retarget: rewrite each item's path and its internal refs, +// leaving Hub refs and arbitrary content untouched. +export function retargetProjectExport( + bundle: ProjectExport, + fromSlug: string, + folder: string +): ProjectExport { + if (folder === fromSlug) return bundle + const map = buildRetargetMap(bundle, fromSlug, folder) + const remap = (p: unknown) => (typeof p === 'string' ? (map.get(p) ?? p) : p) + return { + ...bundle, + scripts: bundle.scripts.map((s) => ({ + ...s, + path: remap(s.path), + content: rewriteContent(s.content ?? '', map) + })), + flows: bundle.flows.map((f) => ({ + ...f, + path: remap(f.path), + value: rewriteVarRefsInValue(rewriteFlowValue(f.value, map), map) + })), + apps: bundle.apps.map((a) => ({ + ...a, + path: remap(a.path), + // Raw apps keep their structure in the `value.raw` JSON string. + value: + a.app_type === 'raw' + ? { + ...a.value, + raw: rewriteRawVarRefs(rewriteRawAppContent(a.value?.raw ?? '', map), map) + } + : rewriteVarRefsInValue(rewriteAppValue(a.value, map), map) + })), + resources: bundle.resources.map((r) => ({ ...r, path: remap(r.path) })), + triggers: bundle.triggers.map((t) => ({ + ...t, + path: remap(t.path), + runnable_path: remap(t.runnable_path), + // Configs hold `$res:` tokens, plain resource paths (kafka_resource_path + // etc.) and whole-string `$var:` values — rewrite all three. + config: t.config ? rewriteVarRefsInValue(rewriteTriggerConfig(t.config, map), map) : t.config + })) + } +} + +// Var relocation for a raw app's `value.raw` JSON string: parse, structurally +// rewrite whole-string var values, re-serialize; leave invalid JSON untouched. +function rewriteRawVarRefs(raw: string, map: Map): string { + const parsed = safeParseRaw(raw) + if (parsed === undefined) return raw + return JSON.stringify(rewriteVarRefsInValue(parsed, map)) +} + +export type ItemKind = 'script' | 'flow' | 'app' | 'raw_app' + +export interface ItemRef { + kind: ItemKind + path: string +} + +export interface FetchedItem { + kind: ItemKind + path: string + summary?: string + description?: string + /** scripts + raw_apps */ + content?: string + /** flows + apps */ + value?: any + /** scripts */ + language?: string + schema?: any + lock?: string + scriptKind?: string +} + +export interface BundleDeps { + /** Fetch a workspace item by ref, or undefined if it doesn't exist. */ + fetchItem: (ref: ItemRef) => Promise + /** Resolve a resource path to its type, or undefined if missing. */ + resolveResourceType: (path: string) => Promise +} + +export interface BundledItem extends FetchedItem { + /** Path the item takes inside the project folder. */ + newPath: string +} + +export interface ResourceStub { + originalPath: string + newPath: string + resource_type: string +} + +export interface ProjectBundle { + items: BundledItem[] + resourceStubs: ResourceStub[] + /** Original -> relocated path for every item and resource (incl. unresolved). */ + pathMap: Map + /** External paths we couldn't fetch/resolve (missing items or untyped resources). */ + unresolved: string[] +} + +function refsForFetched(item: FetchedItem): Ref[] { + if (item.kind === 'script') return extractScriptRefs(item.content ?? '') + if (item.kind === 'flow') return extractFlowRefs(item.value) + if (item.kind === 'app') return extractAppRefs(item.value) + if (item.kind === 'raw_app') return extractRawAppRefs(item.content ?? '') + return [] +} + +// Whole-string `$var:`/`$jsonvar:` paths an item resolves at runtime. Scripts carry +// no variable args; raw apps hold their structure in the `content` JSON string. +function varRefsForFetched(item: FetchedItem): string[] { + if (item.kind === 'flow' || item.kind === 'app') return extractVarRefsFromValue(item.value) + if (item.kind === 'raw_app') return extractVarRefsFromValue(safeParseRaw(item.content)) + return [] +} + +// Walks the transitive closure: scripts referenced by path are pulled in +// recursively, resources become empty stubs, hub refs stay external. +export async function buildProjectBundle( + seed: ItemRef[], + slug: string, + deps: BundleDeps, + extraResourcePaths: string[] = [], + extraVarPaths: string[] = [] +): Promise { + const fetched = new Map() + const queued = new Set() + const resourcePaths = new Set() + const varPaths = new Set() + const unresolved: string[] = [] + + // Resources and variables referenced by triggers (by config value, not `$res:` + // in code) — relocated through the same map so the export stays slug-relative. + for (const p of extraResourcePaths) { + if (classifyPath(p, slug) !== 'hub') resourcePaths.add(p) + } + for (const p of extraVarPaths) varPaths.add(p) + + // Key by `${kind}:${path}`, not bare path: a script and flow can share a path, + // and keying by path alone would silently drop one. + const refKey = (kind: string, path: string) => `${kind}:${path}` + + // Refs at the same BFS depth are independent: fetch each level concurrently. + let level: ItemRef[] = [] + for (const s of seed) { + const key = refKey(s.kind, s.path) + if (!queued.has(key)) { + queued.add(key) + level.push(s) + } + } + while (level.length > 0) { + const results = await Promise.all( + level.map(async (ref) => ({ ref, item: await deps.fetchItem(ref) })) + ) + const next: ItemRef[] = [] + for (const { ref, item } of results) { + if (!item) { + unresolved.push(ref.path) + continue + } + fetched.set(refKey(ref.kind, ref.path), item) + for (const r of refsForFetched(item)) { + if (classifyPath(r.path, slug) === 'hub') continue + if (r.kind === 'resource') { + resourcePaths.add(r.path) + } else if (r.kind === 'script' || r.kind === 'flow') { + const key = refKey(r.kind, r.path) + if (!queued.has(key)) { + queued.add(key) + next.push({ kind: r.kind, path: r.path }) + } + } + } + // Relocate the item's runtime variable refs into the project folder too, so + // the export is slug-relative regardless of the source folder (import then + // materializes them as placeholders). Variables are never hub-hosted. + for (const p of varRefsForFetched(item)) varPaths.add(p) + } + level = next + } + + const fetchedItems = [...fetched.values()] + const itemPaths = fetchedItems.map((it) => it.path) + const map = buildPathMap([...itemPaths, ...resourcePaths, ...varPaths], slug) + + const items: BundledItem[] = fetchedItems.map((it) => { + const rewritten: BundledItem = { ...it, newPath: map.get(it.path) ?? it.path } + if (it.kind === 'script') { + rewritten.content = rewriteContent(it.content ?? '', map) + } else if (it.kind === 'raw_app') { + rewritten.content = rewriteRawVarRefs(rewriteRawAppContent(it.content ?? '', map), map) + } else if (it.kind === 'flow') { + rewritten.value = rewriteVarRefsInValue(rewriteFlowValue(it.value, map), map) + } else if (it.kind === 'app') { + rewritten.value = rewriteVarRefsInValue(rewriteAppValue(it.value, map), map) + } + return rewritten + }) + + const resourceStubs: ResourceStub[] = [] + const resolved = await Promise.all( + [...resourcePaths].map(async (path) => ({ path, type: await deps.resolveResourceType(path) })) + ) + for (const { path, type } of resolved) { + if (!type) { + unresolved.push(path) + continue + } + resourceStubs.push({ originalPath: path, newPath: map.get(path) ?? path, resource_type: type }) + } + + // `unresolved` keys missing items by kind:path but stores the bare path, so a + // missing script and flow (or a runnable and resource) sharing a path can push + // the same string twice. Dedupe: callers use it as a display/blocker list where + // duplicate keys would break keyed rendering. + return { items, resourceStubs, pathMap: map, unresolved: [...new Set(unresolved)] } +} diff --git a/frontend/src/lib/components/workspaceSettings/projectInstall.test.ts b/frontend/src/lib/components/workspaceSettings/projectInstall.test.ts new file mode 100644 index 0000000000..a02d4e264d --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectInstall.test.ts @@ -0,0 +1,78 @@ +import { describe, it, expect } from 'vitest' +import { refContainmentViolation, varContainmentViolation } from './projectInstall' +import type { Ref } from './projectBundle' + +describe('refContainmentViolation', () => { + const folder = 'proj' + const violation = (r: Ref) => refContainmentViolation([r], folder) + + it('allows references relocated into the target folder', () => { + expect(violation({ kind: 'resource', path: 'f/proj/db' })).toBeUndefined() + expect(violation({ kind: 'script', path: 'f/proj/helper' })).toBeUndefined() + expect(violation({ kind: 'flow', path: 'f/proj/sub' })).toBeUndefined() + }) + + it('allows hub script/flow references but never hub resources', () => { + expect(violation({ kind: 'script', path: 'hub/1/x/y' })).toBeUndefined() + expect(violation({ kind: 'flow', path: 'hub/1/a/b' })).toBeUndefined() + // Resources are not hub-hosted, so a hub/ resource path is still an escape. + expect(violation({ kind: 'resource', path: 'hub/1/x/y' })).toBeDefined() + }) + + it('rejects references bound to another namespace', () => { + // The crux: an in-folder runnable pointing its resource at an existing asset. + expect(violation({ kind: 'resource', path: 'u/admin/db' })).toContain('escapes') + expect(violation({ kind: 'script', path: 'f/other/helper' })).toContain('escapes') + expect(violation({ kind: 'flow', path: 'u/admin/sub' })).toContain('escapes') + }) + + it('does not treat a prefix-only folder match as internal', () => { + expect(violation({ kind: 'script', path: 'f/proj2/helper' })).toContain('escapes') + }) + + it('reports the first offending reference and passes a fully-contained set', () => { + expect( + refContainmentViolation( + [ + { kind: 'resource', path: 'f/proj/db' }, + { kind: 'script', path: 'hub/1/x/y' } + ], + folder + ) + ).toBeUndefined() + expect( + refContainmentViolation( + [ + { kind: 'resource', path: 'f/proj/db' }, + { kind: 'resource', path: 'u/admin/secret' } + ], + folder + ) + ).toContain('u/admin/secret') + }) +}) + +describe('varContainmentViolation', () => { + const folder = 'proj' + + it('allows in-folder variable references', () => { + expect(varContainmentViolation({ token: '$var:f/proj/token' }, folder)).toBeUndefined() + expect(varContainmentViolation({ x: 'no refs here' }, folder)).toBeUndefined() + }) + + it('rejects a `$var:` or `$jsonvar:` bound to another namespace', () => { + // The crux: a variable arg the ref extractors miss, resolved under the perms. + expect(varContainmentViolation({ queue_url: '$var:u/admin/token' }, folder)).toContain( + 'u/admin/token' + ) + expect(varContainmentViolation({ cfg: '$jsonvar:f/other/secret' }, folder)).toContain('escapes') + }) + + it('ignores a `$var:` literal embedded in inline code', () => { + const flowValue = { + flow_env: { API: '$var:f/proj/api_key' }, + modules: [{ value: { type: 'rawscript', content: 'return "$var:u/admin/should_not_flag"' } }] + } + expect(varContainmentViolation(flowValue, folder)).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/workspaceSettings/projectInstall.ts b/frontend/src/lib/components/workspaceSettings/projectInstall.ts new file mode 100644 index 0000000000..1827e00283 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectInstall.ts @@ -0,0 +1,405 @@ +// Imports a Hub project export into a workspace: one importer per item kind, +// each item reported individually so one bad item never aborts the rest. +// UI-free — the install page owns folder choice and migration review. + +import { + AppService, + FlowService, + FolderService, + ResourceService, + ScriptService, + VariableService, + WorkspaceService +} from '$lib/gen' +import { + TRIGGER_KINDS, + createWorkspaceTriggerDisabled, + triggerHandlerRefs, + type WorkspaceTrigger, + type WorkspaceTriggerKind +} from '../triggers/workspaceTriggersList' +import { updatePolicy } from '$lib/components/apps/editor/appPolicy' +import { updateRawAppPolicy } from '$lib/sharedUtils' +import type { App } from '$lib/components/apps/types' +import { runScriptAndPollResult } from '$lib/components/jobs/utils' +import { + classifyPath, + collectExportVarPaths, + extractAppRefs, + extractFlowRefs, + extractRawAppRefs, + extractScriptRefs, + extractTriggerConfigResourceRefs, + extractVarRefsFromValue, + retargetProjectExport, + type ExportItem, + type ProjectExport, + type ProjectMigration, + type Ref +} from './projectBundle' + +export interface InstallResult { + path: string + ok: boolean + error?: string +} + +// Guarding an item's own path is not enough: the `$res:`/script/flow refs baked +// into its content are live bindings the backend acts on. A well-formed export +// relocates them all into f// (hub/ script refs stay external); anything +// else points a runnable at an existing asset in another namespace, so refuse the +// item rather than bind it there. Resources are never hub-hosted, so a hub/ path +// there is not a valid escape hatch. Mirrors the trigger-config containment. +export function refContainmentViolation(refs: Ref[], folder: string): string | undefined { + for (const r of refs) { + const cls = classifyPath(r.path, folder) + if (cls === 'internal') continue + if (cls === 'hub' && r.kind !== 'resource') continue + return `reference '${r.path}' escapes the target folder f/${folder}/ — skipped` + } + return undefined +} + +// `$var:`/`$jsonvar:` references (in flow static inputs, flow_env, app runnable +// inputs, trigger config) are resolved at runtime under the imported runnable's +// permissions and are never hub-hosted. Retargeting relocates a project's own refs +// into the target folder; anything still outside it points at another namespace, so +// reject those. Takes the parsed value so inline code carrying a literal is ignored. +export function varContainmentViolation(value: any, folder: string): string | undefined { + for (const p of extractVarRefsFromValue(value)) { + if (classifyPath(p, folder) !== 'internal') { + return `variable '${p}' escapes the target folder f/${folder}/ — skipped` + } + } + return undefined +} + +// Surface the backend's explanation: API errors carry the real message in +// `.body` (plain text for Windmill 4xx), while `.message` is the generic +// status text ("Bad Request"). Prefer the body so e.g. a path/route_path +// collision reads as the actual reason, not just "Bad Request". +function errorMessage(e: any): string { + const body = e?.body + if (typeof body === 'string' && body.trim() !== '') return body + if (body && typeof body === 'object') + return body.error?.message ?? body.message ?? JSON.stringify(body) + return e?.message ?? String(e) +} + +// Recompute an app's execution policy from its (retargeted) value, mirroring +// what the editor does on deploy. `triggerables_v2` is keyed by +// `:rawscript/`; retargeting rewrites that +// content, so a copied or empty policy would leave every inline runnable +// "forbidden by policy" at runtime. Default to publisher (auth required). +async function computeAppPolicy(value: any): Promise { + const policy = (await updatePolicy(value as App, undefined)) as any + if (!policy.execution_mode) policy.execution_mode = 'publisher' + return policy +} +async function computeRawAppPolicy(runnables: Record): Promise { + const policy = (await updateRawAppPolicy(runnables, undefined)) as any + if (!policy.execution_mode) policy.execution_mode = 'publisher' + return policy +} + +function importScript(workspace: string, s: ExportItem): Promise { + return ScriptService.createScript({ + workspace, + requestBody: { + path: s.path, + summary: s.summary ?? '', + description: s.description ?? '', + content: s.content ?? '', + language: s.language, + schema: s.schema ?? undefined, + kind: s.kind ?? 'script', + lock: s.lockfile ?? undefined + } + }) +} + +function importFlow(workspace: string, f: ExportItem): Promise { + return FlowService.createFlow({ + workspace, + requestBody: { + path: f.path, + summary: f.summary ?? '', + description: f.description ?? '', + value: f.value, + schema: f.schema ?? undefined + } + }) +} + +// Stubs only: never overwrite an existing resource's value (updateIfExists +// stays false so a path collision is reported as a failed item instead). +function importResourceStub(workspace: string, r: ExportItem): Promise { + return ResourceService.createResource({ + workspace, + updateIfExists: false, + requestBody: { + path: r.path, + resource_type: r.resource_type, + value: {}, + description: 'Imported stub — fill in the value.' + } + }) +} + +// Variables hold secrets/config, so their values are never shipped. Create an empty +// secret placeholder for a project variable the importer must fill, mirroring the +// resource stubs. Conflict-safe: an already-present variable (the importer filled it, +// or a re-import) is left untouched rather than clobbered. +async function importVariablePlaceholder(workspace: string, path: string): Promise { + if (await VariableService.existsVariable({ workspace, path })) return + await VariableService.createVariable({ + workspace, + requestBody: { + path, + value: '', + is_secret: true, + description: 'Imported placeholder — fill in the value.' + } + }) +} + +async function importApp(workspace: string, a: ExportItem): Promise { + if (a.app_type === 'raw') { + let parsed: any + try { + parsed = JSON.parse(a.value?.raw ?? '{}') + } catch (e: any) { + throw new Error(`invalid raw app bundle: ${e?.message ?? String(e)}`) + } + const files = { ...(parsed.files ?? {}) } + const js = files['/bundle.js'] ?? '' + const css = files['/bundle.css'] ?? '' + delete files['/bundle.js'] + delete files['/bundle.css'] + const runnables = parsed.runnables ?? {} + return AppService.createAppRaw({ + workspace, + formData: { + app: { + path: a.path, + summary: a.summary ?? '', + value: { + files, + runnables, + // Keep the full-code app's explicit data table declaration. + ...(parsed.data !== undefined ? { data: parsed.data } : {}), + ...(parsed.datatables !== undefined ? { datatables: parsed.datatables } : {}) + }, + policy: await computeRawAppPolicy(runnables) + }, + js, + css + } + }) + } + return AppService.createApp({ + workspace, + requestBody: { + path: a.path, + summary: a.summary ?? '', + value: a.value, + policy: await computeAppPolicy(a.value) + } + }) +} + +// Apply one migration to the target data table. If the data table opted into +// migrations, record it (datatable_migrations + _wm_migrations, run only this +// version); otherwise run the SQL once as a preview job (unrecorded). +async function applyOneMigration( + workspace: string, + projectSlug: string, + m: ProjectMigration +): Promise { + let recorded = false + try { + const status = await WorkspaceService.getDatatableMigrationsStatus({ + workspace, + datatableName: m.datatable_name + }) + recorded = !!status.enabled + } catch {} + + if (recorded) { + // Record the shipped down migration (DROP the created tables) so it can be + // rolled back. + const codeDown = (m.sql_down ?? '').trim() + const created = await WorkspaceService.createDatatableMigration({ + workspace, + datatableName: m.datatable_name, + requestBody: { + name: `hub_import_${projectSlug}`, + code_up: m.sql, + code_down: codeDown || undefined + } + }) + await WorkspaceService.runDatatableMigrations({ + workspace, + datatableName: m.datatable_name, + only: created.timestamp + }) + } else { + await runScriptAndPollResult({ + workspace, + requestBody: { + language: 'postgresql', + content: m.sql, + args: { database: `datatable://${m.datatable_name}` } + } + }) + } +} + +/** + * Install a project export into `workspace` under `f//`: create the + * folder, retarget every item, import kind by kind, then apply the (already + * reviewed) migrations. Each item's outcome is reported through `onResult`; + * failures never abort the remaining items. + */ +export async function installProject(args: { + workspace: string + exportData: ProjectExport + folder: string + migrations: ProjectMigration[] + hasEeLicense: boolean + onResult: (r: InstallResult) => void +}): Promise { + const { workspace, exportData, folder, migrations, hasEeLicense, onResult } = args + + const record = (path: string, p: Promise): Promise => + p.then( + () => onResult({ path, ok: true }), + (e: any) => onResult({ path, ok: false, error: errorMessage(e) }) + ) + + try { + await FolderService.createFolder({ workspace, requestBody: { name: folder } }) + } catch {} + + const proj = retargetProjectExport(exportData, exportData.project.slug, folder) + + // The export is remote input: every path it wants to write must stay inside + // the folder the user chose. Anything else (crafted export, or an export + // whose items weren't relocated into f// at publish) is refused + // per-item instead of being created in another namespace. + const prefix = `f/${folder}/` + const guard = (path: unknown, ...also: unknown[]): string | undefined => { + for (const p of [path, ...also]) { + if (typeof p !== 'string' || !p.startsWith(prefix)) { + return `path '${String(p)}' escapes the target folder ${prefix} — skipped` + } + } + return undefined + } + const checked = (path: unknown, run: () => Promise, ...also: unknown[]) => { + const violation = guard(path, ...also) + return violation + ? record(String(path), Promise.reject(new Error(violation))) + : record(String(path), run()) + } + + // `refs` catches structured runnable/`$res:` refs; `varValue` is the parsed item + // walked for `$var:`/`$jsonvar:` argument refs (which the ref extractors miss). + const checkedItem = (path: unknown, refs: Ref[], varValue: any, run: () => Promise) => { + const violation = + guard(path) ?? + refContainmentViolation(refs, folder) ?? + varContainmentViolation(varValue, folder) + return violation + ? record(String(path), Promise.reject(new Error(violation))) + : record(String(path), run()) + } + + for (const s of proj.scripts) { + // `$var:` is resolved in job args (flow inputs, schedule args, trigger config), + // not in script source, so there is no variable arg to contain here. + await checkedItem(s.path, extractScriptRefs(s.content ?? ''), undefined, () => + importScript(workspace, s) + ) + } + for (const f of proj.flows) { + await checkedItem(f.path, extractFlowRefs(f.value), f.value, () => importFlow(workspace, f)) + } + for (const r of proj.resources) { + await checked(r.path, () => importResourceStub(workspace, r)) + } + // Placeholders for the project's internal `$var:`/`$jsonvar:` refs (retargeted + // into this folder). External refs are rejected per-item, so only stub in-folder + // ones; guard again in case an out-of-folder ref slipped through retargeting. + for (const p of collectExportVarPaths(proj)) { + if (!p.startsWith(prefix)) continue + await record(`variable: ${p}`, importVariablePlaceholder(workspace, p)) + } + for (const a of proj.apps) { + const isRaw = a.app_type === 'raw' + const refs = isRaw ? extractRawAppRefs(a.value?.raw ?? '') : extractAppRefs(a.value) + // Raw apps hold their runnables in the `value.raw` JSON string; parse it so the + // walk sees the same structure the backend resolves. Malformed raw fails at import. + let varValue: any = a.value + if (isRaw) { + try { + varValue = JSON.parse(a.value?.raw ?? '{}') + } catch { + varValue = undefined + } + } + await checkedItem(a.path, refs, varValue, () => importApp(workspace, a)) + } + // A trigger's config is a live binding, not inert content: resource fields, + // handler runnables and $res: refs it names are acted on by the backend, so + // every one must stay inside the chosen folder (handlers may also point at + // hub/ scripts). Otherwise a crafted export could bind the trigger to + // existing assets in another namespace. + const triggerConfigViolation = (t: ExportItem): string | undefined => { + const cfg = (t.config ?? {}) as Record + for (const r of triggerHandlerRefs({ kind: t.kind, config: cfg } as WorkspaceTrigger)) { + if (!r.path.startsWith(prefix) && !r.path.startsWith('hub/')) { + return `handler '${r.path}' escapes the target folder ${prefix} — skipped` + } + } + const resourceRefs = new Set(extractTriggerConfigResourceRefs(cfg)) + const field = TRIGGER_KINDS[t.kind as WorkspaceTriggerKind]?.resourceField + const fieldValue = field ? cfg[field] : undefined + if (typeof fieldValue === 'string' && fieldValue !== '') resourceRefs.add(fieldValue) + for (const p of resourceRefs) { + if (!p.startsWith(prefix)) { + return `resource '${p}' escapes the target folder ${prefix} — skipped` + } + } + // Config fields (e.g. SQS queue_url) can carry `$var:`/`$jsonvar:` refs too. + return varContainmentViolation(cfg, folder) + } + for (const t of proj.triggers) { + const violation = guard(t.path, t.runnable_path) ?? triggerConfigViolation(t) + await record( + String(t.path), + violation + ? Promise.reject(new Error(violation)) + : createWorkspaceTriggerDisabled( + workspace, + { + kind: t.kind, + path: t.path, + script_path: t.runnable_path, + is_flow: t.runnable_kind === 'flow', + summary: t.summary ?? null, + config: t.config ?? null + }, + { hasEeLicense } + ) + ) + } + + // Apply the reviewed data table migrations after items exist. + for (const m of migrations) { + await record( + `data table: ${m.datatable_name}`, + applyOneMigration(workspace, exportData.project.slug, m) + ) + } +} diff --git a/frontend/src/lib/components/workspaceSettings/projectMigrations.test.ts b/frontend/src/lib/components/workspaceSettings/projectMigrations.test.ts new file mode 100644 index 0000000000..7d35c51650 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectMigrations.test.ts @@ -0,0 +1,372 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' + +// inferAssets loads WASM; stub it so script detection is deterministic and no +// wasm init runs in the test. +const inferAssetsMock = vi.fn() +vi.mock('$lib/infer', () => ({ inferAssets: (...a: any[]) => inferAssetsMock(...a) })) + +// Only getDatatableFullSchema is used by the generator; stub the whole service. +const getDatatableFullSchemaMock = vi.fn() +vi.mock('$lib/gen', () => ({ + WorkspaceService: { + getDatatableFullSchema: (...a: any[]) => getDatatableFullSchemaMock(...a) + } +})) + +import { detectDatatableTables, generateDatatableMigrations } from './projectMigrations' +import type { FetchedItem } from './projectBundle' + +describe('detectDatatableTables', () => { + beforeEach(() => inferAssetsMock.mockReset()) + + it('collects datatable/table refs from scripts (re-parsed), flows and raw apps', async () => { + inferAssetsMock.mockResolvedValue({ + status: 'ok', + assets: [ + { kind: 'datatable', path: 'main/customers' }, + { kind: 'resource', path: 'u/admin/pg' } // ignored + ] + }) + const items: FetchedItem[] = [ + { kind: 'script', path: 'f/p/s', language: 'duckdb', content: 'select 1' }, + { + kind: 'flow', + path: 'f/p/fl', + value: { + modules: [ + { + id: 'a', + value: { + type: 'rawscript', + language: 'duckdb', + content: '', + assets: [{ kind: 'datatable', path: 'main/orders' }] + } + } + ] + } + }, + { + kind: 'raw_app', + path: 'f/p/app', + content: JSON.stringify({ + runnables: { + r1: { inlineScript: { assets: [{ kind: 'datatable', path: 'analytics/events' }] } } + } + }) + } + ] + const usage = await detectDatatableTables(items) + expect([...(usage.get('main') ?? [])].sort()).toEqual(['customers', 'orders']) + expect([...(usage.get('analytics') ?? [])]).toEqual(['events']) + }) + + it('collects datatable refs from the preprocessor module', async () => { + inferAssetsMock.mockResolvedValue({ status: 'ok', assets: [] }) + const items: FetchedItem[] = [ + { + kind: 'flow', + path: 'f/p/fl', + value: { + modules: [], + preprocessor_module: { + id: 'pre', + value: { + type: 'rawscript', + language: 'duckdb', + content: '', + assets: [{ kind: 'datatable', path: 'main/inbox' }] + } + } + } + } + ] + const usage = await detectDatatableTables(items) + expect([...(usage.get('main') ?? [])]).toEqual(['inbox']) + }) + + it('records a datatable used with no specific table', async () => { + inferAssetsMock.mockResolvedValue({ + status: 'ok', + assets: [{ kind: 'datatable', path: 'main' }] + }) + const usage = await detectDatatableTables([ + { kind: 'script', path: 'f/p/s', language: 'duckdb', content: 'x' } + ]) + expect(usage.has('main')).toBe(true) + expect(usage.get('main')?.size).toBe(0) + }) + + it('reads a full-code app’s explicit data.tables declaration', async () => { + const items: FetchedItem[] = [ + { + kind: 'raw_app', + path: 'f/p/app', + content: JSON.stringify({ + runnables: {}, + data: { + datatable: 'main', + schema: 'app1', + tables: ['main/customers', 'main/app1:orders'] + } + }) + } + ] + const usage = await detectDatatableTables(items) + // public-schema ref keeps the bare name; non-public keeps schema.table. + expect([...(usage.get('main') ?? [])].sort()).toEqual(['app1.orders', 'customers']) + }) +}) + +describe('generateDatatableMigrations', () => { + beforeEach(() => getDatatableFullSchemaMock.mockReset()) + + const schema = { + public: { + customers: { + name: 'customers', + columns: [ + { name: 'id', datatype: 'integer', primary_key: true, nullable: false }, + { name: 'email', datatype: 'text', nullable: true } + ], + foreign_keys: [] + }, + orders: { + name: 'orders', + columns: [ + { name: 'id', datatype: 'integer', primary_key: true, nullable: false }, + { name: 'customer_id', datatype: 'integer', nullable: false } + ], + foreign_keys: [ + { + target_table: 'public.customers', + columns: [{ source_column: 'customer_id', target_column: 'id' }], + on_delete: 'NO ACTION', + on_update: 'NO ACTION' + } + ] + } + } + } + + it('creates referenced tables in FK-dependency order in one transaction, enabled', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['orders', 'customers'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations).toHaveLength(1) + const m = migrations[0] + expect(m.datatable_name).toBe('main') + expect(m.enabled).toBe(true) + expect(m.sql.startsWith('BEGIN;')).toBe(true) + expect(m.sql.trimEnd().endsWith('COMMIT;')).toBe(true) + // customers (FK target) must be created before orders (FK source). + expect(m.sql.indexOf('"public"."customers"')).toBeLessThan(m.sql.indexOf('"public"."orders"')) + // A single wrapping transaction, not one per table. + expect(m.sql.match(/BEGIN;/g)?.length).toBe(1) + // Idempotent: won't abort if a pulled-in parent already exists in the target. + expect(m.sql).toContain('CREATE TABLE IF NOT EXISTS "public"."customers"') + // Down migration lists drops commented out (nothing dropped by default), + // in reverse order: orders (child) before customers (parent). + expect(m.sql_down).toContain('-- DROP TABLE IF EXISTS "public"."orders";') + expect(m.sql_down).toContain('-- DROP TABLE IF EXISTS "public"."customers";') + // No uncommented DROP TABLE anywhere. + expect(/^\s*DROP TABLE/m.test(m.sql_down)).toBe(false) + expect(m.sql_down.indexOf('"public"."orders"')).toBeLessThan( + m.sql_down.indexOf('"public"."customers"') + ) + }) + + it('accepts schema-qualified table refs', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['public.customers'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].enabled).toBe(true) + expect(migrations[0].sql).toContain('"public"."customers"') + }) + + it('leaves a qualified ref unresolved when its schema misses, never another schema\'s table', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['sales.orders'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].sql).toContain('"sales.orders" is referenced but was not found') + expect(migrations[0].sql).not.toContain('CREATE TABLE "') + }) + + it('emits all CREATE TABLEs before any FK constraint so circular FKs work', async () => { + const cyclicSchema = { + public: { + a: { + name: 'a', + columns: [ + { name: 'id', datatype: 'integer', primary_key: true, nullable: false }, + { name: 'b_id', datatype: 'integer', nullable: true } + ], + foreign_keys: [ + { + target_table: 'public.b', + columns: [{ source_column: 'b_id', target_column: 'id' }], + on_delete: 'NO ACTION', + on_update: 'NO ACTION' + } + ] + }, + b: { + name: 'b', + columns: [ + { name: 'id', datatype: 'integer', primary_key: true, nullable: false }, + { name: 'a_id', datatype: 'integer', nullable: true } + ], + foreign_keys: [ + { + target_table: 'public.a', + columns: [{ source_column: 'a_id', target_column: 'id' }], + on_delete: 'NO ACTION', + on_update: 'NO ACTION' + } + ] + } + } + } + getDatatableFullSchemaMock.mockResolvedValue(cyclicSchema) + const usage = new Map([['main', new Set(['a', 'b'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + const sql = migrations[0].sql + expect(sql).toContain('"public"."a"') + expect(sql).toContain('"public"."b"') + // Both FK constraints present, and every CREATE TABLE precedes the first one. + expect(sql.match(/ADD CONSTRAINT/g)?.length).toBe(2) + const lastCreate = sql.lastIndexOf('CREATE TABLE IF NOT EXISTS') + const firstConstraint = sql.indexOf('DO $$') + expect(lastCreate).toBeGreaterThan(-1) + expect(firstConstraint).toBeGreaterThan(lastCreate) + }) + + it('guards FK creation so re-running on an existing table does not abort', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['orders'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + const sql = migrations[0].sql + // The ADD CONSTRAINT must be wrapped in a pg_constraint existence check. + expect(sql).toContain('DO $$') + expect(sql).toContain('SELECT 1 FROM pg_constraint') + expect(sql).toContain(`conrelid = '"public"."orders"'::regclass`) + // No unguarded ALTER TABLE ... ADD at the start of a line. + expect(/^ALTER TABLE .* ADD CONSTRAINT/m.test(sql)).toBe(false) + }) + + it('creates non-public schemas before their tables', async () => { + const appSchema = { + app: { + customers: { + name: 'customers', + columns: [{ name: 'id', datatype: 'integer', primary_key: true, nullable: false }], + foreign_keys: [] + } + } + } + getDatatableFullSchemaMock.mockResolvedValue(appSchema) + const usage = new Map([['main', new Set(['app.customers'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + const sql = migrations[0].sql + expect(sql).toContain('CREATE SCHEMA IF NOT EXISTS "app";') + expect(sql.indexOf('CREATE SCHEMA IF NOT EXISTS "app";')).toBeLessThan( + sql.indexOf('CREATE TABLE IF NOT EXISTS "app"."customers"') + ) + expect(sql).not.toContain('CREATE SCHEMA IF NOT EXISTS "public"') + }) + + it('keeps same-named tables from different schemas both created', async () => { + const twoSchemas = { + public: { + customers: { + name: 'customers', + columns: [{ name: 'id', datatype: 'integer', primary_key: true, nullable: false }], + foreign_keys: [] + } + }, + app: { + customers: { + name: 'customers', + columns: [{ name: 'id', datatype: 'integer', primary_key: true, nullable: false }], + foreign_keys: [] + } + } + } + getDatatableFullSchemaMock.mockResolvedValue(twoSchemas) + const usage = new Map([['main', new Set(['public.customers', 'app.customers'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].sql).toContain('"public"."customers"') + expect(migrations[0].sql).toContain('"app"."customers"') + }) + + it('transitively pulls in FK-referenced tables not directly used', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + // Only `orders` is referenced; `customers` (its FK target) must still be + // created, and before `orders`. + const usage = new Map([['main', new Set(['orders'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + const m = migrations[0] + expect(m.enabled).toBe(true) + expect(m.sql).toContain('"public"."customers"') + expect(m.sql).toContain('"public"."orders"') + expect(m.sql.indexOf('"public"."customers"')).toBeLessThan(m.sql.indexOf('"public"."orders"')) + }) + + it('drops a foreign key whose target is not in the schema', async () => { + // `orders` references a `warehouses` table that no longer exists in the + // schema: the FK must be pruned so the migration still runs. + const schemaWithDanglingFk = { + public: { + orders: { + name: 'orders', + columns: [{ name: 'id', datatype: 'integer', primary_key: true, nullable: false }], + foreign_keys: [ + { + target_table: 'public.warehouses', + columns: [{ source_column: 'id', target_column: 'id' }], + on_delete: 'NO ACTION', + on_update: 'NO ACTION' + } + ] + } + } + } + getDatatableFullSchemaMock.mockResolvedValue(schemaWithDanglingFk) + const usage = new Map([['main', new Set(['orders'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].enabled).toBe(true) + expect(migrations[0].sql).toContain('"public"."orders"') + expect(migrations[0].sql).not.toContain('warehouses') + }) + + it('emits a disabled comment entry when a referenced table is not found', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['nonexistent'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations).toHaveLength(1) + expect(migrations[0].enabled).toBe(false) + expect(migrations[0].sql).toContain('-- Table "nonexistent" is referenced but was not found') + expect(migrations[0].sql).not.toContain('BEGIN;') + }) + + it('keeps found tables and comments the missing ones in one migration', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['customers', 'ghost'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].enabled).toBe(true) + expect(migrations[0].sql).toContain('"public"."customers"') + expect(migrations[0].sql).toContain('-- Table "ghost" is referenced but was not found') + // Comments precede the runnable transaction. + expect(migrations[0].sql.indexOf('-- Table "ghost"')).toBeLessThan( + migrations[0].sql.indexOf('BEGIN;') + ) + }) + + it('comments a data table used with no specific table', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set()]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].enabled).toBe(false) + expect(migrations[0].sql).toContain('no specific table was referenced') + }) +}) diff --git a/frontend/src/lib/components/workspaceSettings/projectMigrations.ts b/frontend/src/lib/components/workspaceSettings/projectMigrations.ts new file mode 100644 index 0000000000..b4e72389c0 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectMigrations.ts @@ -0,0 +1,345 @@ +// Best-effort data table migration generation for the "project = folder" Hub +// bundle. Detects which data tables (and tables within them) a project's +// scripts/flows/raw apps reference via `datatable` assets, then generates a +// `CREATE TABLE` bundle per data table from the source workspace's live schema, +// so importing the project into another workspace can recreate those tables. +// +// Best-effort by design: the generated SQL is shown to the publisher and is +// fully editable before publishing. Low-code (non-raw) apps have no persisted +// asset list and are not scanned. + +import { inferAssets } from '$lib/infer' +import type { SupportedLanguage } from '$lib/common' +import { allFlowModules } from './projectBundle' +import { getFlowModuleAssets } from '$lib/components/assets/lib' +import { extractDataConfig, parseDataTableRef } from '$lib/components/raw_apps/dataTableRefUtils' +import { + apiSchemaToEditorSchema, + generateAddedTableSql, + type DatabaseSchema +} from '$lib/components/datatableSchemaSql' +import { WorkspaceService } from '$lib/gen' +import type { FetchedItem } from './projectBundle' + +export interface GeneratedMigration { + datatable_name: string + /** Up migration: creates the tables. */ + sql: string + /** Down migration: drops the created tables. Best-effort, generated once and + * editable by the publisher (not re-derived from `sql`). */ + sql_down: string + enabled: boolean +} + +// A datatable asset path is `datatable`, `datatable/table`, or +// `datatable/schema.table` (see the SQL asset parser). The first segment is the +// data table name; the remainder identifies a specific table (absent = whole +// data table, no table to create). +function parseDatatableAssetPath(path: string): { datatable: string; table?: string } { + const slash = path.indexOf('/') + if (slash === -1) return { datatable: path } + const datatable = path.slice(0, slash) + const table = path.slice(slash + 1).trim() + return { datatable, table: table || undefined } +} + +function addDatatableTable( + map: Map>, + datatable: string, + table: string | undefined +): void { + if (!datatable) return + const set = map.get(datatable) ?? new Set() + if (table) set.add(table) + map.set(datatable, set) +} + +function addUsage(map: Map>, path: string): void { + const { datatable, table } = parseDatatableAssetPath(path) + addDatatableTable(map, datatable, table) +} + +/** + * Scan a project's fetched items for data table usage and return + * `datatable -> set of table refs` (a table ref is `table` or `schema.table`). + * - scripts: re-parse the code with the asset parser (`inferAssets`) + * - flows: read each module's stored `assets` + * - full-code (raw) apps: read the explicit `data.tables` declaration; fall back + * to `runnables[key].inlineScript.assets` for older apps + */ +export async function detectDatatableTables( + items: FetchedItem[] +): Promise>> { + const map = new Map>() + + for (const item of items) { + if (item.kind === 'script') { + const res = await inferAssets( + item.language as SupportedLanguage | undefined, + item.content ?? '' + ) + if (res.status === 'ok') { + for (const a of res.assets) if (a.kind === 'datatable') addUsage(map, a.path) + } + } else if (item.kind === 'flow') { + for (const mod of allFlowModules(item.value)) { + const assets = getFlowModuleAssets(mod) + if (assets) for (const a of assets) if (a.kind === 'datatable') addUsage(map, a.path) + } + } else if (item.kind === 'raw_app') { + let parsed: any + try { + parsed = JSON.parse(item.content ?? '{}') + } catch { + continue + } + // Full-code apps explicitly declare the data tables/tables they use + // (`data.tables`, refs like `main/customers` or `main/schema:table`), so + // read that rather than parsing assets. + const config = extractDataConfig(parsed) + if (config) { + for (const ref of config.tables) { + const r = parseDataTableRef(ref) + const table = r.table + ? r.schema && r.schema !== 'public' + ? `${r.schema}.${r.table}` + : r.table + : undefined + addDatatableTable(map, r.datatable, table) + } + } + // Older raw apps instead carry datatable usage as inline-script assets. + const runnables = parsed?.runnables ?? {} + for (const key of Object.keys(runnables)) { + const assets = runnables[key]?.inlineScript?.assets + if (Array.isArray(assets)) + for (const a of assets) + if (a?.kind === 'datatable' && typeof a.path === 'string') addUsage(map, a.path) + } + } + } + return map +} + +// Resolve a table ref (`table` or `schema.table`) to a concrete +// `{ schemaName, tableName }` present in the live schema, or undefined if the +// table can't be found (dropped since, typo, …). A schema-qualified ref that +// misses stays unresolved: falling back to a same-named table in another +// schema would generate a migration for an unrelated table while the code +// still references the missing one. +function resolveTable( + schema: DatabaseSchema, + tableRef: string +): { schemaName: string; tableName: string } | undefined { + const dot = tableRef.indexOf('.') + if (dot !== -1) { + const schemaName = tableRef.slice(0, dot) + const tableName = tableRef.slice(dot + 1) + return schema[schemaName]?.[tableName] ? { schemaName, tableName } : undefined + } + // Bare name: find it across every schema, first match wins. + for (const schemaName of Object.keys(schema)) { + if (schema[schemaName][tableRef]) return { schemaName, tableName: tableRef } + } + return undefined +} + +type ResolvedTable = { schemaName: string; tableName: string } + +const tableKey = (t: ResolvedTable) => `${t.schemaName}.${t.tableName}` + +// Grow the set of tables to create so it's closed under foreign keys: a used +// table's FK targets (and their FK targets, transitively) are pulled in, so the +// generated CREATE TABLEs never reference a table that isn't also created. FK +// targets that don't resolve in this schema are left out (their FK is pruned by +// pruneSchemaForTables). +function expandFkClosure(schema: DatabaseSchema, seed: ResolvedTable[]): ResolvedTable[] { + const inSet = new Map(seed.map((t) => [tableKey(t), t])) + const queue = [...seed] + while (queue.length > 0) { + const t = queue.shift()! + const fks = schema[t.schemaName]?.[t.tableName]?.foreignKeys ?? [] + for (const fk of fks) { + const target = resolveTable(schema, fk.targetTable ?? '') + if (target && !inSet.has(tableKey(target))) { + inSet.set(tableKey(target), target) + queue.push(target) + } + } + } + return [...inSet.values()] +} + +// A copy of the schema restricted to `tables`, with each table's foreign keys +// filtered to targets that are also in `tables`. generateAddedTableSql emits every +// FK it finds on a table, so pruning here keeps a stray FK (to a table outside the +// migration) from making the generated SQL fail. +function pruneSchemaForTables(schema: DatabaseSchema, tables: ResolvedTable[]): DatabaseSchema { + const inSet = new Set(tables.map(tableKey)) + const pruned: DatabaseSchema = {} + for (const t of tables) { + const orig = schema[t.schemaName]?.[t.tableName] + if (!orig) continue + ;(pruned[t.schemaName] ??= {})[t.tableName] = { + ...orig, + foreignKeys: (orig.foreignKeys ?? []).filter((fk) => { + const target = resolveTable(schema, fk.targetTable ?? '') + return target != null && inSet.has(tableKey(target)) + }) + } + } + return pruned +} + +// Order tables so a table is created after the in-set tables it references via a +// foreign key. Keyed by schema-qualified name (like the rest of the pipeline) so +// two same-named tables in different schemas aren't collapsed. Falls back to input +// order on a cycle so generation never hangs. +function orderByFkDependency(schema: DatabaseSchema, tables: ResolvedTable[]): ResolvedTable[] { + const inSet = new Set(tables.map(tableKey)) + const deps = new Map>() + for (const t of tables) { + const fks = schema[t.schemaName]?.[t.tableName]?.foreignKeys ?? [] + const targets = new Set() + for (const fk of fks) { + const target = resolveTable(schema, fk.targetTable ?? '') + if (target && tableKey(target) !== tableKey(t) && inSet.has(tableKey(target))) { + targets.add(tableKey(target)) + } + } + deps.set(tableKey(t), targets) + } + const ordered: ResolvedTable[] = [] + const done = new Set() + const visiting = new Set() + const byKey = new Map(tables.map((t) => [tableKey(t), t])) + const visit = (key: string) => { + if (done.has(key) || visiting.has(key)) return + visiting.add(key) + for (const dep of deps.get(key) ?? []) visit(dep) + visiting.delete(key) + done.add(key) + const t = byKey.get(key) + if (t) ordered.push(t) + } + for (const t of tables) visit(tableKey(t)) + return ordered +} + +// Pull a readable one-line message out of an API error for embedding in a SQL +// comment (collapse whitespace so it can't break out of the `--` line). +function errorText(e: any): string { + const body = e?.body + const raw = + typeof body === 'string' && body.trim() + ? body + : body && typeof body === 'object' + ? (body.error?.message ?? body.message ?? JSON.stringify(body)) + : (e?.message ?? String(e)) + return String(raw).replace(/\s+/g, ' ').trim() +} + +/** + * Generate one best-effort migration per used data table. Resolved tables (plus + * the tables they depend on via foreign key, in FK-dependency order) become a + * single CREATE TABLE transaction, enabled by default. Anything that couldn't be + * auto-generated — a table not found in the schema, a data table referenced as a + * whole, or a schema that couldn't be loaded — is written as a `--` SQL comment + * describing the problem, so the publisher sees what's missing instead of a blank + * entry. A migration with no runnable statements (only comments) is left disabled. + */ +export async function generateDatatableMigrations( + workspace: string, + usage: Map> +): Promise { + const out: GeneratedMigration[] = [] + for (const [datatable, tableRefs] of usage) { + let schema: DatabaseSchema + try { + const api = await WorkspaceService.getDatatableFullSchema({ + workspace, + requestBody: { source: `datatable://${datatable}` } + }) + schema = apiSchemaToEditorSchema(api) + } catch (e) { + // Couldn't reach the schema at all: leave a commented stub explaining why, + // so the publisher can fill it in rather than seeing a silent blank. + out.push({ + datatable_name: datatable, + sql: + `-- Could not load the schema of data table "${datatable}": ${errorText(e)}\n` + + `-- Add the CREATE TABLE statement(s) for the tables this project uses.`, + sql_down: '', + enabled: false + }) + continue + } + // Resolve the referenced tables; record a comment for each one we can't find + // so a partial migration still explains what's missing. + const resolved: ResolvedTable[] = [] + const comments: string[] = [] + for (const ref of tableRefs) { + const t = resolveTable(schema, ref) + if (t) resolved.push(t) + else + comments.push( + `-- Table "${ref}" is referenced but was not found in data table "${datatable}"; add its CREATE TABLE manually.` + ) + } + if (tableRefs.size === 0) { + comments.push( + `-- Data table "${datatable}" is used but no specific table was referenced; nothing to generate automatically.` + ) + } + // Pull in the tables the referenced ones depend on via FK, then generate + // against a schema whose FKs are restricted to this set, so the migration + // creates everything it references and never emits a dangling FK. + const closure = expandFkClosure(schema, resolved) + const ordered = orderByFkDependency(schema, closure) + const prunedSchema = pruneSchemaForTables(schema, ordered) + // Every CREATE TABLE is emitted before any FK constraint: circular FKs have + // no valid creation order, so constraints can only run once all tables exist. + const creates: string[] = [] + const constraints: string[] = [] + for (const t of ordered) { + // IF NOT EXISTS: FK closure pulls in shared parent tables (e.g. a + // referenced `orders` drags in `customers`) that often already exist in + // the target, so a plain CREATE would abort the whole transaction. The + // caveat — an existing differently-shaped table is silently left as-is — + // is acceptable for a best-effort, editable migration. + const gen = generateAddedTableSql( + { schemaName: t.schemaName, tableName: t.tableName, kind: 'added' }, + prunedSchema, + { ifNotExists: true } + ) + if (!gen) continue + creates.push(gen.create) + constraints.push(...gen.constraints) + } + const statements = [...creates, ...constraints] + // Comments (the errors) go on top; the CREATE TABLE transaction, if any, + // follows. Enabled only when there's something to run. + const parts: string[] = [] + if (comments.length > 0) parts.push(comments.join('\n')) + if (statements.length > 0) parts.push(`BEGIN;\n${statements.join('\n\n')}\nCOMMIT;`) + // Best-effort down migration: the DROP TABLE statements are commented out + // because the FK closure pulls in shared parent tables that may have + // pre-existed in the target (dropping them would lose data the project never + // created). The publisher uncomments the tables this migration should drop. + const drops = [...ordered] + .reverse() + .map((t) => `-- DROP TABLE IF EXISTS "${t.schemaName}"."${t.tableName}";`) + const sqlDown = + drops.length > 0 + ? `-- Rollback: uncomment the tables this migration should drop (leave shared\n` + + `-- tables that already existed in the workspace commented out).\nBEGIN;\n${drops.join('\n')}\nCOMMIT;` + : '' + out.push({ + datatable_name: datatable, + sql: parts.join('\n\n'), + sql_down: sqlDown, + enabled: statements.length > 0 + }) + } + return out.sort((a, b) => a.datatable_name.localeCompare(b.datatable_name)) +} diff --git a/frontend/src/lib/hubPaths.json b/frontend/src/lib/hubPaths.json index 0f0322676c..0cb2cd5602 100644 --- a/frontend/src/lib/hubPaths.json +++ b/frontend/src/lib/hubPaths.json @@ -1,6 +1,6 @@ { "gitSyncTest": "hub/28184/git-repo-test-read-write-windmill", - "gitInitRepo": "hub/28789/git-sync-init-repository-windmill", + "gitInitRepo": "hub/28795/git-sync-init-repository-windmill", "slackErrorHandler": "hub/28794/workspace-or-schedule-error-handler-slack", "emailErrorHandler": "hub/19795/workspace-or-error-handler-email", "slackRecoveryHandler": "hub/28791/slack/schedule-recovery-handler-slack", diff --git a/frontend/src/lib/userDraftDbSyncer.svelte.ts b/frontend/src/lib/userDraftDbSyncer.svelte.ts index 77e2eecc88..0e8257023d 100644 --- a/frontend/src/lib/userDraftDbSyncer.svelte.ts +++ b/frontend/src/lib/userDraftDbSyncer.svelte.ts @@ -237,6 +237,25 @@ const flushes = new SvelteMap() */ const saveListeners = new Map void>>() +/** + * Global listeners fired whenever ANY draft write lands on the server — + * upserts and deletes alike. This is the invalidation hook for caches keyed + * on persisted draft state (the chat diff snapshot): the moment a save + * commits, the affected item can be marked stale without polling. + */ +type DraftSavedEvent = { workspace: string; itemKind: UserDraftItemKind; path: string } +const anySavedListeners = new Set<(event: DraftSavedEvent) => void>() + +function notifyAnySaved(event: DraftSavedEvent): void { + for (const listener of [...anySavedListeners]) { + try { + listener(event) + } catch (e) { + console.error('UserDraftDbSyncer.onAnySaved listener threw', e) + } + } +} + /** * Best-effort error → readable string. The generated client wraps HTTP * failures as `ApiError` (`body` / `statusText`); raw fetch errors are a @@ -310,6 +329,10 @@ async function postSave(opts: UserDraftDbSyncerSaveOpts): Promise { const listeners = saveListeners.get(key) if (listeners) for (const l of [...listeners]) l() } + // Global subscribers hear deletes too — a removed row invalidates + // cached state the same way an upsert does. Listener errors must never + // make a committed save read as failed. + notifyAnySaved({ workspace: opts.workspace, itemKind: opts.itemKind, path: opts.path }) } catch (e) { console.error('UserDraftDbSyncer.save failed', e) // Leave pending opts in place so the next attempt retries the same @@ -366,8 +389,11 @@ function flushOnPageHide(): void { console.error('UserDraftDbSyncer: keepalive flush failed', e) }) // POST advanced the row past `lastSync` and we can't read the - // response — mark the key so a bfcache restore drops it. + // response — mark the key so a bfcache restore drops it, and notify + // subscribers conservatively (this path bypasses postSave; on a + // bfcache restore a cache must not serve the pre-flush state). staleSyncAfterHideFlush.add(key) + notifyAnySaved({ workspace: opts.workspace, itemKind: opts.itemKind, path: opts.path }) } catch (e) { console.error('UserDraftDbSyncer: keepalive flush threw', e) } @@ -555,6 +581,18 @@ export const UserDraftDbSyncer = { } }, + /** + * Fires when any draft write lands on the server — upserts AND deletes, + * every workspace and key. For caches over persisted draft state that + * must invalidate the affected item the moment a write commits. + */ + onAnySaved(listener: (event: DraftSavedEvent) => void): () => void { + anySavedListeners.add(listener) + return () => { + anySavedListeners.delete(listener) + } + }, + /** Reactive conflict snapshot (if any) for a draft. */ getConflict(query: UserDraftLastSyncQuery): { readonly conflict: DraftConflictInfo | undefined diff --git a/frontend/src/lib/userDraftFlushToggle.test.ts b/frontend/src/lib/userDraftFlushToggle.test.ts new file mode 100644 index 0000000000..6d3ab29f77 --- /dev/null +++ b/frontend/src/lib/userDraftFlushToggle.test.ts @@ -0,0 +1,81 @@ +import { describe, it, expect, afterEach, vi } from 'vitest' + +const updateDraft = vi.fn(async (..._args: any[]) => ({ + status: 'saved' as const, + current_timestamp: '2020-01-01T00:00:00Z' +})) + +vi.mock('./gen', () => ({ + DraftService: { updateDraft: (...a: unknown[]) => updateDraft(...(a as [])) } +})) +vi.mock('./gen/core/OpenAPI', () => ({ OpenAPI: { BASE: '' } })) +vi.mock('./localDraftHints.svelte', () => ({ setLocalDraftHint: vi.fn() })) + +import { UserDraftDbSyncer } from './userDraftDbSyncer.svelte' + +afterEach(() => { + vi.clearAllMocks() + UserDraftDbSyncer.autosaveEnabled = true +}) + +/** + * Read-only consumers (the chat `diff` tool) flush with `honorAutosaveToggle` + * and then read `hasUnsavedDisabledChanges` to know the persisted state is + * stale. Pins the contract pair: a toggle-honoring flush must NOT persist + * auto-save-off edits (and must keep reporting them), while an explicit flush + * persists them and clears the signal. + */ +describe('UserDraftDbSyncer toggle-honoring flush', () => { + it('keeps auto-save-off edits parked and reported; explicit flush clears them', async () => { + const q = { workspace: 'w', itemKind: 'script' as const, path: 'u/me/toggle_off' } + UserDraftDbSyncer.autosaveEnabled = false + await UserDraftDbSyncer.save({ ...q, value: { content: 'x' }, auto: true, canBeDisabled: true }) + + await UserDraftDbSyncer.flush(q, { honorAutosaveToggle: true }) + expect(updateDraft).not.toHaveBeenCalled() + expect(UserDraftDbSyncer.hasUnsavedDisabledChanges(q)).toBe(true) + + await UserDraftDbSyncer.flush(q) + expect(updateDraft).toHaveBeenCalledTimes(1) + expect(UserDraftDbSyncer.hasUnsavedDisabledChanges(q)).toBe(false) + }) +}) + +/** A conflicted save leaves the local payload parked with the conflict + * recorded while the state reads 'none' — the exact triple the diff tool's + * unflushed-edits detection reads. Pins that a conflict never looks like a + * clean sync. */ +describe('UserDraftDbSyncer conflict aftermath', () => { + it('keeps the payload parked and the conflict readable after flush settles', async () => { + const q = { workspace: 'w', itemKind: 'script' as const, path: 'u/me/conflicted' } + updateDraft.mockResolvedValueOnce({ + status: 'conflict', + current_timestamp: '2020-01-02T00:00:00Z' + }) + await UserDraftDbSyncer.save({ ...q, value: { content: 'mine' }, immediate: true }) + + // The triple the diff tool's unflushed detection reads: conflict set, + // state 'none' (not pending/failed), auto-save signal silent. + expect(UserDraftDbSyncer.getConflict(q).conflict).toBeDefined() + expect(UserDraftDbSyncer.getState(q).state).toBe('none') + expect(UserDraftDbSyncer.hasUnsavedDisabledChanges(q)).toBe(false) + }) +}) + +/** The diff snapshot cache invalidates through this hook — it must fire for + * upserts AND deletes, the moment the write lands. */ +describe('UserDraftDbSyncer.onAnySaved', () => { + it('fires for landed upserts and deletes, and unsubscribes cleanly', async () => { + const q = { workspace: 'w', itemKind: 'script' as const, path: 'u/me/hooked' } + const events: string[] = [] + const off = UserDraftDbSyncer.onAnySaved((e) => events.push(`${e.itemKind}:${e.path}`)) + + await UserDraftDbSyncer.save({ ...q, value: { content: 'x' }, immediate: true }) + await UserDraftDbSyncer.save({ ...q, value: null, immediate: true }) + expect(events).toEqual(['script:u/me/hooked', 'script:u/me/hooked']) + + off() + await UserDraftDbSyncer.save({ ...q, value: { content: 'y' }, immediate: true }) + expect(events).toHaveLength(2) + }) +}) diff --git a/frontend/src/lib/utils_draft_deploy.ts b/frontend/src/lib/utils_draft_deploy.ts index 6068753c4c..514ce9cdb2 100644 --- a/frontend/src/lib/utils_draft_deploy.ts +++ b/frontend/src/lib/utils_draft_deploy.ts @@ -39,7 +39,9 @@ import type { DeployResult } from '$lib/utils_workspace_deploy' import { TRIGGER_RUNTIME_IGNORE } from '$lib/utils_deployable' import { deployRawAppDraft } from '$lib/rawAppDeploy' import { canonicalRawAppDiffValue } from '$lib/components/raw_apps/utils' +import { classicAppDraftParts } from '$lib/appDiffSides' import { invalidateWorkspaceDrafts } from '$lib/workspaceDrafts.svelte' +import { invalidateWorkspaceComparison } from '$lib/workspaceComparison' import { setLocalDraftHint } from '$lib/localDraftHints.svelte' import { userStore } from '$lib/stores' import { deployTriggers, type Trigger } from '$lib/components/triggers/utils' @@ -94,21 +96,35 @@ const OVERLAY_GETTERS: Partial< } /** Strip the per-user draft-overlay metadata, returning `{deployed, draft}`. */ -function splitOverlay(r: any): { deployed: any; draft: any } { +function splitOverlay(r: any): { + deployed: any + draft: any + hasDraft: boolean + noDeployed: boolean +} { const { draft, is_draft: _i, draft_saved_at: _c, - no_deployed: _n, + no_deployed, other_drafts_users: _o, ...deployed } = r - return { deployed, draft: draft ?? deployed } + return { + deployed, + draft: draft ?? deployed, + hasDraft: draft != null, + noDeployed: no_deployed === true + } } export interface DraftDiffValues { deployed: unknown draft: unknown + /** False when the overlay carried no draft row (the item's own value was used as the draft side). */ + hasDraft: boolean + /** True when the item has never been deployed (`draft_only` overlay). */ + noDeployed: boolean } // Empty-but-valid "deployed" shapes for draft_only items. A bare `{}` breaks @@ -117,7 +133,56 @@ export interface DraftDiffValues { const EMPTY_DEPLOYED: Partial unknown>> = { script: (draft) => ({ content: '', language: draft?.language, schema: {} }), flow: () => ({ summary: '', value: { modules: [] }, schema: {} }), - app: () => ({ summary: '', value: {}, policy: {} }) + app: () => ({ summary: '', value: {} }) +} + +// Server-managed script-row fields, stripped from BOTH sides of a draft diff: +// never user-edited, they are either identical noise (created_at, workspace_id) +// or spuriously different (lock is recomputed at deploy). The draft-side +// pinned-base `parent_hash` is stripped separately, like the flow `version_id`. +const SCRIPT_ROW_RUNTIME_IGNORE = new Set([ + 'workspace_id', + 'hash', + 'parent_hash', + 'parent_hashes', + 'created_at', + 'created_by', + 'archived', + 'deleted', + 'extra_perms', + 'lock', + 'lock_error_logs', + 'starred', + 'has_draft', + 'draft_only', + 'assets', + 'marked' +]) + +function stripScriptRowRuntime(row: any): Record { + if (!row || typeof row !== 'object') return {} + return Object.fromEntries(Object.entries(row).filter(([k]) => !SCRIPT_ROW_RUNTIME_IGNORE.has(k))) +} + +/** Canonicalize a raw draft value onto the same shape `getDraftDiffValues` + * yields for its draft side, so a value read from an in-memory editor cell + * diffs cleanly against a deployed side (and compares equal to its own + * persisted form instead of differing on stripped fields). */ +export function canonicalDraftSideValue(kind: DraftKind, value: unknown): unknown { + if (kind === 'script') return stripScriptRowRuntime(value) + if (kind === 'raw_app') return canonicalRawAppDiffValue((value ?? {}) as Record) + if (kind === 'app') { + const parts = classicAppDraftParts(value) + return { summary: parts.summary ?? '', value: parts.value } + } + if (kind === 'flow' && value !== null && typeof value === 'object') { + const { version_id: _v, ...rest } = value as Record + return rest + } + // Drawer kinds (variables/resources/schedules/triggers): the editor-state + // shape diverges from the backend row — same canonicalization the overlay + // diff applies. + return canonicalizeDraftDiffValue(kind, value, true) } // Schedule & trigger rows drop the same runtime/server-managed fields as the @@ -193,15 +258,17 @@ export async function getDraftDiffValues( draft, is_draft: _i, draft_saved_at: _c, - no_deployed: _n, + no_deployed, other_drafts_users: _o, hash: _h, ...deployed } = r - const draftValue = draft ?? deployed + const draftValue = stripScriptRowRuntime(draft ?? deployed) return { - deployed: draftOnly ? EMPTY_DEPLOYED.script!(draftValue) : deployed, - draft: draftValue + deployed: draftOnly ? EMPTY_DEPLOYED.script!(draftValue) : stripScriptRowRuntime(deployed), + draft: draftValue, + hasDraft: draft != null, + noDeployed: no_deployed === true } } else if (kind === 'flow') { const r = (await FlowService.getFlowByPath({ workspace, path, getDraft: true })) as any @@ -209,7 +276,7 @@ export async function getDraftDiffValues( draft, is_draft: _i, draft_saved_at: _c, - no_deployed: _n, + no_deployed, other_drafts_users: _o, version_id: _v, ...deployed @@ -217,7 +284,12 @@ export async function getDraftDiffValues( // Strip the draft's pinned base `version_id` (which differs from the deployed // head for a stale draft) so it never renders as a spurious diff line. const { version_id: _dv, ...draftValue } = (draft ?? deployed) as any - return { deployed: draftOnly ? EMPTY_DEPLOYED.flow!(draftValue) : deployed, draft: draftValue } + return { + deployed: draftOnly ? EMPTY_DEPLOYED.flow!(draftValue) : deployed, + draft: draftValue, + hasDraft: draft != null, + noDeployed: no_deployed === true + } } else if (kind === 'app' || kind === 'raw_app') { // A never-deployed raw app has no `app` row; the backend resolves the // draft kind from `rawApp`, so it MUST be set or the lookup 404s. @@ -232,22 +304,36 @@ export async function getDraftDiffValues( // deployed row nests them under `value`, and deployed inline scripts carry // server-recomputed locks. Canonicalize both onto the same shape with the // post-deploy noise stripped — the same module the editor's Diff button uses. + // A staged rename (`draft_path`) changes where deploy lands the app — + // compare it as `path` on both sides so a rename-only draft diffs. + const rawDraftPath = (r.draft?.draft_path as string | undefined) ?? r.path return { - deployed: draftOnly ? canonicalRawAppDiffValue({}) : canonicalRawAppDiffValue(r), - draft: canonicalRawAppDiffValue(r.draft ?? r) + deployed: draftOnly + ? canonicalRawAppDiffValue({}) + : { ...canonicalRawAppDiffValue(r), path: r.path }, + draft: { ...canonicalRawAppDiffValue(r.draft ?? r), path: rawDraftPath }, + hasDraft: r.draft != null, + noDeployed: r.no_deployed === true } } - const deployed = { - summary: r.summary, - value: r.value, - policy: r.policy, - path: r.path, - custom_path: r.custom_path + // Classic app: the editor drafts the bare grid with summary/draft_path + // mirrored into it, while the row keeps summary as a column beside + // `value`. Both sides reduce to `{ summary, value }` with the metadata + // extracted from the grid, so a summary edit diffs as a summary edit and + // the grid never diffs against draft-only markers. + const deployedParts = classicAppDraftParts(r.value) + const draftParts = r.draft != null ? classicAppDraftParts(r.draft) : deployedParts + const deployed = { summary: r.summary ?? '', value: deployedParts.value, path: r.path } + return { + deployed: draftOnly ? EMPTY_DEPLOYED.app!(undefined) : deployed, + draft: { + summary: draftParts.summary ?? r.summary ?? '', + value: draftParts.value, + path: draftParts.draftPath ?? r.path + }, + hasDraft: r.draft != null, + noDeployed: r.no_deployed === true } - // Strip the draft's pinned fork-base `parent_version` (the deployed allowlist - // above already omits it) so it never renders as a spurious diff line. - const { parent_version: _pv, ...draftValue } = (r.draft ?? deployed) as any - return { deployed: draftOnly ? EMPTY_DEPLOYED.app!(draftValue) : deployed, draft: draftValue } } else { // Variables / resources / schedules / triggers: one overlay GET yields // both sides, but the draft side is the editor's state shape while the @@ -258,10 +344,12 @@ export async function getDraftDiffValues( if (!getter) { throw new Error(`Draft diff not supported for kind ${kind}`) } - const { deployed, draft } = splitOverlay(await getter(workspace, path)) + const { deployed, draft, hasDraft, noDeployed } = splitOverlay(await getter(workspace, path)) return { deployed: draftOnly ? {} : canonicalizeDraftDiffValue(kind, deployed, false), - draft: canonicalizeDraftDiffValue(kind, draft, true) + draft: canonicalizeDraftDiffValue(kind, draft, true), + hasDraft, + noDeployed } } } @@ -556,6 +644,10 @@ export async function deployDraft( }) // Mutated the workspace's Server Drafts — refresh every mounted reader. invalidateWorkspaceDrafts(workspace) + // The DEPLOYED state moved: cached fork comparisons involving this + // workspace (as fork or as parent) are no longer trustworthy. Draft-only + // mutations skip this — they never move the deployed tally. + invalidateWorkspaceComparison(workspace) // For script/flow/app the server-side delete bypasses UserDraftDbSyncer, // so the syncer-owned hint won't auto-clear — clear it explicitly. // (Idempotent: the drawer-kind delete above already cleared it.) diff --git a/frontend/src/lib/workspaceComparison.test.ts b/frontend/src/lib/workspaceComparison.test.ts new file mode 100644 index 0000000000..bf9fc960b1 --- /dev/null +++ b/frontend/src/lib/workspaceComparison.test.ts @@ -0,0 +1,163 @@ +import { describe, expect, it, vi, beforeEach } from 'vitest' + +const compareWorkspaces = vi.fn() +vi.mock('$lib/gen', () => ({ + WorkspaceService: { compareWorkspaces: (...a: unknown[]) => compareWorkspaces(...(a as [])) } +})) +vi.mock('$lib/stores', async () => { + const { writable } = await import('svelte/store') + return { usersWorkspaceStore: writable(undefined) } +}) + +import { fetchWorkspaceComparison, invalidateWorkspaceComparison } from './workspaceComparison' +import { usersWorkspaceStore } from '$lib/stores' + +function deferred() { + let resolve!: (v: T) => void + const promise = new Promise((res) => (resolve = res)) + return { promise, resolve } +} + +beforeEach(() => { + compareWorkspaces.mockReset() +}) + +describe('fetchWorkspaceComparison', () => { + it('a freshness-forced call never adopts an older in-flight request', async () => { + const first = deferred() + compareWorkspaces.mockImplementationOnce(() => first.promise) + compareWorkspaces.mockResolvedValueOnce({ summary: { total_diffs: 1 } }) + + // Tolerant caller starts a request (e.g. the fork banner)... + const tolerant = fetchWorkspaceComparison('p', 'f-forced', { maxAgeMs: 30_000 }) + // ...a mutation happens, then a forced caller must get its OWN fetch. + const forced = fetchWorkspaceComparison('p', 'f-forced', { maxAgeMs: 0 }) + expect(compareWorkspaces).toHaveBeenCalledTimes(2) + + first.resolve({ summary: { total_diffs: 0 } }) + expect((await forced).summary.total_diffs).toBe(1) + expect((await tolerant).summary.total_diffs).toBe(0) + }) + + it('a superseded older request never overwrites a newer result, even same-millisecond', async () => { + vi.useFakeTimers() + try { + const older = deferred() + compareWorkspaces.mockImplementationOnce(() => older.promise) + compareWorkspaces.mockResolvedValueOnce({ summary: { total_diffs: 7 } }) + + // Same frozen Date.now() for both requests. + const tolerant = fetchWorkspaceComparison('p', 'f-race', { maxAgeMs: 30_000 }) + const forced = fetchWorkspaceComparison('p', 'f-race', { maxAgeMs: 0 }) + // Newer (forced) resolves FIRST; older resolves after with stale data. + expect((await forced).summary.total_diffs).toBe(7) + older.resolve({ summary: { total_diffs: 0 } }) + await tolerant + + // A tolerant read must see the newer result, not the late stale write. + const reread = await fetchWorkspaceComparison('p', 'f-race', { maxAgeMs: 30_000 }) + expect(reread.summary.total_diffs).toBe(7) + expect(compareWorkspaces).toHaveBeenCalledTimes(2) + } finally { + vi.useRealTimers() + } + }) + + it('invalidation evicts by EITHER side and fences in-flight requests', async () => { + compareWorkspaces.mockResolvedValueOnce({ summary: { total_diffs: 0 } }) + compareWorkspaces.mockResolvedValueOnce({ summary: { total_diffs: 3 } }) + + // Banner prewarms the cache pre-deploy... + await fetchWorkspaceComparison('p-side', 'f-inval', { maxAgeMs: 30_000 }) + // ...a deploy in the PARENT invalidates too... + invalidateWorkspaceComparison('p-side') + // ...so even a first-ever tolerant read cannot reuse the stale tally. + const fresh = await fetchWorkspaceComparison('p-side', 'f-inval', { maxAgeMs: 30_000 }) + expect(fresh.summary.total_diffs).toBe(3) + expect(compareWorkspaces).toHaveBeenCalledTimes(2) + }) + + it('a pre-invalidation in-flight request is not joined and cannot land in the cache', async () => { + const stale = deferred() + compareWorkspaces.mockImplementationOnce(() => stale.promise) + compareWorkspaces.mockResolvedValueOnce({ summary: { total_diffs: 9 } }) + + const preMutation = fetchWorkspaceComparison('p', 'f-fence', { maxAgeMs: 30_000 }) + invalidateWorkspaceComparison('f-fence') + // Tolerant post-mutation read: must NOT join the fenced request. + const post = fetchWorkspaceComparison('p', 'f-fence', { maxAgeMs: 30_000 }) + expect(compareWorkspaces).toHaveBeenCalledTimes(2) + + stale.resolve({ summary: { total_diffs: 0 } }) + await preMutation + expect((await post).summary.total_diffs).toBe(9) + // The stale request's late completion never landed in the cache. + const reread = await fetchWorkspaceComparison('p', 'f-fence', { maxAgeMs: 30_000 }) + expect(reread.summary.total_diffs).toBe(9) + expect(compareWorkspaces).toHaveBeenCalledTimes(2) + }) + + it('a superseded request fenced by invalidation cannot repopulate the cache late', async () => { + const superseded = deferred() + compareWorkspaces.mockImplementationOnce(() => superseded.promise) + compareWorkspaces.mockResolvedValueOnce({ summary: { total_diffs: 5 } }) + compareWorkspaces.mockResolvedValueOnce({ summary: { total_diffs: 6 } }) + + // Request A pends; forced request B replaces it in the inflight map and completes. + const a = fetchWorkspaceComparison('p', 'f-late', { maxAgeMs: 30_000 }) + await fetchWorkspaceComparison('p', 'f-late', { maxAgeMs: 0 }) + // Invalidation happens while A (no longer tracked in inflight) still pends. + invalidateWorkspaceComparison('f-late') + // A resolves late with pre-mutation data — it must NOT land in the cache. + superseded.resolve({ summary: { total_diffs: 0 } }) + await a + const read = await fetchWorkspaceComparison('p', 'f-late', { maxAgeMs: 30_000 }) + expect(read.summary.total_diffs).toBe(6) + }) + + it('tolerant callers join a recent in-flight request', async () => { + const first = deferred() + compareWorkspaces.mockImplementationOnce(() => first.promise) + + const a = fetchWorkspaceComparison('p', 'f-join', { maxAgeMs: 30_000 }) + const b = fetchWorkspaceComparison('p', 'f-join', { maxAgeMs: 30_000 }) + expect(compareWorkspaces).toHaveBeenCalledTimes(1) + + first.resolve({ summary: { total_diffs: 2 } }) + expect((await a).summary.total_diffs).toBe(2) + expect((await b).summary.total_diffs).toBe(2) + }) + it('an account switch clears cached comparisons', async () => { + usersWorkspaceStore.set({ email: 'first@x.dev' } as any) + compareWorkspaces.mockResolvedValueOnce({ summary: { total_diffs: 3 } }) + await fetchWorkspaceComparison('p', 'f-owner', { maxAgeMs: 30_000 }) + await fetchWorkspaceComparison('p', 'f-owner', { maxAgeMs: 30_000 }) + expect(compareWorkspaces).toHaveBeenCalledTimes(1) + + compareWorkspaces.mockResolvedValueOnce({ summary: { total_diffs: 9 } }) + usersWorkspaceStore.set({ email: 'second@x.dev' } as any) + const fresh = await fetchWorkspaceComparison('p', 'f-owner', { maxAgeMs: 30_000 }) + expect(fresh.summary.total_diffs).toBe(9) + expect(compareWorkspaces).toHaveBeenCalledTimes(2) + }) + + it('a request started under the previous account never joins or lands after a switch', async () => { + const old = deferred() + compareWorkspaces.mockImplementationOnce(() => old.promise) + compareWorkspaces.mockResolvedValueOnce({ summary: { total_diffs: 5 } }) + + usersWorkspaceStore.set({ email: 'a@x.dev' } as any) + fetchWorkspaceComparison('p', 'f-switch', { maxAgeMs: 30_000 }) + usersWorkspaceStore.set({ email: 'b@x.dev' } as any) + const after = fetchWorkspaceComparison('p', 'f-switch', { maxAgeMs: 30_000 }) + // The new account must not have joined the old account's request. + expect(compareWorkspaces).toHaveBeenCalledTimes(2) + + old.resolve({ summary: { total_diffs: 0 } }) + expect((await after).summary.total_diffs).toBe(5) + // And the old account's late result must not have landed in the cache. + const reread = await fetchWorkspaceComparison('p', 'f-switch', { maxAgeMs: 30_000 }) + expect(reread.summary.total_diffs).toBe(5) + expect(compareWorkspaces).toHaveBeenCalledTimes(2) + }) +}) diff --git a/frontend/src/lib/workspaceComparison.ts b/frontend/src/lib/workspaceComparison.ts new file mode 100644 index 0000000000..0d1ad8be25 --- /dev/null +++ b/frontend/src/lib/workspaceComparison.ts @@ -0,0 +1,181 @@ +/** + * Shared fetch layer for the fork↔parent workspace comparison + * (`compareWorkspaces`). The comparison is the expensive tally the fork banner, + * the compare page, and the chat `diff` tool all need — routing every consumer + * through this module means concurrent tolerant requests coalesce and a + * consumer that accepts a slightly stale result (`maxAgeMs`) can reuse the + * fetch another surface just made instead of recomputing it. + */ +import { get } from 'svelte/store' +import { WorkspaceService, type WorkspaceComparison } from '$lib/gen' +import { usersWorkspaceStore } from '$lib/stores' + +interface CacheEntry { + fetchedAt: number + generation: number + comparison: WorkspaceComparison +} + +interface InflightEntry { + startedAt: number + generation: number + promise: Promise +} + +const cache = new Map() +const inflight = new Map() +// Millisecond timestamps collide under concurrency — ordering between +// requests rides on this monotonic generation instead. +let requestGeneration = 0 +// Per-WORKSPACE floor: any request started at or below this generation is +// pre-invalidation and must neither be joined nor land in the cache. Keyed by +// workspace id (either side of a pair), so even requests the inflight map no +// longer tracks are fenced. +const invalidGenFloor = new Map() +// Raised when the authenticated identity changes — fences EVERY earlier +// request at once, including ones whose workspace ids nothing tracks anymore. +let globalGenFloor = 0 + +function generationFloor(parentWorkspaceId: string, forkWorkspaceId: string): number { + return Math.max( + globalGenFloor, + invalidGenFloor.get(parentWorkspaceId) ?? 0, + invalidGenFloor.get(forkWorkspaceId) ?? 0 + ) +} + +// Comparisons are permission-filtered per user but keyed only by workspace +// pair — an SPA logout/login must never serve one account's tallies (or let +// its in-flight fetches land) for another. +let cacheOwner: string | undefined = undefined + +function ensureCacheOwner(): void { + const owner = get(usersWorkspaceStore)?.email + if (owner === cacheOwner) return + cacheOwner = owner + cache.clear() + inflight.clear() + invalidGenFloor.clear() + globalGenFloor = requestGeneration +} +// Comparisons are big; keep only the few pairs a session actually browses. +const MAX_CACHE_ENTRIES = 8 + +function key(parentWorkspaceId: string, forkWorkspaceId: string): string { + return `${parentWorkspaceId}:${forkWorkspaceId}` +} + +/** + * Fetch (or reuse) the comparison of `forkWorkspaceId` against its parent. + * `maxAgeMs` (default 0) is the oldest result the caller accepts — applied to + * cached results AND to joining an in-flight request (a request is as old as + * its start). `maxAgeMs: 0` therefore always issues a fresh fetch: a caller + * forcing freshness after a mutation must never adopt a request that began + * before the mutation. + */ +export async function fetchWorkspaceComparison( + parentWorkspaceId: string, + forkWorkspaceId: string, + opts: { maxAgeMs?: number } = {} +): Promise { + return (await fetchWorkspaceComparisonMeta(parentWorkspaceId, forkWorkspaceId, opts)).comparison +} + +export interface WorkspaceComparisonMeta { + comparison: WorkspaceComparison + /** When the underlying request STARTED — a reused result is as old as its + * fetch, not as old as the reuse. Callers layering their own freshness + * window must age from this, or windows compound. */ + fetchedAt: number + /** Pass to `isComparisonCurrent` to learn whether an invalidation has + * outdated this result since. */ + generation: number +} + +/** True while no `invalidateWorkspaceComparison` (or identity change) has + * fenced the request that produced `generation`. */ +export function isComparisonCurrent( + parentWorkspaceId: string, + forkWorkspaceId: string, + generation: number +): boolean { + ensureCacheOwner() + return generation > generationFloor(parentWorkspaceId, forkWorkspaceId) +} + +export async function fetchWorkspaceComparisonMeta( + parentWorkspaceId: string, + forkWorkspaceId: string, + opts: { maxAgeMs?: number } = {} +): Promise { + ensureCacheOwner() + const k = key(parentWorkspaceId, forkWorkspaceId) + const maxAgeMs = opts.maxAgeMs ?? 0 + const cached = cache.get(k) + if (cached && Date.now() - cached.fetchedAt < maxAgeMs) { + return { + comparison: cached.comparison, + fetchedAt: cached.fetchedAt, + generation: cached.generation + } + } + const pending = inflight.get(k) + if ( + pending && + maxAgeMs > 0 && + Date.now() - pending.startedAt < maxAgeMs && + pending.generation > generationFloor(parentWorkspaceId, forkWorkspaceId) + ) { + return { + comparison: await pending.promise, + fetchedAt: pending.startedAt, + generation: pending.generation + } + } + const startedAt = Date.now() + const generation = ++requestGeneration + const run = (async () => { + const comparison = await WorkspaceService.compareWorkspaces({ + workspace: parentWorkspaceId, + targetWorkspaceId: forkWorkspaceId + }) + // A superseded (older-generation) or pre-invalidation request must not + // clobber a newer result. + const existing = cache.get(k) + if ( + generation > generationFloor(parentWorkspaceId, forkWorkspaceId) && + (!existing || existing.generation < generation) + ) { + cache.delete(k) + cache.set(k, { fetchedAt: startedAt, generation, comparison }) + // Insertion-ordered Map: evict the oldest pairs past the cap. + while (cache.size > MAX_CACHE_ENTRIES) { + cache.delete(cache.keys().next().value as string) + } + } + return comparison + })() + inflight.set(k, { startedAt, generation, promise: run }) + try { + return { comparison: await run, fetchedAt: startedAt, generation } + } finally { + if (inflight.get(k)?.promise === run) inflight.delete(k) + } +} + +/** Drop cached comparisons involving this workspace on EITHER side — a + * deploy in a parent moves its forks' tallies too. Also fences in-flight + * requests: nobody new joins them and their late results never land in the + * cache. (Workspace ids cannot contain ':', so the matches are exact.) */ +export function invalidateWorkspaceComparison(workspaceId: string): void { + const matches = (k: string) => k.startsWith(`${workspaceId}:`) || k.endsWith(`:${workspaceId}`) + for (const k of [...cache.keys()]) { + if (matches(k)) cache.delete(k) + } + for (const k of [...inflight.keys()]) { + if (matches(k)) inflight.delete(k) + } + // Fence EVERY request started before this point — including ones the + // inflight map no longer tracks (superseded requests still resolve late). + invalidGenFloor.set(workspaceId, requestGeneration) +} diff --git a/frontend/src/lib/workspaceDrafts.svelte.ts b/frontend/src/lib/workspaceDrafts.svelte.ts index 32345bd103..52e96ce7a1 100644 --- a/frontend/src/lib/workspaceDrafts.svelte.ts +++ b/frontend/src/lib/workspaceDrafts.svelte.ts @@ -47,6 +47,9 @@ export interface DraftItem { * `allUsers` listing surfaces other users' rows as `false` (view-only). * Defaults to true when the field is absent (older backend). */ mine: boolean + /** Server timestamp of the draft row, bumped on every draft update — a + * reliable per-row change marker for caches keyed on draft content. */ + created_at: string /** Only set when listed with a `compareToWorkspace` (a fork comparing against * its parent): true when this draft is identical to the parent's — cloned in * on fork and never edited here. Undefined when no comparison was requested. */ @@ -74,6 +77,7 @@ export async function getDraftItems( can_write: r.can_write ?? true, draft_users: r.draft_users, mine: r.mine ?? true, + created_at: r.created_at, unchanged_from_parent: r.unchanged_from_parent })) } @@ -87,6 +91,13 @@ export function invalidateWorkspaceDrafts(workspace: string | undefined): void { versions[workspace] = (versions[workspace] ?? 0) + 1 } +/** Current invalidation version for a workspace. Non-reactive read — callers + * compare it against a value captured earlier to detect Server-Draft mutations + * (any deploy/discard/draft write that called `invalidateWorkspaceDrafts`). */ +export function getWorkspaceDraftsVersion(workspace: string): number { + return versions[workspace] ?? 0 +} + export interface WorkspaceDraftsHandle { readonly items: DraftItem[] readonly count: number diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index d36569ae95..adf7fb3314 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -228,10 +228,10 @@ // nest the whole experience. Hide it when embedded. const embedded = BROWSER && window.self !== window.top - // AI sessions are still dev-gated (localStorage wm_dev_global_ai=1), same as - // the global chat. The Workspace ⇄ Sessions switch is the only entry point, so - // gate it on the flag too — otherwise it would ship the unfinished experience - // to prod. The /sessions page has its own gate for direct navigation. + // AI sessions (beta) are on unless the user opted out from the banner under + // the session chat. The Workspace ⇄ Sessions switch is the only entry point, + // so it follows the gate; opted-out users get the legacy Ask-AI pane instead. + // The /sessions page has its own gate for direct navigation. const globalAiEnabled = isGlobalAiEnabled() if (page.status == 404) { @@ -975,8 +975,8 @@ shortcut={`${getModifierKey()}k`} /> {#if !globalAiEnabled} - + aiChatManager.toggleOpen()} @@ -1108,8 +1108,8 @@ shortcut={`${getModifierKey()}k`} /> {#if !globalAiEnabled} - + aiChatManager.toggleOpen()} @@ -1302,10 +1302,14 @@
{/if} + {/snippet} -{#if isGlobalAiEnabled()} +{#if import.meta.env.DEV}

diff --git a/frontend/src/routes/(root)/(logged)/folders/+page.svelte b/frontend/src/routes/(root)/(logged)/folders/+page.svelte index 0a55acfe69..0df2235212 100644 --- a/frontend/src/routes/(root)/(logged)/folders/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/folders/+page.svelte @@ -14,17 +14,25 @@ import { sendUserToast } from '$lib/utils' import DataTable from '$lib/components/table/DataTable.svelte' import Cell from '$lib/components/table/Cell.svelte' - import { Pen, Trash, Plus } from 'lucide-svelte' + import { Pen, Trash, Plus, UploadCloud } from 'lucide-svelte' + import DeployToHub from '$lib/components/workspaceSettings/DeployToHub.svelte' import Head from '$lib/components/table/Head.svelte' import Row from '$lib/components/table/Row.svelte' import Badge from '$lib/components/common/badge/Badge.svelte' import { untrack } from 'svelte' + import { DEMO_RESTRICTION_HINT, isDemoWorkspaceRestricted } from '$lib/cloud' type FolderW = Folder & { canWrite: boolean } + let restricted = $derived( + isDemoWorkspaceRestricted($workspaceStore, $userStore?.is_admin, $userStore?.is_super_admin) + ) + let newFolderName: string = $state('') let folders: FolderW[] | undefined = $state(undefined) let folderDrawer: Drawer | undefined = $state() + let hubDrawer: Drawer | undefined = $state() + let publishFolderName: string = $state('') async function loadFolders(): Promise { folders = (await FolderService.listFolders({ workspace: $workspaceStore! })).map((x) => { @@ -83,6 +91,22 @@ + + { + hubDrawer?.closeDrawer() + publishFolderName = '' + }} + > + {#if publishFolderName} + {#key publishFolderName} + + {/key} + {/if} + + + {#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find((_) => _.id === $workspaceStore)?.operator_settings?.folders}

@@ -206,7 +242,7 @@ - + { + publishFolderName = name + hubDrawer?.openDrawer() + } + }, { displayName: `Delete${canWrite ? '' : ' (require owner permissions)'}`, icon: Trash, diff --git a/frontend/src/routes/(root)/(logged)/forks/compare/+page.svelte b/frontend/src/routes/(root)/(logged)/forks/compare/+page.svelte index 13021ed4d1..152948b190 100644 --- a/frontend/src/routes/(root)/(logged)/forks/compare/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/forks/compare/+page.svelte @@ -2,6 +2,7 @@ import CompareWorkspaces from '$lib/components/CompareWorkspaces.svelte' import CompareDrafts from '$lib/components/CompareDrafts.svelte' import { WorkspaceService, type WorkspaceComparison } from '$lib/gen' + import { fetchWorkspaceComparison } from '$lib/workspaceComparison' import { archiveSessionsForWorkspace, deleteSessionsForWorkspace, @@ -194,10 +195,7 @@ } try { - const result = await WorkspaceService.compareWorkspaces({ - workspace: parentWorkspaceId, - targetWorkspaceId: currentWorkspaceId - }) + const result = await fetchWorkspaceComparison(parentWorkspaceId, currentWorkspaceId) comparison = result } catch (e) { diff --git a/frontend/src/routes/(root)/(logged)/global_drafts/+page.svelte b/frontend/src/routes/(root)/(logged)/global_drafts/+page.svelte index 82adf2a459..f645f7e3d6 100644 --- a/frontend/src/routes/(root)/(logged)/global_drafts/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/global_drafts/+page.svelte @@ -6,7 +6,6 @@ listGlobalDrafts } from '$lib/components/copilot/chat/global/userDraftAdapter' import type { WorkspaceItem } from '$lib/components/copilot/chat/global/workspaceItems' - import { isGlobalAiEnabled } from '$lib/components/copilot/chat/global/gate' import { goto } from '$lib/navigation' import { workspaceStore } from '$lib/stores' import { Trash2 } from 'lucide-svelte' @@ -21,8 +20,8 @@ } onMount(() => { - // Dev-only route. Bounce to home when the global mode gate is closed. - enabled = isGlobalAiEnabled() + // Dev tooling, not part of the sessions beta — only reachable on dev builds. + enabled = import.meta.env.DEV if (!enabled) { goto('/') } diff --git a/frontend/src/routes/(root)/(logged)/groups/+page.svelte b/frontend/src/routes/(root)/(logged)/groups/+page.svelte index 0668c4c307..85b4b536fe 100644 --- a/frontend/src/routes/(root)/(logged)/groups/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/groups/+page.svelte @@ -22,9 +22,14 @@ import { untrack } from 'svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' import { Tooltip } from '$lib/components/meltComponents' + import { DEMO_RESTRICTION_HINT, isDemoWorkspaceRestricted } from '$lib/cloud' type GroupW = Group & { canWrite: boolean } + let restricted = $derived( + isDemoWorkspaceRestricted($workspaceStore, $userStore?.is_admin, $userStore?.is_super_admin) + ) + let newGroupName: string = $state('') let groups: GroupW[] | undefined = $state(undefined) let instanceGroups: InstanceGroupWithWorkspaces[] | undefined = $state(undefined) @@ -103,37 +108,49 @@ >
- - {#snippet trigger()} - - {/snippet} - {#snippet content({ close })} -
- handleKeyUp(e, close) - }} - bind:value={newGroupName} - /> - + {:else} + + {#snippet trigger()} + - Create - -
- {/snippet} -
+ {/snippet} + {#snippet content({ close })} +
+ handleKeyUp(e, close) + }} + bind:value={newGroupName} + /> + +
+ {/snippet} + + {/if}
diff --git a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte index 0c1a6eeb8c..6123c07000 100644 --- a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte @@ -43,6 +43,7 @@ assetProducers } from '$lib/components/assets/AssetGraph/graphTraversal' import { runCascade, runSelection } from '$lib/components/assets/AssetGraph/cascadeOrchestrator' + import { DATA_ASSET_KINDS } from '$lib/components/assets/AssetGraph/cascadeRun' import { boundedSet, buildLineageDag, @@ -72,6 +73,11 @@ type PipelineDraft } from '$lib/components/assets/AssetGraph/pipelineAiHelpers' import { PipelineEditorState } from '$lib/components/assets/AssetGraph/pipelineEditorState.svelte' + import { + createPipelineRecording, + finalizePipelineRecording + } from '$lib/components/recording/pipelineRecording.svelte' + import type { PipelineRecording } from '$lib/components/recording/types' import AutosaveIndicator from '$lib/components/AutosaveIndicator.svelte' import { onMount, tick, untrack } from 'svelte' import { aiChatManager } from '$lib/components/copilot/chat/AIChatManager.svelte' @@ -79,6 +85,8 @@ AlertTriangle, ArrowLeft, ChevronDown, + Circle, + Download, Folder, FolderSearch, History, @@ -101,7 +109,7 @@ type ScriptLang } from '$lib/gen' import { resource } from 'runed' - import { emptySchema, sendUserToast } from '$lib/utils' + import { emptySchema, sendUserToast, type Item } from '$lib/utils' import type { Schema } from '$lib/common' import { beforeNavigate, goto } from '$app/navigation' import { fade } from 'svelte/transition' @@ -113,7 +121,7 @@ // Variables and resources are declarative config, not pipeline assets — // they're hub-shaped (referenced by most runnables) and would swamp the // layout without adding lineage information. - const DATA_KINDS = ['s3object', 'ducklake', 'datatable', 'volume'] + const DATA_KINDS = DATA_ASSET_KINDS let folder = $derived(page.params.folder as string) @@ -1404,6 +1412,61 @@ // other's storage writes. let cascadeRunningRoot = $state(undefined) + // Recorder: when armed, the next cascade run captures the resolved graph, the + // per-node status timeline and each node's job stream into a downloadable + // recording that the /pipeline_replay player can rerun offline (parity with the + // flow/script recorders). Job capture (`watchJob`) and status capture + // (`recordStatuses`) no-op unless the store is active, so the cascade run + // paths call them unconditionally. + let pipelineRecording = createPipelineRecording() + let recordingMode = $state(false) + let lastPipelineRecording = $state(undefined) + + // Shared by the overflow-menu Record item and the inline armed pill so their + // wording can't drift — both describe the same armed recorder. + const RECORDING_ARMED_HINT = + 'Recording armed — the next pipeline run will be captured. Click to disarm.' + + function downloadPipelineRecording() { + if (lastPipelineRecording) { + pipelineRecording.download(lastPipelineRecording) + } + } + + // Secondary top-bar controls (recorder, macros) collapse into a single + // overflow (⋮) menu so the bar stays legible on small screens; only primary + // actions stay inline. Recording lives here rather than on the bar at all + // times — while armed it surfaces a compact inline pill (below) instead. + let overflowMenuItems = $derived.by(() => { + const items: Item[] = [] + if (!isOperator && allPipelineScripts.length > 0) { + items.push({ + displayName: recordingMode ? 'Disarm recorder' : 'Record next run', + icon: Circle, + iconColor: recordingMode ? 'rgb(220 38 38)' : undefined, + disabled: !!cascadeRunningRoot, + tooltip: recordingMode + ? RECORDING_ARMED_HINT + : 'Arm the recorder so the next pipeline run is captured for offline replay', + action: () => (recordingMode = !recordingMode) + }) + if (lastPipelineRecording && !cascadeRunningRoot) { + items.push({ + displayName: 'Download last recording', + icon: Download, + action: () => downloadPipelineRecording() + }) + } + } + items.push({ + displayName: 'Macros', + icon: SquareFunction, + tooltip: "Browse the workspace's DuckDB macros (deployed // macros libraries)", + action: () => macroDrawer?.openDrawer() + }) + return items + }) + // Script path → its schedule's configured args, so a manual "Run pipeline" // launches a schedule-triggered script with the same payload a real tick // would (rather than empty args). Schedule is the only trigger that stores a @@ -1711,6 +1774,10 @@ // Claim the running-guard BEFORE the first await so a rapid second click // (which reads `cascadeRunningRoot`) can't slip through and double-launch. cascadeRunningRoot = schedule.roots[0] ?? scripts[0] + if (recordingMode) { + lastPipelineRecording = undefined + pipelineRecording.start(folder, displayGraph) + } let firstJobId: string | undefined try { // Seed schedule-triggered roots with their configured payload. @@ -1720,6 +1787,8 @@ launch: async (path) => { const jobId = await launchCascadeScript(path) activeRunnables.arm(`script:${path}`) + // No-op unless a recording is active; captures the node's stream. + if ($workspaceStore) pipelineRecording.watchJob(jobId, $workspaceStore) if (firstJobId === undefined) { firstJobId = jobId runsPendingJobId = jobId @@ -1727,7 +1796,8 @@ } return jobId }, - waitTerminal: waitJobTerminal + waitTerminal: waitJobTerminal, + onUpdate: (statuses) => pipelineRecording.recordStatuses(statuses) }) const n = res.statuses.size if (res.ok) { @@ -1750,7 +1820,21 @@ ) } } finally { - cascadeRunningRoot = undefined + // Hold the run guard until finalization finishes: finalize keeps writing + // jobs/samples/code through the recorder store, and a second run's + // `start()` would reset those maps mid-write, corrupting both recordings. + // The nested finally still clears the guard if finalize ever rejects, so + // Run can't wedge permanently. + try { + if (pipelineRecording.active) { + lastPipelineRecording = await finalizePipelineRecording( + pipelineRecording, + $workspaceStore + ) + } + } finally { + cascadeRunningRoot = undefined + } } } @@ -2326,6 +2410,20 @@ {/if}
{#if !isOperator && allPipelineScripts.length > 0} + + {#if recordingMode} + + {/if} +{#snippet replayFailed()} +
+ +

+ This recording could not be replayed — it may be malformed or from an incompatible version. +

+ +
+{/snippet} + + +
+ {#if flowRecording} +
+ +
+ setActiveReplay(undefined)}> + + {#snippet failed()}{@render replayFailed()}{/snippet} + + {:else if scriptRecording} +
+ +
+ setActiveReplay(undefined)}> + + {#snippet failed()}{@render replayFailed()}{/snippet} + + {:else if pipelineRecording} +
+ +
+
+ setActiveReplay(undefined)}> + + {#snippet failed()}{@render replayFailed()}{/snippet} + +
+ {:else if downloading} +
+
+ +

Downloading recording…

+ {#if downloadPercent !== undefined} +
+
+
+

{downloadPercent}% · {fmtBytes(downloadedBytes)}

+ {:else} +

{fmtBytes(downloadedBytes)}

+ {/if} +
+
+ {:else} +
+
+

Replay a recording

+

+ Upload a recording JSON file to replay a flow, script or data-pipeline execution offline. +

+ {#if downloadError} +

{downloadError}

+ {/if} + + Drag and drop a recording file + +
+
+ {/if} +
diff --git a/frontend/src/routes/(root)/(logged)/projects/install/+page.svelte b/frontend/src/routes/(root)/(logged)/projects/install/+page.svelte new file mode 100644 index 0000000000..06f2344356 --- /dev/null +++ b/frontend/src/routes/(root)/(logged)/projects/install/+page.svelte @@ -0,0 +1,374 @@ + + +
+ {#if !slug} +

Missing ?hub=<slug>.

+ {:else if loading} +
+ Loading project… +
+ {:else if loadError} +

Failed to load project: {loadError}

+ {:else if data} +

Add “{data.project.name}” to workspace

+

{data.project.summary}

+ +
+

+ Folder in {workspace} +

+ +

+ Items import under f/{folderName.trim() || data.project.slug}/. +

+
+ +
+ {counts?.scripts} scripts + {counts?.flows} flows + {counts?.apps} apps + {counts?.resources} resources + {counts?.triggers} triggers + {#if counts && counts.migrations > 0} + {counts.migrations} data table migrations + {/if} +
+ +
+ Resources are imported as empty stubs — set their values after import; a resource whose path + already exists is reported as failed (existing values are never overwritten). Trigger kinds + are recreated disabled, except GCP and Azure triggers, which manage cloud subscriptions at + creation and must be re-created manually after filling their resource. Kafka, NATS, SQS, GCP + and Azure triggers all require Enterprise. Triggers that reference a resource depend on stubs + imported empty, so fill in the resource value before re-enabling the trigger. +
+ +
+ + {#if done} + + {/if} +
+ + {#if results.length} +
    + {#each results as r} +
  • + {r.ok ? '✓' : '✗'} + {r.path} + {#if !r.ok}— {r.error}{/if} +
  • + {/each} +
+ {/if} + {/if} +
+ + + + + + closeMigrationReview(false)}> + closeMigrationReview(false)}> +
+

+ This project ships migrations that recreate the data tables it uses. Review and edit the + SQL, then choose which to run. A migration runs against the data table of the same name in + {workspace}; if that data table has migrations enabled it is + recorded, otherwise it runs once as a preview job. +

+ {#each reviewList as m (m.datatable_name)} +
+
+ {m.datatable_name} + +
+ {#if m.run} + + {/if} +
+ {/each} +
+ {#snippet actions()} + + + {/snippet} +
+
diff --git a/frontend/src/routes/(root)/(logged)/replay/+page.svelte b/frontend/src/routes/(root)/(logged)/replay/+page.svelte deleted file mode 100644 index d89bcbb9b0..0000000000 --- a/frontend/src/routes/(root)/(logged)/replay/+page.svelte +++ /dev/null @@ -1,79 +0,0 @@ - - -
- {#if flowRecording} -
- -
- - {:else if scriptRecording} -
- -
- - {:else} -
-
-

Replay a recording

-

- Upload a recording JSON file to replay a flow or script execution offline. -

- - Drag and drop a recording file - -
-
- {/if} -
diff --git a/frontend/src/routes/(root)/(logged)/replay/+page.ts b/frontend/src/routes/(root)/(logged)/replay/+page.ts new file mode 100644 index 0000000000..811dac8e18 --- /dev/null +++ b/frontend/src/routes/(root)/(logged)/replay/+page.ts @@ -0,0 +1,9 @@ +import { redirect } from '@sveltejs/kit' +import { base } from '$app/paths' + +// The replay page moved to /pipeline_replay (it now replays data-pipeline +// recordings in addition to flow/script ones). Redirect the old path in `load` +// so existing /replay links and bookmarks still resolve instead of 404-ing. +export function load({ url }: { url: URL }) { + redirect(307, `${base}/pipeline_replay${url.search}`) +} diff --git a/frontend/src/routes/(root)/(logged)/resources/+page.svelte b/frontend/src/routes/(root)/(logged)/resources/+page.svelte index 5c0f538d3f..d03fe118ee 100644 --- a/frontend/src/routes/(root)/(logged)/resources/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/resources/+page.svelte @@ -605,13 +605,22 @@ } }) - onMount(() => { - let hash = page.url.hash - if (hash.startsWith('#/resource/')) { - console.log('hash', hash) - let path = hash.slice(11) - resourceEditor?.initEdit(path) + // Deep link: #/resource/ opens that resource's edit drawer. Reactive + // rather than onMount so a hash change on the already-mounted page (e.g. the + // AI session preview re-pointing its tab) opens the drawer too. Row links + // pre-set handledHash: their onclick already opens the drawer. + let handledHash = '' + $effect(() => { + const hash = page.url.hash + if (!hash.startsWith('#/resource/')) { + // Navigating away from a drawer target must clear the tracker, or + // re-targeting the same item later would be skipped as already handled. + handledHash = '' + return } + if (hash === handledHash || !resourceEditor) return + handledHash = hash + resourceEditor.initEdit(hash.slice(11)) }) let showTable = $derived( @@ -999,7 +1008,7 @@ Resource type Description - +
@@ -1014,8 +1023,17 @@ resourceEditor?.initEdit?.(path)} - >{#if marked}{@html marked}{:else}{path}{/if}{(getLocalDraftHint($workspaceStore, 'resource', path) ?? is_draft) ? '*' : ''} { + handledHash = `#/resource/${path}` + resourceEditor?.initEdit?.(path) + }} + >{#if marked}{@html marked}{:else}{path}{/if}{(getLocalDraftHint( + $workspaceStore, + 'resource', + path + ) ?? is_draft) + ? '*' + : ''} {#if draft_only} @@ -1158,83 +1176,85 @@ {/if} - - {#if path && assetCanBeExplored({ kind: 'resource', path }, { resource_type }) && !$userStore?.operator} - - {/if} - { - shareModal?.openDrawer?.(path, 'resource') - } - }, - { - displayName: 'Edit', - icon: Pen, - disabled: !canWrite || !showCreateButtons, - action: () => { - resourceEditor?.initEdit?.(path) - } - }, - ...(!ws_specific && isDeployable('resource', path, deployUiSettings) - ? [ - { - displayName: 'Deploy to prod/staging', - icon: FileUp, - action: () => { - deploymentDrawer?.openDrawer(path, 'resource') + +
+ {#if path && assetCanBeExplored({ kind: 'resource', path }, { resource_type }) && !$userStore?.operator} + + {/if} + { + shareModal?.openDrawer?.(path, 'resource') + } + }, + { + displayName: 'Edit', + icon: Pen, + disabled: !canWrite || !showCreateButtons, + action: () => { + resourceEditor?.initEdit?.(path) + } + }, + ...(!ws_specific && isDeployable('resource', path, deployUiSettings) + ? [ + { + displayName: 'Deploy to prod/staging', + icon: FileUp, + action: () => { + deploymentDrawer?.openDrawer(path, 'resource') + } } - } - ] - : []), - { - displayName: 'Delete', - disabled: !canWrite || !showCreateButtons, - icon: Trash, - type: 'delete', - action: (event) => { - // TODO - // @ts-ignore - if (event?.shiftKey) { - deleteResource(path, account) - } else { - deleteIsLinked = is_linked ?? false - deleteConfirmedCallback = () => { + ] + : []), + { + displayName: 'Delete', + disabled: !canWrite || !showCreateButtons, + icon: Trash, + type: 'delete', + action: (event) => { + // TODO + // @ts-ignore + if (event?.shiftKey) { deleteResource(path, account) + } else { + deleteIsLinked = is_linked ?? false + deleteConfirmedCallback = () => { + deleteResource(path, account) + } } } - } - }, - ...(account != undefined - ? [ - { - displayName: 'Refresh token', - icon: RotateCw, - action: async () => { - await OauthService.refreshToken({ - workspace: $workspaceStore ?? '', - id: account ?? 0, - requestBody: { - path - } - }) - sendUserToast('Token refreshed') - loadResources() + }, + ...(account != undefined + ? [ + { + displayName: 'Refresh token', + icon: RotateCw, + action: async () => { + await OauthService.refreshToken({ + workspace: $workspaceStore ?? '', + id: account ?? 0, + requestBody: { + path + } + }) + sendUserToast('Token refreshed') + loadResources() + } } - } - ] - : []) - ]} - /> - + ] + : []) + ]} + /> +
{/each} {/if} @@ -1262,7 +1282,7 @@ Name Description - +
@@ -1298,7 +1318,7 @@ {removeMarkdown(truncate(description ?? '', 200))} - + {#if !canWrite} Shared globally diff --git a/frontend/src/routes/(root)/(logged)/schedules/+page.svelte b/frontend/src/routes/(root)/(logged)/schedules/+page.svelte index c519c01a05..e07dfcf59e 100644 --- a/frontend/src/routes/(root)/(logged)/schedules/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/schedules/+page.svelte @@ -199,18 +199,25 @@ }) let scheduleEditor: ScheduleEditor | undefined = $state() - let hashHandled = false + // Deep link: # opens that schedule's edit drawer. Tracks the last + // handled hash (not a one-shot flag) so a hash change on the already-mounted + // page (e.g. the AI session preview re-pointing its tab) opens the drawer + // too. Row links pre-set handledHash: their onclick already opens the drawer. + let handledHash = '' $effect(() => { - if (!hashHandled && schedules.length > 0 && scheduleEditor) { - let hash = $page.url.hash - if (hash.length > 1) { - let path = hash.slice(1) - let schedule = schedules.find((s) => s.path === path) - if (schedule) { - hashHandled = true - scheduleEditor?.openEdit(path, schedule.is_flow) - } - } + const hash = $page.url.hash + if (hash.length <= 1) { + // Navigating away from a drawer target must clear the tracker, or + // re-targeting the same schedule later would be skipped as already handled. + handledHash = '' + return + } + if (hash === handledHash || schedules.length === 0 || !scheduleEditor) return + const path = hash.slice(1) + const schedule = schedules.find((s) => s.path === path) + if (schedule) { + handledHash = hash + scheduleEditor.openEdit(path, schedule.is_flow) } }) @@ -381,13 +388,22 @@ scheduleEditor?.openEdit(path, is_flow)} + onclick={() => { + handledHash = `#${path}` + scheduleEditor?.openEdit(path, is_flow) + }} class="min-w-0 grow hover:underline decoration-gray-400" >
- {summary || script_path}{(getLocalDraftHint($workspaceStore, 'trigger_schedule', path) ?? is_draft) ? '*' : ''} + {summary || script_path}{(getLocalDraftHint( + $workspaceStore, + 'trigger_schedule', + path + ) ?? is_draft) + ? '*' + : ''}
schedule: {path} diff --git a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte index a3f45f95fd..225bbec8ed 100644 --- a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte @@ -35,15 +35,20 @@ import { withWorkspaceParam } from '$lib/components/sessions/sessionMode.svelte' import { enterSessionMode } from '$lib/components/sessions/sessionSwitch.svelte' import type { SessionPreviewTabs } from '$lib/components/sessions/sessionPreviewTabs.svelte' - import { userWorkspaces, usersWorkspaceStore, workspaceStore } from '$lib/stores' + import { userStore, userWorkspaces, usersWorkspaceStore, workspaceStore } from '$lib/stores' import { getOrCreateRuntime, getRuntime, listRuntimes } from '$lib/components/sessions/sessionRuntime.svelte' import { markSessionSeen } from '$lib/components/sessions/sessionUnread.svelte' - import { isGlobalAiEnabled } from '$lib/components/copilot/chat/global/gate' + import { + isGlobalAiEnabled, + setSessionsBetaOptOut + } from '$lib/components/copilot/chat/global/gate' import { setToolCompletionListener } from '$lib/components/copilot/chat/shared' + import { registerToolDisplayActionHandler } from '$lib/components/copilot/chat/createdResourceActions.svelte' + import { previewTargetForSessionTarget } from '$lib/components/sessions/sessionPreviewTabs.svelte' import { base } from '$lib/base' import { artifactKey, @@ -458,6 +463,24 @@ } }) + // Preview cards on create/update tool calls dispatch here. Open + // (or focus, if already shown) the item's preview in the active session's panel — + // the visible chat is always the active session, so `owner` is its panel. Read + // `owner` lazily inside the handler (not in the effect body) so this registers + // once, not on every session switch. A 'focused' open leaves the tab where it is, + // so pulse it to make the click visibly land. + $effect(() => { + return registerToolDisplayActionHandler('open_item_preview', (action) => { + if (action.type !== 'open_item_preview') return + const o = owner + if (!o) return + const target = previewTargetForSessionTarget(action.previewKind, action.path) + if (!target) return + const { status } = o.open(target) + if (status === 'focused') o.pulseFocus(o.activeId) + }) + }) + // Editor-style breadcrumb over the previewed page. We only render clickable // segments when the preview is sitting on a script/flow/app route — for any // other page (home, runs, …) there's no item to drill into, so we fall back @@ -611,10 +634,33 @@ }}>Open sessions
+ {:else if $userStore?.operator} + +
+

AI Sessions are not available for operators

+

Use the Ask AI chat instead.

+ +
{:else if !globalEnabled} -
- Sessions are gated on the global-AI dev flag. Enable with - localStorage.setItem('wm_dev_global_ai', '1') and reload. + +
+

AI Sessions are deactivated

+

You switched back to the legacy chat. Activate AI Sessions (beta) to open this page.

+
{:else if !sessionState.hydrated} +
- + { let owner = isOwner(path, $userStore, $workspaceStore) diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index 54b03d9b9c..49c29cc1a7 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -346,6 +346,12 @@ // their `usr` row is copied from the parent, so forking as an ordinary developer leaves them // unable to bring anyone in to collaborate. Nothing else on this page opens up — the backend // only grants them developer memberships on the fork they created. + // The instance channels are not a valid destination on cloud or on a fork. Never select a tab + // the group does not render: saving would submit a value the API rejects, locking the whole + // error handler behind a 400. + const canUseInstanceAlerts = $derived( + !isCloudHosted() && !currentWorkspace?.parent_workspace_id + ) const isForkOwner = $derived( Boolean(currentWorkspace?.parent_workspace_id) && currentWorkspace?.created_by === $userStore?.email @@ -609,6 +615,12 @@ initialPublicAppRateLimitPerMinute = settings.public_app_execution_limit_per_minute ?? undefined if (emptyString($enterpriseLicense)) { errorHandlerSelected = 'custom' + } else if ( + canUseInstanceAlerts && + !errorHandlerPath && + settings.error_handler_fallback_to_instance_alerts + ) { + errorHandlerSelected = 'instance_alerts' } else { errorHandlerSelected = getHandlerType(errorHandlerScriptPath) } @@ -805,7 +817,8 @@ path: `${errorHandlerItemKind}/${errorHandlerScriptPath}`, extra_args: errorHandlerExtraArgs, muted_on_cancel: errorHandlerMutedOnCancel, - muted_on_user_path: errorHandlerMutedOnUserPath + muted_on_user_path: errorHandlerMutedOnUserPath, + fallback_to_instance_alerts: false } }) sendUserToast(`workspace error handler set to ${errorHandlerScriptPath}`) @@ -816,10 +829,17 @@ path: undefined, extra_args: undefined, muted_on_cancel: undefined, - muted_on_user_path: undefined + muted_on_user_path: undefined, + fallback_to_instance_alerts: errorHandlerSelected === 'instance_alerts' } }) - sendUserToast(`workspace error handler removed`) + sendUserToast( + errorHandlerSelected === 'instance_alerts' + ? `failed jobs will be reported to the instance critical alert channels` + : initialErrorHandlerScriptPath + ? `workspace error handler removed` + : `error handler settings saved` + ) } // Update initial values for dirty detection @@ -1809,6 +1829,7 @@ customScriptTemplate="/scripts/add?hub=hub%2F9083%2Fwindmill%2Fworkspace_error_handler_template" bind:customHandlerKind={errorHandlerItemKind} bind:handlerExtraArgs={errorHandlerExtraArgs} + showInstanceAlerts={canUseInstanceAlerts} > {#snippet customTabTooltip()} @@ -1838,24 +1859,26 @@ {/snippet} - - - - + {#if errorHandlerSelected !== 'instance_alerts'} + + + + + {/if}
-
+
diff --git a/integration_tests/test/git_sync_test.py b/integration_tests/test/git_sync_test.py index c0c9fb47e9..3cc54f2796 100644 --- a/integration_tests/test/git_sync_test.py +++ b/integration_tests/test/git_sync_test.py @@ -943,6 +943,117 @@ class TestGitSyncAutoPull(GitSyncTestBase): "Parent workspace received a commit from a fork branch", ) + def test_fork_of_dev_workspace_branch_deploys_into_fork(self): + """A throwaway fork OF a dev workspace pushes to wm-fork// + (the tracked branch, NOT the dev's label), and the root's sync_forks + poller enumerates wm-fork//* and routes a commit on that branch + into the nested fork — through the root, since only the root holds + auto-pull config. Regression for the assumption that such a fork lives on + wm-fork// and is therefore never collected/reconciled.""" + repo_name, _ = self._create_test_repo() + resource_path = self._setup_git_sync_resource(repo_name) + self._configure_single_repo_sync(resource_path, include_type=["script"]) + + script_path = self._deploy_seed_script("forkofdev") + script_file = self._repo_script_file(repo_name, script_path) + # Seed before the fork branch is created so the branch inherits it. + self._seed_wmill_yaml(repo_name) + + self._configure_auto_pull(resource_path, sync_forks=True) + + # Attach an existing workspace as a dev workspace of the root (label + # "dev"). It carries the same inherited sync repo so a fork beneath it + # resolves against it. + dev_id = f"it-dev-{uuid.uuid4().hex[:8]}" + self._fork_workspaces_to_cleanup.append(dev_id) + dev_client = WindmillClient(workspace=dev_id) + dev_client.create_resource( + path=resource_path, + resource_type="git_repository", + value={ + "url": self._gitea.get_docker_clone_url(repo_name), + "branch": "main", + "is_github_app": False, + }, + update_if_exists=True, + ) + dev_client.configure_git_sync({ + "repositories": [{ + "git_repo_resource_path": f"$res:{resource_path}", + "use_individual_branch": False, + "group_by_folder": False, + "settings": {"include_type": ["script"], "include_path": ["**"]}, + }], + }) + root = self._client._workspace + # Only one dev workspace per root, so clear any left attached by a prior + # test before attaching ours, and detach ours afterward so we don't leak. + existing = self._client._client.get( + f"/api/w/{root}/workspaces/get_dev_workspace" + ) + if existing.status_code == 200 and existing.json(): + self._client._client.post( + f"/api/w/{root}/workspaces/detach_dev_workspace", + json={"dev_workspace_id": existing.json()["id"]}, + ) + self.addCleanup( + lambda: self._client._client.post( + f"/api/w/{root}/workspaces/detach_dev_workspace", + json={"dev_workspace_id": dev_id}, + ) + ) + attach = self._client._client.post( + f"/api/w/{root}/workspaces/attach_dev_workspace", + json={"dev_workspace_id": dev_id, "dev_workspace_label": "dev"}, + ) + self.assertEqual( + attach.status_code // 100, + 2, + f"attach_dev_workspace failed: {attach.content.decode()}", + ) + + # Fork the dev workspace (branch first, then workspace) — its parent is + # the dev, so this is a fork OF a dev workspace. + fork_id = f"wm-fork-{uuid.uuid4().hex[:8]}" + self._fork_workspaces_to_cleanup.append(fork_id) + job_ids = dev_client.create_workspace_fork_branch(fork_id, f"Fork {fork_id}") + if job_ids: + dev_client.wait_for_jobs_by_ids(job_ids, timeout=90) + time.sleep(3) + dev_client.create_workspace_fork(fork_id, f"Fork {fork_id}") + + # The fork branch is named after the tracked branch, not the dev label. + fork_suffix = fork_id[len("wm-fork-"):] + fork_branch = f"wm-fork/main/{fork_suffix}" + branches = self._get_branches(self._clone_repo_all_branches(repo_name)) + self.assertTrue( + any(fork_branch in b for b in branches), + f"expected {fork_branch} in the repo after forking the dev, got: {branches}", + ) + self.assertFalse( + any(f"wm-fork/dev/{fork_suffix}" in b for b in branches), + f"fork-of-dev must not live on a wm-fork//* branch: {branches}", + ) + + self._gitea.create_file( + repo_name, script_file, ts_script("return 'fork only'"), + branch=fork_branch, + ) + + fork_client = WindmillClient(workspace=fork_id) + self._wait_until( + lambda: "fork only" in fork_client.get_script_content(script_path), + timeout=self.PULL_TIMEOUT, + message=f"fork-of-dev workspace {fork_id} did not receive the fork-branch commit", + ) + + # The root (the fork's grandparent, holder of the poller) must not see it. + self.assertNotIn( + "fork only", + self._client.get_script_content(script_path), + "Root workspace received a commit from a fork-of-dev branch", + ) + def test_settings_normalization_and_redaction(self): """Webhook mode on a token repo is persisted as polling; server-owned webhook fields are never exposed; legacy repos gain no auto_pull key.""" diff --git a/lsp/Pipfile b/lsp/Pipfile index 7ed5878119..2c9459087d 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.764.0" +wmill = ">=1.769.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index b28b7184a1..43cc0a5866 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.764.0 + version: 1.769.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 4f21d2832d..cdc502b04f 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.764.0' + ModuleVersion = '1.769.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index acede5f804..f44b602c3c 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.764.0" +version = "1.769.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index fb2e40e6ec..7ab815b644 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -755,16 +755,54 @@ export const PIPELINE_BASE = `# Data pipeline authoring A **data pipeline** is NOT a flow. A flow is one runnable that orchestrates steps internally. A data pipeline is a set of **independent scripts**, each deployed on its own, that form a DAG by reading and writing shared **storage assets** (DuckLake tables, data tables, S3 objects, volumes, resources) and by declaring execution **triggers**. The pipeline is visualized and edited at \`/pipeline/\`; every node is a normal workspace script that happens to carry pipeline annotations. When the user asks for a "data pipeline" (or to "ingest / transform / materialize" data across steps), build pipeline-annotated scripts — do NOT build a flow. +## Default to DuckDB + DuckLake + +A pipeline node that produces a table should almost always be a **\`duckdb\`** node that materializes its output into a **DuckLake** table with \`-- materialize ducklake:///
\` (in a DuckDB node the annotation uses SQL \`--\` comment syntax; write the body as a bare \`SELECT\` and let the runtime do the write). DuckLake is the default lakehouse store for pipelines and is the shape the pipeline editor is built around, so prefer it unless the work specifically calls for something else: + +- \`postgresql\` / data tables — only for row-level, OLTP-style mutations against an existing Postgres data table (frequent single-row upserts/updates, transactional reads that an app queries live). +- \`bun\` / \`python3\` — only for non-tabular work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node still produces tabular data for downstream steps, land it in DuckLake (write it with the wmill SDK / ducklake helpers) rather than inventing a parallel store. + +Do not spread a pipeline across postgres, S3, and DuckLake when one DuckLake lake would do; a consistent DuckLake lakehouse is the goal. + +## Storage prerequisites + +A DuckLake pipeline only runs once the workspace has **object storage** (S3 / Azure Blob / GCS) **and a DuckLake catalog** configured — DuckLake tables and \`s3://\` assets can't be materialized or read without it. Check with the \`list_ducklakes\` tool before you build (it returns the configured DuckLake catalogs, or none). Drafting the annotated scripts does not require storage, but the pipeline can't ingest, materialize, or read its assets until it exists. So if \`list_ducklakes\` returns none (or the user hits "storage not configured" errors), say so and give the right next step **by role**: + +- a workspace **admin** sets it up in Workspace settings → Object Storage (add an S3/Azure/GCS storage), then adds a DuckLake catalog on top of it; +- anyone **without admin rights** should ask a workspace admin to configure object storage + a DuckLake catalog. + +Never hand back a DuckLake pipeline that cannot run without flagging the missing storage and pointing to who sets it up. + ## What makes a script a pipeline node A script joins the pipeline when its source begins with the \`pipeline\` annotation as a top-of-file comment, **written in the script's own comment syntax** — \`//\` for TS/JS (bun), \`--\` for SQL (DuckDB/Postgres), \`#\` for Python/Bash. So it's \`-- pipeline\` in a DuckDB node, \`# pipeline\` in a Python node, \`// pipeline\` in a bun node. Every annotation below uses that same prefix (the \`//\` shown is the TS form). All other wiring is expressed as annotation comments near the top of the file: - \`// on \` — declares an execution-DAG **input** (what triggers/feeds this node). \`\` is either: - - an **asset URI** (the node runs when that asset is produced upstream): \`ducklake://main/orders\`, \`datatable://main/users\`, \`s3://\`, \`$res:f/folder/my_resource\`, \`volume://name/path\`. - - a **native trigger kind**: \`schedule\`, \`webhook\`, \`email\`, \`kafka\`, \`mqtt\`, \`amqp\`, \`nats\`, \`postgres\`, \`sqs\`, \`gcp\`, or \`data_upload\` (a user-uploaded S3 file). For these the actual trigger row (cron, topic, …) is created separately; the annotation only declares the binding. + - an **asset URI** (the node runs when that asset is produced upstream): \`ducklake://main/orders\`, \`datatable://main/users\`, \`$res:f/folder/my_resource\`, \`volume://name/path\`, or an S3 object (see the S3 storage-form rule below). + - a **native trigger kind**: \`schedule\`, \`webhook\`, \`email\`, \`kafka\`, \`mqtt\`, \`amqp\`, \`nats\`, \`postgres\`, \`sqs\`, \`gcp\`, or \`data_upload\` (a user-uploaded S3 file). For these the actual trigger row (cron, topic, …) is created separately; the annotation only declares the binding. **\`data_upload\` is special**: there is no trigger row — the node instead declares an **\`S3Object\` input parameter** fed by the auto-generated upload picker; it never hard-codes a key. Any language can be the \`data_upload\` node: + - Python (has \`import wmill\`): \`def main(file: wmill.S3Object):\` then \`wmill.load_s3_file(file)\`; TS (has \`import * as wmill from "windmill-client"\`): \`export async function main(file: wmill.S3Object)\`. Qualify the type as \`wmill.S3Object\` (or add \`from wmill import S3Object\` / \`import { S3Object } from "windmill-client"\`) — a bare \`S3Object\` is undefined. + - **DuckDB** takes the s3object arg via a \`-- $ (s3object)\` declaration and reads it directly, so a single DuckDB node can ingest **and** materialize: + \`\`\` + -- pipeline + -- on data_upload + -- materialize ducklake://main/raw_uploads + -- $file (s3object) + SELECT * FROM read_csv($file) + \`\`\` - **Outputs** are inferred from what the body writes — a \`CREATE TABLE\`, a \`wmill.writeS3File(...)\`, a DuckLake/datatable write. To declare a managed output explicitly, use \`// materialize \`. - Optional badges: \`// partitioned \`, \`// freshness \` (e.g. \`1h\`), \`// tag \`, \`// retry [delay]\`, \`// data_test ...\`. +## S3 object wiring (storage form matters) + +An \`s3://\` URI's first slashes select the **storage**, not part of the key — get this wrong and the producer/consumer edge silently won't connect: + +- \`s3:///\` (**triple** slash, empty first segment) = the **default** workspace storage. A downstream node reading or triggering on that object uses \`s3:///\` — e.g. DuckDB \`-- on s3:///orders/2024.parquet\` and \`read_parquet('s3:///orders/2024.parquet')\`. +- \`s3:///\` (**double** slash, non-empty first segment) = a **named secondary** storage called \`\` — so \`s3://ingest/x\` means storage \`ingest\`, key \`x\`, NOT key \`ingest/x\`. Only use this when the object genuinely lives in a configured secondary storage; never invent a bucket/storage name for a default-storage object (it breaks the edge). + +To make the producer side visible to lineage, a Python/TS node MUST pass the **\`S3Object\` form**, not a bare key string: Python \`wmill.write_s3_file(wmill.S3Object(s3=""), data)\` (or the import-free dict \`{"s3": ""}\`), TS \`wmill.writeS3File({ s3: "" }, data)\`. That records the default-storage asset \`/\`, which a downstream \`s3:///\` reader connects to (same key both sides). A bare \`write_s3_file("", ...)\` records **no** asset and produces **no** edge. Add \`storage=""\` only for a named secondary storage. + +The key must be a **string literal** — the graph parser is static and cannot follow a variable, f-string, or computed path, so \`write_s3_file(wmill.S3Object(s3=key_var), ...)\` records no edge. Inline the literal (\`s3="events/user_events.parquet"\`) on both the writing and reading node. The same rule applies to every asset URI in an annotation or SDK call (\`ducklake://\`, \`datatable://\`, \`s3://\`): write them literally, not via a variable. + ## Materialize (the managed output) > **\`// materialize\` is DuckDB-only**, and its target must be a DuckLake table (\`ducklake:///
\`). Deploy **rejects** \`// materialize\` on any other language (\`python3\`, \`bun\`, \`postgresql\`) or a non-DuckLake target. For a non-DuckDB node, do **not** use \`// materialize\` — write the output via the SDK (\`wmill.writeS3File(...)\`, a postgresql \`CREATE TABLE\`, ducklake helpers, …) and let it be inferred. Use \`duckdb\` when a node should materialize a DuckLake table. @@ -784,7 +822,7 @@ A script joins the pipeline when its source begins with the \`pipeline\` annotat ## How to build one in chat 1. Put every node in the **same folder**: \`f//\`. The folder is the pipeline. -2. Author each node as a **script draft** with \`write_script\` (or \`edit_script\`), language chosen for the work: \`duckdb\` or \`postgresql\` for SQL-shaped data work, \`bun\`/\`python3\` for general transforms. SQL-heavy lakehouse steps usually use \`duckdb\`. +2. Author each node as a **script draft** with \`write_script\` (or \`edit_script\`). Default to \`duckdb\` materializing into DuckLake (see "Default to DuckDB + DuckLake" above); pick \`postgresql\`, \`bun\`, or \`python3\` only when that section says the work calls for it. 3. Start each body with \`// pipeline\`, then the \`// on\` input declarations, then the transform that writes the output. 4. **Chain nodes by asset URI**: read an upstream node's output asset, then \`// on \` in the downstream node so the edge forms. Reuse exact asset paths from existing nodes rather than inventing parallel ones. 5. Leave nodes as drafts unless the user asks to deploy. A pipeline only "runs" once its scripts are deployed and their triggers exist. @@ -799,7 +837,7 @@ Node \`f/sales/orders_ingest\` (runs on a schedule, materializes a DuckLake tabl -- pipeline -- on schedule -- materialize ducklake://main/orders -SELECT * FROM read_csv('s3://raw/orders/*.csv') +SELECT * FROM read_csv('s3:///raw/orders/*.csv') \`\`\` Node \`f/sales/orders_daily\` (runs when \`orders\` is produced, writes a rollup): @@ -3729,7 +3767,7 @@ being buffered, bypassing the 10000-row return cap. export const LANG_BUN = `# TypeScript (Bun) -Bun runtime with full npm ecosystem and fastest execution. +Bun runtime with full npm ecosystem and fastest execution. **Bun is the default and preferred TypeScript runtime** — choose it for any TypeScript script unless there is a major reason to use Deno for that specific use-case. ## Structure @@ -4016,6 +4054,8 @@ export const LANG_DENO = `# TypeScript (Deno) Deno runtime with npm support via \`npm:\` prefix and native Deno libraries. +**Prefer Bun (\`write-script-bun\`) for TypeScript.** Only use Deno when the script specifically requires the Deno runtime — Deno's standard library or \`deno.land\` URL imports that have no npm equivalent. For all other TypeScript, use Bun instead. + ## Structure Export a single **async** function called \`main\`: diff --git a/system_prompts/auto-generated/script.md b/system_prompts/auto-generated/script.md index 9d393d2c7f..f5a6fd99ba 100644 --- a/system_prompts/auto-generated/script.md +++ b/system_prompts/auto-generated/script.md @@ -215,7 +215,7 @@ being buffered, bypassing the 10000-row return cap. # TypeScript (Bun) -Bun runtime with full npm ecosystem and fastest execution. +Bun runtime with full npm ecosystem and fastest execution. **Bun is the default and preferred TypeScript runtime** — choose it for any TypeScript script unless there is a major reason to use Deno for that specific use-case. ## Structure @@ -502,6 +502,8 @@ public class Script Deno runtime with npm support via `npm:` prefix and native Deno libraries. +**Prefer Bun (`write-script-bun`) for TypeScript.** Only use Deno when the script specifically requires the Deno runtime — Deno's standard library or `deno.land` URL imports that have no npm equivalent. For all other TypeScript, use Bun instead. + ## Structure Export a single **async** function called `main`: diff --git a/system_prompts/auto-generated/skills/write-script-bun/SKILL.md b/system_prompts/auto-generated/skills/write-script-bun/SKILL.md index 44cccc7ba8..21aafa55d0 100644 --- a/system_prompts/auto-generated/skills/write-script-bun/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-bun/SKILL.md @@ -1,6 +1,6 @@ --- name: write-script-bun -description: MUST use when writing Bun/TypeScript scripts. +description: MUST use when writing TypeScript scripts. Bun is the default and preferred TypeScript runtime — pick it for TypeScript unless the script specifically needs Deno. --- ## CLI Commands @@ -50,7 +50,7 @@ Use `wmill resource-type list --schema` to discover available resource types. # TypeScript (Bun) -Bun runtime with full npm ecosystem and fastest execution. +Bun runtime with full npm ecosystem and fastest execution. **Bun is the default and preferred TypeScript runtime** — choose it for any TypeScript script unless there is a major reason to use Deno for that specific use-case. ## Structure diff --git a/system_prompts/auto-generated/skills/write-script-deno/SKILL.md b/system_prompts/auto-generated/skills/write-script-deno/SKILL.md index ea2bfd4421..55c7bf7155 100644 --- a/system_prompts/auto-generated/skills/write-script-deno/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-deno/SKILL.md @@ -1,6 +1,6 @@ --- name: write-script-deno -description: MUST use when writing Deno/TypeScript scripts. +description: Use ONLY when a TypeScript script specifically requires the Deno runtime (Deno stdlib or deno.land URL imports). For all other TypeScript, use write-script-bun instead. --- ## CLI Commands @@ -52,6 +52,8 @@ Use `wmill resource-type list --schema` to discover available resource types. Deno runtime with npm support via `npm:` prefix and native Deno libraries. +**Prefer Bun (`write-script-bun`) for TypeScript.** Only use Deno when the script specifically requires the Deno runtime — Deno's standard library or `deno.land` URL imports that have no npm equivalent. For all other TypeScript, use Bun instead. + ## Structure Export a single **async** function called `main`: diff --git a/system_prompts/base/pipeline-base.md b/system_prompts/base/pipeline-base.md index 55c1c53169..b2b428c401 100644 --- a/system_prompts/base/pipeline-base.md +++ b/system_prompts/base/pipeline-base.md @@ -2,16 +2,54 @@ A **data pipeline** is NOT a flow. A flow is one runnable that orchestrates steps internally. A data pipeline is a set of **independent scripts**, each deployed on its own, that form a DAG by reading and writing shared **storage assets** (DuckLake tables, data tables, S3 objects, volumes, resources) and by declaring execution **triggers**. The pipeline is visualized and edited at `/pipeline/`; every node is a normal workspace script that happens to carry pipeline annotations. When the user asks for a "data pipeline" (or to "ingest / transform / materialize" data across steps), build pipeline-annotated scripts — do NOT build a flow. +## Default to DuckDB + DuckLake + +A pipeline node that produces a table should almost always be a **`duckdb`** node that materializes its output into a **DuckLake** table with `-- materialize ducklake:///
` (in a DuckDB node the annotation uses SQL `--` comment syntax; write the body as a bare `SELECT` and let the runtime do the write). DuckLake is the default lakehouse store for pipelines and is the shape the pipeline editor is built around, so prefer it unless the work specifically calls for something else: + +- `postgresql` / data tables — only for row-level, OLTP-style mutations against an existing Postgres data table (frequent single-row upserts/updates, transactional reads that an app queries live). +- `bun` / `python3` — only for non-tabular work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node still produces tabular data for downstream steps, land it in DuckLake (write it with the wmill SDK / ducklake helpers) rather than inventing a parallel store. + +Do not spread a pipeline across postgres, S3, and DuckLake when one DuckLake lake would do; a consistent DuckLake lakehouse is the goal. + +## Storage prerequisites + +A DuckLake pipeline only runs once the workspace has **object storage** (S3 / Azure Blob / GCS) **and a DuckLake catalog** configured — DuckLake tables and `s3://` assets can't be materialized or read without it. Check with the `list_ducklakes` tool before you build (it returns the configured DuckLake catalogs, or none). Drafting the annotated scripts does not require storage, but the pipeline can't ingest, materialize, or read its assets until it exists. So if `list_ducklakes` returns none (or the user hits "storage not configured" errors), say so and give the right next step **by role**: + +- a workspace **admin** sets it up in Workspace settings → Object Storage (add an S3/Azure/GCS storage), then adds a DuckLake catalog on top of it; +- anyone **without admin rights** should ask a workspace admin to configure object storage + a DuckLake catalog. + +Never hand back a DuckLake pipeline that cannot run without flagging the missing storage and pointing to who sets it up. + ## What makes a script a pipeline node A script joins the pipeline when its source begins with the `pipeline` annotation as a top-of-file comment, **written in the script's own comment syntax** — `//` for TS/JS (bun), `--` for SQL (DuckDB/Postgres), `#` for Python/Bash. So it's `-- pipeline` in a DuckDB node, `# pipeline` in a Python node, `// pipeline` in a bun node. Every annotation below uses that same prefix (the `//` shown is the TS form). All other wiring is expressed as annotation comments near the top of the file: - `// on ` — declares an execution-DAG **input** (what triggers/feeds this node). `` is either: - - an **asset URI** (the node runs when that asset is produced upstream): `ducklake://main/orders`, `datatable://main/users`, `s3://`, `$res:f/folder/my_resource`, `volume://name/path`. - - a **native trigger kind**: `schedule`, `webhook`, `email`, `kafka`, `mqtt`, `amqp`, `nats`, `postgres`, `sqs`, `gcp`, or `data_upload` (a user-uploaded S3 file). For these the actual trigger row (cron, topic, …) is created separately; the annotation only declares the binding. + - an **asset URI** (the node runs when that asset is produced upstream): `ducklake://main/orders`, `datatable://main/users`, `$res:f/folder/my_resource`, `volume://name/path`, or an S3 object (see the S3 storage-form rule below). + - a **native trigger kind**: `schedule`, `webhook`, `email`, `kafka`, `mqtt`, `amqp`, `nats`, `postgres`, `sqs`, `gcp`, or `data_upload` (a user-uploaded S3 file). For these the actual trigger row (cron, topic, …) is created separately; the annotation only declares the binding. **`data_upload` is special**: there is no trigger row — the node instead declares an **`S3Object` input parameter** fed by the auto-generated upload picker; it never hard-codes a key. Any language can be the `data_upload` node: + - Python (has `import wmill`): `def main(file: wmill.S3Object):` then `wmill.load_s3_file(file)`; TS (has `import * as wmill from "windmill-client"`): `export async function main(file: wmill.S3Object)`. Qualify the type as `wmill.S3Object` (or add `from wmill import S3Object` / `import { S3Object } from "windmill-client"`) — a bare `S3Object` is undefined. + - **DuckDB** takes the s3object arg via a `-- $ (s3object)` declaration and reads it directly, so a single DuckDB node can ingest **and** materialize: + ``` + -- pipeline + -- on data_upload + -- materialize ducklake://main/raw_uploads + -- $file (s3object) + SELECT * FROM read_csv($file) + ``` - **Outputs** are inferred from what the body writes — a `CREATE TABLE`, a `wmill.writeS3File(...)`, a DuckLake/datatable write. To declare a managed output explicitly, use `// materialize `. - Optional badges: `// partitioned `, `// freshness ` (e.g. `1h`), `// tag `, `// retry [delay]`, `// data_test ...`. +## S3 object wiring (storage form matters) + +An `s3://` URI's first slashes select the **storage**, not part of the key — get this wrong and the producer/consumer edge silently won't connect: + +- `s3:///` (**triple** slash, empty first segment) = the **default** workspace storage. A downstream node reading or triggering on that object uses `s3:///` — e.g. DuckDB `-- on s3:///orders/2024.parquet` and `read_parquet('s3:///orders/2024.parquet')`. +- `s3:///` (**double** slash, non-empty first segment) = a **named secondary** storage called `` — so `s3://ingest/x` means storage `ingest`, key `x`, NOT key `ingest/x`. Only use this when the object genuinely lives in a configured secondary storage; never invent a bucket/storage name for a default-storage object (it breaks the edge). + +To make the producer side visible to lineage, a Python/TS node MUST pass the **`S3Object` form**, not a bare key string: Python `wmill.write_s3_file(wmill.S3Object(s3=""), data)` (or the import-free dict `{"s3": ""}`), TS `wmill.writeS3File({ s3: "" }, data)`. That records the default-storage asset `/`, which a downstream `s3:///` reader connects to (same key both sides). A bare `write_s3_file("", ...)` records **no** asset and produces **no** edge. Add `storage=""` only for a named secondary storage. + +The key must be a **string literal** — the graph parser is static and cannot follow a variable, f-string, or computed path, so `write_s3_file(wmill.S3Object(s3=key_var), ...)` records no edge. Inline the literal (`s3="events/user_events.parquet"`) on both the writing and reading node. The same rule applies to every asset URI in an annotation or SDK call (`ducklake://`, `datatable://`, `s3://`): write them literally, not via a variable. + ## Materialize (the managed output) > **`// materialize` is DuckDB-only**, and its target must be a DuckLake table (`ducklake:///
`). Deploy **rejects** `// materialize` on any other language (`python3`, `bun`, `postgresql`) or a non-DuckLake target. For a non-DuckDB node, do **not** use `// materialize` — write the output via the SDK (`wmill.writeS3File(...)`, a postgresql `CREATE TABLE`, ducklake helpers, …) and let it be inferred. Use `duckdb` when a node should materialize a DuckLake table. @@ -31,7 +69,7 @@ A script joins the pipeline when its source begins with the `pipeline` annotatio ## How to build one in chat 1. Put every node in the **same folder**: `f//`. The folder is the pipeline. -2. Author each node as a **script draft** with `write_script` (or `edit_script`), language chosen for the work: `duckdb` or `postgresql` for SQL-shaped data work, `bun`/`python3` for general transforms. SQL-heavy lakehouse steps usually use `duckdb`. +2. Author each node as a **script draft** with `write_script` (or `edit_script`). Default to `duckdb` materializing into DuckLake (see "Default to DuckDB + DuckLake" above); pick `postgresql`, `bun`, or `python3` only when that section says the work calls for it. 3. Start each body with `// pipeline`, then the `// on` input declarations, then the transform that writes the output. 4. **Chain nodes by asset URI**: read an upstream node's output asset, then `// on ` in the downstream node so the edge forms. Reuse exact asset paths from existing nodes rather than inventing parallel ones. 5. Leave nodes as drafts unless the user asks to deploy. A pipeline only "runs" once its scripts are deployed and their triggers exist. @@ -46,7 +84,7 @@ Node `f/sales/orders_ingest` (runs on a schedule, materializes a DuckLake table) -- pipeline -- on schedule -- materialize ducklake://main/orders -SELECT * FROM read_csv('s3://raw/orders/*.csv') +SELECT * FROM read_csv('s3:///raw/orders/*.csv') ``` Node `f/sales/orders_daily` (runs when `orders` is produced, writes a rollup): diff --git a/system_prompts/generate.py b/system_prompts/generate.py index d315e3a97b..3ee993147e 100644 --- a/system_prompts/generate.py +++ b/system_prompts/generate.py @@ -832,6 +832,7 @@ WORKSPACE_TOOL_ZOD_SCHEMAS = [ ('NewSqsTrigger', 'sqsTriggerRequestSchema'), ('GcpTriggerData', 'gcpTriggerRequestSchema'), ('AzureTriggerData', 'azureTriggerRequestSchema'), + ('NewEmailTrigger', 'emailTriggerRequestSchema'), ('CreateVariable', 'variableRequestSchema'), ('CreateResource', 'resourceRequestSchema'), ] @@ -847,6 +848,7 @@ WORKSPACE_TOOL_TRIGGER_SCHEMAS = [ ('sqs', 'sqsTriggerRequestSchema'), ('gcp', 'gcpTriggerRequestSchema'), ('azure', 'azureTriggerRequestSchema'), + ('email', 'emailTriggerRequestSchema'), ] WORKSPACE_TOOL_ZOD_OUTPUT_PATH = ( diff --git a/system_prompts/languages/bun.md b/system_prompts/languages/bun.md index c505b6cf97..22aeb2fb74 100644 --- a/system_prompts/languages/bun.md +++ b/system_prompts/languages/bun.md @@ -1,6 +1,6 @@ # TypeScript (Bun) -Bun runtime with full npm ecosystem and fastest execution. +Bun runtime with full npm ecosystem and fastest execution. **Bun is the default and preferred TypeScript runtime** — choose it for any TypeScript script unless there is a major reason to use Deno for that specific use-case. ## Structure diff --git a/system_prompts/languages/deno.md b/system_prompts/languages/deno.md index c84e5e772a..fddd08efb2 100644 --- a/system_prompts/languages/deno.md +++ b/system_prompts/languages/deno.md @@ -2,6 +2,8 @@ Deno runtime with npm support via `npm:` prefix and native Deno libraries. +**Prefer Bun (`write-script-bun`) for TypeScript.** Only use Deno when the script specifically requires the Deno runtime — Deno's standard library or `deno.land` URL imports that have no npm equivalent. For all other TypeScript, use Bun instead. + ## Structure Export a single **async** function called `main`: diff --git a/system_prompts/utils.py b/system_prompts/utils.py index 9666b6cf0c..e2fa77d3b6 100644 --- a/system_prompts/utils.py +++ b/system_prompts/utils.py @@ -87,13 +87,13 @@ CLI_EXCLUDED_FIELDS = [ LANGUAGE_METADATA = { 'bun': { 'name': 'TypeScript (Bun)', - 'description': 'MUST use when writing Bun/TypeScript scripts.', - 'use_cases': 'TypeScript automation, npm packages, data processing, API integrations' + 'description': 'MUST use when writing TypeScript scripts. Bun is the default and preferred TypeScript runtime — pick it for TypeScript unless the script specifically needs Deno.', + 'use_cases': 'TypeScript automation, npm packages, data processing, API integrations — the default choice for TypeScript' }, 'deno': { 'name': 'TypeScript (Deno)', - 'description': 'MUST use when writing Deno/TypeScript scripts.', - 'use_cases': 'TypeScript with Deno stdlib, secure sandboxed execution' + 'description': 'Use ONLY when a TypeScript script specifically requires the Deno runtime (Deno stdlib or deno.land URL imports). For all other TypeScript, use write-script-bun instead.', + 'use_cases': 'TypeScript that specifically needs the Deno runtime (Deno stdlib or deno.land imports); prefer Bun otherwise' }, # 'nativets' is intentionally omitted: it is a legacy duplicate of # 'bunnative' (a Bun script with a leading //native marker). No diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index cceec4de9b..b600205868 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.764.0", + "version": "1.769.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index bafac5a181..40b609e7f5 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.764.0", + "version": "1.769.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/typescript-client#readme", diff --git a/version.txt b/version.txt index 35944d0771..475443dad3 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.764.0 +1.769.0