diff --git a/.github/workflows/backend-test-windows.yml b/.github/workflows/backend-test-windows.yml index e2e7318ab6..f9153c0ce0 100644 --- a/.github/workflows/backend-test-windows.yml +++ b/.github/workflows/backend-test-windows.yml @@ -101,7 +101,6 @@ jobs: - name: Install OpenSSL via vcpkg run: | - vcpkg.exe install openssl-windows:x64-windows vcpkg.exe install openssl:x64-windows-static vcpkg.exe integrate install diff --git a/.github/workflows/build_windows_worker_.yml b/.github/workflows/build_windows_worker_.yml index 84f4366235..46befe42b6 100644 --- a/.github/workflows/build_windows_worker_.yml +++ b/.github/workflows/build_windows_worker_.yml @@ -62,7 +62,6 @@ jobs: - name: Cargo build binary windows timeout-minutes: 180 run: | - vcpkg.exe install openssl-windows:x64-windows vcpkg.exe install openssl:x64-windows-static vcpkg.exe integrate install $env:VCPKGRS_DYNAMIC=1 diff --git a/.github/workflows/git-sync-test.yml b/.github/workflows/git-sync-test.yml index ee732569dd..bd15ec8876 100644 --- a/.github/workflows/git-sync-test.yml +++ b/.github/workflows/git-sync-test.yml @@ -9,6 +9,7 @@ on: - "backend/windmill-api-integration-tests/tests/git_sync*" - "backend/ee-repo-ref.txt" - "backend/windmill-common/src/workspaces.rs" + - "frontend/src/lib/hubPaths.json" - "backend/windmill-worker/src/result_processor.rs" - "backend/windmill-api-workspaces/**" - "cli/src/commands/sync/**" @@ -22,6 +23,7 @@ on: - "backend/windmill-api-integration-tests/tests/git_sync*" - "backend/ee-repo-ref.txt" - "backend/windmill-common/src/workspaces.rs" + - "frontend/src/lib/hubPaths.json" - "backend/windmill-worker/src/result_processor.rs" - "backend/windmill-api-workspaces/**" - "cli/src/commands/sync/**" @@ -59,7 +61,7 @@ jobs: echo "$CHANGED_FILES" # Direct git sync file changes — always relevant. - if echo "$CHANGED_FILES" | grep -qE '^(backend/windmill-git-sync/|backend/windmill-worker/src/result_processor\.rs|backend/windmill-api-workspaces/|backend/windmill-api-integration-tests/tests/git_sync|backend/windmill-common/src/workspaces\.rs|cli/src/commands/sync/|cli/src/utils/git\.ts|integration_tests/test/git_sync|\.github/workflows/git-sync-test\.yml)'; then + if echo "$CHANGED_FILES" | grep -qE '^(backend/windmill-git-sync/|backend/windmill-worker/src/result_processor\.rs|backend/windmill-api-workspaces/|backend/windmill-api-integration-tests/tests/git_sync|backend/windmill-common/src/workspaces\.rs|frontend/src/lib/hubPaths\.json|cli/src/commands/sync/|cli/src/utils/git\.ts|integration_tests/test/git_sync|\.github/workflows/git-sync-test\.yml)'; then echo "should_run=true" >> "$GITHUB_OUTPUT" echo "Relevant: direct git sync file changes" exit 0 diff --git a/.github/workflows/publish_windows_worker.yml b/.github/workflows/publish_windows_worker.yml index d159619fba..b94d5d57c7 100644 --- a/.github/workflows/publish_windows_worker.yml +++ b/.github/workflows/publish_windows_worker.yml @@ -51,7 +51,6 @@ jobs: - name: Cargo build windows timeout-minutes: 180 run: | - vcpkg.exe install openssl-windows:x64-windows vcpkg.exe install openssl:x64-windows-static vcpkg.exe integrate install $env:VCPKGRS_DYNAMIC=1 diff --git a/.release-please-manifest.json b/.release-please-manifest.json index ae93300697..249f8350ae 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.796.0" + ".": "1.803.0" } diff --git a/AGENTS.md b/AGENTS.md index 47919cfeda..71c59b479d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -165,6 +165,15 @@ $NAV --root backend callees "X" # what does X call? - Search for existing code to reuse before writing new code - Follow established patterns in the codebase - Keep changes focused — don't refactor beyond what's asked +- **A simpler design found late is still the design.** Work already spent is not an argument + for a shape, and neither is a clean review round, a passing suite, or a long PR thread. The + signal to stop and re-derive rather than patch again is a change that keeps growing to defend + its own structure: each review finding fixing an assumption the previous fix broke, the same + class of bug reappearing somewhere new, or most of the diff being consequences of one early + choice rather than the thing you set out to do. When that happens, say plainly what the + simpler design is and what switching costs — a migration, a review cycle restarted from zero, + work discarded — and let the user decide. Do not keep paying down the harder one because it + is nearly finished, and do not present the accumulated cost as a reason to continue. - **Ship only the tests the PR needs.** A committed test must pin behavior a future change could plausibly break, and be the smallest setup that exercises the new logic. While developing, write as many exhaustive tests and do as much manual testing as you need to convince yourself the change works — then remove that scaffolding before marking the PR ready, keeping only the essential regression guard(s). A test that merely re-exercises pre-existing behavior, or needs elaborate fixtures to assert something trivial, is scaffolding: delete it. If nothing meaningful is left to guard, ship no test rather than a ceremonial one. - **Comments record constraints, not narration.** Write a comment only for what the code can't show: why a non-obvious approach is required, what breaks if it's "simplified" away. State each invariant once, at the place where someone would break it, in ≤4 lines. Don't describe what the next line does, don't repeat the same rationale at multiple sites, and don't address the PR reviewer (justifying a change belongs in the PR description, not the code). Reference nothing ephemeral — no numbered steps from your dev flow, no "the poller / the test does X" scaffolding, no transient state that won't exist for the next reader; keep only the essential, durable rationale. Describe the code as it is, never its drafting history: "we no longer do X", "unchanged behavior", "instead of the previous approach" are meaningless to a reader who never saw the earlier iteration — before finishing, reread your comments as if the current state is the only state that ever existed. - **Never attribute work to a specific customer, account, or "requested by a customer" in repo-tracked content** (PR descriptions, commit messages, code comments, docs). Describe changes by their technical motivation instead. diff --git a/CHANGELOG.md b/CHANGELOG.md index 947e63f5c6..33e24c0c00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,157 @@ # Changelog +## [1.803.0](https://github.com/windmill-labs/windmill/compare/v1.802.0...v1.803.0) (2026-09-03) + + +### Features + +* expose request headers to scripts invoked via MCP ([#10903](https://github.com/windmill-labs/windmill/issues/10903)) ([e474e88](https://github.com/windmill-labs/windmill/commit/e474e8803ce2ff5c2df09a58dab51d45f5c922ca)) +* reuse an existing workspace resource in the project import wizard ([#10935](https://github.com/windmill-labs/windmill/issues/10935)) ([582761e](https://github.com/windmill-labs/windmill/commit/582761e37c776e92dc1c6ebfee8c4efe7c35d822)) + + +### Bug Fixes + +* bump git sync hub scripts to cli 1.802.1, test the fork ui pull ([#10955](https://github.com/windmill-labs/windmill/issues/10955)) ([ca88009](https://github.com/windmill-labs/windmill/commit/ca8800959aa6a0017cc29bad187c9f49e0d13cc4)) +* **cli:** make a sync push into a fork converge on schedules and inline names ([#10951](https://github.com/windmill-labs/windmill/issues/10951)) ([0f5a1db](https://github.com/windmill-labs/windmill/commit/0f5a1db2abba8df30a2f975f4498e269f13cf93d)) +* fade the home Build with AI placeholder every 10s instead of typing it ([#10953](https://github.com/windmill-labs/windmill/issues/10953)) ([3d089b5](https://github.com/windmill-labs/windmill/commit/3d089b57344f5814086e6176301c5031dc519674)) +* let operators use wmill.datatable() from within running jobs ([#10931](https://github.com/windmill-labs/windmill/issues/10931)) ([9b64a89](https://github.com/windmill-labs/windmill/commit/9b64a89cd46ae718d6c58fa12f925fa041fb1032)) + +## [1.802.0](https://github.com/windmill-labs/windmill/compare/v1.801.0...v1.802.0) (2026-09-02) + + +### Features + +* add retention cleanup for the otel_traces table ([#10949](https://github.com/windmill-labs/windmill/issues/10949)) ([d472193](https://github.com/windmill-labs/windmill/commit/d472193e5bf5f6428e0096a402eb2c9299634fb2)) +* open path links from chat messages in the session preview panel ([#10881](https://github.com/windmill-labs/windmill/issues/10881)) ([f10ac6c](https://github.com/windmill-labs/windmill/commit/f10ac6c2b3644fb16697e650efbc4f7cd3c6944c)) +* restore owner and label filter chips on the homepage ([#10942](https://github.com/windmill-labs/windmill/issues/10942)) ([ccf8476](https://github.com/windmill-labs/windmill/commit/ccf84761dd664b9228dfe2f65867e8c32cd20c21)) +* **sessions:** offer the item you came from when starting a new session ([#10940](https://github.com/windmill-labs/windmill/issues/10940)) ([d3747d6](https://github.com/windmill-labs/windmill/commit/d3747d62555ebcb09c78cfabcaa3b6177758d6ea)) +* workspace setting to hide the AI assistant, agent steps unaffected ([#10941](https://github.com/windmill-labs/windmill/issues/10941)) ([fdd3b36](https://github.com/windmill-labs/windmill/commit/fdd3b36423344a2e1a464674179406581074e926)) + + +### Bug Fixes + +* apply object-storage test SSRF validation to all non-super-admins ([#10933](https://github.com/windmill-labs/windmill/issues/10933)) ([4fef119](https://github.com/windmill-labs/windmill/commit/4fef1195adaa9fa036a219884bd6c996460ca37f)) +* connect to dev server instead of localhost ([#10912](https://github.com/windmill-labs/windmill/issues/10912)) ([337154b](https://github.com/windmill-labs/windmill/commit/337154b8304a5969f35216add627b5c1153c0f6c)) +* preselect first row of AI agent and AI sandbox insert panes ([#10937](https://github.com/windmill-labs/windmill/issues/10937)) ([95b6bbd](https://github.com/windmill-labs/windmill/commit/95b6bbd46ada11d96a914ae5b0e92aba4dd02530)) +* record supplied script lock hashes so importers can skip relocking ([#10915](https://github.com/windmill-labs/windmill/issues/10915)) ([17ba521](https://github.com/windmill-labs/windmill/commit/17ba521c352aec65a8270893752bbadd7f3d6eaa)) +* sandbox script-controlled content types in result_to_response ([#10932](https://github.com/windmill-labs/windmill/issues/10932)) ([419741e](https://github.com/windmill-labs/windmill/commit/419741e5d226c67c51429094fb6ded9474afed99)) + +## [1.801.0](https://github.com/windmill-labs/windmill/compare/v1.800.1...v1.801.0) (2026-09-01) + + +### Features + +* **ai-chat:** make reusable skills ai_skill resources you select per workspace ([#10914](https://github.com/windmill-labs/windmill/issues/10914)) ([cfcfe29](https://github.com/windmill-labs/windmill/commit/cfcfe298dd9ab50196bd64926ef78c4563f58c2c)) +* **ai-sessions:** show a running session across tabs and reload finished turns ([#10916](https://github.com/windmill-labs/windmill/issues/10916)) ([816dc9d](https://github.com/windmill-labs/windmill/commit/816dc9dcd2c310e499d2d210a0abcd403469f29c)) +* edit folders and groups in a drawer that saves once ([#10873](https://github.com/windmill-labs/windmill/issues/10873)) ([5d5ad4e](https://github.com/windmill-labs/windmill/commit/5d5ad4e8974e076ef53a26a5584e4209255a2248)) +* make the home Build with AI composer dismissible, quiet the rest of the home page ([#10930](https://github.com/windmill-labs/windmill/issues/10930)) ([772fafe](https://github.com/windmill-labs/windmill/commit/772fafec8316a1e0c0e76b9a0737cc41d40a9a8c)) + + +### Bug Fixes + +* let a principal without a login account own a draft ([#10925](https://github.com/windmill-labs/windmill/issues/10925)) ([94af8d0](https://github.com/windmill-labs/windmill/commit/94af8d0fb5aceebe83936fd6761c6c1c02c75323)) +* resolve chat path links against the session's operating workspace ([#10924](https://github.com/windmill-labs/windmill/issues/10924)) ([9074de2](https://github.com/windmill-labs/windmill/commit/9074de25ea730ca02653c9a2e2b8b99eda6f3137)) +* tolerate string app_id in GHES app config deserialization ([#10923](https://github.com/windmill-labs/windmill/issues/10923)) ([af8ff38](https://github.com/windmill-labs/windmill/commit/af8ff3868748412cb658c803ebc8a71edc3cd8fb)) + +## [1.800.1](https://github.com/windmill-labs/windmill/compare/v1.800.0...v1.800.1) (2026-09-01) + + +### Bug Fixes + +* add top margin to the home Build with AI section ([#10909](https://github.com/windmill-labs/windmill/issues/10909)) ([bedf5ae](https://github.com/windmill-labs/windmill/commit/bedf5ae57445025729e94e16f1b5f13f6ff38ffa)) +* **ai-chat:** consume an @ mention with the message that carried it ([#10907](https://github.com/windmill-labs/windmill/issues/10907)) ([c512110](https://github.com/windmill-labs/windmill/commit/c512110a1f8d0d3437c20048f6446ef62b10222c)) +* keep a local dbt descriptor under sync pull --keep-deleted ([#10911](https://github.com/windmill-labs/windmill/issues/10911)) ([4b5be38](https://github.com/windmill-labs/windmill/commit/4b5be386ce0f851a087f43c0b0ac6e4b1b055a47)) +* keep windmill-indexer out of builds without tantivy ([#10908](https://github.com/windmill-labs/windmill/issues/10908)) ([db0f004](https://github.com/windmill-labs/windmill/commit/db0f004613e3f90428fea4c824cc53f1b2fc03b0)) + +## [1.800.0](https://github.com/windmill-labs/windmill/compare/v1.799.0...v1.800.0) (2026-08-31) + + +### Features + +* add --keep-deleted flag to wmill sync pull and push ([#10878](https://github.com/windmill-labs/windmill/issues/10878)) ([66123f3](https://github.com/windmill-labs/windmill/commit/66123f3a9b8978c0084b02f50b44cffba125a13a)) +* day-partition the service log index and expire whole chunks ([#10893](https://github.com/windmill-labs/windmill/issues/10893)) ([d91ee46](https://github.com/windmill-labs/windmill/commit/d91ee4614a70f20a194f47e190327129f499ec63)) +* free AI tokens + home search/filter revamp ([#10020](https://github.com/windmill-labs/windmill/issues/10020)) ([716ce2e](https://github.com/windmill-labs/windmill/commit/716ce2ece00cd5cfb8641afc6432636bc4aa46e9)) +* make the service log retention period an instance setting ([#10889](https://github.com/windmill-labs/windmill/issues/10889)) ([815de49](https://github.com/windmill-labs/windmill/commit/815de49e2322f85ca92b1e41a2bcd22591ebe93f)) +* rework the evals dataset drawer and run navigation ([#10884](https://github.com/windmill-labs/windmill/issues/10884)) ([1462f17](https://github.com/windmill-labs/windmill/commit/1462f17643302127b4bd76bacfde80cc03f9d606)) +* serve service log context from parquet and retire the raw log files ([#10892](https://github.com/windmill-labs/windmill/issues/10892)) ([338d75c](https://github.com/windmill-labs/windmill/commit/338d75cc5227e352cb84828c99bfd3b984cf0fa5)) +* serve service log retrieval from a columnar parquet store ([#10886](https://github.com/windmill-labs/windmill/issues/10886)) ([7c1a785](https://github.com/windmill-labs/windmill/commit/7c1a785f756ed27e4425f6534709b19971a73a97)) + + +### Bug Fixes + +* correct the service log ingest flush boundary ([#10898](https://github.com/windmill-labs/windmill/issues/10898)) ([ac56586](https://github.com/windmill-labs/windmill/commit/ac56586c0e56d4022761d3c80306a03d57f8bfcb)) +* harden the service log indexer's recovery and read paths ([#10904](https://github.com/windmill-labs/windmill/issues/10904)) ([831370c](https://github.com/windmill-labs/windmill/commit/831370cdde8e06f4298b17baa1a0041bacdd98c7)) +* keep raw-app editor selection consistent across sidebar and tabs ([#10885](https://github.com/windmill-labs/windmill/issues/10885)) ([b57e231](https://github.com/windmill-labs/windmill/commit/b57e231c2bf5e5fe007f0aa7b958a51e32b47141)) +* register every rotated service log file exactly once ([#10891](https://github.com/windmill-labs/windmill/issues/10891)) ([c817248](https://github.com/windmill-labs/windmill/commit/c8172480b0b1be6c57210212afc71d6ec8711235)) +* show a loading indicator while the initial data table migration is generated ([#10900](https://github.com/windmill-labs/windmill/issues/10900)) ([b998267](https://github.com/windmill-labs/windmill/commit/b998267c91b9dcf02787768f6205cc5aeda494fb)) +* track outstanding service log files on the rows themselves ([#10894](https://github.com/windmill-labs/windmill/issues/10894)) ([aa4a6ff](https://github.com/windmill-labs/windmill/commit/aa4a6ffd66813010a79c07741b01a984ed4e7df6)) + + +### Performance Improvements + +* add service log documents to the index one batch at a time ([#10906](https://github.com/windmill-labs/windmill/issues/10906)) ([0c2eb0a](https://github.com/windmill-labs/windmill/commit/0c2eb0ae3d18f49c21370131d15011e8dd103746)) + +## [1.799.0](https://github.com/windmill-labs/windmill/compare/v1.798.1...v1.799.0) (2026-08-28) + + +### Features + +* enable Anthropic prompt caching on Vertex AI agent steps ([#10876](https://github.com/windmill-labs/windmill/issues/10876)) ([320f400](https://github.com/windmill-labs/windmill/commit/320f4005124202852e6e9c70b394e7f87231d278)) +* instrument AI fill/fix, evals, agents and the debugger ([#10853](https://github.com/windmill-labs/windmill/issues/10853)) ([0bbd559](https://github.com/windmill-labs/windmill/commit/0bbd559ac8a35dba04ba5e8d6f2fd8d1d1124891)) + + +### Bug Fixes + +* **datatables:** stop a fork's pg_dump restore from failing silently ([#10830](https://github.com/windmill-labs/windmill/issues/10830)) ([3ce9bbc](https://github.com/windmill-labs/windmill/commit/3ce9bbc7168b837cb2111aabd533bb67803502b8)) +* key build artifact caches on a runnable's inline modules ([#10819](https://github.com/windmill-labs/windmill/issues/10819)) ([b72ccc3](https://github.com/windmill-labs/windmill/commit/b72ccc35934165b4bad112b947ca5af064aab26f)) +* nested template literals in step inputs, and unresolvable $args tags ([#10856](https://github.com/windmill-labs/windmill/issues/10856)) ([8f349c0](https://github.com/windmill-labs/windmill/commit/8f349c032a0d75fc3350292075e5050a030f6166)) +* pre-fill the test panel JSON args editor and align its placeholder ([#10871](https://github.com/windmill-labs/windmill/issues/10871)) ([fb82f36](https://github.com/windmill-labs/windmill/commit/fb82f36e6d6492dd0740984d8d78ea4eaa30361e)) +* reject a prefixed error_handler_path on triggers ([#10847](https://github.com/windmill-labs/windmill/issues/10847)) ([d334831](https://github.com/windmill-labs/windmill/commit/d33483173526a3b352d2829ac8a2e1e229cc1127)) +* unify billable seat counting and prevent fork subscriptions ([#10818](https://github.com/windmill-labs/windmill/issues/10818)) ([7dd88c4](https://github.com/windmill-labs/windmill/commit/7dd88c470caee5f095dc240667aa7550c55696bc)) + +## [1.798.1](https://github.com/windmill-labs/windmill/compare/v1.798.0...v1.798.1) (2026-08-27) + + +### Bug Fixes + +* allow job tokens to read the automate_username_creation setting ([#10869](https://github.com/windmill-labs/windmill/issues/10869)) ([c2279db](https://github.com/windmill-labs/windmill/commit/c2279db8a96ac76382eafe254627dafd24d173fd)) + +## [1.798.0](https://github.com/windmill-labs/windmill/compare/v1.797.0...v1.798.0) (2026-08-27) + + +### Features + +* a wizard for importing a hub project, and finishing what the import cannot ([#10729](https://github.com/windmill-labs/windmill/issues/10729)) ([2913339](https://github.com/windmill-labs/windmill/commit/29133398f99cd2dd5b33057ee9df4492d82e067a)) + +## [1.797.0](https://github.com/windmill-labs/windmill/compare/v1.796.0...v1.797.0) (2026-08-26) + + +### Features + +* configurable expiry for presigned s3 public url signatures ([#10835](https://github.com/windmill-labs/windmill/issues/10835)) ([8a6dc27](https://github.com/windmill-labs/windmill/commit/8a6dc27236aca67f0efe941d9606b787c2305ea8)) +* **frontend:** flag the fork-compare datatable schema diff as legacy ([#10829](https://github.com/windmill-labs/windmill/issues/10829)) ([07c77ea](https://github.com/windmill-labs/windmill/commit/07c77ead7425f1877372d358d867445a4c525c96)) +* keep a Hub project live while an update is under review ([#10814](https://github.com/windmill-labs/windmill/issues/10814)) ([c04b570](https://github.com/windmill-labs/windmill/commit/c04b5705745c36ecbb3a551ac59459218d2e3807)) + + +### Bug Fixes + +* **cli:** keep svelte component styles in the raw-app bundle ([#10838](https://github.com/windmill-labs/windmill/issues/10838)) ([b8bf539](https://github.com/windmill-labs/windmill/commit/b8bf539c3fe2b4db9c74dd73f04b3029287acdc6)) +* **debugger:** parse bun 1.4's UUID inspector token ([#10828](https://github.com/windmill-labs/windmill/issues/10828)) ([4658224](https://github.com/windmill-labs/windmill/commit/46582245926a7f8ea961bcd125a58fbfba3530cf)) +* force HTTP router rebuild on trigger-change notification ([#10849](https://github.com/windmill-labs/windmill/issues/10849)) ([ffdf17e](https://github.com/windmill-labs/windmill/commit/ffdf17ef8dc5575dd92d62d0d0ba887c1e378576)) +* **frontend:** follow the operating workspace in step input forms ([#10834](https://github.com/windmill-labs/windmill/issues/10834)) ([6b73145](https://github.com/windmill-labs/windmill/commit/6b73145e7220232601538b801ebc9dc73fe79bbb)) +* **frontend:** key the GitHub App installation selector on installation_id ([#10831](https://github.com/windmill-labs/windmill/issues/10831)) ([78331fd](https://github.com/windmill-labs/windmill/commit/78331fda8b290a2d9a5dd92b8362ff32c8b39432)) +* **frontend:** operator menu opens on hover, pins on click ([#10824](https://github.com/windmill-labs/windmill/issues/10824)) ([665f83e](https://github.com/windmill-labs/windmill/commit/665f83e1f438e34d006429889d51a5fb6a6b6176)) +* keep connection string query parameters under token auth ([#10859](https://github.com/windmill-labs/windmill/issues/10859)) ([f131c39](https://github.com/windmill-labs/windmill/commit/f131c3920f50f9fa18cd637eac39609495999aef)) +* migrate slack resource-connect oauth to v2 ([#10836](https://github.com/windmill-labs/windmill/issues/10836)) ([9fa8159](https://github.com/windmill-labs/windmill/commit/9fa8159ad16204cab52fd18a34a48ebf13f800f6)) +* recover from unresolvable AI session links instead of a dead end ([#10854](https://github.com/windmill-labs/windmill/issues/10854)) ([e38c449](https://github.com/windmill-labs/windmill/commit/e38c449007f27b952808cba5aa812441f2ce5946)) +* require admin on workspace tarball settings export ([#10817](https://github.com/windmill-labs/windmill/issues/10817)) ([46c363f](https://github.com/windmill-labs/windmill/commit/46c363ffa4bc72bef6b367ece4bdbeef5e0eadc9)) +* restrict filesystem workspace storage to debug builds ([#10864](https://github.com/windmill-labs/windmill/issues/10864)) ([8b80b09](https://github.com/windmill-labs/windmill/commit/8b80b09f33d311f0881678577ca6004c12d97c22)) + + +### Performance Improvements + +* index the suspended-job resume test instead of filtering it ([#10863](https://github.com/windmill-labs/windmill/issues/10863)) ([69320b2](https://github.com/windmill-labs/windmill/commit/69320b28f615b897a92f580bd5961c41e5c29951)) + ## [1.796.0](https://github.com/windmill-labs/windmill/compare/v1.795.0...v1.796.0) (2026-08-24) diff --git a/CONTEXT.md b/CONTEXT.md index 6efb92b669..64aa1b93d8 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -36,3 +36,22 @@ _Avoid_: argument field, param **Expression input**: Any other place a property can be picked into: the loop iterator, skip and early-stop predicates, the retry condition, a branch predicate, timeout. Its prop picker opens in a popover from the connect button rather than taking a pane. _Avoid_: JS field, code input + +### Permissions + +**Member**: +A user or group granted a role on a folder, a group, or an item's extra ACL. The list of them is +"Members (n)" everywhere it is shown, and one is added with "Add member". +_Avoid_: participant, collaborator, owner, ACL entry, permission (that names the concept, not the people) + +**Role**: +The access level a member holds: viewer, writer or admin on a folder; member or admin on a group. +Viewers read, writers also edit, admins also manage the members. A group role of **manager** — +manages the group without belonging to it — is a legacy state the UI shows and can leave, but +offers no way to enter. +_Avoid_: permission level, access level, rank + +**Owner**: +Reserved for the path prefix that says where an item lives — `u/alice` or `f/team`. A folder's +`owners` column in the database is its admin members; call those admins, never owners, in the UI. +_Avoid_: using "owner" for a folder admin diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index 5c682be5eb..6d80f19197 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -7,6 +7,7 @@ import type { ListableApp, ListableResource, ListableVariable, + Resource, Script } from '../../../frontend/src/lib/gen' import type { @@ -81,6 +82,15 @@ export interface BenchmarkWorkspaceAiProvider { isDefault?: boolean } +/** A plain (non-AI) resource of the benchmark workspace, for cases about referencing a + * credential — passing one as a run argument, say. `value` is what `get_resource` returns. */ +export interface BenchmarkWorkspaceResource { + path: string + resource_type: string + value?: Record + description?: string +} + export interface BenchmarkWorkspaceJob { /** Stable id so a case prompt can reference a specific run (e.g. for get_job_logs). */ id?: string @@ -98,6 +108,7 @@ export interface BenchmarkWorkspaceRunnables { apps?: BenchmarkWorkspaceApp[] variables?: BenchmarkWorkspaceVariable[] aiProviders?: BenchmarkWorkspaceAiProvider[] + resources?: BenchmarkWorkspaceResource[] datatables?: BenchmarkDatatableSeed[] jobs?: BenchmarkWorkspaceJob[] } @@ -284,15 +295,71 @@ export function listBenchmarkAiProviderResources(workspace: string): ListableRes })) } -/** The value of a seeded AI provider resource. Only the endpoint fields are modelled — a key is - * never needed, because no eval run calls the provider through this resource. */ +/** Plain seeded resources of a benchmark workspace, shaped like `ResourceService.listResource` + * rows. Null when the workspace is not a benchmark one. */ +export function listBenchmarkPlainResources(workspace: string): ListableResource[] | null { + const runnables = benchmarkWorkspaceRunnables.get(workspace) + if (!runnables) { + return null + } + return (runnables.resources ?? []).map((seed) => ({ + workspace_id: workspace, + path: seed.path, + resource_type: seed.resource_type, + description: seed.description, + value: null, + is_oauth: false, + is_linked: false, + is_refreshed: false, + extra_perms: {}, + edited_at: BENCHMARK_TIMESTAMP + })) +} + +/** A seeded resource with its value, as `ResourceService.getResource` returns it. Covers both + * seed kinds, so it agrees with `existsResource` and `listResource` — both of those report AI + * providers too, and a case that lists resources and then reads one by path would otherwise get + * a row it cannot fetch. */ +export function getBenchmarkResource(workspace: string, path: string): Resource | null { + const runnables = benchmarkWorkspaceRunnables.get(workspace) + const seed = runnables?.resources?.find((entry) => entry.path === path) + if (seed) { + return { + workspace_id: workspace, + path: seed.path, + resource_type: seed.resource_type, + description: seed.description, + value: seed.value ?? {}, + is_oauth: false, + extra_perms: {} + } as Resource + } + const provider = runnables?.aiProviders?.find((entry) => entry.path === path) + if (!provider) { + return null + } + return { + workspace_id: workspace, + path: provider.path, + resource_type: provider.kind, + value: getBenchmarkResourceValue(workspace, path) ?? {}, + is_oauth: false, + extra_perms: {} + } as Resource +} + +/** The value of a seeded resource. For an AI provider only the endpoint fields are modelled — a + * key is never needed, because no eval run calls the provider through this resource. */ export function getBenchmarkResourceValue( workspace: string, path: string ): Record | null { - const seed = benchmarkWorkspaceRunnables - .get(workspace) - ?.aiProviders?.find((entry) => entry.path === path) + const runnables = benchmarkWorkspaceRunnables.get(workspace) + const plain = runnables?.resources?.find((entry) => entry.path === path) + if (plain) { + return plain.value ?? {} + } + const seed = runnables?.aiProviders?.find((entry) => entry.path === path) if (!seed) { return null } diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index e59ec11ad9..338ed8504c 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -70,7 +70,9 @@ vi.mock('$lib/gen', async () => { getBenchmarkResourceValue, getBenchmarkVariableByPath, hasBenchmarkWorkspace, + getBenchmarkResource, listBenchmarkAiProviderResources, + listBenchmarkPlainResources, listBenchmarkApps, listBenchmarkDatatables, listBenchmarkDrafts, @@ -359,18 +361,24 @@ vi.mock('$lib/gen', async () => { hasBenchmarkWorkspace(data.workspace) ? Boolean(getBenchmarkResourceValue(data.workspace, data.path)) : actual.ResourceService.existsResource(data), - // Only AI provider resources are modelled: they are what an AI agent step references. listResource: async (data: { workspace: string; resourceType?: string }) => { if (!hasBenchmarkWorkspace(data.workspace)) { return actual.ResourceService.listResource(data) } - const seeded = listBenchmarkAiProviderResources(data.workspace) ?? [] + const seeded = [ + ...(listBenchmarkAiProviderResources(data.workspace) ?? []), + ...(listBenchmarkPlainResources(data.workspace) ?? []) + ] const wanted = data.resourceType?.split(',') return wanted ? seeded.filter((r) => wanted.includes(r.resource_type)) : seeded }, getResource: async (data: { workspace: string; path: string }) => { if (hasBenchmarkWorkspace(data.workspace)) { - throw new Error(`Resource "${data.path}" not found in benchmark workspace`) + const resource = getBenchmarkResource(data.workspace, data.path) + if (!resource) { + throw new Error(`Resource "${data.path}" not found in benchmark workspace`) + } + return resource } return actual.ResourceService.getResource(data) }, diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 21f80bef7e..94ecb5c029 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -2365,3 +2365,37 @@ - the step uses the workspace's anthropic resource f/evals/global/anthropic_main - the model is the Opus one the user asked for, taken from the models that resource serves - the diff flow input reaches the agent + +# The failure this pins: passing a resource as `{"$res": ""}` (or as a bare path), which +# reaches the script unresolved because the backend only substitutes a string value that itself +# starts with `$res:`. The mock preview echoes args back and reports success, so nothing in the +# loop corrects a wrong shape — the arg form is the whole test. +- id: global-run-arg-resource-reference + prompt: |- + Run `f/evals/global/github_repo_stats` against the `windmill-labs/windmill` repo, passing our + GitHub credentials at `f/evals/global/github_main` as its `gh_auth` input, and tell me whether + it went through. + initial: ai_evals/fixtures/frontend/global/initial/github_repo_stats_script.json + runtime: + maxTurns: 10 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - test_run_script + forbiddenToolsUsed: + - write_script + - deploy_workspace_item + toolCallArgs: + # Exact: the mock never resolves the reference, so a near-miss path like + # `$res:f/evals/global/github_main_backup` would otherwise pass. + - tool: test_run_script + field: args.gh_auth + stringEqualsAnyOf: + - "$res:f/evals/global/github_main" + # The judge only sees drafts, and this case makes none — the deliverable is the shape of the + # run argument, checked deterministically above. + skipJudge: true + judgeChecklist: + - runs the existing script rather than rewriting it + - passes the GitHub resource as the bare string $res:f/evals/global/github_main diff --git a/ai_evals/core/types.ts b/ai_evals/core/types.ts index c9a4f7830e..4107dc53da 100644 --- a/ai_evals/core/types.ts +++ b/ai_evals/core/types.ts @@ -160,6 +160,13 @@ export interface ToolCallArgumentRule { field: string; stringStartsWithAnyOf?: string[]; stringMustNotStartWithAnyOf?: string[]; + /** + * Universal over calls: every recorded call to `tool` must carry `field` as + * exactly one of these strings. Use when a near-miss would still satisfy a + * prefix — a resource reference like `$res:f/a/b` shares its prefix with the + * wrong `$res:f/a/b_backup`, and the mock never resolves it to catch that. + */ + stringEqualsAnyOf?: string[]; /** * Case-insensitive "contains", existential over calls: at least one recorded * call to `tool` must have `field` containing one of these substrings. Other diff --git a/ai_evals/core/validators.test.ts b/ai_evals/core/validators.test.ts index 9b606c1c85..4b7891f67e 100644 --- a/ai_evals/core/validators.test.ts +++ b/ai_evals/core/validators.test.ts @@ -228,6 +228,43 @@ describe("validateToolExpectations", () => { }); }); + // A resource reference shares its prefix with a wrong sibling path, and the mock + // never resolves it, so only exact matching separates the two. + it("rejects a resource reference whose path merely shares the prefix", () => { + const checks = validateToolExpectations({ + run: { + success: true, + actual: {}, + assistantMessageCount: 1, + toolCallCount: 1, + toolsUsed: ["test_run_script"], + toolCallDetails: [ + { + name: "test_run_script", + arguments: { args: { gh_auth: "$res:f/evals/global/github_main_backup" } }, + }, + ], + skillsInvoked: [], + }, + toolExpect: { + toolCallArgs: [ + { + tool: "test_run_script", + field: "args.gh_auth", + stringEqualsAnyOf: ["$res:f/evals/global/github_main"], + }, + ], + }, + }); + + expect(checks).toContainEqual({ + name: "test_run_script.args.gh_auth matches an accepted value", + passed: false, + details: + 'accepted values: $res:f/evals/global/github_main; values: "$res:f/evals/global/github_main_backup"', + }); + }); + // The whole point of the same-call rule: the per-field rules are existential over // calls, so two single-filter pages would satisfy them while never opening the // combined view the case asks for. diff --git a/ai_evals/core/validators.ts b/ai_evals/core/validators.ts index 7150dab0c3..132a9ff3d0 100644 --- a/ai_evals/core/validators.ts +++ b/ai_evals/core/validators.ts @@ -278,6 +278,20 @@ export function validateToolExpectations(input: { ); } + if (rule.stringEqualsAnyOf && rule.stringEqualsAnyOf.length > 0) { + const invalidValues = values.filter( + (value) => + typeof value !== "string" || !rule.stringEqualsAnyOf!.includes(value) + ); + checks.push( + check( + `${rule.tool}.${rule.field} matches an accepted value`, + invalidValues.length === 0, + `accepted values: ${rule.stringEqualsAnyOf.join(", ")}; values: ${summarizeToolValues(values)}` + ) + ); + } + if (rule.stringMustNotStartWithAnyOf && rule.stringMustNotStartWithAnyOf.length > 0) { const invalidValues = values.filter( (value) => diff --git a/ai_evals/fixtures/frontend/global/initial/github_repo_stats_script.json b/ai_evals/fixtures/frontend/global/initial/github_repo_stats_script.json new file mode 100644 index 0000000000..80e1b960c5 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/github_repo_stats_script.json @@ -0,0 +1,37 @@ +{ + "workspace": { + "resources": [ + { + "path": "f/evals/global/github_main", + "resource_type": "github", + "description": "GitHub credentials", + "value": { "token": "$var:f/evals/global/github_token" } + } + ], + "scripts": [ + { + "path": "f/evals/global/github_repo_stats", + "summary": "Count open issues on a GitHub repository", + "description": "Reads the open issue count for a repository using GitHub credentials.", + "language": "bun", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "gh_auth": { + "type": "object", + "format": "resource-github", + "description": "GitHub credentials" + }, + "repo": { + "type": "string", + "description": "Repository in owner/name form" + } + }, + "required": ["gh_auth", "repo"] + }, + "content": "type Github = { token: string }\n\nexport async function main(gh_auth: Github, repo: string) {\n const res = await fetch(`https://api.github.com/repos/${repo}/issues?state=open`, {\n headers: { Authorization: `Bearer ${gh_auth.token}` }\n })\n const issues = await res.json()\n return { repo, open_issues: issues.length }\n}\n" + } + ] + } +} diff --git a/backend/.sqlx/query-002a606e71364b0581dbc496bf4337f276861dc71d2e277a7aef711543eb14d7.json b/backend/.sqlx/query-002a606e71364b0581dbc496bf4337f276861dc71d2e277a7aef711543eb14d7.json deleted file mode 100644 index abae0ba456..0000000000 --- a/backend/.sqlx/query-002a606e71364b0581dbc496bf4337f276861dc71d2e277a7aef711543eb14d7.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n COUNT(*)::bigint AS \"total!\",\n COUNT(*) FILTER (WHERE name = ANY($2::text[]))::bigint AS \"replacing!\"\n FROM ai_skill\n WHERE workspace_id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "total!", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "replacing!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text", - "TextArray" - ] - }, - "nullable": [ - null, - null - ] - }, - "hash": "002a606e71364b0581dbc496bf4337f276861dc71d2e277a7aef711543eb14d7" -} diff --git a/backend/.sqlx/query-032b51ce97c2f31dc2aea8ddf64e6971818ea3b19b4d4866d16e1bf9f7f2ec6f.json b/backend/.sqlx/query-032b51ce97c2f31dc2aea8ddf64e6971818ea3b19b4d4866d16e1bf9f7f2ec6f.json new file mode 100644 index 0000000000..304efc22d2 --- /dev/null +++ b/backend/.sqlx/query-032b51ce97c2f31dc2aea8ddf64e6971818ea3b19b4d4866d16e1bf9f7f2ec6f.json @@ -0,0 +1,65 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN d.email END) as \"username?\",\n d.created_at as \"draft_saved_at!\"\n FROM draft d\n LEFT JOIN usr u\n ON u.workspace_id = d.workspace_id\n AND u.email = d.email\n LEFT JOIN password p\n ON p.email = d.email\n AND p.super_admin = true\n WHERE d.workspace_id = $1\n AND d.path = $2\n AND d.typ = $3\n AND (d.email IS NULL OR d.email <> $4)\n AND (d.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL)\n ORDER BY d.email NULLS LAST", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "username?", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "draft_saved_at!", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github", + "data_pipeline", + "trigger_amqp" + ] + } + } + }, + "Text" + ] + }, + "nullable": [ + null, + false + ] + }, + "hash": "032b51ce97c2f31dc2aea8ddf64e6971818ea3b19b4d4866d16e1bf9f7f2ec6f" +} diff --git a/backend/.sqlx/query-0a02678e3f062c8854226d6d5eb7e493c229d205048eeac78a7cbe328c689b88.json b/backend/.sqlx/query-0a02678e3f062c8854226d6d5eb7e493c229d205048eeac78a7cbe328c689b88.json new file mode 100644 index 0000000000..8275796e90 --- /dev/null +++ b/backend/.sqlx/query-0a02678e3f062c8854226d6d5eb7e493c229d205048eeac78a7cbe328c689b88.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO draft(workspace_id, path, typ, value, email) VALUES\n ('test-workspace', 'u/ext/s', 'script', '{}'::json, 'ext-jwt@windmill.dev'),\n ('test-workspace', 'u/two/s', 'script', '{\"summary\": \"moving\"}'::json, 'test2@windmill.dev'),\n ('test-workspace', 'u/two/s', 'script', '{\"summary\": \"displaced\"}'::json, 'renamed@windmill.dev'),\n ('test-workspace', 'u/three/s', 'script', '{}'::json, 'test3@windmill.dev')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "0a02678e3f062c8854226d6d5eb7e493c229d205048eeac78a7cbe328c689b88" +} diff --git a/backend/.sqlx/query-94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033.json b/backend/.sqlx/query-0e03dc960c0a22e042e54af719ac90c4b8506acefc85ef5b2f92a7bc451b1c5e.json similarity index 51% rename from backend/.sqlx/query-94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033.json rename to backend/.sqlx/query-0e03dc960c0a22e042e54af719ac90c4b8506acefc85ef5b2f92a7bc451b1c5e.json index 24a5ee62dd..42bac71afd 100644 --- a/backend/.sqlx/query-94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033.json +++ b/backend/.sqlx/query-0e03dc960c0a22e042e54af719ac90c4b8506acefc85ef5b2f92a7bc451b1c5e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval RETURNING file_path, hostname", + "query": "DELETE FROM log_file WHERE (hostname, log_ts) IN (\n SELECT hostname, log_ts FROM log_file\n WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval\n LIMIT $2\n ) RETURNING file_path, hostname", "describe": { "columns": [ { @@ -16,6 +16,7 @@ ], "parameters": { "Left": [ + "Int8", "Int8" ] }, @@ -24,5 +25,5 @@ false ] }, - "hash": "94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033" + "hash": "0e03dc960c0a22e042e54af719ac90c4b8506acefc85ef5b2f92a7bc451b1c5e" } diff --git a/backend/.sqlx/query-1572b7348a05b7e357031f8d44b5bbee155569488352c10b334ce57d83ce1c0a.json b/backend/.sqlx/query-1572b7348a05b7e357031f8d44b5bbee155569488352c10b334ce57d83ce1c0a.json new file mode 100644 index 0000000000..03f11ef137 --- /dev/null +++ b/backend/.sqlx/query-1572b7348a05b7e357031f8d44b5bbee155569488352c10b334ce57d83ce1c0a.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT lockfile_hash FROM lock_hash WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "lockfile_hash", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "1572b7348a05b7e357031f8d44b5bbee155569488352c10b334ce57d83ce1c0a" +} diff --git a/backend/.sqlx/query-16b174aaa944fd94458ce3108f0fec23514ae0419f77b632064b75473b5636c3.json b/backend/.sqlx/query-16b174aaa944fd94458ce3108f0fec23514ae0419f77b632064b75473b5636c3.json new file mode 100644 index 0000000000..693d84173a --- /dev/null +++ b/backend/.sqlx/query-16b174aaa944fd94458ce3108f0fec23514ae0419f77b632064b75473b5636c3.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT trace_id FROM otel_traces ORDER BY trace_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "trace_id", + "type_info": "Bytea" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "16b174aaa944fd94458ce3108f0fec23514ae0419f77b632064b75473b5636c3" +} diff --git a/backend/.sqlx/query-188d024a53b2ef37442412824f73ab5cd81242501d9e7a476698ea7acccd4aef.json b/backend/.sqlx/query-188d024a53b2ef37442412824f73ab5cd81242501d9e7a476698ea7acccd4aef.json new file mode 100644 index 0000000000..73d5000f2d --- /dev/null +++ b/backend/.sqlx/query-188d024a53b2ef37442412824f73ab5cd81242501d9e7a476698ea7acccd4aef.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM draft dest\n WHERE dest.email = $1\n AND EXISTS (SELECT 1 FROM draft src\n WHERE src.email = $2\n AND src.workspace_id = dest.workspace_id\n AND src.path = dest.path\n AND src.typ = dest.typ)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "188d024a53b2ef37442412824f73ab5cd81242501d9e7a476698ea7acccd4aef" +} diff --git a/backend/.sqlx/query-1b244f65ee6a2607ebc1c333d4359fbbf8be5a81276a3050a42770e4a5b5aa5e.json b/backend/.sqlx/query-1b244f65ee6a2607ebc1c333d4359fbbf8be5a81276a3050a42770e4a5b5aa5e.json new file mode 100644 index 0000000000..f35dbc23e4 --- /dev/null +++ b/backend/.sqlx/query-1b244f65ee6a2607ebc1c333d4359fbbf8be5a81276a3050a42770e4a5b5aa5e.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM otel_traces WHERE ctid IN (\n SELECT ctid FROM otel_traces\n WHERE start_time_unix_nano < EXTRACT(\n EPOCH FROM now() - ($1::bigint::text || ' s')::interval\n )::bigint * 1000000000\n LIMIT $2\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "1b244f65ee6a2607ebc1c333d4359fbbf8be5a81276a3050a42770e4a5b5aa5e" +} diff --git a/backend/.sqlx/query-247486558e023ec3adf0c1e8f5664fc5e65995ce81e622c174a89befc1a527e5.json b/backend/.sqlx/query-247486558e023ec3adf0c1e8f5664fc5e65995ce81e622c174a89befc1a527e5.json new file mode 100644 index 0000000000..4ae74ff53c --- /dev/null +++ b/backend/.sqlx/query-247486558e023ec3adf0c1e8f5664fc5e65995ce81e622c174a89befc1a527e5.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT cost_nanos FROM ai_free_token_usage WHERE email = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "cost_nanos", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "247486558e023ec3adf0c1e8f5664fc5e65995ce81e622c174a89befc1a527e5" +} diff --git a/backend/.sqlx/query-43ea7d0cf7f3c50ec1e79b4d2384d49b6c65bc442263f228912a24c1c5740cc8.json b/backend/.sqlx/query-43ea7d0cf7f3c50ec1e79b4d2384d49b6c65bc442263f228912a24c1c5740cc8.json new file mode 100644 index 0000000000..f7c39c21fc --- /dev/null +++ b/backend/.sqlx/query-43ea7d0cf7f3c50ec1e79b4d2384d49b6c65bc442263f228912a24c1c5740cc8.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft SET email = $1 WHERE email = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "43ea7d0cf7f3c50ec1e79b4d2384d49b6c65bc442263f228912a24c1c5740cc8" +} diff --git a/backend/.sqlx/query-44b9400fed2082c0df9d57f940923f37b8d452ea8e199ccc207da238368f1996.json b/backend/.sqlx/query-44b9400fed2082c0df9d57f940923f37b8d452ea8e199ccc207da238368f1996.json new file mode 100644 index 0000000000..2253944269 --- /dev/null +++ b/backend/.sqlx/query-44b9400fed2082c0df9d57f940923f37b8d452ea8e199ccc207da238368f1996.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_free_token_daily_usage (day, cost_nanos, updated_at)\n VALUES ($2::date, $1::bigint, now())\n ON CONFLICT (day) DO UPDATE\n SET cost_nanos = ai_free_token_daily_usage.cost_nanos + $1::bigint,\n updated_at = now()\n RETURNING cost_nanos", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "cost_nanos", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8", + "Date" + ] + }, + "nullable": [ + false + ] + }, + "hash": "44b9400fed2082c0df9d57f940923f37b8d452ea8e199ccc207da238368f1996" +} diff --git a/backend/.sqlx/query-4b93550c7836fd3643180ade3548faa875e471d3f9ca37fc669f359e7a1818bb.json b/backend/.sqlx/query-4b93550c7836fd3643180ade3548faa875e471d3f9ca37fc669f359e7a1818bb.json deleted file mode 100644 index 63283a830a..0000000000 --- a/backend/.sqlx/query-4b93550c7836fd3643180ade3548faa875e471d3f9ca37fc669f359e7a1818bb.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at)\n VALUES ('admins', $1, $2, $3, now())\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET schema = EXCLUDED.schema, description = EXCLUDED.description, edited_at = now()", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Jsonb", - "Text" - ] - }, - "nullable": [] - }, - "hash": "4b93550c7836fd3643180ade3548faa875e471d3f9ca37fc669f359e7a1818bb" -} diff --git a/backend/.sqlx/query-4bc43a5e1c95cb8989962aeb1393a50df05437a2f1909ad5d303e2c2b89a0668.json b/backend/.sqlx/query-4bc43a5e1c95cb8989962aeb1393a50df05437a2f1909ad5d303e2c2b89a0668.json new file mode 100644 index 0000000000..681206a381 --- /dev/null +++ b/backend/.sqlx/query-4bc43a5e1c95cb8989962aeb1393a50df05437a2f1909ad5d303e2c2b89a0668.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM draft WHERE email = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "4bc43a5e1c95cb8989962aeb1393a50df05437a2f1909ad5d303e2c2b89a0668" +} diff --git a/backend/.sqlx/query-624a7dbc6cc951a199b0e70d86c463a0e7b5248c226ee92d95df94c3099cc400.json b/backend/.sqlx/query-624a7dbc6cc951a199b0e70d86c463a0e7b5248c226ee92d95df94c3099cc400.json new file mode 100644 index 0000000000..3713af1486 --- /dev/null +++ b/backend/.sqlx/query-624a7dbc6cc951a199b0e70d86c463a0e7b5248c226ee92d95df94c3099cc400.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE log_file SET indexed_at = now()\n FROM unnest($1::text[], $2::text[]) AS c(hostname, file_path)\n WHERE log_file.indexed_at IS NULL\n AND log_file.hostname = c.hostname\n AND log_file.file_path = c.file_path", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "624a7dbc6cc951a199b0e70d86c463a0e7b5248c226ee92d95df94c3099cc400" +} diff --git a/backend/.sqlx/query-62ed1fe52bc2c22d199101309cbbadb9842318c4c7a1d2526ac567ce41b7fbdc.json b/backend/.sqlx/query-62ed1fe52bc2c22d199101309cbbadb9842318c4c7a1d2526ac567ce41b7fbdc.json new file mode 100644 index 0000000000..67b672228f --- /dev/null +++ b/backend/.sqlx/query-62ed1fe52bc2c22d199101309cbbadb9842318c4c7a1d2526ac567ce41b7fbdc.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO otel_traces (trace_id, span_id, name, kind, start_time_unix_nano, end_time_unix_nano)\n VALUES ($1, $2, 'GET /', 3, $3, $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Bytea", + "Bytea", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "62ed1fe52bc2c22d199101309cbbadb9842318c4c7a1d2526ac567ce41b7fbdc" +} diff --git a/backend/.sqlx/query-67a83afb708c90b2132cba81a0701dfc8b5e7aedf2f26fc1fe5be8f685cf709c.json b/backend/.sqlx/query-67a83afb708c90b2132cba81a0701dfc8b5e7aedf2f26fc1fe5be8f685cf709c.json new file mode 100644 index 0000000000..4481fa3dea --- /dev/null +++ b/backend/.sqlx/query-67a83afb708c90b2132cba81a0701dfc8b5e7aedf2f26fc1fe5be8f685cf709c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT max(log_ts) FROM log_file\n WHERE hostname = $1 AND log_ts < (SELECT max(log_ts) FROM log_file WHERE hostname = $1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "max", + "type_info": "Timestamp" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "67a83afb708c90b2132cba81a0701dfc8b5e7aedf2f26fc1fe5be8f685cf709c" +} diff --git a/backend/.sqlx/query-b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a.json b/backend/.sqlx/query-6bbcb27a3bb70302076c559c8394b14b842f595f68dd885248abaaeabd2d0bf1.json similarity index 86% rename from backend/.sqlx/query-b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a.json rename to backend/.sqlx/query-6bbcb27a3bb70302076c559c8394b14b842f595f68dd885248abaaeabd2d0bf1.json index ba9f3814e8..13d1c9b1e8 100644 --- a/backend/.sqlx/query-b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a.json +++ b/backend/.sqlx/query-6bbcb27a3bb70302076c559c8394b14b842f595f68dd885248abaaeabd2d0bf1.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n hostname,\n mode::text,\n worker_group,\n log_ts,\n file_path,\n ok_lines,\n err_lines,\n json_fmt\n FROM log_file\n WHERE log_ts > $1\n ORDER BY log_ts ASC LIMIT $2", + "query": "SELECT\n hostname,\n mode::text,\n worker_group,\n log_ts,\n file_path,\n ok_lines,\n err_lines,\n json_fmt\n FROM log_file\n WHERE indexed_at IS NULL\n ORDER BY log_ts ASC, hostname ASC LIMIT $1", "describe": { "columns": [ { @@ -46,7 +46,6 @@ ], "parameters": { "Left": [ - "Timestamp", "Int8" ] }, @@ -61,5 +60,5 @@ true ] }, - "hash": "b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a" + "hash": "6bbcb27a3bb70302076c559c8394b14b842f595f68dd885248abaaeabd2d0bf1" } diff --git a/backend/.sqlx/query-734781e8e55e95c55f72e094e96297aa852e20a0f0d20db4b993947792f6b0a8.json b/backend/.sqlx/query-734781e8e55e95c55f72e094e96297aa852e20a0f0d20db4b993947792f6b0a8.json deleted file mode 100644 index d55722ed24..0000000000 --- a/backend/.sqlx/query-734781e8e55e95c55f72e094e96297aa852e20a0f0d20db4b993947792f6b0a8.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO ai_skill (workspace_id, name, description, instructions, edited_at, edited_by)\n VALUES ($1, $2, $3, $4, now(), $5)\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET description = EXCLUDED.description,\n instructions = EXCLUDED.instructions,\n edited_at = now(),\n edited_by = EXCLUDED.edited_by", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Text", - "Text", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "734781e8e55e95c55f72e094e96297aa852e20a0f0d20db4b993947792f6b0a8" -} diff --git a/backend/.sqlx/query-872be44352d4a27a2005f6bdc38f927ac4886755fd17ec755037763aa92f6c0e.json b/backend/.sqlx/query-872be44352d4a27a2005f6bdc38f927ac4886755fd17ec755037763aa92f6c0e.json new file mode 100644 index 0000000000..98e08b8167 --- /dev/null +++ b/backend/.sqlx/query-872be44352d4a27a2005f6bdc38f927ac4886755fd17ec755037763aa92f6c0e.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH moved AS (\n UPDATE log_file SET indexed_at = CASE\n WHEN log_ts > NOW() - make_interval(secs => $1) THEN NULL\n ELSE now() END\n WHERE indexed_at = 'epoch' RETURNING 1)\n SELECT count(*) AS \"n!\" FROM moved", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "n!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Float8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "872be44352d4a27a2005f6bdc38f927ac4886755fd17ec755037763aa92f6c0e" +} diff --git a/backend/.sqlx/query-87e8e034b1cf5ea7ce43870d77d33fa0fd05b067454f0d4686d8b155fe562ffe.json b/backend/.sqlx/query-87e8e034b1cf5ea7ce43870d77d33fa0fd05b067454f0d4686d8b155fe562ffe.json new file mode 100644 index 0000000000..76456ea8fb --- /dev/null +++ b/backend/.sqlx/query-87e8e034b1cf5ea7ce43870d77d33fa0fd05b067454f0d4686d8b155fe562ffe.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE log_file SET indexed_at = now()\n FROM unnest($1::text[], $2::timestamp[]) AS c(hostname, log_ts)\n WHERE log_file.hostname = c.hostname AND log_file.log_ts = c.log_ts", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "TimestampArray" + ] + }, + "nullable": [] + }, + "hash": "87e8e034b1cf5ea7ce43870d77d33fa0fd05b067454f0d4686d8b155fe562ffe" +} diff --git a/backend/.sqlx/query-1ea97f9085ec018f779e77e0fdbda3d4ecd67b3fbee9a58228ef577f846607ae.json b/backend/.sqlx/query-8ad79b80033b38ebddf6c8cd4d8cb160d41bac4c45a0fc74d9c9e96d3ef4486a.json similarity index 63% rename from backend/.sqlx/query-1ea97f9085ec018f779e77e0fdbda3d4ecd67b3fbee9a58228ef577f846607ae.json rename to backend/.sqlx/query-8ad79b80033b38ebddf6c8cd4d8cb160d41bac4c45a0fc74d9c9e96d3ef4486a.json index d453eb430d..3b9a2e2f34 100644 --- a/backend/.sqlx/query-1ea97f9085ec018f779e77e0fdbda3d4ecd67b3fbee9a58228ef577f846607ae.json +++ b/backend/.sqlx/query-8ad79b80033b38ebddf6c8cd4d8cb160d41bac4c45a0fc74d9c9e96d3ef4486a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3)", + "query": "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4))", "describe": { "columns": [ { @@ -13,12 +13,14 @@ "Left": [ "Text", "Jsonb", - "Text" + "Text", + "Text", + "Bool" ] }, "nullable": [ null ] }, - "hash": "1ea97f9085ec018f779e77e0fdbda3d4ecd67b3fbee9a58228ef577f846607ae" + "hash": "8ad79b80033b38ebddf6c8cd4d8cb160d41bac4c45a0fc74d9c9e96d3ef4486a" } diff --git a/backend/.sqlx/query-8d207cc9ed101ff116b617d25a94633c1531170ded1fdf09114718b941f5e1db.json b/backend/.sqlx/query-8d207cc9ed101ff116b617d25a94633c1531170ded1fdf09114718b941f5e1db.json deleted file mode 100644 index d0b96444ee..0000000000 --- a/backend/.sqlx/query-8d207cc9ed101ff116b617d25a94633c1531170ded1fdf09114718b941f5e1db.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n hostname,\n mode::text,\n worker_group,\n log_ts,\n file_path,\n ok_lines,\n err_lines,\n json_fmt\n FROM log_file\n WHERE log_ts > NOW() - make_interval(secs => $1)\n ORDER BY log_ts ASC LIMIT $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "hostname", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "mode", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "worker_group", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "log_ts", - "type_info": "Timestamp" - }, - { - "ordinal": 4, - "name": "file_path", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "ok_lines", - "type_info": "Int8" - }, - { - "ordinal": 6, - "name": "err_lines", - "type_info": "Int8" - }, - { - "ordinal": 7, - "name": "json_fmt", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Float8", - "Int8" - ] - }, - "nullable": [ - false, - null, - true, - false, - false, - true, - true, - true - ] - }, - "hash": "8d207cc9ed101ff116b617d25a94633c1531170ded1fdf09114718b941f5e1db" -} diff --git a/backend/.sqlx/query-8da59f1ace46dc9830cb0fc5a640df68f552e38d4b587839e0e41285a2d55455.json b/backend/.sqlx/query-8da59f1ace46dc9830cb0fc5a640df68f552e38d4b587839e0e41285a2d55455.json new file mode 100644 index 0000000000..610030cffc --- /dev/null +++ b/backend/.sqlx/query-8da59f1ace46dc9830cb0fc5a640df68f552e38d4b587839e0e41285a2d55455.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT ON (path) path, lock FROM script\n WHERE workspace_id = $1 AND NOT archived AND NOT deleted AND lock IS NOT NULL\n ORDER BY path, created_at DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "lock", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "8da59f1ace46dc9830cb0fc5a640df68f552e38d4b587839e0e41285a2d55455" +} diff --git a/backend/.sqlx/query-972df41db505fbbd20a558b200a2e2e8bc43633707d8365f73130c5bca3923b9.json b/backend/.sqlx/query-972df41db505fbbd20a558b200a2e2e8bc43633707d8365f73130c5bca3923b9.json new file mode 100644 index 0000000000..3624a32893 --- /dev/null +++ b/backend/.sqlx/query-972df41db505fbbd20a558b200a2e2e8bc43633707d8365f73130c5bca3923b9.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, edited_at)\n VALUES ('admins', $1, $2, $3, $4, now())\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET schema = EXCLUDED.schema, description = EXCLUDED.description,\n -- A fileset is a set of files, so it cannot also be one file.\n -- Create and update reject the pair; this writer bypasses both, so\n -- it declines the extension rather than persisting the forbidden\n -- combination onto a same-named local fileset.\n format_extension = CASE\n WHEN resource_type.is_fileset THEN NULL\n WHEN $5 THEN EXCLUDED.format_extension\n ELSE resource_type.format_extension END,\n edited_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Jsonb", + "Text", + "Varchar", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "972df41db505fbbd20a558b200a2e2e8bc43633707d8365f73130c5bca3923b9" +} diff --git a/backend/.sqlx/query-97691b88d43442e1f1984562372590428895b2bab67687c12efefe0b0d48881e.json b/backend/.sqlx/query-97691b88d43442e1f1984562372590428895b2bab67687c12efefe0b0d48881e.json new file mode 100644 index 0000000000..c7db67f4d6 --- /dev/null +++ b/backend/.sqlx/query-97691b88d43442e1f1984562372590428895b2bab67687c12efefe0b0d48881e.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO draft(workspace_id, path, typ, value, email) VALUES\n ('test-workspace', 'u/ext/s', 'script', '{}'::json, 'ext-jwt@windmill.dev'),\n ('test-workspace', 'u/two/s', 'script', '{}'::json, 'test2@windmill.dev'),\n ('test-workspace', 'u/three/s', 'script', '{}'::json, 'test3@windmill.dev')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "97691b88d43442e1f1984562372590428895b2bab67687c12efefe0b0d48881e" +} diff --git a/backend/.sqlx/query-9e503b65fe8bb1509f0d486829ce13ec9d93bb94192ddd85c3dacb9ff16cd032.json b/backend/.sqlx/query-9e503b65fe8bb1509f0d486829ce13ec9d93bb94192ddd85c3dacb9ff16cd032.json new file mode 100644 index 0000000000..0a0f8b4732 --- /dev/null +++ b/backend/.sqlx/query-9e503b65fe8bb1509f0d486829ce13ec9d93bb94192ddd85c3dacb9ff16cd032.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT email, value->>'summary' AS summary FROM draft WHERE path = 'u/two/s'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "summary", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + true, + null + ] + }, + "hash": "9e503b65fe8bb1509f0d486829ce13ec9d93bb94192ddd85c3dacb9ff16cd032" +} diff --git a/backend/.sqlx/query-a7d450e34084d561f69e588bd76fd56e616ee79d895b7dcc37ad9442789e1574.json b/backend/.sqlx/query-a7d450e34084d561f69e588bd76fd56e616ee79d895b7dcc37ad9442789e1574.json new file mode 100644 index 0000000000..a83c34e0db --- /dev/null +++ b/backend/.sqlx/query-a7d450e34084d561f69e588bd76fd56e616ee79d895b7dcc37ad9442789e1574.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH retired AS (\n UPDATE log_file SET indexed_at = now()\n WHERE indexed_at IS NULL\n AND log_ts <= NOW() - make_interval(secs => $1) RETURNING 1)\n SELECT count(*) AS \"n!\" FROM retired", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "n!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Float8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a7d450e34084d561f69e588bd76fd56e616ee79d895b7dcc37ad9442789e1574" +} diff --git a/backend/.sqlx/query-abdf62ef0e4eeb8c3213d2e8837e7032f710fe20dd272b7840c9bfbdb92554db.json b/backend/.sqlx/query-abdf62ef0e4eeb8c3213d2e8837e7032f710fe20dd272b7840c9bfbdb92554db.json new file mode 100644 index 0000000000..c3ef22f973 --- /dev/null +++ b/backend/.sqlx/query-abdf62ef0e4eeb8c3213d2e8837e7032f710fe20dd272b7840c9bfbdb92554db.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM script WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "abdf62ef0e4eeb8c3213d2e8837e7032f710fe20dd272b7840c9bfbdb92554db" +} diff --git a/backend/.sqlx/query-acf4a5f4b67ebd06157819677fee6be4cc3adc6aa859c3dbaa57fa50b63741cf.json b/backend/.sqlx/query-acf4a5f4b67ebd06157819677fee6be4cc3adc6aa859c3dbaa57fa50b63741cf.json new file mode 100644 index 0000000000..a741ee1c0d --- /dev/null +++ b/backend/.sqlx/query-acf4a5f4b67ebd06157819677fee6be4cc3adc6aa859c3dbaa57fa50b63741cf.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_free_token_daily_usage (day, cost_nanos, updated_at)\n VALUES ($2::date, GREATEST(0, $1::bigint), now())\n ON CONFLICT (day) DO UPDATE\n SET cost_nanos = GREATEST(0, ai_free_token_daily_usage.cost_nanos + $1::bigint),\n updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Date" + ] + }, + "nullable": [] + }, + "hash": "acf4a5f4b67ebd06157819677fee6be4cc3adc6aa859c3dbaa57fa50b63741cf" +} diff --git a/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json b/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json index b336210daf..b892061f56 100644 --- a/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json +++ b/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json @@ -98,12 +98,12 @@ null, null, null, - true, + false, null, null, null, - true, - true + false, + false ] }, "hash": "b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384" diff --git a/backend/.sqlx/query-c29d815cc38493f17950f12e9e5fe34b6c4ed1b2a49ed2b61dc9fb494026e956.json b/backend/.sqlx/query-c29d815cc38493f17950f12e9e5fe34b6c4ed1b2a49ed2b61dc9fb494026e956.json new file mode 100644 index 0000000000..7078c32f1a --- /dev/null +++ b/backend/.sqlx/query-c29d815cc38493f17950f12e9e5fe34b6c4ed1b2a49ed2b61dc9fb494026e956.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_free_token_usage (email, cost_nanos, updated_at)\n VALUES ($1, $2::bigint, now())\n ON CONFLICT (email) DO UPDATE\n SET cost_nanos = ai_free_token_usage.cost_nanos + $2::bigint,\n updated_at = now()\n RETURNING cost_nanos", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "cost_nanos", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "c29d815cc38493f17950f12e9e5fe34b6c4ed1b2a49ed2b61dc9fb494026e956" +} diff --git a/backend/.sqlx/query-c84087a0669d0b71829b0765c7274ca0a03fb823a781fb46d2b2b6cfc535a16b.json b/backend/.sqlx/query-c84087a0669d0b71829b0765c7274ca0a03fb823a781fb46d2b2b6cfc535a16b.json deleted file mode 100644 index ca39442f59..0000000000 --- a/backend/.sqlx/query-c84087a0669d0b71829b0765c7274ca0a03fb823a781fb46d2b2b6cfc535a16b.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT name, description FROM ai_skill WHERE workspace_id = $1 ORDER BY name", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "name", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "description", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - false - ] - }, - "hash": "c84087a0669d0b71829b0765c7274ca0a03fb823a781fb46d2b2b6cfc535a16b" -} diff --git a/backend/.sqlx/query-cc507843e00c83a42cc4a463999656ce9a8b0499d6b9282a3ecfae3164b17c2a.json b/backend/.sqlx/query-cc507843e00c83a42cc4a463999656ce9a8b0499d6b9282a3ecfae3164b17c2a.json new file mode 100644 index 0000000000..19dc4781a6 --- /dev/null +++ b/backend/.sqlx/query-cc507843e00c83a42cc4a463999656ce9a8b0499d6b9282a3ecfae3164b17c2a.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO lock_hash (workspace_id, path, lockfile_hash)\n SELECT $1, * FROM UNNEST($2::text[], $3::bigint[])\n ON CONFLICT (workspace_id, path) DO UPDATE SET lockfile_hash = EXCLUDED.lockfile_hash\n WHERE lock_hash.lockfile_hash IS DISTINCT FROM EXCLUDED.lockfile_hash", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "TextArray", + "Int8Array" + ] + }, + "nullable": [] + }, + "hash": "cc507843e00c83a42cc4a463999656ce9a8b0499d6b9282a3ecfae3164b17c2a" +} diff --git a/backend/.sqlx/query-f0070b36f7c4fc84dc9c23bb6c73d8ba80993a28b2c2e5df70968acf6d7cebe4.json b/backend/.sqlx/query-cfdd5ac1dfc7276fc37d49ddfe1b8880eaafb2d3fe71d75b676f1719e26f660f.json similarity index 72% rename from backend/.sqlx/query-f0070b36f7c4fc84dc9c23bb6c73d8ba80993a28b2c2e5df70968acf6d7cebe4.json rename to backend/.sqlx/query-cfdd5ac1dfc7276fc37d49ddfe1b8880eaafb2d3fe71d75b676f1719e26f660f.json index b31e532fc2..2947dd8413 100644 --- a/backend/.sqlx/query-f0070b36f7c4fc84dc9c23bb6c73d8ba80993a28b2c2e5df70968acf6d7cebe4.json +++ b/backend/.sqlx/query-cfdd5ac1dfc7276fc37d49ddfe1b8880eaafb2d3fe71d75b676f1719e26f660f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, job_perms, concurrency_key, log_file, metrics", + "query": "VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, job_perms, concurrency_key, log_file, metrics, otel_traces", "describe": { "columns": [], "parameters": { @@ -8,5 +8,5 @@ }, "nullable": [] }, - "hash": "f0070b36f7c4fc84dc9c23bb6c73d8ba80993a28b2c2e5df70968acf6d7cebe4" + "hash": "cfdd5ac1dfc7276fc37d49ddfe1b8880eaafb2d3fe71d75b676f1719e26f660f" } diff --git a/backend/.sqlx/query-daa5b57290cd1f821a53eebe96434f1befe6b16eee363c843faa1f836d53ca8d.json b/backend/.sqlx/query-daa5b57290cd1f821a53eebe96434f1befe6b16eee363c843faa1f836d53ca8d.json new file mode 100644 index 0000000000..d64f007d7b --- /dev/null +++ b/backend/.sqlx/query-daa5b57290cd1f821a53eebe96434f1befe6b16eee363c843faa1f836d53ca8d.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT mode::text AS \"mode!\", log_ts FROM log_file WHERE hostname = $1 AND file_path = $2 ORDER BY log_ts DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "mode!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "log_ts", + "type_info": "Timestamp" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null, + false + ] + }, + "hash": "daa5b57290cd1f821a53eebe96434f1befe6b16eee363c843faa1f836d53ca8d" +} diff --git a/backend/.sqlx/query-dd15827cbac128ec53cf03fa305e4cad34e540a2cc09c92e262491145a0de05a.json b/backend/.sqlx/query-dd15827cbac128ec53cf03fa305e4cad34e540a2cc09c92e262491145a0de05a.json new file mode 100644 index 0000000000..eba1b0da99 --- /dev/null +++ b/backend/.sqlx/query-dd15827cbac128ec53cf03fa305e4cad34e540a2cc09c92e262491145a0de05a.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM lock_hash WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "dd15827cbac128ec53cf03fa305e4cad34e540a2cc09c92e262491145a0de05a" +} diff --git a/backend/.sqlx/query-df13e7bb9c14aa19604c40754509f66af26042464ba199586838e073c318c53a.json b/backend/.sqlx/query-df13e7bb9c14aa19604c40754509f66af26042464ba199586838e073c318c53a.json new file mode 100644 index 0000000000..6ffd197e90 --- /dev/null +++ b/backend/.sqlx/query-df13e7bb9c14aa19604c40754509f66af26042464ba199586838e073c318c53a.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT is_fileset, format_extension FROM resource_type\n WHERE name = $1 AND workspace_id = $2 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "is_fileset", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "format_extension", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "df13e7bb9c14aa19604c40754509f66af26042464ba199586838e073c318c53a" +} diff --git a/backend/.sqlx/query-e3c4922e315b75951b5ea07cdfca4cfb32747b52dc62c9e3eccacf9c69e29b3a.json b/backend/.sqlx/query-e3c4922e315b75951b5ea07cdfca4cfb32747b52dc62c9e3eccacf9c69e29b3a.json new file mode 100644 index 0000000000..fc42c451f8 --- /dev/null +++ b/backend/.sqlx/query-e3c4922e315b75951b5ea07cdfca4cfb32747b52dc62c9e3eccacf9c69e29b3a.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT email FROM draft WHERE path = 'u/two/s'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + true + ] + }, + "hash": "e3c4922e315b75951b5ea07cdfca4cfb32747b52dc62c9e3eccacf9c69e29b3a" +} diff --git a/backend/.sqlx/query-e50afd5156b07e550202fb9b33354dce71b37f89f68d78577b250979daa1a87d.json b/backend/.sqlx/query-e50afd5156b07e550202fb9b33354dce71b37f89f68d78577b250979daa1a87d.json deleted file mode 100644 index 121d7fe04a..0000000000 --- a/backend/.sqlx/query-e50afd5156b07e550202fb9b33354dce71b37f89f68d78577b250979daa1a87d.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT name, description, instructions FROM ai_skill WHERE workspace_id = $1 AND name = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "name", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "description", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "instructions", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - false, - false - ] - }, - "hash": "e50afd5156b07e550202fb9b33354dce71b37f89f68d78577b250979daa1a87d" -} diff --git a/backend/.sqlx/query-f08ef43b4523c74fcbfc7879c147e1bf2c9a0d11e9841caafca62d85c9fb4c8f.json b/backend/.sqlx/query-f08ef43b4523c74fcbfc7879c147e1bf2c9a0d11e9841caafca62d85c9fb4c8f.json new file mode 100644 index 0000000000..32f7e8cf1f --- /dev/null +++ b/backend/.sqlx/query-f08ef43b4523c74fcbfc7879c147e1bf2c9a0d11e9841caafca62d85c9fb4c8f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_free_token_usage (email, cost_nanos, updated_at)\n VALUES ($1, GREATEST(0, $2::bigint), now())\n ON CONFLICT (email) DO UPDATE\n SET cost_nanos = GREATEST(0, ai_free_token_usage.cost_nanos + $2::bigint),\n updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "f08ef43b4523c74fcbfc7879c147e1bf2c9a0d11e9841caafca62d85c9fb4c8f" +} diff --git a/backend/.sqlx/query-92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b.json b/backend/.sqlx/query-f277db0459ff311d8a396aa4e03876dac75ba6df5f8eaa9f185300483e3ee36f.json similarity index 51% rename from backend/.sqlx/query-92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b.json rename to backend/.sqlx/query-f277db0459ff311d8a396aa4e03876dac75ba6df5f8eaa9f185300483e3ee36f.json index 7df22ca7b7..976cdfec06 100644 --- a/backend/.sqlx/query-92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b.json +++ b/backend/.sqlx/query-f277db0459ff311d8a396aa4e03876dac75ba6df5f8eaa9f185300483e3ee36f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt)\n VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8)\n ON CONFLICT (hostname, log_ts) DO UPDATE SET ok_lines = log_file.ok_lines + $6, err_lines = log_file.err_lines + $7", + "query": "INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt)\n VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8)\n ON CONFLICT (hostname, log_ts) DO UPDATE SET ok_lines = log_file.ok_lines + $6, err_lines = log_file.err_lines + $7", "describe": { "columns": [], "parameters": { @@ -17,5 +17,5 @@ }, "nullable": [] }, - "hash": "92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b" + "hash": "f277db0459ff311d8a396aa4e03876dac75ba6df5f8eaa9f185300483e3ee36f" } diff --git a/backend/.sqlx/query-e99fe5cd3283f1701d3a361ef31869da89fd10099b76669b9526201c85f71f61.json b/backend/.sqlx/query-f47cafb6e9a6ec58ccafb06cf5e806e3fe749119214863b9111b58fff0bb9583.json similarity index 51% rename from backend/.sqlx/query-e99fe5cd3283f1701d3a361ef31869da89fd10099b76669b9526201c85f71f61.json rename to backend/.sqlx/query-f47cafb6e9a6ec58ccafb06cf5e806e3fe749119214863b9111b58fff0bb9583.json index 29ce0c6f2c..cce644b300 100644 --- a/backend/.sqlx/query-e99fe5cd3283f1701d3a361ef31869da89fd10099b76669b9526201c85f71f61.json +++ b/backend/.sqlx/query-f47cafb6e9a6ec58ccafb06cf5e806e3fe749119214863b9111b58fff0bb9583.json @@ -1,23 +1,22 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM ai_skill WHERE workspace_id = $1 AND name = $2 RETURNING name", + "query": "SELECT plan FROM workspace_settings WHERE workspace_id = $1", "describe": { "columns": [ { "ordinal": 0, - "name": "name", + "name": "plan", "type_info": "Varchar" } ], "parameters": { "Left": [ - "Text", "Text" ] }, "nullable": [ - false + true ] }, - "hash": "e99fe5cd3283f1701d3a361ef31869da89fd10099b76669b9526201c85f71f61" + "hash": "f47cafb6e9a6ec58ccafb06cf5e806e3fe749119214863b9111b58fff0bb9583" } diff --git a/backend/.sqlx/query-f496f06f117f5c91104fc759df35b883070da938dea7c7975332afb876cc4691.json b/backend/.sqlx/query-f496f06f117f5c91104fc759df35b883070da938dea7c7975332afb876cc4691.json new file mode 100644 index 0000000000..24f957a4e2 --- /dev/null +++ b/backend/.sqlx/query-f496f06f117f5c91104fc759df35b883070da938dea7c7975332afb876cc4691.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM draft ORDER BY path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "f496f06f117f5c91104fc759df35b883070da938dea7c7975332afb876cc4691" +} diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 0d9639c6c0..7bd84dfca3 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -131,9 +131,11 @@ minimal explicit set for dev. ## Workspace object storage in dev — use the local filesystem For a dev workspace you don't need MinIO/S3: use the built-in **`FilesystemStorage`** large-file -storage (a root path on local disk). It is intentionally hidden from the settings-UI storage -dropdown (dev-only), so set it via the API. Requires the backend built with `parquet` (+ `private` -for the real S3 helpers, + `enterprise` if you want advanced permission rules enforced): +storage (a root path on local disk). It is a **debug-build affordance only** — every site that +builds a filesystem object store calls `ensure_filesystem_storage_allowed`, so release builds +refuse it, and the settings UI never offers it — so set it via the API on a `cargo run`/`cargo +test` binary. Requires the backend built with `parquet` (+ `private` for the real S3 helpers, ++ `enterprise` if you want advanced permission rules enforced): ```bash curl -X POST "$BASE/api/w//workspaces/edit_large_file_storage_config" \ diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 2721fc7283..d314168b35 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -23,7 +23,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "generic-array", ] @@ -261,13 +261,13 @@ dependencies = [ [[package]] name = "argon2" -version = "0.5.3" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" +checksum = "134c52ddac6d63c576bef8168db10c83c49c26444ecbc68060fef078925a901c" dependencies = [ "base64ct", - "blake2", - "cpufeatures 0.2.17", + "blake2 0.11.0", + "cpufeatures 0.3.1", "password-hash", ] @@ -430,7 +430,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 2.14.0", + "indexmap 2.14.1", "lexical-core", "memchr", "num", @@ -970,9 +970,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.18.0" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e" +checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -981,9 +981,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.44.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09fae7be8bb3174e05c6afdb34199e6dc0c7c04ba9fa237b1967adfbde27483" +checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" dependencies = [ "cc", "cmake", @@ -1338,7 +1338,7 @@ dependencies = [ "http 1.5.0", "http-body 0.4.6", "hyper 0.14.32", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-rustls 0.24.2", "hyper-rustls 0.27.9", "hyper-util", @@ -1532,7 +1532,7 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "itoa", "matchit 0.8.4", @@ -1629,7 +1629,7 @@ dependencies = [ "addr2line", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.8.9", "object 0.37.3", "rustc-demangle", "windows-link 0.2.1", @@ -1729,7 +1729,7 @@ dependencies = [ "clang-sys", "itertools 0.13.0", "log", - "prettyplease", + "prettyplease 0.2.37", "proc-macro2", "quote", "regex", @@ -1749,7 +1749,7 @@ dependencies = [ "clang-sys", "itertools 0.13.0", "log", - "prettyplease", + "prettyplease 0.2.37", "proc-macro2", "quote", "regex", @@ -1824,6 +1824,15 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "blake2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b5d4d889834ee8ecfc0f8426ad30faf7cdcb10f741a8e6d7224d95325479f6f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "blake3" version = "1.8.7" @@ -1834,7 +1843,7 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq 0.4.2", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", ] [[package]] @@ -1856,6 +1865,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "block-modes" version = "0.8.1" @@ -1908,7 +1926,7 @@ dependencies = [ "hex", "http 1.5.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-named-pipe", "hyper-util", "hyperlocal", @@ -1940,34 +1958,32 @@ dependencies = [ [[package]] name = "bon" -version = "3.9.3" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a602c73c7b0148ec6d12af6fd5cc7a46e2eacc8878271a999abac56eed12f561" +checksum = "9e3fac94a66da67200398458a25412bcc3f9b6443b5119a6cad9cf3ccfcd8cc6" dependencies = [ "bon-macros", - "rustversion", ] [[package]] name = "bon-macros" -version = "3.9.3" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" +checksum = "d4654961ad0494e4774c5c60b4cb4cd0ae9b9d92d039d901638b1dba97ebebf5" dependencies = [ - "darling 0.23.0", + "darling 0.24.1", "ident_case", - "prettyplease", + "prettyplease 0.3.0", "proc-macro2", "quote", - "rustversion", - "syn 2.0.119", + "syn 3.0.4", ] [[package]] name = "borsh" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88b7ea17d208c4193f2c1e6de3c35fe71f98c96982d5ced308bdcc749ff6e1f" +checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a" dependencies = [ "borsh-derive", "bytes", @@ -1976,15 +1992,15 @@ dependencies = [ [[package]] name = "borsh-derive" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8f347189c62a579b8cd5f80714efa178f52e461dc2e6d701d264f5ff22e566c" +checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56" dependencies = [ "once_cell", "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.4", ] [[package]] @@ -2334,12 +2350,12 @@ checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] name = "chacha20" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", "rand_core 0.10.1", ] @@ -2392,7 +2408,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "inout", ] @@ -2456,6 +2472,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "cms" version = "0.2.3" @@ -2476,9 +2498,9 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "combine" -version = "4.6.7" +version = "4.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" dependencies = [ "bytes", "memchr", @@ -2678,9 +2700,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" dependencies = [ "libc", ] @@ -2820,6 +2842,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "csv" version = "1.4.0" @@ -2850,6 +2881,15 @@ dependencies = [ "cipher 0.4.4", ] +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "curl-sys" version = "0.4.90+curl-8.21.0" @@ -3207,7 +3247,7 @@ dependencies = [ "base64 0.22.1", "half", "hashbrown 0.14.5", - "indexmap 2.14.0", + "indexmap 2.14.1", "libc", "log", "object_store", @@ -3386,7 +3426,7 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", - "indexmap 2.14.0", + "indexmap 2.14.1", "paste", "recursive", "serde_json", @@ -3401,7 +3441,7 @@ checksum = "422ac9cf3b22bbbae8cdf8ceb33039107fde1b5492693168f13bd566b1bcc839" dependencies = [ "arrow", "datafusion-common", - "indexmap 2.14.0", + "indexmap 2.14.1", "itertools 0.14.0", "paste", ] @@ -3415,7 +3455,7 @@ dependencies = [ "arrow", "arrow-buffer", "base64 0.22.1", - "blake2", + "blake2 0.10.6", "blake3", "chrono", "datafusion-common", @@ -3555,7 +3595,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-physical-expr", - "indexmap 2.14.0", + "indexmap 2.14.1", "itertools 0.14.0", "log", "recursive", @@ -3578,7 +3618,7 @@ dependencies = [ "datafusion-physical-expr-common", "half", "hashbrown 0.14.5", - "indexmap 2.14.0", + "indexmap 2.14.1", "itertools 0.14.0", "log", "paste", @@ -3640,7 +3680,7 @@ dependencies = [ "futures", "half", "hashbrown 0.14.5", - "indexmap 2.14.0", + "indexmap 2.14.1", "itertools 0.14.0", "log", "parking_lot", @@ -3682,7 +3722,7 @@ dependencies = [ "bigdecimal", "datafusion-common", "datafusion-expr", - "indexmap 2.14.0", + "indexmap 2.14.1", "log", "recursive", "regex", @@ -3691,9 +3731,9 @@ dependencies = [ [[package]] name = "datasketches" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745" +checksum = "46c4cf71a36b46dcfc00e5014c0c20ccad2b1b6a008304d7d57d2749b2d41b3d" [[package]] name = "debug-helper" @@ -3813,7 +3853,7 @@ dependencies = [ "deno_path_util", "deno_unsync", "futures", - "indexmap 2.14.0", + "indexmap 2.14.1", "libc", "parking_lot", "percent-encoding", @@ -3955,7 +3995,7 @@ dependencies = [ "hickory-resolver", "http 1.5.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-rustls 0.27.9", "hyper-util", "ipnet", @@ -4082,7 +4122,7 @@ version = "0.228.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bf8dbe5abf37d270bb853c5dfe45fbe3b1b6c453877cc11d7fe84e9862a6dbc" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.1", "proc-macro-rules", "proc-macro2", "quote", @@ -4158,7 +4198,7 @@ dependencies = [ "deno_error 0.6.1", "deno_tls", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-rustls 0.27.9", "hyper-util", "log", @@ -4457,10 +4497,21 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "const-oid", - "crypto-common", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "dirs" version = "4.0.0" @@ -5032,13 +5083,13 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" dependencies = [ "crc32fast", "libz-sys", - "miniz_oxide", + "miniz_oxide 0.9.1", "zlib-rs", ] @@ -5124,6 +5175,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "frostem" +version = "1.20260821.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "481ace7f781f5ae54a5c0a6d6d8edb30adba737cfa1230fbd5632d63ba8dfd80" + [[package]] name = "fs3" version = "0.5.0" @@ -5698,9 +5755,9 @@ dependencies = [ [[package]] name = "gosyn" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9758a950dc61a15bc65162f72f5bec7e8efde91f102dc7dce7317f67019a6ba9" +checksum = "28f4ed3cbcb66ddad22553bb3b94087573cf492a34501bc025a23167ed57e856" dependencies = [ "anyhow", "strum", @@ -5740,7 +5797,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.14.0", + "indexmap 2.14.1", "slab", "tokio", "tokio-util", @@ -5759,7 +5816,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.5.0", - "indexmap 2.14.0", + "indexmap 2.14.1", "slab", "tokio", "tokio-util", @@ -5841,7 +5898,7 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd1246c0e5493286aeb2dde35b1f4eb9c4ce00e628641210a5e553fc001a1f26" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.1", "proc-macro2", "quote", "syn 2.0.119", @@ -5894,9 +5951,9 @@ checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "hermit-abi" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" [[package]] name = "hex" @@ -6111,7 +6168,7 @@ dependencies = [ "futures", "http 1.5.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-rustls 0.26.0", "hyper-tls", "hyper-tungstenite", @@ -6135,6 +6192,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "0.14.32" @@ -6161,9 +6227,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" dependencies = [ "atomic-waker", "bytes", @@ -6191,7 +6257,7 @@ dependencies = [ "futures-util", "headers", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-rustls 0.27.9", "hyper-util", "pin-project-lite", @@ -6207,7 +6273,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fab3637d6b04a8037af8a266fdf6cf92ea957e8c53981a2bf6136572531025bf" dependencies = [ "hex", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "pin-project-lite", "tokio", @@ -6238,7 +6304,7 @@ checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" dependencies = [ "futures-util", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "log", "rustls 0.22.4", @@ -6256,7 +6322,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "log", "rustls 0.23.35", @@ -6273,7 +6339,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "pin-project-lite", "tokio", @@ -6288,7 +6354,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "native-tls", "tokio", @@ -6303,7 +6369,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a343d17fe7885302ed7252767dc7bb83609a874b6ff581142241ec4b73957ad" dependencies = [ "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "pin-project-lite", "tokio", @@ -6323,7 +6389,7 @@ dependencies = [ "futures-util", "http 1.5.0", "http-body 1.1.0", - "hyper 1.11.0", + "hyper 1.11.1", "ipnet", "libc", "percent-encoding", @@ -6344,7 +6410,7 @@ checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" dependencies = [ "hex", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "pin-project-lite", "tokio", @@ -6504,9 +6570,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.14.0" +version = "2.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "07aa2048142242915a31d35844fb311e0e53fcca590c3a0a40dcf1b841fa09eb" dependencies = [ "equivalent", "hashbrown 0.17.1", @@ -6892,7 +6958,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.1", ] [[package]] @@ -6948,7 +7014,7 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-http-proxy", "hyper-rustls 0.27.9", "hyper-timeout", @@ -7196,9 +7262,9 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.20" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d0a00925a9f930d679b6789b721e3a7f9ed110f41b86d2497caa780c3a070a" +checksum = "8d8f1ea3f21fd3405dcaf6c9b5c1630af9afc422d9073ea39c5f6d6c772e08ed" dependencies = [ "bitflags 2.13.1", "libc", @@ -7321,18 +7387,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.16.4" +version = "0.18.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" -dependencies = [ - "hashbrown 0.16.1", -] - -[[package]] -name = "lru" -version = "0.18.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d2f2f9b4ba7e6b24d95e7e899329d35be83bcded72c8540cdd5368932d1d90a" +checksum = "ff9840bcc50b71349309900da0ce7279aa336ae71d73250b07998932c7d97c25" dependencies = [ "hashbrown 0.17.1", ] @@ -7363,9 +7420,9 @@ dependencies = [ [[package]] name = "lz4_flex" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" [[package]] name = "lzma-sys" @@ -7703,6 +7760,15 @@ name = "miniz_oxide" version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" dependencies = [ "adler2", "simd-adler32", @@ -7710,9 +7776,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" dependencies = [ "libc", "wasi 0.11.1+wasi-snapshot-preview1", @@ -7780,9 +7846,9 @@ dependencies = [ [[package]] name = "murmurhash32" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" +checksum = "a8afc6df942f4c022d70c5725e18df1a705773870e5b7f96fde94ca0334ce77a" [[package]] name = "mysql-common-derive" @@ -7804,10 +7870,11 @@ dependencies = [ [[package]] name = "mysql_async" -version = "0.37.0" +version = "0.37.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3519e91b0d254ac1ffa495bc42053286cb2172ad7241d5b3b1b9f8a891f21ee2" +checksum = "40d11da0e2d9fad4640c9f9198ee431c6d68444568f83ef1f10f3367270071e4" dependencies = [ + "arc-swap", "bytes", "crossbeam-queue", "crossbeam-utils", @@ -7816,7 +7883,7 @@ dependencies = [ "futures-sink", "futures-util", "keyed_priority_queue", - "lru 0.18.2", + "lru 0.18.4", "mysql_common", "native-tls", "pem 3.0.6", @@ -8058,7 +8125,7 @@ dependencies = [ "dirs-sys 0.4.1", "fancy-regex 0.14.0", "heck", - "indexmap 2.14.0", + "indexmap 2.14.1", "log", "lru 0.12.5", "miette", @@ -8236,7 +8303,7 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" dependencies = [ - "hermit-abi 0.5.2", + "hermit-abi 0.5.3", "libc", ] @@ -8321,7 +8388,7 @@ dependencies = [ "http-body-util", "httparse", "humantime", - "hyper 1.11.0", + "hyper 1.11.1", "itertools 0.14.0", "md-5 0.10.6", "parking_lot", @@ -8773,16 +8840,16 @@ checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "ownedbytes" version = "0.9.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "stable_deref_trait", ] [[package]] name = "owo-colors" -version = "4.3.0" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" +checksum = "13c45bb4a6ae1280ec0803b1ef9d3455eb50f01efbbe1447ab020f1d54fba9d8" [[package]] name = "p12-keystore" @@ -8920,13 +8987,12 @@ dependencies = [ [[package]] name = "password-hash" -version = "0.5.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" +checksum = "aab41826031698d6ffcd9cff78ef56ef998e39dc7e5067cdfebe373842d4723b" dependencies = [ - "base64ct", - "rand_core 0.6.4", - "subtle", + "getrandom 0.4.3", + "phc", ] [[package]] @@ -9058,7 +9124,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset", - "indexmap 2.14.0", + "indexmap 2.14.1", ] [[package]] @@ -9070,6 +9136,17 @@ dependencies = [ "phf 0.11.3", ] +[[package]] +name = "phc" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44dc769b75f93afdddd8c7fa12d685292ddeff1e66f7f0f3a234cf1818afe892" +dependencies = [ + "base64ct", + "ctutils", + "getrandom 0.4.3", +] + [[package]] name = "phf" version = "0.11.3" @@ -9313,7 +9390,7 @@ checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" dependencies = [ "cfg-if", "concurrent-queue", - "hermit-abi 0.5.2", + "hermit-abi 0.5.3", "pin-project-lite", "rustix 1.1.4", "windows-sys 0.61.2", @@ -9442,6 +9519,16 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "prettyplease" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" +dependencies = [ + "proc-macro2", + "syn 3.0.4", +] + [[package]] name = "primeorder" version = "0.13.6" @@ -9533,7 +9620,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3ef4f2f0422f23a82ec9f628ea2acd12871c81a9362b02c43c1aa86acfc3ba1" dependencies = [ "futures", - "indexmap 2.14.0", + "indexmap 2.14.1", "nix 0.30.1", "tokio", "tracing", @@ -10204,7 +10291,7 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-rustls 0.27.9", "hyper-tls", "hyper-util", @@ -10252,7 +10339,7 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-rustls 0.27.9", "hyper-util", "js-sys", @@ -10308,7 +10395,7 @@ dependencies = [ "futures", "getrandom 0.2.17", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "reqwest 0.13.4", "reqwest-middleware", "retry-policies", @@ -10437,9 +10524,9 @@ dependencies = [ [[package]] name = "rmcp-macros" -version = "3.1.4" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a85d45508e9b4ba024fe996c2638799635d75b6dd0ba8f32ccf08f8026f0c780" +checksum = "cdf1c49bd4d52014b94db0877410db273c2008f01628b0252a2e9460ad9b7fda" dependencies = [ "darling 0.24.1", "proc-macro2", @@ -10479,7 +10566,7 @@ dependencies = [ "convert_case 0.10.0", "fnv", "ident_case", - "indexmap 2.14.0", + "indexmap 2.14.1", "proc-macro-crate", "proc-macro2", "quote", @@ -10571,16 +10658,6 @@ dependencies = [ "walkdir", ] -[[package]] -name = "rust-stemmers" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" -dependencies = [ - "serde", - "serde_derive", -] - [[package]] name = "rust_decimal" version = "1.42.1" @@ -11322,7 +11399,7 @@ version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.1", "itoa", "memchr", "serde", @@ -11416,7 +11493,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.14.0", + "indexmap 2.14.1", "jiff", "schemars 0.9.0", "schemars 1.2.2", @@ -11444,7 +11521,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.1", "itoa", "ryu", "serde", @@ -11457,7 +11534,7 @@ version = "0.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59e2dd588bf1597a252c3b920e0143eb99b0f76e4e082f4c92ce34fbc9e71ddd" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.1", "itoa", "libyml", "memchr", @@ -11677,9 +11754,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.2" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f" dependencies = [ "serde", ] @@ -11877,7 +11954,7 @@ dependencies = [ "futures-util", "hashbrown 0.15.5", "hashlink", - "indexmap 2.14.0", + "indexmap 2.14.1", "log", "memchr", "once_cell", @@ -12234,7 +12311,7 @@ checksum = "72e90b52ee734ded867104612218101722ad87ff4cf74fe30383bd244a533f97" dependencies = [ "anyhow", "bytes-str", - "indexmap 2.14.0", + "indexmap 2.14.1", "serde", "serde_json", "swc_config_macro", @@ -12368,7 +12445,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c6f1b8f4232e7a7f614ff7c0f6ccb89c2d028cdf7629f79ad710cff5b28b62c" dependencies = [ "better_scoped_tls", - "indexmap 2.14.0", + "indexmap 2.14.1", "once_cell", "par-core", "phf 0.11.3", @@ -12434,7 +12511,7 @@ checksum = "69ea0052ac23b5b9fbc85bbdb1791b36b918f9d55f594b0ed8e25babb4c32d16" dependencies = [ "base64 0.22.1", "bytes-str", - "indexmap 2.14.0", + "indexmap 2.14.1", "once_cell", "rustc-hash 2.1.3", "serde", @@ -12474,7 +12551,7 @@ version = "21.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83259addd99ed4022aa9fc4d39428c008d3d42533769e1a005529da18cde4568" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.1", "num_cpus", "once_cell", "par-core", @@ -12714,12 +12791,12 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" [[package]] name = "tantivy" -version = "0.26.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.27.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "aho-corasick", "arc-swap", - "base64 0.22.1", + "base64 0.23.1", "bitpacking", "bon", "byteorder", @@ -12730,20 +12807,20 @@ dependencies = [ "downcast-rs", "fastdivide", "fnv", + "frostem", "fs4", "htmlescape", "itertools 0.14.0", "levenshtein_automata", "log", - "lru 0.16.4", - "lz4_flex 0.13.1", + "lru 0.18.4", + "lz4_flex 0.14.0", "measure_time", "memmap2", "once_cell", "oneshot", "rayon", "regex", - "rust-stemmers", "rustc-hash 2.1.3", "serde", "serde_json", @@ -12760,22 +12837,23 @@ dependencies = [ "thiserror 2.0.20", "time", "typetag", + "unwrap-infallible", "uuid", "winapi", ] [[package]] name = "tantivy-bitpacker" -version = "0.9.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.10.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "bitpacking", ] [[package]] name = "tantivy-columnar" -version = "0.6.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.7.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "downcast-rs", "fastdivide", @@ -12789,8 +12867,8 @@ dependencies = [ [[package]] name = "tantivy-common" -version = "0.10.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.11.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "async-trait", "byteorder", @@ -12812,8 +12890,8 @@ dependencies = [ [[package]] name = "tantivy-query-grammar" -version = "0.25.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.26.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "fnv", "nom", @@ -12824,8 +12902,8 @@ dependencies = [ [[package]] name = "tantivy-sstable" -version = "0.6.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.7.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "futures-util", "itertools 0.14.0", @@ -12837,8 +12915,8 @@ dependencies = [ [[package]] name = "tantivy-stacker" -version = "0.6.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.7.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "murmurhash32", "tantivy-common", @@ -12846,8 +12924,8 @@ dependencies = [ [[package]] name = "tantivy-tokenizer-api" -version = "0.6.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.7.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "serde", ] @@ -13472,7 +13550,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.1", "serde", "serde_spanned", "toml_datetime 0.6.11", @@ -13485,7 +13563,7 @@ version = "0.25.13+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.1", "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", "winnow 1.0.4", @@ -13516,7 +13594,7 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-timeout", "hyper-util", "percent-encoding", @@ -13548,7 +13626,7 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-timeout", "hyper-util", "percent-encoding", @@ -13593,7 +13671,7 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", - "indexmap 2.14.0", + "indexmap 2.14.1", "pin-project-lite", "slab", "sync_wrapper", @@ -13808,9 +13886,9 @@ dependencies = [ [[package]] name = "tree-sitter-language" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782" +checksum = "ca0d1bf6fdd806e43ae5198f82f527056d359def39e54e67a0f478ac09dac081" [[package]] name = "tree-sitter-r" @@ -13901,9 +13979,9 @@ dependencies = [ [[package]] name = "twox-hash" -version = "2.1.3" +version = "2.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" +checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a" [[package]] name = "typed-path" @@ -14140,7 +14218,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "subtle", ] @@ -14168,6 +14246,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "unwrap-infallible" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e497bb1f828cc9fb236722c2eaa100dcf201563f38f4da6252357a59037adf31" + [[package]] name = "ureq" version = "2.12.1" @@ -14251,9 +14335,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.25.0" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f053576934f05a761a402421fbbe3d425d9366f75f978806a037b3ca481abecc" +checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -14272,7 +14356,7 @@ dependencies = [ "fslock", "gzip-header", "home", - "miniz_oxide", + "miniz_oxide 0.8.9", "paste", "which 6.0.3", ] @@ -14663,7 +14747,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "async-nats", @@ -14748,7 +14832,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.796.0" +version = "1.803.0" dependencies = [ "async-stream", "async-trait", @@ -14781,7 +14865,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.796.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14794,7 +14878,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "argon2", @@ -14824,8 +14908,8 @@ dependencies = [ "hex", "hmac", "http 1.5.0", - "hyper 1.11.0", - "indexmap 2.14.0", + "hyper 1.11.1", + "indexmap 2.14.1", "itertools 0.14.0", "jsonwebtoken 8.3.0", "lazy_static", @@ -14934,12 +15018,12 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.796.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "lazy_static", "quick_cache", "serde", @@ -14957,7 +15041,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.796.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14974,7 +15058,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15000,7 +15084,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.796.0" +version = "1.803.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15010,7 +15094,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.796.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15027,7 +15111,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.796.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -15049,7 +15133,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15072,7 +15156,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.796.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15088,11 +15172,11 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.796.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", - "hyper 1.11.0", + "hyper 1.11.1", "serde", "serde_json", "sql-builder", @@ -15110,7 +15194,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.796.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15131,7 +15215,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.796.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15145,7 +15229,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "async-nats", @@ -15180,14 +15264,14 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "axum 0.8.9", "base64 0.22.1", "chrono", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "lazy_static", "serde", "serde_json", @@ -15205,7 +15289,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.796.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15233,12 +15317,12 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "axum 0.8.9", "http 1.5.0", - "indexmap 2.14.0", + "indexmap 2.14.1", "itertools 0.14.0", "lazy_static", "serde", @@ -15255,7 +15339,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.796.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15275,13 +15359,13 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.796.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", "futures", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "itertools 0.14.0", "lazy_static", "prometheus", @@ -15313,7 +15397,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15341,7 +15425,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.796.0" +version = "1.803.0" dependencies = [ "lazy_static", "serde", @@ -15353,13 +15437,13 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.796.0" +version = "1.803.0" dependencies = [ "argon2", "axum 0.8.9", "chrono", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "lazy_static", "serde", "serde_json", @@ -15377,7 +15461,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.796.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15391,14 +15475,14 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.796.0" +version = "1.803.0" dependencies = [ "axum 0.8.9", "chrono", "futures", "hex", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "lazy_static", "magic-crypt", "regex", @@ -15426,7 +15510,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.796.0" +version = "1.803.0" dependencies = [ "chrono", "lazy_static", @@ -15440,7 +15524,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15459,7 +15543,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.796.0" +version = "1.803.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15496,8 +15580,8 @@ dependencies = [ "globset", "hex", "hmac", - "hyper 1.11.0", - "indexmap 2.14.0", + "hyper 1.11.1", + "indexmap 2.14.1", "itertools 0.14.0", "jsonwebtoken 8.3.0", "lazy_static", @@ -15563,9 +15647,10 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.796.0" +version = "1.803.0" dependencies = [ "chrono", + "futures", "itertools 0.14.0", "lazy_static", "serde", @@ -15582,7 +15667,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.796.0" +version = "1.803.0" dependencies = [ "regex", "serde", @@ -15597,16 +15682,18 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "astral-tokio-tar", "bytes", "chrono", "const_format", + "datafusion", "flume", "futures", "lazy_static", + "object_store", "serde", "serde_json", "sqlx", @@ -15614,6 +15701,7 @@ dependencies = [ "tempfile", "tokio", "tracing", + "url", "uuid", "windmill-common", "windmill-object-store", @@ -15621,7 +15709,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "futures", @@ -15638,7 +15726,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.796.0" +version = "1.803.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15654,7 +15742,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -15675,7 +15763,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -15706,7 +15794,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "arc-swap", @@ -15731,7 +15819,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "async-stream", @@ -15765,7 +15853,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "futures", @@ -15783,7 +15871,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.796.0" +version = "1.803.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15792,7 +15880,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -15804,7 +15892,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "serde_json", @@ -15816,7 +15904,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "gosyn", @@ -15828,7 +15916,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -15840,7 +15928,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "serde_json", @@ -15852,7 +15940,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "nu-parser", @@ -15863,7 +15951,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15874,7 +15962,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15886,7 +15974,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15897,7 +15985,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "async-recursion", @@ -15919,7 +16007,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "serde_json", @@ -15931,7 +16019,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -15945,7 +16033,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15962,7 +16050,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -15975,7 +16063,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "serde", @@ -15987,7 +16075,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -16005,7 +16093,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16021,7 +16109,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16037,7 +16125,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -16051,7 +16139,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "async-recursion", @@ -16090,7 +16178,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "const_format", @@ -16130,7 +16218,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.796.0" +version = "1.803.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16141,7 +16229,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "async-recursion", @@ -16152,7 +16240,7 @@ dependencies = [ "futures", "hex", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "lazy_static", "magic-crypt", "quick_cache", @@ -16176,7 +16264,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16200,14 +16288,14 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", "axum 0.8.9", "chrono", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "itertools 0.14.0", "lazy_static", "rand 0.9.0", @@ -16233,7 +16321,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16260,7 +16348,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16293,7 +16381,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16313,7 +16401,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16347,7 +16435,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16359,7 +16447,7 @@ dependencies = [ "hex", "hmac", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "itertools 0.14.0", "lazy_static", "matchit 0.7.3", @@ -16383,7 +16471,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16406,7 +16494,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16430,7 +16518,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "async-nats", @@ -16454,7 +16542,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16489,7 +16577,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16517,7 +16605,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "async-trait", @@ -16542,7 +16630,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "bitflags 2.13.1", @@ -16561,7 +16649,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "async-once-cell", @@ -16678,7 +16766,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.796.0" +version = "1.803.0" dependencies = [ "bytes", "futures", @@ -17477,7 +17565,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c42e33efc22a0650c311c2ef19115ce232583abbe80850bc8b66509ebef02de0" dependencies = [ "crc32fast", - "indexmap 2.14.0", + "indexmap 2.14.1", "memchr", "typed-path", ] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 6dd9e452b8..f432f022c5 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.796.0" +version = "1.803.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.796.0" +version = "1.803.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -115,7 +115,7 @@ strip = "none" [features] default = [] -private = ["windmill-api/private", "windmill-api-agent-workers?/private", "windmill-autoscaling/private", "windmill-common/private", "windmill-dep-map/private", "windmill-object-store/private", "windmill-git-sync/private", "windmill-indexer/private", "windmill-operator?/private", "windmill-queue/private", "windmill-worker/private", "windmill-test-utils/private"] +private = ["windmill-api/private", "windmill-api-agent-workers?/private", "windmill-autoscaling/private", "windmill-common/private", "windmill-dep-map/private", "windmill-object-store/private", "windmill-git-sync/private", "windmill-indexer?/private", "windmill-operator?/private", "windmill-queue/private", "windmill-worker/private", "windmill-test-utils/private"] agent_worker_server = ["windmill-api/agent_worker_server", "dep:windmill-api-agent-workers", "windmill-test-utils/agent_worker_server"] enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "windmill-api-agent-workers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise", "windmill-object-store/enterprise", "license"] local_reports = ["windmill-common/local_reports"] @@ -131,6 +131,10 @@ quickjs = ["windmill-worker/quickjs", "windmill-api/quickjs"] openidconnect = ["windmill-api/openidconnect", "windmill-common/openidconnect", "windmill-object-store/openidconnect"] cloud = ["windmill-queue/cloud", "windmill-worker/cloud", "windmill-common/cloud", "windmill-api/cloud"] jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"] +# `tantivy` must stay the only feature that enables windmill-indexer: it is the only one that +# also gives it `enterprise` + `parquet`, and its EE sources gate nearly everything on that +# pair, so a bare windmill-indexer is a crate of dead code that `-D warnings` rejects. Any +# other feature wanting one of its features has to use the optional `windmill-indexer?/` form. tantivy = ["dep:windmill-indexer", "windmill-api/tantivy", "windmill-indexer/enterprise", "windmill-indexer/parquet", "windmill-common/tantivy", "enterprise", "parquet"] sqlx = ["windmill-worker/sqlx"] deno_core = ["windmill-worker/deno_core", "dep:windmill-runtime-nativets", "windmill-test-utils/deno_core"] @@ -477,7 +481,10 @@ rust-embed = { version = "^6", features = ["interpolate-folder-path"] } mime_guess = "^2" hex = "^0" sql-builder = "^3" -argon2 = "^0" +# Minor-pinned rather than the `^0` used elsewhere in this file: argon2's 0.x +# minors are API-breaking (0.6 moved `SaltString` into `phc`, put `rand_core` +# behind a feature and changed `hash_password`), so a float breaks the build. +argon2 = "0.6" quick_cache = "^0" rand = "=0.9.0" rand_core = { version = "^0", features = ["std"] } @@ -697,7 +704,7 @@ tikv-jemalloc-ctl = { version = "^0.5" } triomphe = "^0" pin-project-lite = "^0" -tantivy = { git="https://github.com/windmill-labs/tantivy", rev="6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" } +tantivy = { git="https://github.com/windmill-labs/tantivy", rev="ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" } backon = "1.3.0" diff --git a/backend/THREAT_MODEL.md b/backend/THREAT_MODEL.md index 0468ebf721..f5725d65a3 100644 --- a/backend/THREAT_MODEL.md +++ b/backend/THREAT_MODEL.md @@ -85,7 +85,7 @@ published advisory history (73 GHSA advisories, several rated 9.9 critical). | EP9 Worker sandbox | nsjail / unshare / dind / rootless podman isolating user code | user code → host & cross-tenant filesystem/network | Worker host, isolation, downstream | | EP10 Worker code generation / wrappers | Entrypoint override, env-var names, workspace env interpolated into generated wrapper code | user-controlled identifier → executable code | Worker host, isolation | | EP11 OAuth / OIDC / SAML / MCP-OAuth / logout | Login callbacks, MCP OAuth client registration, logout `rd` redirect | untrusted IdP / redirect input → session | Session tokens, accounts | -| EP12 Stored-content rendering | App builder HTML component, markdown, S3 download response headers | stored user content → admin browser (same origin) | Admin session, account takeover | +| EP12 Stored-content rendering | App builder HTML component, markdown, S3 download response headers, script-controlled `wm_content_type`/`wm_headers` on `run_wait_result` and sync HTTP-route responses | stored user content → admin browser (same origin) | Admin session, account takeover | | EP13 Log/file reading & export endpoints | `service_logs`, `jobs_u/getupdate` log file read (symlinks), workspace/tarball export | authed/unauth request → arbitrary file or admin-only config | Arbitrary files, global settings | | EP14 Secret-value & resource-value caches | In-memory caches in `windmill-store` keyed (historically un-keyed) by path | cache lookup crossing identity/folder boundary | Secret variables, resource creds | | EP15 Deployment & runtime config | docker-compose defaults: dind, debugger (`REQUIRE_SIGNED_DEBUG_REQUESTS` now defaults to `true`; can still be overridden to `false`), CORS `Any`, default admin/`changeme`, exposed Postgres, `SUPERADMIN_SECRET`, `ENABLE_NSJAIL=false`, privileged containers | operator/infra default → full instance | All assets | @@ -106,7 +106,7 @@ published advisory history (73 GHSA advisories, several rated 9.9 critical). | T8 | Unauthenticated RCE via the Debugger WebSocket: `/ws_debug/*` exposed by the gateway/ingress with the debugger service as the auth boundary; signature gate was bypassable via `program`-mode launches (read+exec an arbitrary server-side file path, never signed) even with signing on, and the WS handshake had no Origin check (CSWSH) | remote_unauth | EP15 | Worker host, all assets | critical | possible | partially_mitigated | `program`-mode launches now rejected when `REQUIRE_SIGNED_DEBUG_REQUESTS` is on (signing covers every launch, not just inline `code`); shipped `docker-compose` now defaults `REQUIRE_SIGNED_DEBUG_REQUESTS=true`; opt-in `DEBUG_ALLOWED_ORIGINS` allowlist rejects cross-origin handshakes. Residual: code default is secure but operators can still set `=false`; origin allowlist is opt-in | GHSA-725h-99vx-9xr4 | | T9 | Supply-chain compromise via cached hub scripts, GitHub workflow command injection, or vulnerable base-image deps | supply_chain | EP16 | Worker host, build integrity | critical | possible | partially_mitigated | hub-script re-pin to patched versions; HUB_BASE_URL override | GHSA-w2m9-q5f7-3gpq, edf340c4d4, GHSA-8rq7-w7g6-8wvr, GHSA-vch9-39v5-4wg7 (CVE-2024-37371) | | T10 | Unauthenticated disclosure of job results, args, logs, and admin config via missing-authz public endpoints | remote_unauth | EP2, EP13 | Job results/args/logs, global settings, scripts | high | likely | partially_mitigated | anonymous-job checks, log-endpoint authz hardening | GHSA-qfg7-x243-5hg4, GHSA-v448-fmm4-52fp, 108a88a180, bb90f4ce83 | -| T11 | Stored XSS leading to admin/account takeover via app HTML component, markdown, or S3 download content-type | remote_auth | EP12 | Admin session, accounts | high | likely | partially_mitigated | DOMPurify markdown sanitization, `X-Content-Type-Options: nosniff` + CSP sandbox on downloads | GHSA-9c5c-hh3c-r9mc, GHSA-qxj7-hpx3-r892, GHSA-cf2x-rg8c-v63v, bb78b1c06d, 625b67dff0 | +| T11 | Stored XSS leading to admin/account takeover via app HTML component, markdown, S3 download content-type, or a script-chosen `text/html` content type on `run_wait_result` / sync HTTP-route responses (GET-reachable with the `SameSite=Lax` session cookie) | remote_auth | EP12 | Admin session, accounts | high | likely | partially_mitigated | DOMPurify markdown sanitization, `X-Content-Type-Options: nosniff` + CSP sandbox on downloads and on every `result_to_response` composite result (inserted after `wm_headers`; hop-by-hop names such as `Connection` rejected so a proxy cannot strip them) | GHSA-9c5c-hh3c-r9mc, GHSA-qxj7-hpx3-r892, GHSA-cf2x-rg8c-v63v, bb78b1c06d, 625b67dff0, WIN-2471 | | T12 | Webhook authentication bypass / signature replay forges trigger invocations and approvals | remote_unauth | EP3 | Job execution integrity, approvals | high | likely | partially_mitigated | HMAC verification on some triggers; signing-oracle fix | GHSA-jw8c-h45c-xpjw, GHSA-hh9x-rcf8-xjr2, GHSA-q9g3-q6fj-hc2x, GHSA-8jc4-wj2p-2vmp, ab2a15b2a8 | | T13 | Path traversal / arbitrary file read via log-reading and MCP path endpoints (incl. symlink following) | remote_auth | EP13 | Arbitrary files on server, global settings | high | likely | partially_mitigated | traversal checks + no-symlink-follow added | GHSA-4hrf-mgvv-xp9x, bb90f4ce83, df451aa64f, ad5ec293b5, 5f2d3e6812 | | T14 | Privilege escalation via token rescope/refresh, script-issued JWTs, or operator-permission gaps | remote_auth | EP17, EP5 | Tokens, isolation, accounts | high | likely | partially_mitigated | monotonic-privilege enforcement on token lifecycle; SECURITY DEFINER triggers | GHSA-p62p-67xp-v775, GHSA-vv9w-wx3c-q3x2, 2ddf93de96, 865ab70c89, 33fb08cf3d | diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 8f96f52924..9f37cbedb0 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -d6aef91c0f7ba556befbf4addeb7674d4a9dd819 \ No newline at end of file +f5b783d2f7608e1ff3a817caa8b719e06f8b8981 diff --git a/backend/migrations/20260622072905_ai_free_token_usage.down.sql b/backend/migrations/20260622072905_ai_free_token_usage.down.sql new file mode 100644 index 0000000000..95879034e4 --- /dev/null +++ b/backend/migrations/20260622072905_ai_free_token_usage.down.sql @@ -0,0 +1,2 @@ +DROP TABLE ai_free_token_daily_usage; +DROP TABLE ai_free_token_usage; diff --git a/backend/migrations/20260622072905_ai_free_token_usage.up.sql b/backend/migrations/20260622072905_ai_free_token_usage.up.sql new file mode 100644 index 0000000000..ae0cf47e45 --- /dev/null +++ b/backend/migrations/20260622072905_ai_free_token_usage.up.sql @@ -0,0 +1,19 @@ +-- One-time grant of the Windmill-provided free AI tier, measured as cost in nano-dollars +-- (1e-9 USD) rather than raw tokens — a prompt-cache hit costs a fraction of a fresh input +-- token, so a token count wildly overstates the real bill. The grant never resets: once +-- spent, the user must bring their own API key. Keyed by normalized email so the allowance +-- is shared across a user's workspaces (and is resistant to +tag / gmail-dot aliasing). +CREATE TABLE ai_free_token_usage ( + email VARCHAR(255) PRIMARY KEY, + cost_nanos BIGINT NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- Instance-wide daily cost ceiling (nano-dollars) for the free tier — a kill-switch +-- independent of the per-user grant, bounding the blast radius of a bad day. One row per +-- UTC day. +CREATE TABLE ai_free_token_daily_usage ( + day DATE PRIMARY KEY, + cost_nanos BIGINT NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); diff --git a/backend/migrations/20260826202939_queue_suspended_resume_at_index.down.sql b/backend/migrations/20260826202939_queue_suspended_resume_at_index.down.sql new file mode 100644 index 0000000000..fe727f6761 --- /dev/null +++ b/backend/migrations/20260826202939_queue_suspended_resume_at_index.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS queue_suspended_v2; diff --git a/backend/migrations/20260826202939_queue_suspended_resume_at_index.up.sql b/backend/migrations/20260826202939_queue_suspended_resume_at_index.up.sql new file mode 100644 index 0000000000..6d91a64d5f --- /dev/null +++ b/backend/migrations/20260826202939_queue_suspended_resume_at_index.up.sql @@ -0,0 +1,20 @@ +-- Serves the suspended-job pull in windmill-common/src/worker.rs, whose resume test is the +-- indexed CASE expression. Two things about the shape are load-bearing: +-- * (priority DESC NULLS LAST, created_at) leads, so the scan yields that query's ORDER BY +-- and stops at the first match rather than sorting. +-- * the index is dropped before it is built rather than relying on IF NOT EXISTS. The +-- OVERRIDDEN_MIGRATIONS rewrite in windmill-api/src/db.rs runs these CONCURRENTLY, and an +-- interrupted concurrent build leaves the index present but invalid, which IF NOT EXISTS +-- would then skip rebuilding. Retiring the index this replaces is left to the migration +-- that follows, so this one can only ever be replayed while that index is still there to +-- cover the rebuild. +DROP INDEX IF EXISTS queue_suspended_v2; + +CREATE INDEX IF NOT EXISTS queue_suspended_v2 + ON v2_job_queue ( + priority DESC NULLS LAST, + created_at, + (CASE WHEN suspend <= 0 THEN '-infinity'::timestamptz ELSE suspend_until END), + tag + ) + WHERE suspend_until IS NOT NULL; diff --git a/backend/migrations/20260826214706_queue_suspended_drop_legacy_index.down.sql b/backend/migrations/20260826214706_queue_suspended_drop_legacy_index.down.sql new file mode 100644 index 0000000000..011105927a --- /dev/null +++ b/backend/migrations/20260826214706_queue_suspended_drop_legacy_index.down.sql @@ -0,0 +1,3 @@ +CREATE INDEX IF NOT EXISTS queue_suspended + ON v2_job_queue (priority DESC NULLS LAST, created_at, suspend_until, suspend, tag) + WHERE suspend_until IS NOT NULL; diff --git a/backend/migrations/20260826214706_queue_suspended_drop_legacy_index.up.sql b/backend/migrations/20260826214706_queue_suspended_drop_legacy_index.up.sql new file mode 100644 index 0000000000..b8851b99e0 --- /dev/null +++ b/backend/migrations/20260826214706_queue_suspended_drop_legacy_index.up.sql @@ -0,0 +1,6 @@ +-- Retires the index queue_suspended_v2 replaces. Separate from the migration that builds it +-- so that one is only ever replayed while this index still exists: sqlx records a migration +-- only after all its statements run, so a process that dies before the record is written +-- replays the build, and its leading DROP would otherwise be destroying the sole usable +-- index rather than an interrupted build. +DROP INDEX IF EXISTS queue_suspended; diff --git a/backend/migrations/20260830085453_log_file_indexed_at.down.sql b/backend/migrations/20260830085453_log_file_indexed_at.down.sql new file mode 100644 index 0000000000..ac3c92f3eb --- /dev/null +++ b/backend/migrations/20260830085453_log_file_indexed_at.down.sql @@ -0,0 +1,4 @@ +DROP INDEX IF EXISTS index_log_file_premigration; +DROP INDEX IF EXISTS index_log_file_pending_path; +DROP INDEX IF EXISTS index_log_file_pending; +ALTER TABLE log_file DROP COLUMN IF EXISTS indexed_at; diff --git a/backend/migrations/20260830085453_log_file_indexed_at.up.sql b/backend/migrations/20260830085453_log_file_indexed_at.up.sql new file mode 100644 index 0000000000..3421a5082a --- /dev/null +++ b/backend/migrations/20260830085453_log_file_indexed_at.up.sql @@ -0,0 +1,35 @@ +-- The service log ingest walked `log_file` with a cursor over `log_ts`, which is when a line +-- was written rather than when its row appeared. Rows do not arrive in that order — an upload +-- retried after a failure, a host that has just started, a batch the row limit cut mid-minute — +-- and a row that becomes visible behind the cursor is never read: it stays in `log_file` and its +-- lines stay out of search until retention drops them. +-- +-- No ordering fixes this. A cursor over arrival order fails the same way, because `nextval` is +-- allocated before its INSERT commits: a row can be assigned a lower value and commit after a +-- higher one has already moved the cursor past it. Which rows are outstanding is a property of +-- the rows, so it is recorded on them. +ALTER TABLE log_file ADD COLUMN indexed_at TIMESTAMPTZ; + +-- Rows that already existed are marked, not queued: on a 14-day window most were ingested long +-- ago and their raw files are gone. A sentinel rather than a timestamp, because the indexer has to +-- tell them apart from rows registered since — those start NULL — and it puts the window's worth of +-- them back on the queue on its first pass, keeping only what the columnar store can vouch for. +-- +-- Not split here on the cursor the old ingest had reached. Below that cursor sits every row it +-- skipped, which is the loss this migration exists to stop; recording those as done would carry the +-- bug into its own fix. +UPDATE log_file SET indexed_at = 'epoch' WHERE indexed_at IS NULL; + +-- The work queue, and the only index the ingest query needs: outstanding rows are a small +-- fraction of the table, so this stays proportional to what is left to do rather than to the +-- retention window. +CREATE INDEX index_log_file_pending ON log_file (log_ts) WHERE indexed_at IS NULL; + +-- A rebuild takes rows out of the queue by the file it just read out of the store, which is +-- the one lookup that arrives without a `log_ts`: the primary key is `(hostname, log_ts)`, so +-- nothing else covers it and each batch would scan every outstanding row instead. +CREATE INDEX index_log_file_pending_path ON log_file (hostname, file_path) WHERE indexed_at IS NULL; + +-- Reached once per pass while pre-migration rows survive, and never again after the first +-- conversion clears them. +CREATE INDEX index_log_file_premigration ON log_file (log_ts) WHERE indexed_at = 'epoch'; diff --git a/backend/migrations/20260831120907_skills_resource_type.down.sql b/backend/migrations/20260831120907_skills_resource_type.down.sql new file mode 100644 index 0000000000..5cc32da37d --- /dev/null +++ b/backend/migrations/20260831120907_skills_resource_type.down.sql @@ -0,0 +1,13 @@ +-- The up migration only ever added: `ai_skill` still holds every skill it copied, +-- so there is nothing to restore and nothing to delete. Removing the resources +-- would destroy any a user has since edited or created, and removing a folder +-- would take whatever else was put in it. +-- +-- The seeded resource type goes. `created_by` only distinguishes this migration's +-- row from one a user created by hand: a hub sync updates the schema in place and +-- leaves `created_by` alone, so a synced-over row is still removed here and the +-- next sync puts it back. +DELETE FROM resource_type +WHERE workspace_id = 'admins' + AND name = 'ai_skill' + AND created_by = 'system'; diff --git a/backend/migrations/20260831120907_skills_resource_type.up.sql b/backend/migrations/20260831120907_skills_resource_type.up.sql new file mode 100644 index 0000000000..4e8f9cfa95 --- /dev/null +++ b/backend/migrations/20260831120907_skills_resource_type.up.sql @@ -0,0 +1,88 @@ +-- AI chat skills move from the `ai_skill` table onto ordinary resources, so they +-- gain folder ACLs, version history, workspace export and git-sync. An `ai_skill` +-- resource holds the SKILL.md body in `value.content`; its description lives in +-- the resource's own `description` column and its name is the path basename. +-- +-- Nothing here is destructive. `ai_skill` is left in place, unread, for a later +-- release to drop once operators have confirmed the copy. That is what lets every +-- step below skip on conflict rather than resolve one: a skipped row is still in +-- the table, so it is not lost, and the migration needs no record of what it did +-- in order to be reversible. + +-- `format_extension` makes the resource editor render `value.content` as a plain +-- .md file. Seeded under 'admins' so every workspace sees it. +INSERT INTO resource_type (workspace_id, name, schema, description, created_by, format_extension, edited_at) +VALUES ( + 'admins', + 'ai_skill', + '{"type": "object", "properties": {"content": {"type": "string"}}}', + 'A reusable instruction set for the AI chat, in the SKILL.md format. The resource description is what the assistant sees when deciding whether the skill applies; the file body is the instructions it follows.', + 'system', + 'md', + now() +) +ON CONFLICT (workspace_id, name) DO NOTHING; + +-- Shared home matching the admin-only upload these skills had. A workspace that +-- already has a `skills` folder keeps it untouched, ACL and all: adopting one +-- would hand its own grants — possibly write for everyone — over a set of +-- instructions the assistant follows. +INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) +SELECT DISTINCT workspace_id, 'skills', 'Skills', ARRAY[]::TEXT[], '{"g/all": false}'::jsonb +FROM ai_skill +ON CONFLICT (workspace_id, name) DO NOTHING; + +-- Copied only where the destination is free and the folder matches the one above +-- exactly, owners included: a pre-existing folder carrying the same ACL but an +-- owner would hand that owner update and delete over skills the removed API let +-- only workspace admins touch. Anything else stays in `ai_skill` for an operator +-- to place deliberately. +-- +-- What was actually inserted is recorded rather than inferred. Inferring it from +-- "is there an ai_skill resource at the destination" reports nothing when the +-- blocker is itself an ai_skill with different instructions — the one case where +-- the skipped skill is least likely to be noticed. +CREATE TEMP TABLE ai_skill_copied AS +WITH inserted AS ( + INSERT INTO resource (workspace_id, path, value, description, resource_type, created_by, edited_at) + SELECT + s.workspace_id, + 'f/skills/' || s.name, + jsonb_build_object('content', s.instructions), + s.description, + 'ai_skill', + s.edited_by, + s.edited_at + FROM ai_skill s + JOIN folder f + ON f.workspace_id = s.workspace_id + AND f.name = 'skills' + AND f.extra_perms = '{"g/all": false}'::jsonb + AND cardinality(f.owners) = 0 + ON CONFLICT (workspace_id, path) DO NOTHING + RETURNING workspace_id, path +) +SELECT workspace_id, path FROM inserted; + +-- Anything not copied is still in `ai_skill`, but nothing reads that table any +-- more, so from the app's side the skill is missing until an operator places it. +-- Name them rather than leaving that to be discovered. +DO $$ +DECLARE + leftover RECORD; +BEGIN + FOR leftover IN + SELECT s.workspace_id, s.name + FROM ai_skill s + WHERE NOT EXISTS ( + SELECT 1 FROM ai_skill_copied c + WHERE c.workspace_id = s.workspace_id + AND c.path = 'f/skills/' || s.name + ) + LOOP + RAISE WARNING 'ai_skill %/% was not copied to a resource (its destination or the f/skills folder is already taken); it remains in the ai_skill table', + leftover.workspace_id, leftover.name; + END LOOP; +END $$; + +DROP TABLE ai_skill_copied; diff --git a/backend/migrations/20260901130041_drop_draft_password_fkey.down.sql b/backend/migrations/20260901130041_drop_draft_password_fkey.down.sql new file mode 100644 index 0000000000..e62fc02975 --- /dev/null +++ b/backend/migrations/20260901130041_drop_draft_password_fkey.down.sql @@ -0,0 +1,12 @@ +-- Drafts owned by a principal with no login account cannot exist under the constraint; drop them +-- before restoring it. +DELETE FROM draft +WHERE email IS NOT NULL + AND NOT EXISTS (SELECT 1 FROM password WHERE password.email = draft.email); + +ALTER TABLE draft + ADD CONSTRAINT draft_password_fkey + FOREIGN KEY (email) + REFERENCES password(email) + ON DELETE CASCADE + ON UPDATE CASCADE; diff --git a/backend/migrations/20260901130041_drop_draft_password_fkey.up.sql b/backend/migrations/20260901130041_drop_draft_password_fkey.up.sql new file mode 100644 index 0000000000..65a6686e95 --- /dev/null +++ b/backend/migrations/20260901130041_drop_draft_password_fkey.up.sql @@ -0,0 +1,3 @@ +-- The delete and rename this cascaded are now explicit, at the sites that remove or rename an +-- account; `windmill_common::user_drafts::delete_drafts_of_email` carries the reasoning. +ALTER TABLE draft DROP CONSTRAINT IF EXISTS draft_password_fkey; diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 2f2fafedf4..395e692623 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.796.0" +version = "1.803.0" dependencies = [ "aho-corasick", "anyhow", @@ -6274,7 +6274,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.796.0" +version = "1.803.0" dependencies = [ "proc-macro2", "quote", @@ -6286,7 +6286,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.796.0" +version = "1.803.0" dependencies = [ "convert_case", "serde", @@ -6295,7 +6295,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -6307,7 +6307,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "serde_json", @@ -6319,7 +6319,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "gosyn", @@ -6331,7 +6331,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -6343,7 +6343,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "serde_json", @@ -6355,7 +6355,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "nu-parser", @@ -6366,7 +6366,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6377,7 +6377,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6389,7 +6389,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6400,7 +6400,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "async-recursion", @@ -6422,7 +6422,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "serde_json", @@ -6434,7 +6434,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -6448,7 +6448,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "convert_case", @@ -6465,7 +6465,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -6478,7 +6478,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "serde", @@ -6490,7 +6490,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -6508,7 +6508,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6524,7 +6524,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6540,7 +6540,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6572,7 +6572,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "lazy_static", @@ -6586,7 +6586,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.796.0" +version = "1.803.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 826e279c39..45c8b0d94e 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.796.0" +version = "1.803.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/src/main.rs b/backend/src/main.rs index f8b150383a..f84812151d 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -7,14 +7,14 @@ */ use anyhow::Context; use monitor::{ - load_base_url, load_otel, reload_critical_alerts_on_db_oversize, - reload_delete_logs_periodically_setting, reload_indexer_config, - reload_instance_python_version_setting, reload_maven_repos_setting, + flush_pending_log_files_to_object_store, load_base_url, load_otel, + reload_critical_alerts_on_db_oversize, reload_delete_logs_periodically_setting, + reload_indexer_config, reload_instance_python_version_setting, reload_maven_repos_setting, reload_maven_settings_xml_setting, reload_no_default_maven_setting, reload_nuget_config_setting, reload_powershell_repo_pat_setting, reload_powershell_repo_url_setting, reload_ruby_repos_setting, reload_timeout_wait_result_setting, reload_workspace_registries_setting, - send_current_log_file_to_object_store, send_logs_to_object_store, WORKERS_NAMES, + send_logs_to_object_store, WORKERS_NAMES, }; use rand::Rng; use sqlx::{Pool, Postgres}; @@ -53,16 +53,17 @@ use windmill_common::{ KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MAVEN_SETTINGS_XML_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING, NPM_CONFIG_REGISTRY_SETTING, NSJAIL_TMPFS_SIZE_MB_SETTING, NSJAIL_TMP_BACKING_SETTING, - NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, - PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, - PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, + NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, OTEL_TRACES_RETENTION_SECS_SETTING, + OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, + POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RESTART_COORDINATION_SETTING, RETENTION_PERIOD_SECS_OVERRIDES_SETTING, RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, SANDBOX_IMAGE_PULL_POLICY_SETTING, SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, - SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, - UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, + SERVICE_LOG_RETENTION_SECS_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, + TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, + UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, WORKSPACE_MAX_QUEUED_JOBS_SETTING, WORKSPACE_REGISTRIES_SETTING, @@ -139,11 +140,12 @@ use crate::monitor::{ reload_instance_events_webhook_setting, reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting, reload_license_key, reload_npm_config_registry_setting, reload_nsjail_tmp_backing_setting, - reload_nsjail_tmpfs_size_setting, reload_otel_tracing_proxy_setting, - reload_pip_index_url_setting, reload_retention_period_setting, - reload_sandbox_image_cache_max_setting, reload_sandbox_image_default_registry_setting, - reload_sandbox_image_max_size_setting, reload_sandbox_image_pull_policy_setting, - reload_sandbox_registry_auth_setting, reload_scim_token_setting, reload_smtp_config, + reload_nsjail_tmpfs_size_setting, reload_otel_traces_retention_secs_setting, + reload_otel_tracing_proxy_setting, reload_pip_index_url_setting, + reload_retention_period_setting, reload_sandbox_image_cache_max_setting, + reload_sandbox_image_default_registry_setting, reload_sandbox_image_max_size_setting, + reload_sandbox_image_pull_policy_setting, reload_sandbox_registry_auth_setting, + reload_scim_token_setting, reload_service_log_retention_secs_setting, reload_smtp_config, reload_store_audit_logs_s3_setting, reload_uv_exclude_newer_setting, reload_uv_index_strategy_setting, reload_uv_python_install_mirror_setting, reload_worker_config, MonitorIteration, @@ -380,6 +382,7 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { "", &mut None, &None, + None, ) .await { @@ -403,8 +406,12 @@ struct HubResourceTypeRaw { pub schema: Option, pub app: String, pub description: Option, + /// Absent from hubs predating the column, and from caches written before it. + #[serde(default)] + pub format_extension: Option, } + /// Processed resource type with parsed schema #[derive(serde::Deserialize, serde::Serialize, Clone)] pub struct HubResourceType { @@ -413,6 +420,18 @@ pub struct HubResourceType { pub schema: Option, pub app: String, pub description: Option, + /// Doubly optional on purpose. A cache written before this column has no key at + /// all (`None`) and must leave the stored extension alone; one written since + /// always writes the key, so an explicit null (`Some(None)`) is the hub genuinely + /// dropping it and must clear. A single `Option` conflates the two, and picking + /// either meaning breaks the other — as does plain serde, which folds `null` + /// into the outer `None`, hence the wrapping deserializer. + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option", + skip_serializing_if = "Option::is_none" + )] + pub format_extension: Option>, } const HUB_RT_CACHE_FILE: &str = "resource_types.json"; @@ -459,6 +478,7 @@ async fn cache_hub_resource_types() -> anyhow::Result<()> { schema, app: rt.app, description: rt.description, + format_extension: Some(rt.format_extension), }) }) .collect(); @@ -500,9 +520,17 @@ pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyh tracing::info!("Found {} cached resource types", cached_types.len()); - // Get existing resource types in admins workspace - let existing_types: Vec<(String, Option, Option)> = sqlx::query_as( - "SELECT name, schema, description FROM resource_type WHERE workspace_id = 'admins'", + // Get existing resource types in admins workspace. `format_extension` is part of + // the comparison below, so a type whose only change is gaining or losing it is + // not mistaken for unchanged; `is_fileset` decides whether it may take one. + let existing_types: Vec<( + String, + Option, + Option, + Option, + bool, + )> = sqlx::query_as( + "SELECT name, schema, description, format_extension, is_fileset FROM resource_type WHERE workspace_id = 'admins'", ) .fetch_all(db) .await @@ -510,19 +538,42 @@ pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyh let existing_map: std::collections::HashMap< String, - (Option, Option), + (Option, Option, Option, bool), > = existing_types .into_iter() - .map(|(name, schema, desc)| (name, (schema, desc))) + .map(|(name, schema, desc, format_extension, is_fileset)| { + (name, (schema, desc, format_extension, is_fileset)) + }) .collect(); let mut synced_count = 0; let mut skipped_count = 0; for rt in cached_types { - // Check if resource type already exists with same schema and description - if let Some((existing_schema, existing_desc)) = existing_map.get(&rt.name) { - if existing_schema == &rt.schema && existing_desc == &rt.description { + let existing = existing_map.get(&rt.name); + let is_fileset = existing.map(|(_, _, _, f)| *f).unwrap_or(false); + let stored_extension = existing.and_then(|(_, _, e, _)| e.clone()); + // A fileset is a set of files, so it cannot also be one file. Create, update + // and the manual sync all reject the pair; this writer would otherwise + // persist it onto a same-named local fileset. + // + // A cache with no key at all leaves the stored value alone, so the target is + // what is already there — which is also what makes the comparison below + // agree with the write instead of re-upserting the row on every boot. + let format_extension = if is_fileset { + None + } else { + match &rt.format_extension { + Some(from_cache) => from_cache.clone(), + None => stored_extension.clone(), + } + }; + + if let Some((existing_schema, existing_desc, _, _)) = existing { + if existing_schema == &rt.schema + && existing_desc == &rt.description + && stored_extension == format_extension + { skipped_count += 1; continue; } @@ -530,14 +581,19 @@ pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyh // Insert or update resource type sqlx::query( - "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at) - VALUES ('admins', $1, $2, $3, now()) + // `format_extension` is resolved above rather than coalesced here: a + // COALESCE could never clear one, so a hub that dropped an extension + // would leave the stale value behind forever. + "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, edited_at) + VALUES ('admins', $1, $2, $3, $4, now()) ON CONFLICT (workspace_id, name) DO UPDATE - SET schema = EXCLUDED.schema, description = EXCLUDED.description, edited_at = now()", + SET schema = EXCLUDED.schema, description = EXCLUDED.description, + format_extension = EXCLUDED.format_extension, edited_at = now()", ) .bind(&rt.name) .bind(&rt.schema) .bind(&rt.description) + .bind(&format_extension) .execute(db) .await .with_context(|| format!("Failed to upsert resource type {}", rt.name))?; @@ -1261,10 +1317,12 @@ Windmill Community Edition {GIT_VERSION} #[cfg(all(feature = "tantivy", feature = "parquet"))] let log_indexer_f = { let log_indexer_rx = killpill_rx.resubscribe(); - let log_index_writer2 = log_index_writer.clone(); + // Moved, not cloned: sealing a chunk takes sole ownership of its + // tantivy writer, which a second live handle would silently prevent. + let moved_log_index_writer = log_index_writer; async { if let Some(db) = conn.as_sql() { - if let Some(log_index_writer) = log_index_writer2 { + if let Some(log_index_writer) = moved_log_index_writer { windmill_indexer::service_logs_oss::run_indexer( db.clone(), log_index_writer, @@ -1661,7 +1719,7 @@ Windmill Community Edition {GIT_VERSION} } else { tracing::info!("Nothing to do, exiting."); } - send_current_log_file_to_object_store(&conn, &hostname, &mode).await; + flush_pending_log_files_to_object_store(&conn, &hostname, &mode).await; if let Some(db) = conn.as_sql() { tracing::info!("Exiting connection pool"); @@ -1952,6 +2010,12 @@ async fn process_notify_event( } TIMEOUT_WAIT_RESULT_SETTING => reload_timeout_wait_result_setting(conn).await, RETENTION_PERIOD_SECS_SETTING => reload_retention_period_setting(conn).await, + SERVICE_LOG_RETENTION_SECS_SETTING => { + reload_service_log_retention_secs_setting(conn).await + } + OTEL_TRACES_RETENTION_SECS_SETTING => { + reload_otel_traces_retention_secs_setting(conn).await + } RETENTION_PERIOD_SECS_OVERRIDES_SETTING => { if let Err(e) = load_retention_period_overrides(db).await { tracing::error!("Error loading per-workspace retention overrides: {e:#}"); diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 430f3659c3..86e75b7de9 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -64,24 +64,24 @@ use windmill_common::{ JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING, NSJAIL_TMPFS_SIZE_MB_SETTING, NSJAIL_TMP_BACKING_SETTING, NUGET_CONFIG_SETTING, - OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, - POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, - REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, - RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, + OTEL_SETTING, OTEL_TRACES_RETENTION_SECS_SETTING, OTEL_TRACING_PROXY_SETTING, + PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, + PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, + REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, + SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, SANDBOX_IMAGE_PULL_POLICY_SETTING, SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, - SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TIMEOUT_WAIT_RESULT_SETTING, - UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, - WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING, - WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, - WORKSPACE_MAX_QUEUED_JOBS_SETTING, + SERVICE_LOG_RETENTION_SECS_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, + TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, + UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, + WORKSPACE_FAIRNESS_ENABLED_SETTING, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, + WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, WORKSPACE_MAX_QUEUED_JOBS_SETTING, }, indexer::load_indexer_config, jobs::delete_jobs, jwt::JWT_SECRET, oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH, server::load_smtp_config, - tracing_init::JSON_FMT, users::truncate_token, utils::{empty_as_none, now_from_db, report_critical_error, Mode, HUB_API_SECRET}, worker::{ @@ -98,10 +98,10 @@ use windmill_common::{ KillpillSender, AUDIT_LOG_RETENTION_DAYS, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERTS_ON_TOKEN_EXPIRY, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, - HUB_BASE_URL, JOB_RETENTION_SECS, JOB_RETENTION_SECS_OVERRIDES, - JOB_RETENTION_SECS_OVERRIDES_LOADED, METRICS_DEBUG_ENABLED, METRICS_ENABLED, - MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, - SERVICE_LOG_RETENTION_SECS, STORE_AUDIT_LOGS_S3, + DEFAULT_OTEL_TRACES_RETENTION_SECS, DEFAULT_SERVICE_LOG_RETENTION_SECS, HUB_BASE_URL, + JOB_RETENTION_SECS, JOB_RETENTION_SECS_OVERRIDES, JOB_RETENTION_SECS_OVERRIDES_LOADED, + METRICS_DEBUG_ENABLED, METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, + OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, STORE_AUDIT_LOGS_S3, }; use windmill_common::{ client::AuthedClient, @@ -476,6 +476,19 @@ pub async fn initial_load( |v: Option| async move { HUB_API_SECRET.store(std::sync::Arc::new(v)) }, ); + // Outside the `server_mode` guard below: every mode reads this. A worker registers its + // rotated log files against the cutoff, and a dedicated indexer trims the search index to a + // window derived from it — neither is a server. + pass.setting(SERVICE_LOG_RETENTION_SECS_SETTING, true, |v| async move { + windmill_common::set_service_log_retention_secs(parse_setting_value::( + v, + SERVICE_LOG_RETENTION_SECS_SETTING, + "SERVICE_LOG_RETENTION_SECS", + DEFAULT_SERVICE_LOG_RETENTION_SECS, + |x| x, + )) + }); + if server_mode { pass.setting(RETENTION_PERIOD_SECS_SETTING, true, |v| async move { JOB_RETENTION_SECS.store( @@ -502,6 +515,15 @@ pub async fn initial_load( Ordering::Relaxed, ) }); + pass.setting(OTEL_TRACES_RETENTION_SECS_SETTING, true, |v| async move { + windmill_common::set_otel_traces_retention_secs(parse_setting_value::( + v, + OTEL_TRACES_RETENTION_SECS_SETTING, + "OTEL_TRACES_RETENTION_SECS", + DEFAULT_OTEL_TRACES_RETENTION_SECS, + |x| x, + )) + }); pass.setting(STORE_AUDIT_LOGS_S3_SETTING, true, |v| async move { STORE_AUDIT_LOGS_S3.store( parse_setting_value::( @@ -1221,32 +1243,60 @@ async fn sleep_until_next_minute_start_plus_one_s() { } use windmill_common::tracing_init::TMP_WINDMILL_LOGS_SERVICE; -async fn find_two_highest_files(hostname: &str) -> (Option, Option) { + +/// The minutely rolling appender names each file `.log.<%Y-%m-%d-%H-%M>`; +/// anything else in the directory is not a rotated log file. +fn parse_log_file_ts(file_name: &str) -> Option { + NaiveDateTime::parse_from_str( + file_name.rsplit('.').next()?, + windmill_common::tracing_init::LOG_TIMESTAMP_FMT, + ) + .ok() +} + +/// Oldest first. Readdir order is filesystem-dependent — tmpfs hands back the +/// newest entry first, ext4 hashes the names — so the listing has to be sorted +/// before anything picks a file out of it. +fn sorted_log_files(file_names: impl Iterator) -> Vec<(NaiveDateTime, String)> { + let mut files = file_names + .filter_map(|name| parse_log_file_ts(&name).map(|ts| (ts, name))) + .collect::>(); + files.sort(); + files +} + +/// Every log file but the newest one: that one is still being appended to, every +/// older one is final. +fn rotated_log_files(file_names: impl Iterator) -> Vec<(NaiveDateTime, String)> { + let mut files = sorted_log_files(file_names); + files.pop(); + files +} + +async fn read_log_file_names(hostname: &str) -> Vec { let log_dir = format!("{}/{}/", *TMP_WINDMILL_LOGS_SERVICE, hostname); - let rd_dir = tokio::fs::read_dir(log_dir).await; - if let Ok(mut log_files) = rd_dir { - let mut highest_file: Option = None; - let mut second_highest_file: Option = None; - while let Ok(Some(file)) = log_files.next_entry().await { - let file_name = file - .file_name() - .to_str() - .map(|x| x.to_string()) - .unwrap_or_default(); - if file_name > highest_file.clone().unwrap_or_default() { - second_highest_file = highest_file; - highest_file = Some(file_name); - } + let mut rd_dir = match tokio::fs::read_dir(&log_dir).await { + Ok(rd_dir) => rd_dir, + Err(e) => { + tracing::error!("Error reading log files: {}, {:#?}", log_dir, e); + return vec![]; + } + }; + let mut file_names = vec![]; + while let Ok(Some(file)) = rd_dir.next_entry().await { + if let Some(file_name) = file.file_name().to_str() { + file_names.push(file_name.to_string()); } - (highest_file, second_highest_file) - } else { - tracing::error!( - "Error reading log files: {}, {:#?}", - *TMP_WINDMILL_LOGS_SERVICE, - rd_dir.unwrap_err() - ); - (None, None) } + file_names +} + +async fn list_log_files(hostname: &str) -> Vec<(NaiveDateTime, String)> { + sorted_log_files(read_log_file_names(hostname).await.into_iter()) +} + +async fn list_rotated_log_files(hostname: &str) -> Vec<(NaiveDateTime, String)> { + rotated_log_files(read_log_file_names(hostname).await.into_iter()) } fn get_worker_group(mode: &Mode) -> Option { @@ -1266,133 +1316,188 @@ pub fn send_logs_to_object_store(conn: &Connection, hostname: &str, mode: &Mode) tokio::spawn(async move { let mut interval = tokio::time::interval(Duration::from_secs(10)); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + init_last_log_file_sent(&conn, &hostname).await; sleep_until_next_minute_start_plus_one_s().await; loop { interval.tick().await; - let (_, snd_highest_file) = find_two_highest_files(&hostname).await; - send_log_file_to_object_store( - &hostname, - &mode, - &worker_group, - &conn, - snd_highest_file, - false, - ) - .await; + let files = list_rotated_log_files(&hostname).await; + send_log_files_to_object_store(&hostname, &mode, &worker_group, &conn, files).await; } }); } -pub async fn send_current_log_file_to_object_store(conn: &Connection, hostname: &str, mode: &Mode) { - tracing::info!("Sending current log file to object store"); - let (highest_file, _) = find_two_highest_files(hostname).await; +pub async fn flush_pending_log_files_to_object_store( + conn: &Connection, + hostname: &str, + mode: &Mode, +) { + tracing::info!("Sending pending log files to object store"); let worker_group = get_worker_group(&mode); - send_log_file_to_object_store(hostname, mode, &worker_group, conn, highest_file, true).await; -} - -fn get_now_and_str() -> (NaiveDateTime, String) { - let ts = Utc::now().naive_utc(); - ( - ts, - ts.format(windmill_common::tracing_init::LOG_TIMESTAMP_FMT) - .to_string(), - ) + // Nothing rotates after this, so the file still being appended to is registered + // here, along with any rotated one the loop had not reached yet. Bounded like the + // pool close that follows: a backlog against a slow object store would otherwise + // hold the process past its termination grace period. Whatever is left over is + // registered by the next run's catch-up. + let flush = async { + let files = list_log_files(hostname).await; + send_log_files_to_object_store(hostname, mode, &worker_group, conn, files).await; + }; + if timeout(Duration::from_secs(15), flush).await.is_err() { + tracing::warn!("Could not send all pending log files in time (15s). Exiting anyway."); + } } lazy_static::lazy_static! { static ref LAST_LOG_FILE_SENT: Arc>> = Arc::new(Mutex::new(None)); + /// Serializes the periodic uploader against the shutdown flush. The uploader is a + /// detached task that keeps ticking while the flush runs and both walk the same + /// files, so without this both can clear the watermark for one file and count its + /// lines twice through the additive upsert. + static ref SENDING_LOG_FILES: tokio::sync::Mutex<()> = tokio::sync::Mutex::new(()); } +fn last_log_file_sent() -> Option { + LAST_LOG_FILE_SENT.lock().ok().and_then(|ts| *ts) +} + +/// Resume from what this host already registered, so a previous run's leftovers reach +/// the object store rather than being dropped. Their line counts come out zero, this +/// run having counted none of them, which only flattens their bars in the UI. +/// +/// The newest registered minute is left out on purpose: the shutdown flush registers +/// the file that was still open and the appender reopens that minute in append mode, +/// so a restart inside it would otherwise strand everything written afterwards. +/// +/// A row rewritten this way restores the object and sums the counters, but it keeps the +/// `indexed_at` it already had, so one the indexers have taken is not offered again and +/// the lines added by the rewrite stay out of search. +async fn init_last_log_file_sent(conn: &Connection, hostname: &str) { + let Some(db) = conn.as_sql() else { + return; + }; + match sqlx::query_scalar!( + "SELECT max(log_ts) FROM log_file + WHERE hostname = $1 AND log_ts < (SELECT max(log_ts) FROM log_file WHERE hostname = $1)", + hostname + ) + .fetch_one(db) + .await + { + Ok(Some(ts)) => { + if let Err(e) = LAST_LOG_FILE_SENT.lock().map(|mut last_log_file_sent| { + last_log_file_sent.replace(ts); + }) { + tracing::error!("Error initializing last log file sent: {:?}", e); + } + } + Ok(None) => {} + Err(e) => tracing::error!("Error loading last log file sent: {:?}", e), + } +} + +async fn send_log_files_to_object_store( + hostname: &str, + mode: &Mode, + worker_group: &Option, + conn: &Connection, + files: Vec<(NaiveDateTime, String)>, +) { + let _guard = SENDING_LOG_FILES.lock().await; + let retention_cutoff = Utc::now().naive_utc() + - chrono::Duration::seconds(windmill_common::service_log_retention_secs()); + for (ts, file_name) in files { + if last_log_file_sent().is_some_and(|last| last >= ts) { + continue; + } + // A run coming back from a long outage still finds its predecessor's files on + // disk. Registering one past the retention cutoff inserts a row + // `delete_expired_items` drops on its next pass, once the indexers have already + // paid to parse it. + if ts < retention_cutoff { + continue; + } + // Stop at the first failure rather than moving on, so a file is never + // registered before an older one that has not made it to the store yet. + // The indexers do not depend on that ordering — every row is offered until + // it is marked — but a gap here would still be visible while it lasts. + if !send_log_file_to_object_store(hostname, mode, worker_group, conn, &file_name, ts).await + { + break; + } + } +} + +/// Returns whether the file ended up registered in `log_file`. async fn send_log_file_to_object_store( hostname: &str, mode: &Mode, worker_group: &Option, conn: &Connection, - snd_highest_file: Option, - use_now: bool, -) { - if let Some(highest_file) = snd_highest_file { - //parse datetime frome file xxxx.yyyy-MM-dd-HH-mm - let (ts, ts_str) = if use_now { - get_now_and_str() - } else { - highest_file - .split(".") - .last() - .and_then(|x| { - NaiveDateTime::parse_from_str( - x, - windmill_common::tracing_init::LOG_TIMESTAMP_FMT, - ) - .ok() - .map(|y| (y, x.to_string())) - }) - .unwrap_or_else(get_now_and_str) - }; + file_name: &str, + ts: NaiveDateTime, +) -> bool { + #[cfg(feature = "parquet")] + if let Some(s3_client) = windmill_object_store::get_object_store().await { + let path = std::path::Path::new(&*TMP_WINDMILL_LOGS_SERVICE) + .join(hostname) + .join(file_name); - let exists = LAST_LOG_FILE_SENT.lock().map(|last_log_file_sent| { - last_log_file_sent - .map(|last_log_file_sent| last_log_file_sent >= ts) - .unwrap_or(false) - }); - - if exists.unwrap_or(false) { - return; - } - - #[cfg(feature = "parquet")] - let s3_client = windmill_object_store::get_object_store().await; - #[cfg(feature = "parquet")] - if let Some(s3_client) = s3_client { - let path = std::path::Path::new(&*TMP_WINDMILL_LOGS_SERVICE) - .join(hostname) - .join(&highest_file); - - //read file as byte stream - let bytes = tokio::fs::read(&path).await; - if let Err(e) = bytes { + //read file as byte stream + let bytes = match tokio::fs::read(&path).await { + Ok(bytes) => bytes, + Err(e) => { tracing::error!("Error reading log file: {:?}", e); - return; + return false; } - let path = windmill_object_store::object_store_reexports::Path::from_url_path(format!( - "{}{hostname}/{highest_file}", - windmill_common::tracing_init::LOGS_SERVICE - )); - if let Err(e) = path { + }; + let path = windmill_object_store::object_store_reexports::Path::from_url_path(format!( + "{}{hostname}/{file_name}", + windmill_common::tracing_init::LOGS_SERVICE + )); + let path = match path { + Ok(path) => path, + Err(e) => { tracing::error!("Error creating log file path: {:?}", e); - return; - } - if let Err(e) = s3_client.put(&path.unwrap(), bytes.unwrap().into()).await { - tracing::error!("Error sending logs to object store: {:?}", e); + return false; } + }; + if let Err(e) = s3_client.put(&path, bytes.into()).await { + tracing::error!("Error sending logs to object store: {:?}", e); + return false; } + } - let (ok_lines, err_lines) = read_log_counters(ts_str); + let ts_str = ts + .format(windmill_common::tracing_init::LOG_TIMESTAMP_FMT) + .to_string(); + let (ok_lines, err_lines) = read_log_counters(ts_str); - if let Some(db) = conn.as_sql() { - match timeout(Duration::from_secs(10), sqlx::query!("INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt) - VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8) - ON CONFLICT (hostname, log_ts) DO UPDATE SET ok_lines = log_file.ok_lines + $6, err_lines = log_file.err_lines + $7", - hostname, mode.to_string(), worker_group.clone(), ts, highest_file, ok_lines as i64, err_lines as i64, *JSON_FMT) - .execute(db)).await { - Ok(Ok(_)) => { - if let Err(e) = LAST_LOG_FILE_SENT.lock().map(|mut last_log_file_sent| { - last_log_file_sent.replace(ts); - }) { - tracing::error!("Error updating last log file sent: {:?}", e); - } - tracing::info!("Log file sent: {}", highest_file); - } - Ok(Err(e)) => { - tracing::error!("Error inserting log file: {:?}", e); - } - Err(e) => { - tracing::error!("Error inserting log file, timeout elapsed: {:?}", e); - } + let Some(db) = conn.as_sql() else { + // not sending log file to object store in agent mode + return false; + }; + + match timeout(Duration::from_secs(10), sqlx::query!("INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt) + VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8) + ON CONFLICT (hostname, log_ts) DO UPDATE SET ok_lines = log_file.ok_lines + $6, err_lines = log_file.err_lines + $7", + hostname, mode.to_string(), worker_group.clone(), ts, file_name, ok_lines as i64, err_lines as i64, true) + .execute(db)).await { + Ok(Ok(_)) => { + if let Err(e) = LAST_LOG_FILE_SENT.lock().map(|mut last_log_file_sent| { + last_log_file_sent.replace(ts); + }) { + tracing::error!("Error updating last log file sent: {:?}", e); } - } else { - // tracing::warn!("Not sending log file to object store in agent mode"); - () + tracing::info!("Log file sent: {}", file_name); + true + } + Ok(Err(e)) => { + tracing::error!("Error inserting log file: {:?}", e); + false + } + Err(e) => { + tracing::error!("Error inserting log file, timeout elapsed: {:?}", e); + false } } } @@ -1580,6 +1685,64 @@ pub async fn trim_resource_versions(db: &DB) -> () { } } +/// Matches the batch the settings-page cleanup uses for the same table. +const SERVICE_LOG_DELETE_BATCH: i64 = 2_000; +/// Batches per pass. `monitor_db` runs under a 600s timeout that cancels every maintenance +/// future in the same `join!` and reports a critical error, so a large backlog has to drain +/// across ticks rather than inside one, the way the neighbouring sweeps already do. +const SERVICE_LOG_DELETE_MAX_BATCHES: usize = 10; + +/// One span per HTTP request made from a job script, so the table grows far faster than the +/// job table it is keyed against; batched for the same reason the service log sweep is. +const OTEL_TRACES_DELETE_BATCH: i64 = 10_000; +const OTEL_TRACES_DELETE_MAX_BATCHES: usize = 10; + +/// Delete HTTP request tracing spans older than `retention_secs`, returning how many went. +/// +/// `retention_secs` is a parameter rather than a read of the process-wide setting so a test can +/// pin a window without writing state the other tests in this binary run against concurrently. +async fn delete_expired_otel_traces(db: &DB, retention_secs: i64) -> u64 { + // `start_time_unix_nano` is the proto field stored verbatim, so the cutoff is built in that + // unit rather than compared against `now()`. Truncating the epoch to whole seconds first + // keeps the multiplication inside `bigint`. + // + // Batched on `ctid`, not on the `(trace_id, span_id)` primary key: with the key the planner + // hashes the LIMITed subquery and Seq Scans the whole table to probe it, which at the size + // this table reaches is the cost the batching exists to avoid. `ctid` plans as a Tid Scan, so + // each batch touches only the rows it deletes. Safe because the subquery and the delete share + // one snapshot, and spans are never updated after insert. + let mut deleted = 0; + for _ in 0..OTEL_TRACES_DELETE_MAX_BATCHES { + let batch = sqlx::query!( + "DELETE FROM otel_traces WHERE ctid IN ( + SELECT ctid FROM otel_traces + WHERE start_time_unix_nano < EXTRACT( + EPOCH FROM now() - ($1::bigint::text || ' s')::interval + )::bigint * 1000000000 + LIMIT $2 + )", + retention_secs, + OTEL_TRACES_DELETE_BATCH, + ) + .execute(db) + .await; + + match batch { + Ok(res) => { + deleted += res.rows_affected(); + if (res.rows_affected() as i64) < OTEL_TRACES_DELETE_BATCH { + break; + } + } + Err(e) => { + tracing::error!("Error deleting expired otel trace spans: {:?}", e); + break; + } + } + } + deleted +} + pub async fn delete_expired_items(db: &DB) -> () { let expired_tokens_r = sqlx::query_as!( TokenRow, @@ -1662,23 +1825,54 @@ pub async fn delete_expired_items(db: &DB) -> () { Err(e) => tracing::error!("Error deleting cache resource {}", e.to_string()), } - match sqlx::query_as!( - LogFile, - "DELETE FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval RETURNING file_path, hostname", - SERVICE_LOG_RETENTION_SECS, - ) - .fetch_all(db) - .await - { - Ok(log_files_to_delete) => { + // Batched: every process rotates a log file a minute, so lowering the retention makes one + // ordinary setting change expire millions of rows at once. An unbounded `DELETE ... + // RETURNING` would materialize all of them, and their deletion futures, in this one tick. + for _ in 0..SERVICE_LOG_DELETE_MAX_BATCHES { + let batch = sqlx::query_as!( + LogFile, + "DELETE FROM log_file WHERE (hostname, log_ts) IN ( + SELECT hostname, log_ts FROM log_file + WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval + LIMIT $2 + ) RETURNING file_path, hostname", + windmill_common::service_log_retention_secs(), + SERVICE_LOG_DELETE_BATCH, + ) + .fetch_all(db) + .await; + + match batch { + Ok(log_files_to_delete) => { + if log_files_to_delete.is_empty() { + break; + } + let n = log_files_to_delete.len(); let paths = log_files_to_delete .iter() .map(|f| format!("{}/{}", f.hostname, f.file_path)) .collect(); - delete_log_files_from_disk_and_store(paths, &*TMP_WINDMILL_LOGS_SERVICE, windmill_common::tracing_init::LOGS_SERVICE).await; - + delete_log_files_from_disk_and_store( + paths, + &*TMP_WINDMILL_LOGS_SERVICE, + windmill_common::tracing_init::LOGS_SERVICE, + ) + .await; + if (n as i64) < SERVICE_LOG_DELETE_BATCH { + break; + } + } + Err(e) => { + tracing::error!("Error deleting log file: {:?}", e); + break; + } } - Err(e) => tracing::error!("Error deleting log file: {:?}", e), + } + + let deleted_spans = + delete_expired_otel_traces(db, windmill_common::otel_traces_retention_secs()).await; + if deleted_spans > 0 { + tracing::info!("deleted {} expired otel trace spans", deleted_spans); } let audit_retention_days = audit_log_retention_days().await; @@ -2785,6 +2979,36 @@ pub async fn reload_retention_period_setting(conn: &Connection) { } } +pub async fn reload_service_log_retention_secs_setting(conn: &Connection) { + match load_setting_value::( + conn, + SERVICE_LOG_RETENTION_SECS_SETTING, + "SERVICE_LOG_RETENTION_SECS", + DEFAULT_SERVICE_LOG_RETENTION_SECS, + |x| x, + ) + .await + { + Ok(v) => windmill_common::set_service_log_retention_secs(v), + Err(e) => tracing::error!("Error reloading service log retention period: {:?}", e), + } +} + +pub async fn reload_otel_traces_retention_secs_setting(conn: &Connection) { + match load_setting_value::( + conn, + OTEL_TRACES_RETENTION_SECS_SETTING, + "OTEL_TRACES_RETENTION_SECS", + DEFAULT_OTEL_TRACES_RETENTION_SECS, + |x| x, + ) + .await + { + Ok(v) => windmill_common::set_otel_traces_retention_secs(v), + Err(e) => tracing::error!("Error reloading otel traces retention period: {:?}", e), + } +} + pub async fn reload_audit_log_retention_days_setting(conn: &Connection) { match load_setting_value::( conn, @@ -4697,7 +4921,7 @@ async fn poll_git_fork_branches( } async fn vacuuming_tables(db: &Pool) -> error::Result<()> { - sqlx::query!("VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, job_perms, concurrency_key, log_file, metrics") + sqlx::query!("VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, job_perms, concurrency_key, log_file, metrics, otel_traces") .execute(db) .await?; Ok(()) @@ -6832,3 +7056,94 @@ mod zombie_worker_memory_pct_tests { ); } } + +#[cfg(test)] +mod otel_traces_retention_tests { + use super::{delete_expired_otel_traces, DB}; + + async fn insert_span(db: &DB, id: u8, age_secs: i64) { + sqlx::query!( + "INSERT INTO otel_traces (trace_id, span_id, name, kind, start_time_unix_nano, end_time_unix_nano) + VALUES ($1, $2, 'GET /', 3, $3, $3)", + &[id; 16][..], + &[id; 8][..], + (chrono::Utc::now() - chrono::Duration::seconds(age_secs)) + .timestamp_nanos_opt() + .unwrap(), + ) + .execute(db) + .await + .unwrap(); + } + + /// The cutoff crosses two units: a retention configured in seconds against a column holding + /// nanoseconds. Getting that conversion wrong is silent in both directions — a window a + /// billion times too wide never deletes anything, one a billion times too narrow deletes + /// every span on the next tick — so pin it on either side of the boundary. + #[sqlx::test(migrations = "./migrations")] + async fn deletes_only_spans_past_the_window(db: DB) -> anyhow::Result<()> { + let day = 60 * 60 * 24; + insert_span(&db, 1, 60).await; + insert_span(&db, 2, 6 * day).await; + insert_span(&db, 3, 8 * day).await; + + assert_eq!(delete_expired_otel_traces(&db, 7 * day).await, 1); + + let kept = sqlx::query_scalar!("SELECT trace_id FROM otel_traces ORDER BY trace_id") + .fetch_all(&db) + .await?; + assert_eq!(kept, vec![vec![1u8; 16], vec![2u8; 16]]); + Ok(()) + } +} + +#[cfg(test)] +mod log_file_listing_tests { + use super::{rotated_log_files, sorted_log_files}; + + fn names(files: Vec<(chrono::NaiveDateTime, String)>) -> Vec { + files.into_iter().map(|(_, n)| n).collect() + } + + /// A directory read newest-entry-first is what tmpfs actually hands back. + #[test] + fn orders_by_minute_whatever_order_readdir_used() { + let newest_first = [ + "h.log.2026-08-29-06-49", + "h.log.2026-08-29-06-46", + "h.log.2026-08-29-06-48", + "h.log.2026-08-29-06-47", + ]; + assert_eq!( + names(sorted_log_files(newest_first.iter().map(|x| x.to_string()))), + vec![ + "h.log.2026-08-29-06-46", + "h.log.2026-08-29-06-47", + "h.log.2026-08-29-06-48", + "h.log.2026-08-29-06-49", + ] + ); + assert_eq!( + names(rotated_log_files( + newest_first.iter().map(|x| x.to_string()) + )), + vec![ + "h.log.2026-08-29-06-46", + "h.log.2026-08-29-06-47", + "h.log.2026-08-29-06-48", + ] + ); + } + + #[test] + fn drops_names_that_are_not_rotated_log_files() { + let files = sorted_log_files( + ["h.log", "not-a-log-file", "h.log.2026-08-29-06-46"] + .iter() + .map(|x| x.to_string()), + ); + assert_eq!(files.len(), 1); + assert_eq!(files[0].1, "h.log.2026-08-29-06-46"); + assert_eq!(files[0].0.to_string(), "2026-08-29 06:46:00"); + } +} diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index a9865cebd3..dfd0305e57 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -38,6 +38,8 @@ account: workspace_id(char), id(int), expires_at(ts), refresh_token(char), clien FK: (workspace_id) -> workspace(id) agent_token_blacklist: token(char), expires_at(ts), blacklisted_at(ts), blacklisted_by(char) ai_agent_memory: workspace_id(char), conversation_id(uuid), step_id(char), messages(jsonb), created_at(ts), updated_at(ts) +ai_free_token_daily_usage: day(date), cost_nanos(bigint), updated_at(ts) +ai_free_token_usage: email(char), cost_nanos(bigint), updated_at(ts) ai_token_usage: workspace_id(char), day(date), email(char), provider(char), model(char), session_id(char), input_tokens(bigint), cache_read_tokens(bigint), cache_write_tokens(bigint), output_tokens(bigint), reported_cost_nano_usd(bigint), requests(bigint), updated_at(ts) FK: (workspace_id) -> workspace(id) alerts: id(int), alert_type(char), message(text), created_at(ts), acknowledged(bool), workspace_id(text), acknowledged_workspace(bool), resource(text) @@ -128,7 +130,7 @@ job_stats: workspace_id(char), job_id(uuid), metric_id(char), metric_name(char), kafka_pending_commits: id(bigint), workspace_id(char), kafka_trigger_path(char), topic(char), partition(int), offset(bigint), created_at(ts) FK: (workspace_id, kafka_trigger_path) -> kafka_trigger(workspace_id, path) kafka_trigger: path(char), kafka_resource_path(char), topics(char), group_id(char), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), server_id(char), last_server_ping(ts), error(text), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), mode(trigger_mode), filters(jsonb[]), auto_commit(bool), labels(text[]) -log_file: hostname(char), log_ts(ts), ok_lines(bigint), err_lines(bigint), mode(log_mode), worker_group(char), file_path(char), json_fmt(bool) +log_file: hostname(char), log_ts(ts), ok_lines(bigint), err_lines(bigint), mode(log_mode), worker_group(char), file_path(char), json_fmt(bool), indexed_at(ts) macro_definition: workspace_id(char), name(char), provider_path(char), params(text), body(text), is_table_macro(bool), created_at(ts) FK: (workspace_id) -> workspace(id) macro_usage: workspace_id(char), consumer_path(char), macro_name(char) diff --git a/backend/tests/fixtures/inline_preview_auth.sql b/backend/tests/fixtures/inline_preview_auth.sql index 59fe2dc917..67ca598bda 100644 --- a/backend/tests/fixtures/inline_preview_auth.sql +++ b/backend/tests/fixtures/inline_preview_auth.sql @@ -2,7 +2,9 @@ -- Layered on top of `base` (which provides test-workspace and the non-operator -- `test-user-2`/SECRET_TOKEN_2). Adds an Operator member so we can assert that -- Operators cannot reach the arbitrary-code inline preview path --- (`POST /jobs/run_inline/preview`). +-- (`POST /jobs/run_inline/preview`) with their own token, plus two deployed script +-- jobs of the operator: one running, so we can assert that its WM_TOKEN can, and +-- one queued but not yet pulled, so we can assert that "queued" is not enough. INSERT INTO password(email, password_hash, login_type, super_admin, verified, name) VALUES ('operator@windmill.dev', 'not-a-real-hash', 'password', false, true, 'Operator User'); @@ -12,3 +14,11 @@ INSERT INTO usr(workspace_id, email, username, is_admin, operator, role) VALUES INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin) VALUES (encode(sha256('OPERATOR_TOKEN'::bytea), 'hex'), 'OPERATOR_T', 'OPERATOR_TOKEN', 'operator@windmill.dev', 'operator token', false); + +INSERT INTO v2_job(id, workspace_id, kind, runnable_path, created_by, permissioned_as, permissioned_as_email) VALUES + ('2aa0c0de-0000-4000-8000-000000000001', 'test-workspace', 'script', 'u/test-user/deployed', 'operator-user', 'u/operator-user', 'operator@windmill.dev'), + ('2aa0c0de-0000-4000-8000-000000000002', 'test-workspace', 'script', 'u/test-user/deployed', 'operator-user', 'u/operator-user', 'operator@windmill.dev'); + +INSERT INTO v2_job_queue(id, workspace_id, scheduled_for, running) VALUES + ('2aa0c0de-0000-4000-8000-000000000001', 'test-workspace', now(), true), + ('2aa0c0de-0000-4000-8000-000000000002', 'test-workspace', now(), false); diff --git a/backend/tests/inline_preview_auth.rs b/backend/tests/inline_preview_auth.rs index 97b70fb545..8841719962 100644 --- a/backend/tests/inline_preview_auth.rs +++ b/backend/tests/inline_preview_auth.rs @@ -9,16 +9,33 @@ //! was the incomplete-fix residual of CVE-2026-22683, whose v1.615.0 patch only //! covered the entity-CRUD endpoints and left this direct inline-exec sink open. //! +//! The guard on both routes has one exemption: `wmill.datatable()` called from +//! inside a job the operator is running. Operators can only run deployed code, +//! so a request the job's WM_TOKEN authenticates comes from code a non-operator +//! authored, and the exemption is limited to the request shape the helper sends +//! (PostgreSQL against a `datatable://` database) so a leaked WM_TOKEN cannot +//! be replayed to run anything else. +//! //! This test pins down: -//! - an Operator is rejected by the operator guard (the core fix; pre-fix this -//! reached the inline executor instead of returning 401), and +//! - an Operator's own token is rejected by the operator guard (the core fix; +//! pre-fix this reached the inline executor instead of returning 401), //! - a regular non-operator passes the guard (the fix must not over-block the //! legitimate inline preview flow): in the test harness the worker inline //! utils are not registered, so a caller past the guard gets the distinct -//! "worker inline functions" error rather than the operator rejection. +//! "worker inline functions" error rather than the operator rejection, +//! - an Operator's job token passes the guard for a datatable query while its +//! job is running, on the inline route and on the `/jobs/run/preview` +//! fallback the SDKs use when the worker has no internal server, +//! - the same token is rejected for any other payload (in-process DuckDB, or a +//! `-- database` directive redirecting the query, whether written literally or +//! reached through a `WM_INTERNAL_DB` marker) and for a deferred run, +//! - an Operator's job token for a job that is not running, whether finished or +//! merely queued, is rejected. use serde_json::json; use sqlx::{Pool, Postgres}; +use windmill_common::auth::create_jwt_token; +use windmill_common::db::Authed; use windmill_test_utils::*; fn client() -> reqwest::Client { @@ -38,11 +55,65 @@ fn inline_preview_body() -> serde_json::Value { }) } +/// The request `wmill.datatable("main")` sends: PostgreSQL against `datatable://main`. +fn datatable_query_body() -> serde_json::Value { + json!({ + "language": "postgresql", + "content": "SELECT 1 AS x;", + "args": { "database": "datatable://main" } + }) +} + +/// Mint the WM_TOKEN a job hands its own code: an internally-signed job JWT +/// (note the `job_id` claim) for the fixture's operator, exactly as the worker +/// issues it when the operator runs a deployed script. +async fn operator_job_token(job_id: uuid::Uuid) -> String { + let authed = Authed { + email: "operator@windmill.dev".to_string(), + username: "operator-user".to_string(), + is_admin: false, + is_operator: true, + groups: vec![], + folders: vec![], + scopes: None, + token_prefix: None, + }; + create_jwt_token( + authed, + "test-workspace", + 3600, + Some(job_id), + Some("ephemeral-script".to_string()), + None, + None, + ) + .await + .expect("mint operator job token") +} + const OPERATOR_GUARD_MSG: &str = "Operators cannot run preview jobs"; +/// The fixture's deployed-script jobs of the operator: one running, one queued. +const RUNNING_JOB_ID: &str = "2aa0c0de-0000-4000-8000-000000000001"; +const QUEUED_JOB_ID: &str = "2aa0c0de-0000-4000-8000-000000000002"; + +async fn post(url: &str, token: &str, body: &serde_json::Value) -> (u16, String) { + let resp = authed(client().post(url), token) + .json(body) + .send() + .await + .expect("request"); + let status = resp.status().as_u16(); + let body = resp.text().await.expect("body"); + (status, body) +} + #[sqlx::test(fixtures("base", "inline_preview_auth"))] async fn test_inline_preview_authorization(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; + // The server decodes WM_TOKENs with the same in-process JWT secret, so + // setting it once lets us mint valid ones below. + set_jwt_secret().await; let server = ApiServer::start(db.clone()).await?; let port = server.addr.port(); @@ -51,12 +122,7 @@ async fn test_inline_preview_authorization(db: Pool) -> anyhow::Result // 1. CORE REGRESSION: an Operator must be rejected by the operator guard. // Pre-fix this fell through to the inline executor (arbitrary code // execution); post-fix it returns 401 with the operator guard message. - let resp = authed(client().post(&url), "OPERATOR_TOKEN") - .json(&inline_preview_body()) - .send() - .await?; - let status = resp.status(); - let body = resp.text().await?; + let (status, body) = post(&url, "OPERATOR_TOKEN", &inline_preview_body()).await; assert_eq!( status, 401, "Operator must be rejected from inline preview (got {status}): {body}" @@ -71,12 +137,7 @@ async fn test_inline_preview_authorization(db: Pool) -> anyhow::Result // the worker inline utils, so the request proceeds past the guard and // fails later with the distinct "worker inline functions" error — proving // the operator guard did not reject it. - let resp = authed(client().post(&url), "SECRET_TOKEN_2") - .json(&inline_preview_body()) - .send() - .await?; - let status = resp.status(); - let body = resp.text().await?; + let (status, body) = post(&url, "SECRET_TOKEN_2", &inline_preview_body()).await; assert_ne!( status, 401, "non-operator must not be blocked by the operator guard (got {status}): {body}" @@ -86,5 +147,113 @@ async fn test_inline_preview_authorization(db: Pool) -> anyhow::Result "non-operator must not hit the operator guard, got: {body}" ); + // 3. The WM_TOKEN of a deployed-script job the Operator is running passes the + // guard for a datatable query: this is `wmill.datatable()` called from + // inside that job. As in 2, the harness then fails with the "worker inline + // functions" error. + let running_job_token = + operator_job_token(uuid::Uuid::parse_str(RUNNING_JOB_ID).unwrap()).await; + let (status, body) = post(&url, &running_job_token, &datatable_query_body()).await; + assert_ne!( + status, 401, + "operator job token of a running job must pass the guard for a datatable query (got {status}): {body}" + ); + assert!( + !body.contains(OPERATOR_GUARD_MSG), + "operator job token of a running job must not hit the operator guard, got: {body}" + ); + + // 4. The same token is rejected for any other payload: the exemption covers + // the datatable request shape only, never in-process DuckDB, and never a + // `-- database` directive, which the executor honors over `args.database`. + let mut redirected = datatable_query_body(); + redirected["content"] = json!("-- database u/test-user/other_db\nSELECT 1 AS x;"); + let mut to_s3 = datatable_query_body(); + to_s3["content"] = json!("-- s3\nSELECT 1 AS x;"); + let mut resource_db = datatable_query_body(); + resource_db["args"]["database"] = json!("$res:u/test-user/other_db"); + // A marker is a single line the directive regexes cannot match; the directive only + // appears once the executor expands it, so the guard must check the expansion. + let mut marker = datatable_query_body(); + marker["content"] = json!(concat!( + r#"-- WM_INTERNAL_DB_SELECT {"table":"t","columnDefs":[{"field":"id","datatype":"int4"}],"#, + r#""whereClause":"true\n-- database u/test-user/other_db\n AND true"}"# + )); + for (label, payload) in [ + ("DuckDB", inline_preview_body()), + ("database directive", redirected), + ("s3 directive", to_s3), + ("resource database", resource_db), + ("marker-expanded database directive", marker), + ] { + let (status, body) = post(&url, &running_job_token, &payload).await; + assert_eq!( + status, 401, + "operator job token must be rejected for a {label} payload (got {status}): {body}" + ); + assert!( + body.contains(OPERATOR_GUARD_MSG), + "rejection for a {label} payload must be the operator guard, got: {body}" + ); + } + + // 5. An Operator's job token whose job is not running is rejected like the + // operator's own token, whether the job is over (no queue row) or merely + // queued: a WM_TOKEN that leaked through logs cannot be replayed once the + // job is over. + for (label, job_id) in [ + ("finished", uuid::Uuid::new_v4()), + ("queued", uuid::Uuid::parse_str(QUEUED_JOB_ID).unwrap()), + ] { + let token = operator_job_token(job_id).await; + let (status, body) = post(&url, &token, &datatable_query_body()).await; + assert_eq!( + status, 401, + "operator job token of a {label} job must be rejected (got {status}): {body}" + ); + assert!( + body.contains(OPERATOR_GUARD_MSG), + "rejection for a {label} job must be the operator guard, got: {body}" + ); + } + + // 6. The SDKs fall back to `/jobs/run/preview` when the worker has no internal + // server (agent workers). The same exemption applies there: the running + // job's token queues the datatable query (201 with the job id), the + // operator's own token is still refused. + let fallback_url = format!("http://localhost:{port}/api/w/test-workspace/jobs/run/preview"); + let (status, body) = post(&fallback_url, &running_job_token, &datatable_query_body()).await; + assert_eq!( + status, 201, + "operator job token of a running job must queue a datatable preview (got {status}): {body}" + ); + let (status, body) = post(&fallback_url, "OPERATOR_TOKEN", &datatable_query_body()).await; + assert_eq!( + status, 401, + "Operator must be rejected from the preview fallback (got {status}): {body}" + ); + assert!( + body.contains(OPERATOR_GUARD_MSG), + "rejection must be the operator guard, got: {body}" + ); + + // 7. A deferred run on the fallback would outlive the running job the + // exemption keys off, so the running job's token cannot schedule one. + for deferral in [ + "scheduled_in_secs=86400", + "scheduled_for=2099-01-01T00:00:00Z", + ] { + let deferred_url = format!("{fallback_url}?{deferral}"); + let (status, body) = post(&deferred_url, &running_job_token, &datatable_query_body()).await; + assert_eq!( + status, 401, + "operator job token must not schedule a deferred preview with {deferral} (got {status}): {body}" + ); + assert!( + body.contains(OPERATOR_GUARD_MSG), + "rejection for {deferral} must be the operator guard, got: {body}" + ); + } + Ok(()) } diff --git a/backend/tests/object_storage_test_ssrf.rs b/backend/tests/object_storage_test_ssrf.rs new file mode 100644 index 0000000000..aa9eaaf74c --- /dev/null +++ b/backend/tests/object_storage_test_ssrf.rs @@ -0,0 +1,97 @@ +//! `POST /api/settings/test_object_storage_config` runs the probe on the API server and reflects the +//! upstream response, so every non-super-admin must be rejected for private/loopback endpoints and +//! the Filesystem backend on every deployment (`CLOUD_HOSTED` is unset here), while a super admin's +//! Filesystem probe still round-trips. Requires the `parquet` feature, like the route. +#![cfg(feature = "parquet")] + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use windmill_test_utils::*; + +const SUPER_ADMIN_TOKEN: &str = "SECRET_TOKEN"; +const USER_TOKEN: &str = "SECRET_TOKEN_2"; + +async fn test_object_storage( + url: &str, + token: &str, + body: serde_json::Value, +) -> anyhow::Result<(u16, String)> { + let resp = reqwest::Client::new() + .post(url) + .header("Authorization", format!("Bearer {token}")) + .json(&body) + .send() + .await?; + Ok((resp.status().as_u16(), resp.text().await?)) +} + +#[sqlx::test(fixtures("base"))] +async fn object_storage_test_is_restricted_for_non_super_admins_off_cloud( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let url = format!( + "http://localhost:{}/api/settings/test_object_storage_config", + server.addr.port() + ); + + // A loopback "S3 endpoint" standing in for an internal service: the probe must be rejected + // before the server opens a connection to it. + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let internal_port = listener.local_addr()?.port(); + let connected = Arc::new(AtomicBool::new(false)); + tokio::spawn({ + let connected = connected.clone(); + async move { + while listener.accept().await.is_ok() { + connected.store(true, Ordering::SeqCst); + } + } + }); + let internal_s3 = json!({ + "type": "S3", + "bucket": "bucket", + "region": "us-east-1", + "access_key": "key", + "secret_key": "secret", + "endpoint": format!("http://127.0.0.1:{internal_port}"), + "allow_http": true, + "path_style": true, + }); + let (status, body) = test_object_storage(&url, USER_TOKEN, internal_s3).await?; + assert_eq!( + status, 401, + "non-super-admin must be rejected for a loopback endpoint (got {status}): {body}" + ); + assert!( + body.contains("requires a super admin"), + "unexpected rejection: {body}" + ); + assert!( + !connected.load(Ordering::SeqCst), + "the server must not connect to the rejected endpoint" + ); + + let tmp = tempfile::tempdir()?; + let filesystem = json!({ "type": "Filesystem", "root_path": tmp.path().to_str().unwrap() }); + let (status, body) = test_object_storage(&url, USER_TOKEN, filesystem.clone()).await?; + assert_eq!( + status, 401, + "non-super-admin must be rejected for a Filesystem backend (got {status}): {body}" + ); + assert!( + body.contains("requires a super admin"), + "unexpected rejection: {body}" + ); + + // Super admins keep the unrestricted path. + let (status, body) = test_object_storage(&url, SUPER_ADMIN_TOKEN, filesystem).await?; + assert_eq!( + status, 200, + "super admin must be able to test a Filesystem backend (got {status}): {body}" + ); + Ok(()) +} diff --git a/backend/tests/suspended_pull_index.rs b/backend/tests/suspended_pull_index.rs new file mode 100644 index 0000000000..4b31f4d45e --- /dev/null +++ b/backend/tests/suspended_pull_index.rs @@ -0,0 +1,77 @@ +//! Pins the plan of the suspended-job pull. Its resume test degrades silently: once the +//! query expression and `queue_suspended_v2` stop matching, Postgres still returns the right +//! job, just by falling back to a heap filter and fetching one tuple per suspended row on +//! every worker poll. No functional test can see that, so assert on the plan instead. + +use serde_json::Value; +use sqlx::{Pool, Postgres}; +use windmill_common::worker::make_suspended_pull_query; + +/// Depth-first walk of an `EXPLAIN (FORMAT JSON)` plan tree. +fn nodes(plan: &Value, out: &mut Vec) { + out.push(plan.clone()); + for child in plan["Plans"].as_array().unwrap_or(&vec![]) { + nodes(child, out); + } +} + +#[sqlx::test(fixtures("base"))] +async fn suspended_pull_tests_resume_time_inside_the_index( + db: Pool, +) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, created_at, scheduled_for, running, suspend, suspend_until, tag) + SELECT gen_random_uuid(), 'test-workspace', now() - make_interval(secs => i), + now(), true, 1 + (i % 3), now() + interval '7 day', 'flow' + FROM generate_series(1, 2000) i", + ) + .execute(&db) + .await?; + sqlx::query("ANALYZE v2_job_queue").execute(&db).await?; + + // Both plans are cheap on a 2000-row table, and which one wins there says nothing + // about a queue with a large suspended backlog. Force the index path, which is the + // one production takes, and assert on how it evaluates the resume test. + let mut conn = db.acquire().await?; + sqlx::query("SET enable_seqscan = off") + .execute(&mut *conn) + .await?; + let version: String = sqlx::query_scalar("SELECT version()") + .fetch_one(&mut *conn) + .await?; + // FORMAT JSON rather than the default: `Index Cond` and `Filter` are separate keys on the + // node, so this does not ride on EXPLAIN's line layout staying put across a major bump. + let explained: Value = sqlx::query_scalar(&format!( + "EXPLAIN (FORMAT JSON) {}", + make_suspended_pull_query(&["flow".to_string()]) + )) + .bind("test-worker") + .fetch_one(&mut *conn) + .await?; + + let mut all = vec![]; + nodes(&explained[0]["Plan"], &mut all); + let pretty = serde_json::to_string_pretty(&explained)?; + let scan = all + .iter() + .find(|n| n["Index Name"] == "queue_suspended_v2") + .unwrap_or_else(|| { + panic!("suspended pull did not scan queue_suspended_v2 on {version}:\n{pretty}") + }); + // Only `Index Cond` is checked against the index tuple, so that is where the resume test + // has to land — as a `Filter` it would cost a heap fetch per suspended row. The residual + // `suspend_until IS NOT NULL` filter is not that: it is always true for rows the partial + // index holds, and only ever runs on the row LIMIT 1 already fetched. + let cond = scan["Index Cond"].as_str().unwrap_or_else(|| { + panic!("no Index Cond on the suspended pull scan on {version}:\n{pretty}") + }); + assert!( + cond.contains("CASE WHEN"), + "resume test is not an index condition on {version}:\n{pretty}" + ); + assert!( + !scan["Filter"].as_str().unwrap_or("").contains("CASE WHEN"), + "resume test fell back to a heap filter on {version}:\n{pretty}" + ); + Ok(()) +} diff --git a/backend/tests/wm_token_confinement.rs b/backend/tests/wm_token_confinement.rs index 5eb885fba9..2d3f1373fe 100644 --- a/backend/tests/wm_token_confinement.rs +++ b/backend/tests/wm_token_confinement.rs @@ -318,6 +318,33 @@ async fn test_wm_token_is_confined_to_its_workspace(db: Pool) -> anyho resp.text().await? ); } + // ...and the one `settings/global` key on the allowlist, which the CLI reads before + // creating a user on a git-sync push. `ws_base_url` is the control: the handler leaves + // it as ungated as `automate_username_creation`, so only the allowlist stops it. + let resp = authed( + client().get(format!("{api}/settings/global/automate_username_creation")), + &user_wm, + ) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "WM_TOKEN must still read automate_username_creation: {}", + resp.text().await? + ); + let resp = authed( + client().get(format!("{api}/settings/global/ws_base_url")), + &user_wm, + ) + .send() + .await?; + assert_eq!( + resp.status(), + 403, + "WM_TOKEN must not read any other global setting: {}", + resp.text().await? + ); let resp = authed(client().post(format!("{api}/schedules/preview")), &user_wm) .json(&json!({ "schedule": "0 0 12 * * *", "timezone": "UTC" })) .send() diff --git a/backend/windmill-ai/src/ai_providers.rs b/backend/windmill-ai/src/ai_providers.rs index bf16877f7b..a4dcbc4f35 100644 --- a/backend/windmill-ai/src/ai_providers.rs +++ b/backend/windmill-ai/src/ai_providers.rs @@ -24,6 +24,16 @@ lazy_static::lazy_static! { .ok() .map(|v| v == "true" || v == "1") .unwrap_or(false); + /// Drops the cache breakpoints from agent-step requests on every Anthropic platform, + /// not just the one that motivates it: a Google Cloud project can have explicit prompt + /// caching turned off (by request to Cloud support), and Vertex then rejects any request + /// carrying breakpoints. An instance that sets this to unblock such a project also gives + /// up caching on its direct-Anthropic and Foundry resources. + pub static ref DISABLE_ANTHROPIC_PROMPT_CACHING: bool = + std::env::var("DISABLE_ANTHROPIC_PROMPT_CACHING") + .ok() + .map(|v| v == "true" || v == "1") + .unwrap_or(false); } pub const OPENAI_BASE_URL: &str = "https://api.openai.com/v1"; diff --git a/backend/windmill-ai/src/providers/anthropic.rs b/backend/windmill-ai/src/providers/anthropic.rs index f22f28de14..486b08b739 100644 --- a/backend/windmill-ai/src/providers/anthropic.rs +++ b/backend/windmill-ai/src/providers/anthropic.rs @@ -1,7 +1,7 @@ use super::{anthropic_model_rejects_sampling_params, REASONING_OFF_SENTINEL}; use crate::{ ai_google::parse_data_url, - ai_providers::{AIPlatform, AIProvider}, + ai_providers::{AIPlatform, AIProvider, DISABLE_ANTHROPIC_PROMPT_CACHING}, image_handler::prepare_messages_for_api, proxy::{ add_user_to_body, common_outbound_headers, credential_header, ProxyBuildArgs, ProxyRequest, @@ -632,15 +632,13 @@ impl AnthropicQueryBuilder { } } + let caching = !*DISABLE_ANTHROPIC_PROMPT_CACHING; + let system = collect_system_prompt(&prepared_messages, args.system_prompt).map(|text| { vec![AnthropicSystemContent { r#type: "text".to_string(), text, - cache_control: if self.is_vertex() { - None - } else { - Some(CacheControl::ephemeral()) - }, + cache_control: caching.then(CacheControl::ephemeral), }] }); @@ -665,7 +663,7 @@ impl AnthropicQueryBuilder { let max_tokens = Some(args.max_tokens.unwrap_or(64000)); // Apply cache_control on the last custom tool - if !self.is_vertex() { + if caching { if let Some(ref mut tools_vec) = tools_option { if let Some(AnthropicTool::Custom(ref mut custom)) = tools_vec.last_mut() { custom.cache_control = Some(CacheControl::ephemeral()); @@ -674,7 +672,7 @@ impl AnthropicQueryBuilder { } // Apply cache_control on the last content block of the last message - if !self.is_vertex() { + if caching { if let Some(last_msg) = anthropic_messages.last_mut() { if let Some(last_block) = last_msg.content.last_mut() { match last_block { @@ -882,10 +880,15 @@ mod tests { } } - async fn build_text_body(messages: &[OpenAIMessage], system_prompt: Option<&str>) -> String { + async fn build_text_body_on( + platform: AIPlatform, + messages: &[OpenAIMessage], + system_prompt: Option<&str>, + tools: Option<&[ToolDef]>, + ) -> String { let args = BuildRequestArgs { messages, - tools: None, + tools, model: "claude-sonnet-4", temperature: None, reasoning_effort: None, @@ -899,12 +902,16 @@ mod tests { prompt_cache_key: None, }; - AnthropicQueryBuilder::new(AIProvider::Anthropic, AIPlatform::Standard) + AnthropicQueryBuilder::new(AIProvider::Anthropic, platform) .build_request(&args, &authed_client(), "test-workspace") .await .unwrap() } + async fn build_text_body(messages: &[OpenAIMessage], system_prompt: Option<&str>) -> String { + build_text_body_on(AIPlatform::Standard, messages, system_prompt, None).await + } + /// The worker prepends the system prompt as a system message *and* passes it as /// `system_prompt`; the request must still carry it exactly once. #[tokio::test] @@ -946,6 +953,35 @@ mod tests { assert!(request.get("system").is_none()); } + /// Vertex serves the same Messages API and honours `cache_control` breakpoints, so + /// its requests must carry the same three the standard platform gets. + #[tokio::test] + async fn sets_cache_breakpoints_on_every_platform() { + let messages = vec![message("system", SYSTEM_PROMPT), message("user", "hi")]; + let tools = vec![ToolDef { + r#type: "function".to_string(), + function: ToolDefFunction { + name: "get_weather".to_string(), + description: None, + parameters: RawValue::from_string("{}".to_string()).unwrap(), + }, + }]; + let ephemeral = serde_json::json!({ "type": "ephemeral" }); + + for platform in [AIPlatform::Standard, AIPlatform::GoogleVertexAi] { + let body = + build_text_body_on(platform, &messages, Some(SYSTEM_PROMPT), Some(&tools)).await; + let request: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert_eq!(request["system"][0]["cache_control"], ephemeral); + let sent_tools = request["tools"].as_array().unwrap(); + assert_eq!(sent_tools.last().unwrap()["cache_control"], ephemeral); + let sent = request["messages"].as_array().unwrap(); + let content = sent.last().unwrap()["content"].as_array().unwrap(); + assert_eq!(content.last().unwrap()["cache_control"], ephemeral); + } + } + fn has_header(headers: &[(String, String)], name: &str, value: &str) -> bool { headers .iter() diff --git a/backend/windmill-api-auth/src/scopes.rs b/backend/windmill-api-auth/src/scopes.rs index 7109a04425..05de1183cd 100644 --- a/backend/windmill-api-auth/src/scopes.rs +++ b/backend/windmill-api-auth/src/scopes.rs @@ -288,7 +288,6 @@ pub enum ScopeDomain { Configs, OAuth, AI, - AiSkills, AiEvals, // AI agent eval datasets Indexer, @@ -349,7 +348,6 @@ impl ScopeDomain { Self::Configs => "configs", Self::OAuth => "oauth", Self::AI => "ai", - Self::AiSkills => "ai_skills", Self::AiEvals => "ai_evals", Self::Capture => "capture", Self::Drafts => "drafts", @@ -405,7 +403,6 @@ impl ScopeDomain { "configs" => Some(Self::Configs), "oauth" => Some(Self::OAuth), "ai" => Some(Self::AI), - "ai_skills" => Some(Self::AiSkills), "ai_evals" => Some(Self::AiEvals), "indexer" | "srch" => Some(Self::Indexer), "teams" => Some(Self::Teams), @@ -996,6 +993,13 @@ fn scope_grants_access( /// the caller's own row; `email` and `allowed_domain_auto_invite` are derived from the /// token itself and touch no table. /// +/// `settings/global/automate_username_creation` is the one instance setting on the list. +/// `get_global_setting` exempts a handful of keys from its own super-admin gate, that one +/// among them, so the boolean is already readable by every authenticated user; it is here +/// because the CLI reads it before creating a user during a git-sync push, which runs as a +/// job. The other ungated keys have no such caller, so they stay confined — being ungated +/// earns a key nothing on its own. +/// /// Deliberately absent, as each crosses that line: `users/list_invites` (returns the /// workspace ids the identity was invited to), `users/tokens/list` (credential metadata /// of the borrowed identity), `users/exists/{email}` (an oracle over arbitrary @@ -1011,6 +1015,7 @@ fn is_global_read_open_to_job_token(route_path: &str) -> bool { | "/api/users/usage" | "/api/users/tutorial_progress" | "/api/workspaces/allowed_domain_auto_invite" + | "/api/settings/global/automate_username_creation" | "/api/docs/search" | "/api/docs/page" | "/api/integrations/hub/list" @@ -1194,12 +1199,6 @@ mod tests { assert_eq!(domain, ScopeDomain::FlowConversations); assert_eq!(kind, None); assert_eq!(route_suffix, Some("flow_conversations/list".to_string())); - - let (domain, kind, route_suffix) = - extract_domain_from_route("/api/w/test_workspace/ai_skills/list").unwrap(); - assert_eq!(domain, ScopeDomain::AiSkills); - assert_eq!(kind, None); - assert_eq!(route_suffix, Some("ai_skills/list".to_string())); } #[test] @@ -1360,11 +1359,6 @@ mod tests { ScopeDomain::from_str("flow_conversations"), Some(ScopeDomain::FlowConversations) ); - assert_eq!( - ScopeDomain::from_str("ai_skills"), - Some(ScopeDomain::AiSkills) - ); - // Test canonical string conversion assert_eq!(ScopeDomain::Acls.as_str(), "acls"); assert_eq!(ScopeDomain::RawApps.as_str(), "raw_apps"); @@ -1373,41 +1367,6 @@ mod tests { ScopeDomain::FlowConversations.as_str(), "flow_conversations" ); - assert_eq!(ScopeDomain::AiSkills.as_str(), "ai_skills"); - } - - #[test] - fn test_ai_skills_scope_access() { - let read_scopes = vec!["ai_skills:read".to_string()]; - assert!( - check_route_access(&read_scopes, "/api/w/test_workspace/ai_skills/list", "GET").is_ok() - ); - assert!(check_route_access( - &read_scopes, - "/api/w/test_workspace/ai_skills/get/foo", - "GET" - ) - .is_ok()); - assert!(check_route_access( - &read_scopes, - "/api/w/test_workspace/ai_skills/upload", - "POST" - ) - .is_err()); - - let write_scopes = vec!["ai_skills:write".to_string()]; - assert!(check_route_access( - &write_scopes, - "/api/w/test_workspace/ai_skills/upload", - "POST" - ) - .is_ok()); - assert!(check_route_access( - &write_scopes, - "/api/w/test_workspace/ai_skills/delete/foo", - "DELETE" - ) - .is_ok()); } #[test] diff --git a/backend/windmill-api-debug/src/lib.rs b/backend/windmill-api-debug/src/lib.rs index 7b2f569821..ab89df988d 100644 --- a/backend/windmill-api-debug/src/lib.rs +++ b/backend/windmill-api-debug/src/lib.rs @@ -468,6 +468,9 @@ async fn sign_debug_request( // Parse the language let script_lang: ScriptLang = request.language.parse().unwrap_or(ScriptLang::Bun); + // Taken from the parsed language, not the request's string: the telemetry key vocabulary has + // to stay the closed set of languages rather than whatever a caller sent. + let lang_key = script_lang.as_str(); // Hash the code (we don't include full code in JWT to keep it small) let mut hasher = Sha256::new(); @@ -578,6 +581,8 @@ async fn sign_debug_request( tx.commit().await?; + windmill_common::feature_usage::log_feature_usage("debugger", "session", lang_key); + Ok(Json(SignedDebugPayload { token, code: request.code, diff --git a/backend/windmill-api-flows/src/flows.rs b/backend/windmill-api-flows/src/flows.rs index bee8f6d3d5..26966fbcde 100644 --- a/backend/windmill-api-flows/src/flows.rs +++ b/backend/windmill-api-flows/src/flows.rs @@ -157,7 +157,8 @@ async fn list_flows( FROM draft d \ LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email \ LEFT JOIN password p ON p.email = d.email AND p.super_admin = true \ - WHERE d.workspace_id = o.workspace_id AND d.path = o.path AND d.typ = 'flow') as draft_users", + WHERE d.workspace_id = o.workspace_id AND d.path = o.path AND d.typ = 'flow' \ + AND (d.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL)) as draft_users", "folder_labels(o.workspace_id, o.path) as inherited_labels" ]) .left() diff --git a/backend/windmill-api-groups/src/granular_acls.rs b/backend/windmill-api-groups/src/granular_acls.rs index 5a7af05cbe..c6f88dace5 100644 --- a/backend/windmill-api-groups/src/granular_acls.rs +++ b/backend/windmill-api-groups/src/granular_acls.rs @@ -183,9 +183,9 @@ async fn add_granular_acl( if kind == "folder" { let change_type = if write.unwrap_or(false) { - "grant_read" - } else { "grant_write" + } else { + "grant_read" }; crate::folders::log_folder_permission_change( &mut *tx, diff --git a/backend/windmill-api-integration-tests/tests/mcp_preprocessor_headers.rs b/backend/windmill-api-integration-tests/tests/mcp_preprocessor_headers.rs new file mode 100644 index 0000000000..3e1035ff82 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/mcp_preprocessor_headers.rs @@ -0,0 +1,186 @@ +//! Request headers reaching a runnable's preprocessor over MCP. +//! +//! The property this pins is structural rather than a filter: the model writes +//! the tool's arguments, which become `event.body`, while the server writes +//! `event.headers`. A model that guesses a header's name can only ever land in +//! `body`, so an identity read from `headers` is one prompt injection cannot +//! forge. Nothing else in the suite exercises MCP argument shaping end to end. +//! +//! Requires: bun runtime, live database (migrations applied by sqlx::test). +#![cfg(feature = "mcp")] + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const SCRIPT_PATH: &str = "u/test-user/mcp_hdr_probe"; + +/// Echoes the two halves of the event separately, so the assertions can tell +/// which one a value arrived in. +const PREPROCESSOR_SCRIPT: &str = r#" +export async function preprocessor(event: any) { + return { + kind: event.kind, + from_headers: event.headers?.["x-user-id"] ?? "", + from_body: event.body?.x_user_id ?? "", + header_names: Object.keys(event.headers ?? {}).sort(), + }; +} + +export async function main(kind: string, from_headers: string, from_body: string, header_names: string[]) { + return { kind, from_headers, from_body, header_names }; +} +"#; + +async fn insert_mcp_token(db: &Pool) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin, scopes) + VALUES (encode(sha256('MCP_TOKEN'::bytea), 'hex'), 'MCP_TOK', 'MCP_TOKEN', 'test@windmill.dev', 'mcp token', true, ARRAY['mcp:all'])", + ) + .execute(db) + .await?; + Ok(()) +} + +/// POST one JSON-RPC message. The endpoint answers either `application/json` or +/// a single-event SSE stream, so strip the `data: ` framing before parsing. +async fn mcp_post(port: u16, headers: &[(&str, &str)], body: Value) -> anyhow::Result { + let mut req = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/mcp/w/test-workspace/mcp" + )) + .header("Authorization", "Bearer MCP_TOKEN") + .header("Accept", "application/json, text/event-stream") + .json(&body); + for (k, v) in headers { + req = req.header(*k, *v); + } + let text = req.send().await?.text().await?; + let payload = text + .lines() + .find_map(|l| l.strip_prefix("data: ")) + .unwrap_or(text.trim()); + serde_json::from_str(payload).map_err(|e| anyhow::anyhow!("unparseable MCP body {text:?}: {e}")) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_mcp_preprocessor_receives_the_callers_headers( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + set_jwt_secret().await; + insert_mcp_token(&db).await?; + let server = ApiServer::start_mcp(db.clone()).await?; + let port = server.addr.port(); + + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "path": SCRIPT_PATH, + "summary": "mcp header probe", + "description": "", + "content": PREPROCESSOR_SCRIPT, + "language": "bun", + "lock": "", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { "x_user_id": { "type": "string" } }, + "required": [] + } + })) + .send() + .await?; + assert_eq!( + resp.status(), + 201, + "create script: {}", + resp.text().await.unwrap_or_default() + ); + + // A script counts as deployed once it has a lock, which normally arrives from + // a dependency job. Planting an empty one keeps the test to the path under + // test instead of a bun resolution whose timing it does not control. + sqlx::query("UPDATE script SET lock = '' WHERE path = $1 AND workspace_id = 'test-workspace'") + .bind(SCRIPT_PATH) + .execute(&db) + .await?; + + let tools = mcp_post( + port, + &[], + json!({"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}}), + ) + .await?; + let tool_name = tools["result"]["tools"] + .as_array() + .and_then(|list| { + list.iter() + .filter_map(|t| t["name"].as_str()) + .find(|n| n.contains("mcp__hdr__probe")) + }) + .ok_or_else(|| anyhow::anyhow!("the deployed script was not listed as a tool: {tools}"))? + .to_string(); + + let result = in_test_worker( + db.clone(), + async { + mcp_post( + port, + // Every name the withheld list covers has to be on the wire, or + // asserting its absence proves nothing. `Authorization` is already + // set by `mcp_post`, and `extract_token` reads it before the + // cookie, so sending one does not disturb auth. + &[ + ("X-User-Id", "alice@corp.example"), + ("Cookie", "session=secret"), + ("Proxy-Authorization", "Basic Zm9v"), + ], + json!({ + "jsonrpc": "2.0", "id": 2, "method": "tools/call", + // The model names the header it wants to spoof. Its value is an + // argument, so it can only ever reach `event.body`. + "params": { "name": tool_name, "arguments": { "x_user_id": "attacker@evil.test" } } + }), + ) + .await + }, + port, + ) + .await?; + + let text = result["result"]["content"][0]["text"] + .as_str() + .ok_or_else(|| anyhow::anyhow!("tool call returned no text content: {result}"))?; + let out: Value = serde_json::from_str(text)?; + + assert_eq!(out["kind"], "mcp", "preprocessor event kind: {out}"); + assert_eq!( + out["from_headers"], "alice@corp.example", + "the caller's header must reach event.headers: {out}" + ); + assert_eq!( + out["from_body"], "attacker@evil.test", + "the model's argument must land in event.body, not overwrite the header: {out}" + ); + + let names: Vec<&str> = out["header_names"] + .as_array() + .map(|a| a.iter().filter_map(|v| v.as_str()).collect()) + .unwrap_or_default(); + assert!( + names.contains(&"x-user-id"), + "event.headers must carry the request's own headers: {names:?}" + ); + for withheld in ["authorization", "cookie", "proxy-authorization"] { + assert!( + !names.contains(&withheld), + "{withheld} is withheld from a preprocessor: {names:?}" + ); + } + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/scripts.rs b/backend/windmill-api-integration-tests/tests/scripts.rs index 8c88c53454..3a146add27 100644 --- a/backend/windmill-api-integration-tests/tests/scripts.rs +++ b/backend/windmill-api-integration-tests/tests/scripts.rs @@ -38,6 +38,85 @@ fn new_script(path: &str, summary: &str, content: &str) -> serde_json::Value { }) } +/// A supplied lock queues no dependency job, so if the create does not record its hash nothing +/// ever will, and every importer of this script relocks on each of its deploys forever after. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_create_script_persists_supplied_lock_hash(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let path = "u/test-user/supplied_lock"; + let lock = r#"{"version":"4","remote":{}}"#; + let mut script = new_script( + path, + "Supplied lock", + "export async function main() { return 42; }", + ); + script["lock"] = json!(lock); + + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&script) + .send() + .await?; + assert_eq!(resp.status(), 201, "create: {}", resp.text().await?); + + let stored_hash = sqlx::query_scalar!( + "SELECT lockfile_hash FROM lock_hash WHERE workspace_id = $1 AND path = $2", + "test-workspace", + path, + ) + .fetch_one(&db) + .await?; + assert_eq!(stored_hash, windmill_common::scripts::hash_script(lock)); + + // A script deployed before the create recorded hashes has no row, and pushing it unchanged + // creates no version to hang one off. Without the write on that path it would keep its + // importers relocking until someone edited it. + sqlx::query!( + "DELETE FROM lock_hash WHERE workspace_id = $1 AND path = $2", + "test-workspace", + path, + ) + .execute(&db) + .await?; + + // The no-op comparison covers every field, so the push has to carry what the first deploy + // filled in by itself; `auto_parent` both resolves the parent and keeps the hash distinct. + script["auto_parent"] = json!(true); + script["ws_error_handler_muted"] = json!(false); + script["assets"] = json!([]); + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create?skip_if_noop=true" + ))) + .json(&script) + .send() + .await?; + assert_eq!(resp.status(), 201, "no-op push: {}", resp.text().await?); + + let versions: i64 = sqlx::query_scalar!( + "SELECT COUNT(*) FROM script WHERE workspace_id = $1 AND path = $2", + "test-workspace", + path, + ) + .fetch_one(&db) + .await? + .unwrap_or_default(); + assert_eq!(versions, 1, "no-op push must not create a version"); + + let repaired_hash = sqlx::query_scalar!( + "SELECT lockfile_hash FROM lock_hash WHERE workspace_id = $1 AND path = $2", + "test-workspace", + path, + ) + .fetch_one(&db) + .await?; + assert_eq!(repaired_hash, windmill_common::scripts::hash_script(lock)); + + Ok(()) +} + #[sqlx::test(migrations = "../migrations", fixtures("base"))] async fn test_script_endpoints(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; @@ -797,10 +876,12 @@ async fn test_update_script_reports_losing_to_a_concurrent_deploy( // What a deploy leaves behind: the old head archived, a new one live at the path. // Copied through a temp table so this does not have to restate every column. - sqlx::query("CREATE TEMP TABLE superseding ON COMMIT DROP AS SELECT * FROM script WHERE hash = $1") - .bind(head) - .execute(&mut *winner) - .await?; + sqlx::query( + "CREATE TEMP TABLE superseding ON COMMIT DROP AS SELECT * FROM script WHERE hash = $1", + ) + .bind(head) + .execute(&mut *winner) + .await?; sqlx::query("UPDATE superseding SET hash = $1, archived = false, parent_hashes = ARRAY[$2]") .bind(head + 1) .bind(head) @@ -818,7 +899,10 @@ async fn test_update_script_reports_losing_to_a_concurrent_deploy( let resp = tokio::time::timeout(std::time::Duration::from_secs(20), update).await??; let status = resp.status(); let body = resp.text().await?; - assert_eq!(status, 400, "losing the race should not read as success: {body}"); + assert_eq!( + status, 400, + "losing the race should not read as success: {body}" + ); assert!( body.contains("deployed to concurrently"), "the loser must say it was superseded, not that the script is missing: {body}" diff --git a/backend/windmill-api-integration-tests/tests/users.rs b/backend/windmill-api-integration-tests/tests/users.rs index fb64d9eb77..65d44509d7 100644 --- a/backend/windmill-api-integration-tests/tests/users.rs +++ b/backend/windmill-api-integration-tests/tests/users.rs @@ -308,14 +308,17 @@ async fn test_user_endpoints(db: Pool) -> anyhow::Result<()> { let auth_base = format!("http://localhost:{port}/api/auth"); // --- login (will fail: password hash in fixture is fake) --- + // An unparseable stored hash must read as a failed login, not as a server error + // relaying the hash parser's message to an unauthenticated caller. let resp = client() .post(format!("{auth_base}/login")) .json(&json!({"email": "test@windmill.dev", "password": "wrong-password"})) .send() .await .unwrap(); - assert!( - resp.status() == 400 || resp.status() == 401 || resp.status() == 500, + assert_eq!( + resp.status(), + 400, "login: unexpected status {}", resp.status() ); @@ -804,12 +807,16 @@ async fn test_change_user_email_leaves_group_identities(db: Pool) -> a let server = ApiServer::start(db.clone()).await?; let global_base = format!("http://localhost:{}/api/users", server.addr.port()); - sqlx::query!("UPDATE password SET email = 'group-ops@windmill.dev' WHERE email = 'test2@windmill.dev'") - .execute(&db) - .await?; - sqlx::query!("UPDATE usr SET email = 'group-ops@windmill.dev' WHERE email = 'test2@windmill.dev'") - .execute(&db) - .await?; + sqlx::query!( + "UPDATE password SET email = 'group-ops@windmill.dev' WHERE email = 'test2@windmill.dev'" + ) + .execute(&db) + .await?; + sqlx::query!( + "UPDATE usr SET email = 'group-ops@windmill.dev' WHERE email = 'test2@windmill.dev'" + ) + .execute(&db) + .await?; sqlx::query!( "INSERT INTO group_(workspace_id, name, summary, extra_perms) VALUES ('test-workspace', 'ops', '', '{}')" ) @@ -910,3 +917,79 @@ async fn test_change_user_email_leaves_group_identities(db: Pool) -> a Ok(()) } + +/// An address with no `password` row can own a draft, and the account paths carry the delete and +/// rename that no foreign key does any more. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_drafts_follow_their_owner_without_a_fkey(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let global_base = format!("http://localhost:{port}/api/users"); + + // The destination of the rename below already holds a draft of the same item — it belongs to + // an accountless principal, so `change_email`'s "address is free" check does not see it. + sqlx::query!( + "INSERT INTO draft(workspace_id, path, typ, value, email) VALUES + ('test-workspace', 'u/ext/s', 'script', '{}'::json, 'ext-jwt@windmill.dev'), + ('test-workspace', 'u/two/s', 'script', '{\"summary\": \"moving\"}'::json, 'test2@windmill.dev'), + ('test-workspace', 'u/two/s', 'script', '{\"summary\": \"displaced\"}'::json, 'renamed@windmill.dev'), + ('test-workspace', 'u/three/s', 'script', '{}'::json, 'test3@windmill.dev')" + ) + .execute(&db) + .await?; + + // A null username is how the legacy workspace-level row is encoded, so an owner nobody can + // name must be absent from the owner circles rather than pose as one. + let resp = authed(client().get(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/list?all_users=true" + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let listed = resp.json::().await?; + let ext = listed + .as_array() + .unwrap() + .iter() + .find(|d| d["path"] == "u/ext/s") + .expect("the accountless owner's draft is listed"); + assert_eq!(ext.get("draft_users"), None); + + let resp = authed(client().post(format!("{global_base}/change_email/test2@windmill.dev"))) + .json(&json!({ "new_email": "renamed@windmill.dev" })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "change_email: {}", resp.text().await?); + let moved = sqlx::query!( + "SELECT email, value->>'summary' AS summary FROM draft WHERE path = 'u/two/s'" + ) + .fetch_all(&db) + .await?; + assert_eq!( + moved + .iter() + .map(|r| (r.email.as_deref(), r.summary.as_deref())) + .collect::>(), + vec![(Some("renamed@windmill.dev"), Some("moving"))], + "the moving account's draft wins the unique index it now collides on" + ); + + let resp = authed(client().delete(format!("{global_base}/delete/test3@windmill.dev"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "delete_user: {}", resp.text().await?); + let remaining = sqlx::query_scalar!("SELECT path FROM draft ORDER BY path") + .fetch_all(&db) + .await?; + assert_eq!( + remaining, + vec!["u/ext/s".to_string(), "u/two/s".to_string()], + "the deleted account's draft goes, the accountless owner's stays" + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/workspaces.rs b/backend/windmill-api-integration-tests/tests/workspaces.rs index 65fe4241cc..25312cae27 100644 --- a/backend/windmill-api-integration-tests/tests/workspaces.rs +++ b/backend/windmill-api-integration-tests/tests/workspaces.rs @@ -889,6 +889,53 @@ async fn test_get_copilot_info_ignores_empty_instance_ai_row( Ok(()) } +/// A workspace with no provider of its own is served the instance config, but the +/// `copilot_disabled` flag must still come from the workspace's own row. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_get_copilot_info_keeps_workspace_copilot_disabled_over_instance_fallback( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + + sqlx::query("UPDATE workspace_settings SET ai_config = $1 WHERE workspace_id = $2") + .bind(json!({ "copilot_disabled": true })) + .bind("test-workspace") + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO global_settings (name, value) VALUES ($1, $2) \ + ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value", + ) + .bind("ai_config") + .bind(json!({ + "providers": { + "openai": { + "resource_path": "u/test-user/openai_instance", + "models": ["gpt-4o-mini"] + } + } + })) + .execute(&db) + .await?; + + let resp = authed(client().get(format!("{base}/get_copilot_info"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let settings = resp.json::().await?; + assert_eq!( + settings["providers"]["openai"]["models"][0], "gpt-4o-mini", + "instance providers are still served" + ); + assert_eq!(settings["copilot_disabled"], true); + + Ok(()) +} + #[sqlx::test(migrations = "../migrations", fixtures("base"))] async fn test_error_handler_instance_alerts_fallback(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; @@ -941,7 +988,12 @@ async fn test_error_handler_instance_alerts_fallback(db: Pool) -> anyh .send() .await .unwrap(); - assert_eq!(resp.status(), 200, "disable on fork: {}", resp.text().await?); + assert_eq!( + resp.status(), + 200, + "disable on fork: {}", + resp.text().await? + ); assert!(!stored().await?); Ok(()) @@ -1044,9 +1096,11 @@ async fn test_create_service_account_drops_orphaned_group_memberships( .await?; // Same username, different workspace, and very much alive — must not be touched. - sqlx::query("INSERT INTO workspace (id, name, owner) VALUES ('other-workspace', 'other', 'svc_acct')") - .execute(&db) - .await?; + sqlx::query( + "INSERT INTO workspace (id, name, owner) VALUES ('other-workspace', 'other', 'svc_acct')", + ) + .execute(&db) + .await?; sqlx::query( "INSERT INTO group_ (workspace_id, name, summary) VALUES ('other-workspace', 'all', 'All users'), diff --git a/backend/windmill-api-jobs/src/execution.rs b/backend/windmill-api-jobs/src/execution.rs index fbfd656cdf..e897bf5eaf 100644 --- a/backend/windmill-api-jobs/src/execution.rs +++ b/backend/windmill-api-jobs/src/execution.rs @@ -416,11 +416,31 @@ pub fn result_to_response(result: Box, success: bool) -> error::Result let mut headers = HeaderMap::new(); + // A reverse proxy consumes hop-by-hop headers instead of forwarding them and + // drops every header named by `Connection`, so a script could use one to strip + // the sandbox headers this function adds before they reach the browser. + const HOP_BY_HOP_HEADERS: [&str; 9] = [ + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "proxy-connection", + "te", + "trailer", + "transfer-encoding", + "upgrade", + ]; + if let Some(windmill_headers) = windmill_headers { for (k, v) in windmill_headers { let k = HeaderName::from_str(k.as_str()).map_err(|err| { Error::internal_err(format!("Invalid header name {k}: {err}")) })?; + if HOP_BY_HOP_HEADERS.contains(&k.as_str()) { + return Err(Error::ExecutionErr(format!( + "windmill_headers cannot set the hop-by-hop header \"{k}\"" + ))); + } let v = HeaderValue::from_str(v.as_str()).map_err(|err| { Error::internal_err(format!("Invalid header value {v}: {err}")) })?; @@ -428,6 +448,22 @@ pub fn result_to_response(result: Box, success: bool) -> error::Result } } + // The script controls the content type and body, and run_wait_result and sync + // HTTP routes are reachable by top-level GET navigation with the session cookie: + // sandbox the document into an opaque origin so HTML can never run with the + // viewer's session. Inserted after `wm_headers` so a script cannot override it. + headers.insert( + http::header::X_CONTENT_TYPE_OPTIONS, + HeaderValue::from_static("nosniff"), + ); + headers.insert( + http::header::CONTENT_SECURITY_POLICY, + HeaderValue::from_static( + "sandbox allow-scripts allow-forms allow-popups \ + allow-popups-to-escape-sandbox allow-downloads allow-modals", + ), + ); + if let Some(content_type) = windmill_content_type { let serialized_json_result = result_value .map(|val| val.get().to_owned()) @@ -1104,6 +1140,56 @@ mod result_to_response_tests { resp.headers().get(http::header::CONTENT_TYPE).unwrap(), "text/html" ); + assert_sandboxed(resp.headers()); assert_eq!(body_bytes(resp).await, b"

hi

"); } + + fn assert_sandboxed(headers: &HeaderMap) { + assert_eq!( + headers.get(http::header::X_CONTENT_TYPE_OPTIONS).unwrap(), + "nosniff" + ); + let csp = headers + .get(http::header::CONTENT_SECURITY_POLICY) + .expect("content-security-policy") + .to_str() + .unwrap(); + assert!(csp.starts_with("sandbox "), "csp: {csp}"); + assert!(!csp.contains("allow-same-origin"), "csp: {csp}"); + } + + #[tokio::test] + async fn custom_headers_cannot_override_sandbox() { + // wm_headers is script-controlled: a content-type set there replaces the JSON + // one even without wm_content_type, and the sandbox headers must survive an + // attempt to override them. + let resp = result_to_response( + raw( + r#"{"wm_headers":{"content-type":"text/html","content-security-policy":"default-src *","x-content-type-options":"none"},"result":"

hi

"}"#, + ), + true, + ) + .expect("response"); + + assert_eq!( + resp.headers().get(http::header::CONTENT_TYPE).unwrap(), + "text/html" + ); + assert_sandboxed(resp.headers()); + } + + #[tokio::test] + async fn hop_by_hop_custom_headers_are_rejected() { + // A proxy drops every header named by `Connection`, which would strip the + // sandbox headers on the way to the browser. + for name in ["connection", "Connection", "transfer-encoding", "upgrade"] { + let res = result_to_response( + raw(&format!( + r#"{{"wm_content_type":"text/html","wm_headers":{{"{name}":"content-security-policy, x-content-type-options"}},"result":"

hi

"}}"# + )), + true, + ); + assert!(res.is_err(), "hop-by-hop header must be rejected: {name}"); + } + } } diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index 0874c6a849..1f457e8ddf 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -39,7 +39,7 @@ use sqlx::{FromRow, Postgres, Transaction}; use std::{collections::HashMap, sync::Arc}; use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; use windmill_audit::ActionKind; -use windmill_dep_map::process_relative_imports; +use windmill_dep_map::{lock_hash::record_lock_hashes, process_relative_imports}; use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap; use windmill_common::{ @@ -216,12 +216,15 @@ async fn list_scripts( // a member of has no `usr` row, so fall back to their instance-derived username // (`password.username`), or their email when derivation is disabled — this keeps the // raw email out of the payload whenever a derived username exists. The genuine - // NULL-email legacy row stays None (no `usr`/`password` match, `d.email` is NULL). + // NULL-email legacy row stays None (no `usr`/`password` match, `d.email` is NULL), + // which is why an owner that resolves to no name at all — an external JWT's subject + // has neither row — is dropped: None is read as "legacy" downstream. "(SELECT json_agg(json_build_object('username', COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN d.email END)) ORDER BY COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN d.email END) NULLS LAST) \ FROM draft d \ LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email \ LEFT JOIN password p ON p.email = d.email AND p.super_admin = true \ - WHERE d.workspace_id = o.workspace_id AND d.path = o.path AND d.typ = 'script') as draft_users", + WHERE d.workspace_id = o.workspace_id AND d.path = o.path AND d.typ = 'script' \ + AND (d.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL)) as draft_users", "folder_labels(o.workspace_id, o.path) as inherited_labels" ]) .left() @@ -1070,6 +1073,14 @@ fn modules_eq( } } +/// Recorded for the empty lock a codebase or a language with no lock generation carries as well as +/// for a real one: the worker writes `hash_script("")` in the same situation, and a path going from +/// a real lock to an empty one has to stop matching what its importers recorded, or they wrongly +/// skip rather than merely relock too often. +fn lock_hash_entry(path: &str, lock: &str) -> [(String, i64); 1] { + [(path.to_string(), hash_script(lock))] +} + async fn create_script_internal<'c>( mut ns: NewScript, w_id: String, @@ -1337,6 +1348,12 @@ async fn create_script_internal<'c>( parent_hash = %p_hash.0, "Skipping no-op script deploy (identical to parent)" ); + // The version is unchanged, but the row recording its lock's hash may never have + // been written — nothing else writes it for a supplied lock, and a path only ever + // pushed unchanged would otherwise keep its importers relocking forever. + if let Some(lock) = ps.lock.as_deref() { + record_lock_hashes(&mut tx, &w_id, &lock_hash_entry(&ns.path, lock)).await?; + } return Ok((p_hash.clone(), tx, None, Vec::new())); } @@ -1884,6 +1901,13 @@ async fn create_script_internal<'c>( .execute(&mut *tx) .await?; + // A lock that is not left to a dependency job queues none, so this is the only place its hash + // can be recorded. `try_skip_relock` treats a missing hash for an imported script as changed, + // so leaving the row out makes every importer of this path relock on every deploy of it. + if let Some(lock) = lock.as_deref() { + record_lock_hashes(&mut tx, &w_id, &lock_hash_entry(&ns.path, lock)).await?; + } + // Update ci_test_reference table for test scripts // Delete by both new and old path to handle renames let old_path = parent_hashes_and_perms.as_ref().map(|x| x.p_path.as_str()); diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index cead2f4b27..18a8d52ac0 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -284,15 +284,28 @@ pub async fn test_s3_bucket( use bytes::Bytes; use futures::StreamExt; - // The probe executes on the API server itself. On multi-tenant Cloud that is a shared control - // plane, so we constrain untrusted callers to remove the SSRF / credential-exfiltration / - // local-filesystem surface (see validate_object_storage_test). On self-hosted instances the - // object store usually lives on the local/private network and all authenticated users are - // trusted, so testing there stays unrestricted. Super admins keep the unrestricted path too. + // The probe executes on the API server itself and reflects the upstream response into the + // error, so any authenticated caller could otherwise use it as an SSRF / port-scan primitive + // against the server's network, exfiltrate its ambient credentials, or write to its local + // disk (see validate_object_storage_test). That holds on self-hosted instances as much as on + // Cloud, so only super admins get the unrestricted path. let is_super_admin = windmill_api_auth::is_super_admin_authed(&db, &authed).await?; - let restrict = !is_super_admin && *CLOUD_HOSTED; + let restrict = !is_super_admin; if restrict { - validate_object_storage_test(&test_s3_bucket).await?; + validate_object_storage_test(&test_s3_bucket) + .await + .map_err(|e| match e { + // A job token never counts as a super admin (it is capped at workspace admin), so + // a super admin calling this route from a script is told why rather than that + // they lack a privilege they hold. + error::Error::NotAuthorized(msg) if authed.job_id.is_some() => { + error::Error::NotAuthorized(format!( + "{msg} A job token ($WM_TOKEN) is never treated as a super admin; call \ + this route with a user token instead." + )) + } + e => e, + })?; } let client = build_object_store_from_settings(test_s3_bucket, Some(&db)) @@ -355,8 +368,8 @@ pub async fn test_s3_bucket( } } -// Hardening for the object-storage connectivity test by an untrusted (non-super-admin) caller on -// Cloud. The probe runs on the shared API server, so without these constraints an authenticated +// Hardening for the object-storage connectivity test by an untrusted (non-super-admin) caller. +// The probe runs on the API server, so without these constraints an authenticated // user could coerce the server into connecting to arbitrary internal endpoints (SSRF), signing // requests with the instance role (credential exfiltration), or reading/writing the server's local // disk (filesystem object store). @@ -366,6 +379,11 @@ async fn validate_object_storage_test(settings: &ObjectSettings) -> error::Resul opt.as_ref().is_some_and(|s| !s.is_empty()) } + // Every refusal names the way out: the resource usually works in jobs (workers reach the + // endpoint directly), so without it the refusal reads as a broken resource. + const ALTERNATIVE: &str = + "Ask a super admin to run it, or test the resource from a script, which runs on a worker."; + // Reject backends that rely on the server's identity or local filesystem, require explicit // credentials for the rest (so the server never falls back to its own ambient credentials), and // resolve the host the client will actually connect to. We derive the *effective* endpoint here @@ -376,20 +394,25 @@ async fn validate_object_storage_test(settings: &ObjectSettings) -> error::Resul let effective_endpoint: Option = match settings { ObjectSettings::Filesystem(_) => { return Err(error::Error::NotAuthorized( - "Testing a local filesystem object store requires a super admin".to_string(), + "Testing a local filesystem object store requires a super admin: it runs on the \ + Windmill server and reads and writes the server's local disk. Ask a super admin \ + to run it." + .to_string(), )); } ObjectSettings::AwsOidc(_) => { - return Err(error::Error::NotAuthorized( - "Testing OIDC-based object storage requires a super admin".to_string(), - )); + return Err(error::Error::NotAuthorized(format!( + "Testing OIDC-based object storage requires a super admin: it runs on the \ + Windmill server with the server's own identity. {ALTERNATIVE}" + ))); } ObjectSettings::S3(s3) => { if !(non_empty(&s3.access_key) && non_empty(&s3.secret_key)) { - return Err(error::Error::NotAuthorized( - "Testing S3 storage without explicit credentials requires a super admin" - .to_string(), - )); + return Err(error::Error::NotAuthorized(format!( + "Testing S3 storage without an explicit access key and secret key requires a \ + super admin: it runs on the Windmill server, which would use its own ambient \ + credentials. {ALTERNATIVE}" + ))); } let region = s3 .region @@ -413,10 +436,11 @@ async fn validate_object_storage_test(settings: &ObjectSettings) -> error::Resul } ObjectSettings::Azure(azure) => { if !non_empty(&azure.access_key) { - return Err(error::Error::NotAuthorized( - "Testing Azure storage without an explicit access key requires a super admin" - .to_string(), - )); + return Err(error::Error::NotAuthorized(format!( + "Testing Azure storage without an explicit access key requires a super admin: \ + it runs on the Windmill server, which would use its own ambient credentials. \ + {ALTERNATIVE}" + ))); } Some( azure @@ -432,10 +456,11 @@ async fn validate_object_storage_test(settings: &ObjectSettings) -> error::Resul // otherwise an untrusted caller could probe with the server's identity (the very // SSRF/credential-exfil this function guards against). if windmill_object_store::gcs_service_account_key_is_blank(&gcs.service_account_key) { - return Err(error::Error::NotAuthorized( - "Testing GCS storage without a service account key requires a super admin" - .to_string(), - )); + return Err(error::Error::NotAuthorized(format!( + "Testing GCS storage without a service account key requires a super admin: \ + it runs on the Windmill server, which would use its own ambient credentials. \ + {ALTERNATIVE}" + ))); } // The service-account-key JSON can override the data-plane URL (`gcs_base_url`) and the // OAuth token endpoint (`token_uri`); the GCS client connects to whatever they point at. @@ -492,10 +517,15 @@ async fn validate_public_endpoint(endpoint: &str) -> error::Result<()> { // attempts (a name resolving to both a public and a private address). for addr in addrs { if is_forbidden_ip(addr.ip()) { - return Err(error::Error::NotAuthorized( - "Testing object storage at a private, loopback, or link-local endpoint requires a super admin" - .to_string(), - )); + // The resolved address stays out of the message: it is the server's resolver's + // answer, and this message is only ever shown to the caller being constrained. + return Err(error::Error::NotAuthorized(format!( + "Testing object storage at '{host}', which resolves to a private, loopback, or \ + link-local address, requires a super admin: this test runs on the Windmill \ + server, which is not allowed to probe internal addresses for non-super-admins. \ + Ask a super admin to run it, or test the resource from a script, which runs on \ + a worker." + ))); } } Ok(()) @@ -2004,6 +2034,15 @@ struct CachedResourceType { #[allow(dead_code)] app: String, description: Option, + /// Doubly optional, and read through a wrapping deserializer: this struct also + /// decodes the on-disk cache, where an absent key means "written before the + /// column, leave the stored extension alone" and an explicit null means the hub + /// dropped it. Plain serde folds both into `None`. + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option" + )] + format_extension: Option>, } #[derive(serde::Deserialize)] @@ -2013,6 +2052,8 @@ struct HubResourceTypeRaw { schema: Option, app: String, description: Option, + #[serde(default)] + format_extension: Option, } async fn fetch_resource_types_from_hub() -> error::Result> { @@ -2054,6 +2095,7 @@ async fn fetch_resource_types_from_hub() -> error::Result = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3)", + "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4))", &rt.name, rt.schema.as_ref(), rt.description.as_deref(), + rt.format_extension.clone().flatten(), + rt.format_extension.is_some(), ) .fetch_one(&db) .await?; @@ -2120,13 +2164,27 @@ async fn sync_cached_resource_types( } sqlx::query!( - "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at) - VALUES ('admins', $1, $2, $3, now()) + // Whether the payload carried the key at all is what decides: present + // (even as null) is authoritative and may clear, absent means a cache + // written before the column and must leave the stored value alone. + "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, edited_at) + VALUES ('admins', $1, $2, $3, $4, now()) ON CONFLICT (workspace_id, name) DO UPDATE - SET schema = EXCLUDED.schema, description = EXCLUDED.description, edited_at = now()", + SET schema = EXCLUDED.schema, description = EXCLUDED.description, + -- A fileset is a set of files, so it cannot also be one file. + -- Create and update reject the pair; this writer bypasses both, so + -- it declines the extension rather than persisting the forbidden + -- combination onto a same-named local fileset. + format_extension = CASE + WHEN resource_type.is_fileset THEN NULL + WHEN $5 THEN EXCLUDED.format_extension + ELSE resource_type.format_extension END, + edited_at = now()", &rt.name, rt.schema.as_ref(), rt.description.as_deref(), + rt.format_extension.clone().flatten(), + rt.format_extension.is_some(), ) .execute(&db) .await?; diff --git a/backend/windmill-api-settings/src/log_cleanup.rs b/backend/windmill-api-settings/src/log_cleanup.rs index ecbc38fcb7..82080cb61c 100644 --- a/backend/windmill-api-settings/src/log_cleanup.rs +++ b/backend/windmill-api-settings/src/log_cleanup.rs @@ -32,7 +32,7 @@ use windmill_common::tracing_init::{LOGS_SERVICE, TMP_WINDMILL_LOGS_SERVICE}; use windmill_common::worker::WINDMILL_DIR; use windmill_common::{ DB, INSTANCE_NAME, JOB_RETENTION_SECS, JOB_RETENTION_SECS_OVERRIDES, - JOB_RETENTION_SECS_OVERRIDES_LOADED, SERVICE_LOG_RETENTION_SECS, + JOB_RETENTION_SECS_OVERRIDES_LOADED, }; use windmill_object_store::object_store_reexports::{ @@ -249,7 +249,7 @@ async fn cleanup_service_logs( // Count candidates upfront for progress reporting. let total: i64 = sqlx::query_scalar!( "SELECT COUNT(*) FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval", - SERVICE_LOG_RETENTION_SECS, + windmill_common::service_log_retention_secs(), ) .fetch_one(db) .await? @@ -274,7 +274,7 @@ async fn cleanup_service_logs( WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval LIMIT $2 ) RETURNING file_path, hostname", - SERVICE_LOG_RETENTION_SECS, + windmill_common::service_log_retention_secs(), SERVICE_LOG_BATCH, ) .fetch_all(db) @@ -680,9 +680,10 @@ async fn cleanup_s3_orphans( ) -> error::Result<()> { let job_retention_secs = JOB_RETENTION_SECS.load(std::sync::atomic::Ordering::Relaxed); let now = Utc::now(); - // Service logs always have a retention (hardcoded SERVICE_LOG_RETENTION_SECS), - // so we scan for service-log orphans regardless of JOB_RETENTION_SECS. - let service_cutoff = now - chrono::Duration::seconds(SERVICE_LOG_RETENTION_SECS); + // Service logs always have a retention, so we scan for service-log orphans regardless of + // JOB_RETENTION_SECS. + let service_cutoff = + now - chrono::Duration::seconds(windmill_common::service_log_retention_secs()); // Job-log orphans are only considered once past a job's effective retention window. That window // is the instance one OR, for an override workspace (EE), its own — and jobs orphan their logs as diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index 8ea3bd0995..47b520736a 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -19,7 +19,7 @@ use windmill_api_auth::ApiAuthed; pub use windmill_api_auth::Tokened; -use argon2::{Argon2, PasswordHash, PasswordVerifier}; +use argon2::{Argon2, PasswordVerifier}; use axum::{ extract::{Extension, Path, Query}, response::{IntoResponse, Response}, @@ -1239,6 +1239,7 @@ async fn leave_instance(Extension(db): Extension, authed: ApiAuthed) -> Resu sqlx::query!("DELETE FROM password WHERE email = $1", &authed.email) .execute(&mut *tx) .await?; + windmill_common::user_drafts::delete_drafts_of_email(&mut *tx, &authed.email).await?; audit_log( &mut *tx, @@ -1661,6 +1662,7 @@ async fn delete_user( sqlx::query!("DELETE FROM password WHERE email = $1", &email_to_delete) .execute(&mut *tx) .await?; + windmill_common::user_drafts::delete_drafts_of_email(&mut *tx, &email_to_delete).await?; let usernames = sqlx::query_scalar!( "DELETE FROM usr WHERE email = $1 RETURNING username", @@ -1869,7 +1871,7 @@ async fn change_user_email( .execute(&mut *tx) .await?; - // ---- account ---- (draft.email follows through its ON UPDATE CASCADE fkey) + // ---- account ---- sqlx::query!( "UPDATE password SET email = $1 WHERE email = $2", &new_email, @@ -1883,6 +1885,7 @@ async fn change_user_email( } _ => e.into(), })?; + windmill_common::user_drafts::rename_drafts_of_email(&mut *tx, &old_email, &new_email).await?; sqlx::query!( "UPDATE usr SET email = $1 WHERE email = $2", @@ -2680,10 +2683,8 @@ async fn login( .await?; if let Some((email, hash, super_admin)) = email_w_h { - let parsed_hash = - PasswordHash::new(&hash).map_err(|e| Error::internal_err(e.to_string()))?; if argon2 - .verify_password(password.as_bytes(), &parsed_hash) + .verify_password(password.as_bytes(), hash.as_str()) .is_err() { audit_log( @@ -3541,6 +3542,9 @@ async fn overwrite_global_users( require_super_admin(&db, &authed).await?; forbid_superadmin_job_token(&db, &authed.email, job_id).await?; let mut tx = db.begin().await?; + // Replaces the account table, so — unlike the paths that remove one account — it deliberately + // does not call `delete_drafts_of_email`: the addresses are about to be reinstated, and + // dropping every draft on the instance to restore accounts would be pure collateral. sqlx::query!("DELETE FROM password") .execute(&mut *tx) .await?; @@ -3710,3 +3714,23 @@ async fn request_password_reset( } // NOTE: reset_password is in windmill-api (depends on users_oss::hash_password EE dispatch) + +#[cfg(test)] +mod tests { + use super::*; + + /// Stored hashes outlive the hashing crate: every instance still holds hashes minted by + /// older argon2 releases, and an upgrade that stopped reading them locks their users out. + #[test] + fn verifies_a_hash_minted_by_an_older_argon2() { + // The seeded admin hash from migration 20220508150023, m=4096,t=3,p=1. + let seeded = "$argon2id$v=19$m=4096,t=3,p=1$oLJo/lPn/gezXCuFOEyaNw$i0T2tCkw3xUFsrBIKZwr8jVNHlIfoxQe+HfDnLtd12I"; + + assert!(Argon2::default() + .verify_password(b"changeme", seeded) + .is_ok()); + assert!(Argon2::default() + .verify_password(b"not-the-password", seeded) + .is_err()); + } +} diff --git a/backend/windmill-api-workspaces/src/datatable_migrations.rs b/backend/windmill-api-workspaces/src/datatable_migrations.rs index bf0b7dd213..bb4310d1bc 100644 --- a/backend/windmill-api-workspaces/src/datatable_migrations.rs +++ b/backend/windmill-api-workspaces/src/datatable_migrations.rs @@ -10,7 +10,10 @@ //! management, and the workspace-merge diff helper. Split out of `workspaces.rs` //! to keep that file focused on core workspace configuration. -use crate::workspaces::{pg_dump_database, ItemComparison}; +use crate::workspaces::{ + is_instance_datatable, pg_dump_database, strip_unreplayable_dump_lines, ItemComparison, + PgDumpOptions, +}; use axum::{ extract::{Extension, Path, Query}, @@ -1408,18 +1411,25 @@ async fn generate_initial_datatable_migration( let pg_db: PgDatabase = serde_json::from_value(db_resource) .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?; - // Snapshot the schema, excluding Windmill's own migration bookkeeping table. - let dump_file = pg_dump_database(&pg_db, true, &["_wm_migrations"]).await?; + // Snapshot the schema without `_wm_migrations`, Windmill's own bookkeeping table, and + // without what a replay elsewhere cannot run: the replaying user owns none of this + // database's objects, and the grants Windmill plants in an instance database (`ALTER + // DEFAULT PRIVILEGES FOR ROLE ...`) fail even replaying onto the same server. + let no_acl = is_instance_datatable(&db, &w_id, &datatable_name).await?; + let dump_file = pg_dump_database( + &pg_db, + PgDumpOptions { + schema_only: true, + exclude_tables: &["_wm_migrations"], + no_owner: true, + no_acl, + }, + ) + .await?; let raw_dump = tokio::fs::read_to_string(&dump_file.path) .await .map_err(|e| Error::internal_err(format!("Failed to read schema dump: {}", e)))?; - // pg_dump emits psql meta-commands (\restrict / \unrestrict) that aren't - // valid SQL; drop them so the migration body can run via a plain query. - let code_up: String = raw_dump - .lines() - .filter(|line| !line.trim_start().starts_with('\\')) - .collect::>() - .join("\n"); + let code_up = strip_unreplayable_dump_lines(&raw_dump); // Record the definition first, then mark it installed. If marking fails we // delete the definition, so a failure leaves no phantom "initial" (rather diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index e04e1c1ab9..cb5c639d46 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -12,6 +12,7 @@ use windmill_api_auth::{ }; use windmill_api_users::users::WorkspaceInvite; use windmill_common::email_oss::send_email_if_possible; +use windmill_dep_map::lock_hash::record_lock_hashes_for_workspace; use windmill_common::usernames::{get_instance_username_or_create_pending, VALID_USERNAME}; use windmill_common::webhook::WebhookShared; use windmill_common::{BASE_URL, DB}; @@ -117,6 +118,7 @@ pub fn workspaced_service() -> Router { get(get_secondary_storage_names), ) .route("/is_premium", get(is_premium)) + .route("/billable_seats", get(get_billable_seats)) .route("/edit_error_handler", post(edit_error_handler)) .route("/edit_success_handler", post(edit_success_handler)) .route( @@ -686,6 +688,48 @@ async fn is_premium( Ok(Json(premium)) } +#[derive(Serialize)] +struct BillableSeatsResponse { + /// Both omitted when the seats counted are another workspace's: a fork member need not be a + /// member of the billing root, so the root's headcount is not theirs to read. The total is, + /// since it is the divisor of the quota their own executions draw on. + #[serde(skip_serializing_if = "Option::is_none")] + developers: Option, + #[serde(skip_serializing_if = "Option::is_none")] + operators: Option, + seats: i64, +} + +async fn get_billable_seats( + _authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult { + // Readable by any workspace member, like `is_premium`: this is what the sidebar usage meter + // divides by, and that meter is shown to non-admin developers too. + // + // On cloud a fork draws its plan, quota and bill from the root, so the seats its usage is + // measured against are the root's. Resolved here rather than by the caller: a fork member need + // not be a member of that root, and so cannot count its seats from the member list. Off cloud + // a fork is not billed through a root at all, so the workspace answers for itself. + #[cfg(feature = "cloud")] + let billing_w_id = if *CLOUD_HOSTED { + windmill_common::workspaces::get_billing_workspace_id(&db, &w_id).await? + } else { + w_id.clone() + }; + #[cfg(not(feature = "cloud"))] + let billing_w_id = w_id.clone(); + + let counted = windmill_common::workspaces::billable_seats(&db, &billing_w_id).await?; + let own = billing_w_id == w_id; + Ok(Json(BillableSeatsResponse { + developers: own.then_some(counted.developers), + operators: own.then_some(counted.operators), + seats: counted.seats, + })) +} + async fn exists_workspace( authed: ApiAuthed, Extension(user_db): Extension, @@ -1970,6 +2014,23 @@ async fn edit_large_file_storage_config( ))); } + if !windmill_common::workspaces::filesystem_storage_allowed() { + let named = std::iter::once(("primary storage", &lfs_config.large_file_storage)).chain( + lfs_config + .secondary_storage + .iter() + .map(|(name, storage)| (name.as_str(), storage)), + ); + for (name, storage) in named { + if matches!(storage, LargeFileStorage::FilesystemStorage(_)) { + return Err(Error::BadRequest(format!( + "{name}: {}", + windmill_common::workspaces::FILESYSTEM_STORAGE_DEV_ONLY_MSG + ))); + } + } + } + let serialized_lfs_config = serde_json::to_value::(lfs_config) .map_err(|err| Error::internal_err(err.to_string()))?; @@ -2594,6 +2655,66 @@ fn truncate_column_default(default: String) -> String { mod tests { use super::*; + /// The header of a pg_dump, followed by an object whose body also holds a `SET`. + const DUMP: &str = "--\n\ + -- PostgreSQL database dump\n\ + --\n\ + \n\ + \\restrict aBcD\n\ + \n\ + SET statement_timeout = 0;\n\ + SET transaction_timeout = 0;\n\ + SET client_encoding = 'UTF8';\n\ + SELECT pg_catalog.set_config('search_path', '', false);\n\ + \n\ + SET default_table_access_method = heap;\n\ + \n\ + CREATE FUNCTION public.f() RETURNS void LANGUAGE plpgsql AS $$\n\ + BEGIN\n\ + SET transaction_timeout = 0;\n\ + END;\n\ + $$;\n"; + + #[test] + fn replayable_dump_keeps_everything_but_meta_commands_and_session_timeouts() { + let replayable = strip_unreplayable_dump_lines(DUMP); + + assert!(!replayable.contains("\\restrict")); + assert!(!replayable.contains("SET statement_timeout")); + assert!(!replayable.contains("SET transaction_timeout = 0;\nSET client_encoding")); + assert!(replayable.contains("SET client_encoding = 'UTF8';")); + assert!(replayable.contains("SET default_table_access_method = heap;")); + // Past the preamble the dump is an object's own text: left exactly as it is. + assert!(replayable.contains("BEGIN\nSET transaction_timeout = 0;\nEND;")); + } + + #[tokio::test] + async fn dump_preamble_only_drops_settings_the_server_lacks() { + let dump_file = DumpFile::new().unwrap(); + tokio::fs::write(&dump_file.path, DUMP).await.unwrap(); + let supported = [ + "statement_timeout", + "client_encoding", + "default_table_access_method", + ] + .map(String::from) + .into_iter() + .collect(); + + comment_out_unsupported_settings(&dump_file, &supported) + .await + .unwrap(); + + let patched = tokio::fs::read_to_string(&dump_file.path).await.unwrap(); + // Rewriting the header must not shift the rest of the dump. + assert_eq!(patched.len(), DUMP.len()); + assert!(patched.contains("-- transaction_timeout = 0;")); + assert!(patched.contains("SET statement_timeout = 0;")); + assert!(patched.contains("SET default_table_access_method = heap;")); + // The `SET` inside the function body is past the preamble: never touched. + assert!(patched.contains("BEGIN\nSET transaction_timeout = 0;\nEND;")); + } + #[test] fn compact_column_type_truncates_multibyte_defaults_safely() { let default = "é".repeat(31); @@ -2696,6 +2817,35 @@ pub(crate) async fn resolve_pg_source_checked( .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e))) } +/// Whether the data table `name` is backed by the Windmill instance's own PostgreSQL +/// rather than a user resource. +pub(crate) async fn is_instance_datatable(db: &DB, w_id: &str, name: &str) -> Result { + let config = sqlx::query_scalar!( + "SELECT datatable->'datatables'->$2 FROM workspace_settings WHERE workspace_id = $1", + w_id, + name + ) + .fetch_optional(db) + .await? + .flatten(); + Ok(config + .and_then(|v| { + v.get("database") + .and_then(|d| d.get("resource_type")) + .and_then(|r| r.as_str()) + .map(|s| s == "instance") + }) + .unwrap_or(false)) +} + +/// Same, for the `datatable://` / `$res:` form the import endpoints take. +async fn is_instance_datatable_source(db: &DB, w_id: &str, source: &str) -> Result { + match source.strip_prefix("datatable://") { + Some(name) => is_instance_datatable(db, w_id, name).await, + None => Ok(false), + } +} + /// A temporary file for pg_dump output that is automatically deleted when dropped. pub(crate) struct DumpFile { pub(crate) path: std::path::PathBuf, @@ -2743,12 +2893,21 @@ impl Drop for DumpFile { } } +#[derive(Default)] +pub(crate) struct PgDumpOptions<'a> { + pub(crate) schema_only: bool, + pub(crate) exclude_tables: &'a [&'a str], + /// Leave out `ALTER ... OWNER TO`. + pub(crate) no_owner: bool, + /// Leave out `GRANT`, `REVOKE` and `ALTER DEFAULT PRIVILEGES`. + pub(crate) no_acl: bool, +} + /// Run pg_dump against a PgDatabase, writing output to a temp file on disk. /// Returns a DumpFile handle; the file is deleted when the handle is dropped. pub(crate) async fn pg_dump_database( pg_db: &PgDatabase, - schema_only: bool, - exclude_tables: &[&str], + opts: PgDumpOptions<'_>, ) -> Result { let dump_file = DumpFile::new()?; @@ -2759,10 +2918,16 @@ pub(crate) async fn pg_dump_database( let mut cmd = tokio::process::Command::new("pg_dump"); cmd.arg("--format=plain").arg("--file").arg(&dump_file.path); - if schema_only { + if opts.schema_only { cmd.arg("--schema-only"); } - for table in exclude_tables { + if opts.no_owner { + cmd.arg("--no-owner"); + } + if opts.no_acl { + cmd.arg("--no-privileges"); + } + for table in opts.exclude_tables { cmd.arg(format!("--exclude-table={table}")); } cmd.arg("--host") @@ -2794,37 +2959,179 @@ pub(crate) async fn pg_dump_database( Ok(dump_file) } -/// Import a pg_dump file into a target database using psql. -async fn pg_import_dump(target_db: &PgDatabase, dump_file: &DumpFile) -> Result<()> { - let host = &target_db.host; - let port = target_db.port.unwrap_or(5432).to_string(); - let user = target_db.login_name(); - let dbname = &target_db.dbname; +/// Whether `line` still belongs to the preamble pg_dump emits before the first +/// dumped object: comments, blank lines, psql meta-commands and the session `SET`s. +fn is_dump_preamble_line(line: &[u8]) -> bool { + let line = line.trim_ascii_start(); + line.is_empty() + || line.starts_with(b"--") + || line.starts_with(b"\\") + || line.starts_with(b"SET ") + || line.starts_with(b"SELECT pg_catalog.set_config(") +} +/// The GUCs pg_dump's preamble sets only to keep the dumping session out of the way. +/// They are also the ones that come and go across versions (`transaction_timeout` is +/// PG 17+), so they are what a dump replayed on an older server trips over first. +const DUMP_SESSION_TIMEOUTS: [&str; 4] = [ + "statement_timeout", + "lock_timeout", + "idle_in_transaction_session_timeout", + "transaction_timeout", +]; + +/// Turn a dump into SQL that can be replayed on another database: drop pg_dump's psql +/// meta-commands (`\restrict` / `\unrestrict`, not valid SQL) and the session timeouts +/// its preamble sets, which the replaying server may not have as GUCs at all. Only the +/// preamble is filtered, so an object's body keeps whatever it holds. +pub(crate) fn strip_unreplayable_dump_lines(dump: &str) -> String { + let mut in_preamble = true; + dump.lines() + .filter(|line| { + in_preamble = in_preamble && is_dump_preamble_line(line.as_bytes()); + if line.trim_start().starts_with('\\') { + return false; + } + !(in_preamble + && preamble_setting_name(line.as_bytes()) + .is_some_and(|name| DUMP_SESSION_TIMEOUTS.contains(&name))) + }) + .collect::>() + .join("\n") +} + +/// The GUC a preamble `SET = ...;` line assigns, if the line is one. +fn preamble_setting_name(line: &[u8]) -> Option<&str> { + let name = line.strip_prefix(b"SET ")?.split(|c| *c == b' ').next()?; + std::str::from_utf8(name).ok() +} + +/// The preamble Windmill's postgres client writes can set GUCs an older server does not +/// have — harmless session tuning, but one failing statement aborts a restore that stops +/// on the first error. Comment those out in place, three bytes each, so the data +/// section's offsets stay put. +async fn comment_out_unsupported_settings( + dump_file: &DumpFile, + supported_settings: &HashSet, +) -> Result<()> { + use tokio::io::{AsyncBufReadExt, AsyncWriteExt}; + + let file = tokio::fs::File::open(&dump_file.path) + .await + .map_err(|e| Error::internal_err(format!("Failed to open dump file: {}", e)))?; + let mut reader = tokio::io::BufReader::new(file); + + let mut preamble: Vec = Vec::new(); + let mut patched = false; + loop { + let start = preamble.len(); + let read = reader + .read_until(b'\n', &mut preamble) + .await + .map_err(|e| Error::internal_err(format!("Failed to read dump file: {}", e)))?; + if read == 0 { + break; + } + let line = &preamble[start..]; + if !is_dump_preamble_line(line) { + preamble.truncate(start); + break; + } + if preamble_setting_name(line).is_some_and(|name| !supported_settings.contains(name)) { + preamble[start..start + 3].copy_from_slice(b"-- "); + patched = true; + } + } + if !patched { + return Ok(()); + } + + let mut file = tokio::fs::OpenOptions::new() + .write(true) + .open(&dump_file.path) + .await + .map_err(|e| Error::internal_err(format!("Failed to open dump file: {}", e)))?; + file.write_all(&preamble) + .await + .map_err(|e| Error::internal_err(format!("Failed to rewrite dump preamble: {}", e)))?; + file.flush() + .await + .map_err(|e| Error::internal_err(format!("Failed to rewrite dump preamble: {}", e)))?; + Ok(()) +} + +/// A psql invocation against `pg_db`, carrying the connection settings the CLI reads +/// from the environment. +fn psql_command(pg_db: &PgDatabase) -> tokio::process::Command { let mut cmd = tokio::process::Command::new("psql"); cmd.arg("--host") - .arg(host) + .arg(&pg_db.host) .arg("--port") - .arg(&port) + .arg(pg_db.port.unwrap_or(5432).to_string()) .arg("--username") - .arg(user) + .arg(pg_db.login_name()) .arg("--dbname") - .arg(dbname) + .arg(&pg_db.dbname) .arg("--no-psqlrc") - .arg("--file") - .arg(&dump_file.path) .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()); - if let Some(ref password) = target_db.password { + if let Some(ref password) = pg_db.password { cmd.env("PGPASSWORD", password); } - - if let Some(ref sslmode) = target_db.sslmode { + if let Some(ref sslmode) = pg_db.sslmode { cmd.env("PGSSLMODE", sslmode); } + cmd +} - let output = cmd +/// GUC names the server backing `pg_db` knows about. +/// +/// Asked through psql rather than a tokio-postgres connection so the lookup reaches +/// exactly the servers the restore itself can: libpq negotiates TLS for `sslmode=prefer` +/// and an unset mode, where `PgDatabase::connect` would hand a TLS-only server a +/// plaintext socket and fail before the import ever starts. +async fn server_setting_names(pg_db: &PgDatabase) -> Result> { + let output = psql_command(pg_db) + .arg("--tuples-only") + .arg("--no-align") + .arg("--command") + .arg("SELECT name FROM pg_settings") + .output() + .await + .map_err(|e| Error::internal_err(format!("Failed to execute psql: {}", e)))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(Error::internal_err(format!( + "Failed to list the settings of the target server: {}", + stderr + ))); + } + + Ok(String::from_utf8_lossy(&output.stdout) + .lines() + .map(|name| name.trim().to_string()) + .filter(|name| !name.is_empty()) + .collect()) +} + +/// Import a pg_dump file into a target database using psql. +/// +/// Left to its defaults psql reports a failed statement, carries on and still exits 0, +/// so a dump that breaks partway through imports partially and reads as a success. +/// ON_ERROR_STOP surfaces the failure and --single-transaction makes the restore +/// all-or-nothing, leaving the target as it was and the import retryable. +async fn pg_import_dump(target_db: &PgDatabase, dump_file: &DumpFile) -> Result<()> { + let supported_settings = server_setting_names(target_db).await?; + comment_out_unsupported_settings(dump_file, &supported_settings).await?; + + let output = psql_command(target_db) + .arg("--set") + .arg("ON_ERROR_STOP=1") + .arg("--single-transaction") + .arg("--file") + .arg(&dump_file.path) .output() .await .map_err(|e| Error::internal_err(format!("Failed to execute psql: {}", e)))?; @@ -2869,29 +3176,7 @@ async fn create_pg_database( } } - // Determine if this is an instance or resource-backed datatable - let is_instance_datatable = if let Some(dt_name) = req.source.strip_prefix("datatable://") { - let config = sqlx::query_scalar!( - "SELECT datatable->'datatables'->$2 FROM workspace_settings WHERE workspace_id = $1", - &w_id, - dt_name - ) - .fetch_optional(&db) - .await? - .flatten(); - config - .and_then(|v| { - v.get("database") - .and_then(|d| d.get("resource_type")) - .and_then(|r| r.as_str()) - .map(|s| s == "instance") - }) - .unwrap_or(false) - } else { - false - }; - - if is_instance_datatable { + if is_instance_datatable_source(&db, &w_id, &req.source).await? { windmill_common::create_custom_instance_database(&db, &req.target_dbname, "datatable") .await?; } else { @@ -2989,7 +3274,18 @@ async fn import_pg_database( } windmill_common::validate_dbname(&target_pg.dbname)?; - let dump_file = pg_dump_database(&source_pg, schema_only, &[]).await?; + // Ownership never replays: the restore runs as the target's own connection user, and + // what it creates it owns. Grants do, except around an instance data table — Windmill + // plants `custom_instance_user` grants in one, which nothing else can replay. Elsewhere + // the ACLs are user intent (`REVOKE ... FROM PUBLIC`) and dropping them widens access. + let no_acl = is_instance_datatable_source(&db, &w_id, &req.target).await? + || is_instance_datatable_source(&db, &w_id, &req.source).await?; + + let dump_file = pg_dump_database( + &source_pg, + PgDumpOptions { schema_only, no_owner: true, no_acl, ..Default::default() }, + ) + .await?; pg_import_dump(&target_pg, &dump_file).await?; Ok(format!( @@ -3011,7 +3307,11 @@ async fn export_pg_schema( Json(req): Json, ) -> Result { let pg = resolve_pg_source_checked(&db, &user_db, &authed, &w_id, &req.source).await?; - let dump_file = pg_dump_database(&pg, true, &[]).await?; + let dump_file = pg_dump_database( + &pg, + PgDumpOptions { schema_only: true, ..Default::default() }, + ) + .await?; tokio::fs::read_to_string(&dump_file.path) .await .map_err(|e| Error::internal_err(format!("Failed to read dump file: {}", e))) @@ -5500,9 +5800,8 @@ async fn clone_workspace_data( // Clone the forker's own per-user drafts (plus the legacy NULL-email // workspace draft, if any) so they keep their pending edits in the // fork. Other users' drafts are intentionally NOT cloned — they don't - // own a `usr` row in the fork (see `clone_workspace_full`) so their - // drafts would dangle and the home-page `draft_users` aggregate would - // surface them as duplicate legacy entries. + // own a `usr` row in the fork (see `clone_workspace_full`), so those + // drafts would belong to someone the fork holds no membership for. clone_drafts(tx, source_workspace_id, target_workspace_id, &authed.email).await?; // Clone workspace runnable dependencies and dependency map @@ -6841,7 +7140,16 @@ async fn clone_workspace_runnable_dependencies( .execute(&mut **tx) .await?; - // Clone dependency_map to preserve import relationships + // Recorded so the clone's own relocks have something to match; with no row they record NULL + // and nothing in it ever skips. Hashed from the locks the clone holds rather than copied from + // the source's rows, which are only as current as the last write to them: one left stale by a + // supplied lock deployed before this was recorded names a lock the clone no longer has, and an + // importer that resolved against the real one would then skip a relock it needed. + record_lock_hashes_for_workspace(tx, target_workspace_id).await?; + + // Deliberately without `imported_lockfile_hash`: it records what an importer resolved against + // when it was last locked, which nothing here can establish for the version the clone got. + // Left NULL, every importer relocks once and re-anchors both sides to what the clone holds. sqlx::query!( "INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) SELECT $1, importer_path, importer_kind, imported_path, importer_node_id @@ -7118,6 +7426,76 @@ async fn enforce_cloud_fork_cap(db: &DB, parent_workspace_id: &str) -> Result<() enforce_cloud_fork_count(db, &root, 1).await } +/// Cloud: refuse to attach a workspace that already has a paid plan of its own. +/// +/// Once attached it draws the root's plan and meters its usage there, so a subscription of its own +/// bills a second time for one plan. Only an attach can reach this state: a fork is created as a +/// fresh workspace and never had a plan to keep. +/// +/// Asked only of a candidate joining this family, never of one already under the same root: that +/// one is already in the double-billed state, where the settings page surfaces the leftover +/// subscription and the portal that cancels it, and refusing there would block re-designating a +/// renamed dev workspace over a billing problem the attach did not cause. +#[cfg(feature = "cloud")] +async fn reject_attach_of_subscribed_workspace(db: &DB, dev_w_id: &str) -> Result<()> { + let plan = sqlx::query_scalar!( + "SELECT plan FROM workspace_settings WHERE workspace_id = $1", + dev_w_id + ) + .fetch_optional(db) + .await? + .flatten(); + // Any plan, not just `'team'`: the column is written by the subscription webhook, and a plan + // value it does not write yet would otherwise walk straight past this. An enterprise + // arrangement is deliberately not covered — it sets `premium` without a plan and has no + // self-serve portal, so refusing there would be a dead end rather than something to act on. + if plan.is_some() { + return Err(Error::BadRequest(format!( + "Workspace {dev_w_id} is on a paid plan of its own. A dev or fork workspace runs on its parent's plan and is never invoiced separately, so cancel that subscription from its own billing settings before attaching it." + ))); + } + Ok(()) +} + +#[cfg(all(test, feature = "cloud"))] +mod attach_billing_guard_tests { + use super::reject_attach_of_subscribed_workspace; + use sqlx::{Pool, Postgres}; + + async fn workspace_on_plan(db: &Pool, id: &str, plan: Option<&str>) { + sqlx::query("INSERT INTO workspace (id, name, owner) VALUES ($1, $1, 'test-user')") + .bind(id) + .execute(db) + .await + .expect("insert workspace"); + sqlx::query("INSERT INTO workspace_settings (workspace_id, plan) VALUES ($1, $2)") + .bind(id) + .bind(plan) + .execute(db) + .await + .expect("insert workspace_settings"); + } + + #[sqlx::test(migrations = "../migrations")] + async fn refuses_a_candidate_that_still_pays_for_itself(db: Pool) { + workspace_on_plan(&db, "subscribed", Some("team")).await; + workspace_on_plan(&db, "cancelled", None).await; + + let err = reject_attach_of_subscribed_workspace(&db, "subscribed") + .await + .expect_err("a workspace on a paid plan of its own must not be attachable"); + assert!(err.to_string().contains("paid plan of its own"), "{err}"); + + // Cancelling clears `plan` but keeps `customer_id`, so the plan column is what decides. + reject_attach_of_subscribed_workspace(&db, "cancelled") + .await + .expect("a workspace with no plan is attachable"); + reject_attach_of_subscribed_workspace(&db, "no-settings-row") + .await + .expect("a workspace with no settings row is attachable"); + } +} + /// General guardrail (all builds): reject creating a fork/dev under `parent` when it would nest deeper /// than `MAX_FORK_DEPTH`. `added_subtree_height` is the height of the subtree grafted below the new /// node — 0 for a plain fork, or the candidate's own subtree height for an attach. @@ -7660,6 +8038,17 @@ async fn attach_dev_workspace( ))); } + // Deliberately below the admin-of-candidate check, unlike the cap enforcement above: the + // refusal names the candidate's plan, so running it earlier would tell any admin of any + // premium workspace whether an arbitrary workspace id is on a team plan. + #[cfg(feature = "cloud")] + if *CLOUD_HOSTED { + let root = windmill_common::workspaces::get_billing_workspace_id(&db, &prod_w_id).await?; + if windmill_common::workspaces::get_billing_workspace_id(&db, &dev_w_id).await? != root { + reject_attach_of_subscribed_workspace(&db, &dev_w_id).await?; + } + } + let mut tx = db.begin().await?; // Everything above ran outside a transaction, so prod's eligibility and the chain's labels could // have changed under us: re-decide both here, under the pairing lock. diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 99a6cd90f7..ad7f12453c 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -10,7 +10,7 @@ path = "src/lib.rs" [features] default = [] -private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-assets/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-amqp?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-azure?/private", "windmill-trigger-email?/private", "windmill-git-sync/private", "windmill-autoscaling?/private", "windmill-object-store/private", "windmill-api-npm-proxy/private"] +private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-assets/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-amqp?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-azure?/private", "windmill-trigger-email?/private", "windmill-git-sync/private", "windmill-autoscaling?/private", "windmill-object-store/private", "windmill-api-npm-proxy/private", "windmill-indexer?/private"] enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker?/enterprise", "windmill-api-auth/enterprise", "windmill-store/enterprise", "windmill-api-jobs/enterprise", "windmill-api-scripts/enterprise", "windmill-api-flows/enterprise", "windmill-api-users/enterprise", "windmill-api-workspaces/enterprise", "windmill-api-groups/enterprise", "windmill-api-configs/enterprise", "windmill-api-settings/enterprise", "windmill-api-schedule/enterprise", "windmill-api-debug/enterprise", "windmill-api-agent-workers?/enterprise", "windmill-trigger/enterprise", "windmill-trigger-kafka?/enterprise", "windmill-trigger-postgres?/enterprise", "windmill-trigger-mqtt?/enterprise", "windmill-trigger-amqp?/enterprise", "windmill-trigger-websocket?/enterprise", "windmill-trigger-email?/enterprise", "windmill-trigger-nats?/enterprise", "windmill-trigger-sqs?/enterprise", "windmill-trigger-gcp?/enterprise", "windmill-trigger-azure?/enterprise", "windmill-trigger-http?/enterprise", "windmill-native-triggers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-api-npm-proxy/enterprise", "license"] stripe = [] run_inline = ["dep:windmill-worker", "windmill-api-configs/run_inline"] @@ -18,10 +18,12 @@ agent_worker_server = ["dep:windmill-worker", "dep:windmill-api-agent-workers"] enterprise_saml = ["dep:samael", "dep:libxml"] benchmark = [] embedding = ["windmill-api-embeddings/embedding"] -parquet = ["dep:datafusion", "windmill-common/parquet", "windmill-object-store/parquet", "windmill-worker?/parquet", "windmill-api-users/parquet", "windmill-api-settings/parquet", "windmill-api-workspaces/parquet", "windmill-api-npm-proxy/parquet", "dep:aws-sigv4", "dep:aws-sdk-config", "dep:quick-xml"] +parquet = ["dep:datafusion", "windmill-common/parquet", "windmill-object-store/parquet", "windmill-worker?/parquet", "windmill-indexer?/parquet", "windmill-api-users/parquet", "windmill-api-settings/parquet", "windmill-api-workspaces/parquet", "windmill-api-npm-proxy/parquet", "dep:aws-sigv4", "dep:aws-sdk-config", "dep:quick-xml"] prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker?/prometheus", "windmill-api-scripts/prometheus"] openidconnect = ["dep:openidconnect", "windmill-common/openidconnect", "windmill-store/openidconnect"] -tantivy = ["dep:windmill-indexer"] +# The service log search handler reads the columnar store, so tantivy alone is +# not enough for this crate to build on its own. +tantivy = ["dep:windmill-indexer", "parquet"] kafka = ["dep:windmill-trigger-kafka", "windmill-store/kafka"] kafka-gssapi = ["kafka", "windmill-trigger-kafka/kafka-gssapi"] nats = ["dep:windmill-trigger-nats", "windmill-store/nats"] diff --git a/backend/windmill-api/openapi-deref.json b/backend/windmill-api/openapi-deref.json index 92eeaeaedb..3cba2b7da6 100644 --- a/backend/windmill-api/openapi-deref.json +++ b/backend/windmill-api/openapi-deref.json @@ -16532,202 +16532,6 @@ } } }, - "/w/{workspace}/ai_skills/list": { - "get": { - "summary": "list the workspace AI chat skills (name + description only)", - "operationId": "listAiSkills", - "tags": [ - "workspace" - ], - "parameters": [ - { - "$ref": "#/components/parameters/WorkspaceId" - } - ], - "responses": { - "200": { - "description": "skill listing", - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "type": "object", - "required": [ - "name", - "description" - ], - "properties": { - "name": { - "type": "string" - }, - "description": { - "type": "string" - } - } - } - } - } - } - } - } - } - }, - "/w/{workspace}/ai_skills/get/{name}": { - "get": { - "summary": "get a workspace AI chat skill including its instructions", - "operationId": "getAiSkill", - "tags": [ - "workspace" - ], - "parameters": [ - { - "$ref": "#/components/parameters/WorkspaceId" - }, - { - "name": "name", - "in": "path", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "skill", - "content": { - "application/json": { - "schema": { - "type": "object", - "required": [ - "name", - "description", - "instructions" - ], - "properties": { - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "instructions": { - "type": "string" - } - } - } - } - } - } - } - } - }, - "/w/{workspace}/ai_skills/upload": { - "post": { - "summary": "upsert workspace AI chat skills (admin only)", - "operationId": "uploadAiSkills", - "tags": [ - "workspace" - ], - "parameters": [ - { - "$ref": "#/components/parameters/WorkspaceId" - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "object", - "required": [ - "skills" - ], - "properties": { - "skills": { - "type": "array", - "maxItems": 50, - "items": { - "type": "object", - "required": [ - "name", - "description", - "instructions" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "maxLength": 64, - "pattern": "^[a-z0-9-]+$" - }, - "description": { - "type": "string", - "minLength": 1, - "maxLength": 1024 - }, - "instructions": { - "type": "string", - "minLength": 1, - "maxLength": 65536 - } - } - } - } - } - } - } - } - }, - "responses": { - "200": { - "description": "uploaded", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - } - } - } - } - } - }, - "/w/{workspace}/ai_skills/delete/{name}": { - "delete": { - "summary": "delete a workspace AI chat skill (admin only)", - "operationId": "deleteAiSkill", - "tags": [ - "workspace" - ], - "parameters": [ - { - "$ref": "#/components/parameters/WorkspaceId" - }, - { - "name": "name", - "in": "path", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "deleted", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - } - } - } - } - } - }, "/w/{workspace}/apps/get_data/v/{secretWithExtension}": { "get": { "summary": "get raw app data by", diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index 235c80ccf9..d5a14a717d 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -17063,141 +17063,6 @@ paths: text/plain: schema: type: string - /w/{workspace}/ai_skills/list: - get: - summary: list the workspace AI chat skills (name + description only) - operationId: listAiSkills - tags: - - workspace - parameters: - - name: workspace - in: path - required: true - schema: *ref_4 - responses: - '200': - description: skill listing - content: - application/json: - schema: - type: array - items: - type: object - required: - - name - - description - properties: - name: - type: string - description: - type: string - /w/{workspace}/ai_skills/get/{name}: - get: - summary: get a workspace AI chat skill including its instructions - operationId: getAiSkill - tags: - - workspace - parameters: - - name: workspace - in: path - required: true - schema: *ref_4 - - name: name - in: path - required: true - schema: - type: string - responses: - '200': - description: skill - content: - application/json: - schema: - type: object - required: - - name - - description - - instructions - properties: - name: - type: string - description: - type: string - instructions: - type: string - /w/{workspace}/ai_skills/upload: - post: - summary: upsert workspace AI chat skills (admin only) - operationId: uploadAiSkills - tags: - - workspace - parameters: - - name: workspace - in: path - required: true - schema: *ref_4 - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - skills - properties: - skills: - type: array - maxItems: 50 - items: - type: object - required: - - name - - description - - instructions - properties: - name: - type: string - minLength: 1 - maxLength: 64 - pattern: ^[a-z0-9-]+$ - description: - type: string - minLength: 1 - maxLength: 1024 - instructions: - type: string - minLength: 1 - maxLength: 65536 - responses: - '200': - description: uploaded - content: - text/plain: - schema: - type: string - /w/{workspace}/ai_skills/delete/{name}: - delete: - summary: delete a workspace AI chat skill (admin only) - operationId: deleteAiSkill - tags: - - workspace - parameters: - - name: workspace - in: path - required: true - schema: *ref_4 - - name: name - in: path - required: true - schema: - type: string - responses: - '200': - description: deleted - content: - text/plain: - schema: - type: string /w/{workspace}/apps/get_data/v/{secretWithExtension}: get: summary: get raw app data by diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 1f278e565d..89e2020e23 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.796.0 + version: 1.803.0 title: Windmill API contact: @@ -3788,6 +3788,38 @@ paths: schema: type: boolean + /w/{workspace}/workspaces/billable_seats: + get: + summary: get the billable seats of the workspace the plan is billed on + operationId: getBillableSeats + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + + responses: + "200": + description: billable seats + content: + application/json: + schema: + type: object + properties: + developers: + type: integer + description: >- + Omitted when the seats counted are another workspace's, as they are for a + fork resolving to its billing root. + operators: + type: integer + description: >- + Omitted when the seats counted are another workspace's, as they are for a + fork resolving to its billing root. + seats: + type: integer + required: + - seats + /w/{workspace}/workspaces/premium_info: get: summary: get premium info @@ -10467,7 +10499,7 @@ paths: summary: run script by path operationId: runScriptByPath x-mcp-tool: true - x-mcp-instructions: "You should first use getScriptByPath to retrieve the script's schema and understand what arguments are expected." + x-mcp-instructions: "You should first use getScriptByPath to retrieve the script's schema and understand what arguments are expected. A resource argument (schema format `resource-`) is the bare string `$res:` as that whole argument value -- not a wrapper object, not a plain path. A variable is `$var:`." x-mcp-tool-include-query-params: [] tags: - job @@ -12545,133 +12577,6 @@ paths: type: boolean description: more buckets matched than were returned, so summing them under-reports - /w/{workspace}/ai_skills/list: - get: - summary: list the workspace AI chat skills (name + description only) - operationId: listAiSkills - tags: - - workspace - parameters: - - $ref: "#/components/parameters/WorkspaceId" - responses: - "200": - description: skill listing - content: - application/json: - schema: - type: array - items: - type: object - required: - - name - - description - properties: - name: - type: string - description: - type: string - - /w/{workspace}/ai_skills/get/{name}: - get: - summary: get a workspace AI chat skill including its instructions - operationId: getAiSkill - tags: - - workspace - parameters: - - $ref: "#/components/parameters/WorkspaceId" - - name: name - in: path - required: true - schema: - type: string - responses: - "200": - description: skill - content: - application/json: - schema: - type: object - required: - - name - - description - - instructions - properties: - name: - type: string - description: - type: string - instructions: - type: string - - /w/{workspace}/ai_skills/upload: - post: - summary: upsert workspace AI chat skills (admin only) - operationId: uploadAiSkills - tags: - - workspace - parameters: - - $ref: "#/components/parameters/WorkspaceId" - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - skills - properties: - skills: - type: array - maxItems: 50 - items: - type: object - required: - - name - - description - - instructions - properties: - name: - type: string - minLength: 1 - maxLength: 64 - pattern: "^[a-z0-9-]+$" - description: - type: string - minLength: 1 - maxLength: 1024 - instructions: - type: string - minLength: 1 - maxLength: 65536 - responses: - "200": - description: uploaded - content: - text/plain: - schema: - type: string - - /w/{workspace}/ai_skills/delete/{name}: - delete: - summary: delete a workspace AI chat skill (admin only) - operationId: deleteAiSkill - tags: - - workspace - parameters: - - $ref: "#/components/parameters/WorkspaceId" - - name: name - in: path - required: true - schema: - type: string - responses: - "200": - description: deleted - content: - text/plain: - schema: - type: string - /w/{workspace}/apps/get_data/v/{secretWithExtension}: get: summary: get raw app data by @@ -13952,7 +13857,7 @@ paths: summary: run flow by path operationId: runFlowByPath x-mcp-tool: true - x-mcp-instructions: "You should first use getFlowByPath to retrieve the flow's schema and understand what arguments are expected." + x-mcp-instructions: "You should first use getFlowByPath to retrieve the flow's schema and understand what arguments are expected. A resource argument (schema format `resource-`) is the bare string `$res:` as that whole argument value -- not a wrapper object, not a plain path. A variable is `$var:`." x-mcp-tool-include-query-params: [] tags: - job @@ -23901,7 +23806,7 @@ paths: items: type: string hits: - description: log files that matched the query + description: the log lines that matched the query, newest first type: array items: $ref: "#/components/schemas/LogSearchHit" @@ -25330,6 +25235,68 @@ paths: schema: type: string + /w/{workspace}/hub/projects/{slug}/withdraw: + post: + summary: take a hub project submission back out of review + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. Everything pushed for the + submission is kept, so it can be fixed and submitted again. + operationId: withdrawHubProject + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: slug + in: path + required: true + description: hub project slug (3-50 chars, lowercase alphanumeric and hyphens, no leading/trailing hyphen) + schema: + type: string + minLength: 3 + maxLength: 50 + pattern: "^[a-z0-9][a-z0-9-]*[a-z0-9]$" + - $ref: "#/components/parameters/HubPublishFolder" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/projects/{slug}/discard_update: + post: + summary: discard the pending update to a published hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. The published project is + left untouched. + operationId: discardHubProjectUpdate + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: slug + in: path + required: true + description: hub project slug (3-50 chars, lowercase alphanumeric and hyphens, no leading/trailing hyphen) + schema: + type: string + minLength: 3 + maxLength: 50 + pattern: "^[a-z0-9][a-z0-9-]*[a-z0-9]$" + - $ref: "#/components/parameters/HubPublishFolder" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + /w/{workspace}/hub/project: get: summary: get the hub project linked to a workspace folder @@ -27205,10 +27172,35 @@ components: type: integer minimum: 1 maximum: 2000000 + free_tier: + $ref: "#/components/schemas/FreeTierInfo" model_pricing: type: object additionalProperties: $ref: "#/components/schemas/ModelPriceOverride" + copilot_disabled: + type: boolean + description: >- + Hides the Windmill AI assistant (chat, sessions, code generation, completion, + fixes) from the workspace UI. Read from the workspace's own settings even when + the providers served fall back to the instance config. AI agent steps and the + AI sandbox in flows are unaffected. + + FreeTierInfo: + type: object + description: >- + Read-only. Present when the workspace has no AI provider of its own and is running + on Windmill's free tier. Ignored on write. + properties: + exhausted: + type: boolean + description: The one-time grant is spent; no provider is served and the user must add their own API key. + used_ratio: + type: number + description: Fraction of the grant consumed, 0 to 1. + required: + - exhausted + - used_ratio ModelPriceOverride: type: object @@ -27971,7 +27963,10 @@ components: type: string args: $ref: "#/components/schemas/ScriptArgs" - result: {} + result: + description: | + For large results, this may be the placeholder string 'WINDMILL_TOO_BIG'. + Use the completed job result endpoint to retrieve the full result. logs: type: string deleted: @@ -29540,6 +29535,12 @@ components: type: string is_fileset: type: boolean + format_extension: + type: string + nullable: true + description: >- + File extension for a type whose value is one file rather than a set + of fields. Omit to leave it unchanged; send null to clear it. TriggerHistoryEntry: type: object @@ -30245,7 +30246,9 @@ components: description: If true, passes the request body as a raw string instead of parsing as JSON error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -30334,7 +30337,9 @@ components: description: If true, passes the request body as a raw string instead of parsing as JSON error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -30430,7 +30435,9 @@ components: description: If true, passes the request body as a raw string instead of parsing as JSON error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -30622,7 +30629,9 @@ components: description: Optional periodic heartbeat message configuration error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: description: Arguments to pass to the error handler $ref: "#/components/schemas/ScriptArgs" @@ -30687,7 +30696,9 @@ components: description: Optional periodic heartbeat message configuration error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: description: Arguments to pass to the error handler $ref: "#/components/schemas/ScriptArgs" @@ -30763,7 +30774,9 @@ components: description: Optional periodic heartbeat message configuration error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: description: Arguments to pass to the error handler $ref: "#/components/schemas/ScriptArgs" @@ -30891,7 +30904,9 @@ components: description: Last error message if the trigger failed error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -30942,7 +30957,9 @@ components: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -31006,7 +31023,9 @@ components: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -31089,7 +31108,9 @@ components: description: Last error message if the trigger failed error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -31130,7 +31151,9 @@ components: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -31184,7 +31207,9 @@ components: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -31582,7 +31607,9 @@ components: description: Last error message if the trigger failed error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -31684,7 +31711,9 @@ components: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -31740,7 +31769,9 @@ components: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -31868,7 +31899,9 @@ components: description: Timestamp of last server heartbeat (internal) error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -31908,7 +31941,9 @@ components: description: Configuration for creating/managing the publication (tables, operations) error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -31960,7 +31995,9 @@ components: description: Configuration for creating/managing the publication (tables, operations) error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -32037,7 +32074,9 @@ components: description: Last error message if the trigger failed error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -32101,7 +32140,9 @@ components: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -32176,7 +32217,9 @@ components: description: True if script_path points to a flow, false if it points to a script error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -32239,7 +32282,9 @@ components: description: Last error message if the trigger failed error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -32287,7 +32332,9 @@ components: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -32346,7 +32393,9 @@ components: description: True if script_path points to a flow, false if it points to a script error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -34229,8 +34278,34 @@ components: LogSearchHit: type: object properties: - dancer: + ts: + description: timestamp of the log line itself, not of the file containing it type: string + format: date-time + host: + type: string + level: + type: string + enum: [TRACE, DEBUG, INFO, WARN, ERROR] + target: + description: the tracing target that emitted the line + type: string + nullable: true + message: + type: string + file_path: + description: the log file the line came from + type: string + line_no: + description: offset of the line within its file + type: integer + required: + - ts + - host + - level + - message + - file_path + - line_no AutoscalingEvent: type: object diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 1a5f9dba87..e70cc19544 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -409,6 +409,19 @@ impl ExpiringProviderCredentials { } } +/// Set on the copilot config when the workspace has no AI provider of its own and is +/// running on Windmill's free tier, so the client can label the lent model as free, warn +/// before the grant runs out, and tell the user to add their own key once it has — rather +/// than showing the same "no provider configured" state a never-configured workspace gets. +#[derive(Serialize, Deserialize, Debug, Default, Clone)] +pub struct FreeTierInfo { + /// The grant is spent: no provider is served and the user must bring their own key. + pub exhausted: bool, + /// Fraction of the grant consumed, 0.0..=1.0. A ratio, not a dollar amount — the + /// pricing model stays server-side. + pub used_ratio: f64, +} + #[derive(Serialize, Deserialize, Debug, Default)] pub struct AIConfig { #[serde(skip_serializing_if = "Option::is_none")] @@ -423,10 +436,21 @@ pub struct AIConfig { pub custom_prompts: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub max_tokens_per_model: Option>, + /// Response-only: this same struct is the request body for saving a workspace's AI + /// config, and `skip_deserializing` is what stops a client from storing a forged + /// free-tier marker. Only the server sets it, per-request. + #[serde(skip_serializing_if = "Option::is_none", skip_deserializing)] + pub free_tier: Option, /// Per-model price overrides, keyed `provider:model` like `max_tokens_per_model`. /// Only models whose rates differ from the built-in table are stored. #[serde(skip_serializing_if = "Option::is_none")] pub model_pricing: Option>, + /// Hides the Windmill AI assistant (chat, sessions, generation, completion, fixes) from + /// the workspace UI. Only the workspace's own row is consulted: the flag holds even when + /// the providers served come from the instance config or the free tier. AI agent steps + /// and the AI sandbox are unaffected, so the providers stay in force. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub copilot_disabled: bool, } /// Negotiated rates in USD per million tokens. An unset cache rate is read as the @@ -1013,83 +1037,119 @@ async fn proxy( check_scopes(&authed, || format!("resources:read:{}", resource_path))?; } - let mut credentials = match workspace_cache { - Some(request_cache) if !request_cache.is_expired() && forced_resource_path.is_none() => { - request_cache.credentials - } - _ => { - let (resource_path, save_to_cache, resource_workspace, instance_ai_config_revision) = - if let Some(resource_path) = forced_resource_path { - // forced resource path - (resource_path, false, w_id.clone(), None) - } else { - let workspace_ai_config = sqlx::query_scalar!( - "SELECT ai_config FROM workspace_settings WHERE workspace_id = $1", - &w_id - ) - .fetch_one(&db) - .await?; + // Set when serving the request through Windmill's free AI tier (the lent key). Holds + // the per-user concurrency lock and drives response metering. + let mut free_lease: Option = None; + let mut credentials = 'cred: { + match workspace_cache { + Some(request_cache) + if !request_cache.is_expired() && forced_resource_path.is_none() => + { + request_cache.credentials + } + _ => { + let (resource_path, save_to_cache, resource_workspace, instance_ai_config_revision) = + if let Some(resource_path) = forced_resource_path { + // forced resource path + (resource_path, false, w_id.clone(), None) + } else { + let workspace_ai_config = sqlx::query_scalar!( + "SELECT ai_config FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_one(&db) + .await?; - let (ai_config_value, resource_workspace, instance_ai_config_revision) = { - let ws_has_config = workspace_ai_config - .as_ref() - .and_then(|v| serde_json::from_value::(v.clone()).ok()) - .is_some_and(|config| config.has_providers()); + let (ai_config_value, resource_workspace, instance_ai_config_revision) = { + let ws_has_config = workspace_ai_config + .as_ref() + .and_then(|v| serde_json::from_value::(v.clone()).ok()) + .is_some_and(|config| config.has_providers()); - if ws_has_config { - (workspace_ai_config.unwrap(), w_id.clone(), None) - } else { - let instance_config = sqlx::query_scalar!( - "SELECT value FROM global_settings WHERE name = 'ai_config'" - ) - .fetch_optional(&db) - .await?; + if ws_has_config { + (workspace_ai_config.unwrap(), w_id.clone(), None) + } else { + let instance_config = sqlx::query_scalar!( + "SELECT value FROM global_settings WHERE name = 'ai_config'" + ) + .fetch_optional(&db) + .await?; - match instance_config { - Some(config) => ( - config, - "admins".to_string(), - Some(current_instance_ai_config_revision()), - ), - None => { - return Err(Error::internal_err( - "AI resource not configured".to_string(), - )); + let instance_has_config = + instance_config.as_ref().is_some_and(|v| { + serde_json::from_value::(v.clone()) + .ok() + .is_some_and(|c| c.has_providers()) + }); + match instance_config { + // An instance `ai_config` row with no usable provider (e.g. `{}` + // or `{"providers":{}}`) is treated as unconfigured, exactly as + // build_copilot_settings_state does — otherwise its mere presence + // would suppress the free-tier fallback below. + Some(config) if instance_has_config => ( + config, + "admins".to_string(), + Some(current_instance_ai_config_revision()), + ), + _ => { + // Nothing configured: fall back to Windmill's free AI tier + // (EE-only) if a lent key is set and both the user's + // one-time grant and the instance's daily cap have room. + // Errors once the grant is spent, the day is capped, or the + // user already has a request in flight; None otherwise. + // Ineligible identities (e.g. service accounts) are refused + // inside the helper, so every path treats them alike. + let free = + crate::ai_free_tier_oss::resolve_free_tier_credentials( + &provider, + &db, + &ai_path, + &authed.email, + &body, + ) + .await?; + if let Some((free_credentials, lease)) = free { + free_lease = Some(lease); + break 'cred free_credentials; + } + return Err(Error::internal_err( + "AI resource not configured".to_string(), + )); + } } } + }; + + let mut ai_config = serde_json::from_value::(ai_config_value) + .map_err(|e| Error::BadRequest(e.to_string()))?; + + let provider_config = ai_config + .providers + .as_mut() + .and_then(|providers| providers.remove(&provider)) + .ok_or_else(|| { + Error::BadRequest(format!("Provider {:?} not configured", provider)) + })?; + + if provider_config.resource_path.is_empty() { + return Err(Error::BadRequest("Resource path is empty".to_string())); } + + ( + provider_config.resource_path, + true, + resource_workspace, + instance_ai_config_revision, + ) }; - let mut ai_config = serde_json::from_value::(ai_config_value) - .map_err(|e| Error::BadRequest(e.to_string()))?; - - let provider_config = ai_config - .providers - .as_mut() - .and_then(|providers| providers.remove(&provider)) - .ok_or_else(|| { - Error::BadRequest(format!("Provider {:?} not configured", provider)) - })?; - - if provider_config.resource_path.is_empty() { - return Err(Error::BadRequest("Resource path is empty".to_string())); - } - - ( - provider_config.resource_path, - true, - resource_workspace, - instance_ai_config_revision, - ) - }; - - // For user-specified resources, fetch through an RLS-scoped - // connection so PostgreSQL row-level security enforces the same - // folder/group boundaries as the regular resource API. For the - // workspace/instance ai_config path, the resource_path was already - // validated by an admin/devops user when configuring the workspace, - // so the raw pool is used. - let resource = if is_user_specified_resource { + // For user-specified resources, fetch through an RLS-scoped + // connection so PostgreSQL row-level security enforces the same + // folder/group boundaries as the regular resource API. For the + // workspace/instance ai_config path, the resource_path was already + // validated by an admin/devops user when configuring the workspace, + // so the raw pool is used. + let resource = if is_user_specified_resource { let mut tx = user_db.clone().begin(&authed).await?; let res = sqlx::query_scalar::<_, Option>>>( "SELECT value FROM resource WHERE path = $1 AND workspace_id = $2", @@ -1112,38 +1172,45 @@ async fn proxy( .ok_or_else(|| Error::NotFound(format!("Could not find the resource {}, update the resource path in the workspace settings", resource_path)))? .ok_or_else(|| Error::BadRequest(format!("Empty resource value for {}", resource_path)))?; - let resource = serde_json::from_str::(resource.0.get()) - .map_err(|e| Error::BadRequest(e.to_string()))?; + let resource = serde_json::from_str::(resource.0.get()) + .map_err(|e| Error::BadRequest(e.to_string()))?; - // Enforce RLS on $var: resolution when the resource path was - // user-specified (X-Resource-Path header) so users can only read - // variables they have permission to access. - let enforce_authed = if is_user_specified_resource { - Some(&authed) - } else { - None - }; - let credentials = resolve_provider_credentials( - &provider, - &db, - &resource_workspace, - resource, - enforce_authed, - ) - .await?; - if save_to_cache { - AI_REQUEST_CACHE.insert( - (w_id.clone(), provider.clone()), - ExpiringProviderCredentials::new( - credentials.clone(), - instance_ai_config_revision, - ), - ); + // Enforce RLS on $var: resolution when the resource path was + // user-specified (X-Resource-Path header) so users can only read + // variables they have permission to access. + let enforce_authed = if is_user_specified_resource { + Some(&authed) + } else { + None + }; + let credentials = resolve_provider_credentials( + &provider, + &db, + &resource_workspace, + resource, + enforce_authed, + ) + .await?; + if save_to_cache { + AI_REQUEST_CACHE.insert( + (w_id.clone(), provider.clone()), + ExpiringProviderCredentials::new( + credentials.clone(), + instance_ai_config_revision, + ), + ); + } + credentials } - credentials } }; + // Free tier: pin the model and clamp max_tokens server-side before forwarding, + // since the request body is otherwise client-controlled. + if free_lease.is_some() { + body = crate::ai_free_tier_oss::enforce_free_tier_body(&body)?; + } + if let Some(fim_transform) = maybe_transform_fim_request(&provider, &ai_path, &credentials.base_url, &body)? { @@ -1291,8 +1358,32 @@ async fn proxy( let status_code = response.status(); let headers = response.headers().clone(); + let is_sse = is_sse_response(&headers); + + // Free tier: reconcile the cost reserved up-front against what the response actually + // used, holding the per-user lock (via the lease) until it is recorded. The chat + // streams (SSE), where the usage report only arrives in the final chunk; the + // non-streaming JSON path is handled for completeness. + if let Some(lease) = free_lease { + let body = if is_sse { + axum::body::Body::from_stream(inject_keepalives( + Box::pin(crate::ai_free_tier_oss::meter_usage( + response.bytes_stream(), + db.clone(), + lease, + )), + Duration::from_secs(KEEPALIVE_INTERVAL_SECS), + )) + } else { + let bytes = response.bytes().await.map_err(to_anyhow)?; + crate::ai_free_tier_oss::record_json_usage(db.clone(), lease, &bytes); + axum::body::Body::from(bytes) + }; + return Ok((status_code, headers, body)); + } + let stream = response.bytes_stream(); - let body = if is_sse_response(&headers) { + let body = if is_sse { axum::body::Body::from_stream(inject_keepalives( stream, Duration::from_secs(KEEPALIVE_INTERVAL_SECS), diff --git a/backend/windmill-api/src/ai_evals/datasets.rs b/backend/windmill-api/src/ai_evals/datasets.rs index da901276b0..8e1acb3d2a 100644 --- a/backend/windmill-api/src/ai_evals/datasets.rs +++ b/backend/windmill-api/src/ai_evals/datasets.rs @@ -190,6 +190,8 @@ pub async fn create_dataset( } tx.commit().await?; + windmill_common::feature_usage::log_feature_usage("ai_agent_eval", "dataset_created", ""); + Ok(format!("Created eval dataset {}", payload.path)) } diff --git a/backend/windmill-api/src/ai_evals/run.rs b/backend/windmill-api/src/ai_evals/run.rs index e102abaf4c..5d282c7e5b 100644 --- a/backend/windmill-api/src/ai_evals/run.rs +++ b/backend/windmill-api/src/ai_evals/run.rs @@ -707,6 +707,17 @@ pub async fn run_experiment( .await?; return Err(e); } + // Which state of the agent was measured is the whole key vocabulary: it is what separates + // running what is deployed from measuring edits or an older version. + windmill_common::feature_usage::log_feature_usage( + "ai_agent_eval", + "run", + match subject.kind { + EvalSubjectKind::Agent => "agent", + EvalSubjectKind::AgentDraft => "agent_draft", + EvalSubjectKind::AgentVersion => "agent_version", + }, + ); Ok(experiment_id.to_string()) } diff --git a/backend/windmill-api/src/ai_free_tier_oss.rs b/backend/windmill-api/src/ai_free_tier_oss.rs new file mode 100644 index 0000000000..7a183ff8ed --- /dev/null +++ b/backend/windmill-api/src/ai_free_tier_oss.rs @@ -0,0 +1,65 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::ai_free_tier_ee::*; + +// Open-source build: Windmill's free AI tier does not exist. These stubs make the +// callers in `ai.rs` / `workspaces.rs` compile while disabling the feature entirely — +// `resolve_free_tier_credentials` never opts in, so the proxy falls through to its +// normal "AI resource not configured" path and the copilot stays hidden. +// +// Caller contract (enforced by the private impl, restated here for parity): the `email` +// passed to `resolve_free_tier_credentials` / `free_tier_copilot_config` MUST be the +// authenticated caller's own identity (an `ApiAuthed` email), never a client-supplied one — +// it selects whose lent-key grant is spent and whose usage is read. + +#[cfg(not(feature = "private"))] +use crate::ai::AIConfig; +#[cfg(not(feature = "private"))] +use crate::db::DB; +#[cfg(not(feature = "private"))] +use axum::body::Bytes; +#[cfg(not(feature = "private"))] +use windmill_ai::ai_providers::AIProvider; +#[cfg(not(feature = "private"))] +use windmill_ai::credentials::ProviderCredentials; +#[cfg(not(feature = "private"))] +use windmill_common::error::Result; + +#[cfg(not(feature = "private"))] +pub struct FreeTierLease; + +#[cfg(not(feature = "private"))] +pub async fn resolve_free_tier_credentials( + _provider: &AIProvider, + _db: &DB, + _ai_path: &str, + _email: &str, + _body: &Bytes, +) -> Result> { + Ok(None) +} + +#[cfg(not(feature = "private"))] +pub fn enforce_free_tier_body(body: &Bytes) -> Result { + Ok(body.clone()) +} + +#[cfg(not(feature = "private"))] +pub async fn free_tier_copilot_config(_db: &DB, _email: &str) -> Result> { + Ok(None) +} + +#[cfg(not(feature = "private"))] +pub fn record_json_usage(_db: DB, _lease: FreeTierLease, _bytes: &[u8]) {} + +#[cfg(not(feature = "private"))] +pub fn meter_usage( + upstream: S, + _db: DB, + _lease: FreeTierLease, +) -> impl futures::Stream> +where + S: futures::Stream> + Unpin, +{ + upstream +} diff --git a/backend/windmill-api/src/ai_skills.rs b/backend/windmill-api/src/ai_skills.rs deleted file mode 100644 index 11d99d55da..0000000000 --- a/backend/windmill-api/src/ai_skills.rs +++ /dev/null @@ -1,394 +0,0 @@ -/* - * Author: Ruben Fiszel - * Copyright: Windmill Labs, Inc 2026 - * This file and its contents are licensed under the AGPLv3 License. - * Please see the included NOTICE for copyright information and - * LICENSE-AGPL for a copy of the license. - */ - -use crate::db::{ApiAuthed, DB}; -use axum::{ - extract::{Extension, Json, Path}, - routing::{delete, get, post}, - Router, -}; -use serde::{Deserialize, Serialize}; -use std::collections::HashSet; -use windmill_audit::audit_oss::audit_log; -use windmill_audit::ActionKind; -use windmill_common::{ - db::UserDB, - error::{Error, JsonResult, Result}, - utils::require_admin, -}; - -pub fn workspaced_service() -> Router { - Router::new() - .route("/list", get(list_skills)) - .route("/get/{name}", get(get_skill)) - .route("/upload", post(upload_skills)) - .route("/delete/{name}", delete(delete_skill)) -} - -/// Cheap listing surfaced in the AI chat system prompt — no `instructions` body. -#[derive(Serialize)] -pub struct SkillListItem { - pub name: String, - pub description: String, -} - -/// Full skill, including the SKILL.md body, fetched on demand by `read_skill`. -#[derive(Serialize)] -pub struct Skill { - pub name: String, - pub description: String, - pub instructions: String, -} - -#[derive(Deserialize)] -pub struct UploadSkills { - pub skills: Vec, -} - -#[derive(Deserialize)] -pub struct SkillUpload { - pub name: String, - pub description: String, - pub instructions: String, -} - -const MAX_SKILLS_PER_UPLOAD: usize = 50; -// Every stored skill's name + description is advertised in the global AI chat -// system prompt, so bound the total a workspace can accumulate across uploads. -const MAX_SKILLS_PER_WORKSPACE: usize = 100; -// `name` and `description` follow the Claude SKILL.md spec -// (https://platform.claude.com/docs/en/agents-and-tools/agent-skills): both are -// loaded into the AI chat system prompt and `name` is the model-facing skill id, -// so matching the upstream limits keeps skills portable with Claude Code. -const MAX_SKILL_NAME_CHARS: usize = 64; -const MAX_SKILL_DESCRIPTION_CHARS: usize = 1_024; -// Not a spec field — a payload bound on the SKILL.md body, so measured in bytes. -const MAX_SKILL_INSTRUCTIONS_BYTES: usize = 64 * 1024; - -fn validate_skill(skill: &SkillUpload) -> Result<()> { - let name = skill.name.trim(); - if name.is_empty() || name.chars().count() > MAX_SKILL_NAME_CHARS { - return Err(Error::BadRequest(format!( - "skill name must be between 1 and {MAX_SKILL_NAME_CHARS} characters, got {:?}", - skill.name - ))); - } - if !name - .chars() - .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') - { - return Err(Error::BadRequest(format!( - "skill name {name:?} must only contain lowercase letters, digits or '-'" - ))); - } - if skill.description.trim().is_empty() { - return Err(Error::BadRequest(format!( - "skill {name:?} is missing a description (the SKILL.md frontmatter `description`)" - ))); - } - if skill.description.chars().count() > MAX_SKILL_DESCRIPTION_CHARS { - return Err(Error::BadRequest(format!( - "skill {name:?} description must be at most {MAX_SKILL_DESCRIPTION_CHARS} characters" - ))); - } - if skill.instructions.trim().is_empty() { - return Err(Error::BadRequest(format!( - "skill {name:?} has an empty SKILL.md body" - ))); - } - if skill.instructions.len() > MAX_SKILL_INSTRUCTIONS_BYTES { - return Err(Error::BadRequest(format!( - "skill {name:?} instructions must be at most {MAX_SKILL_INSTRUCTIONS_BYTES} bytes" - ))); - } - Ok(()) -} - -/// Collect the trimmed skill names, rejecting duplicates within a single upload. -/// The insert upserts by name, so a duplicate would silently keep only the last -/// and make the reported/audited count wrong. -fn collect_upload_names(skills: &[SkillUpload]) -> Result> { - let mut names = Vec::with_capacity(skills.len()); - let mut seen = HashSet::with_capacity(skills.len()); - for skill in skills { - let name = skill.name.trim().to_string(); - if !seen.insert(name.clone()) { - return Err(Error::BadRequest(format!( - "duplicate skill name {name:?} in upload" - ))); - } - names.push(name); - } - Ok(names) -} - -/// Reject an upload that would push the workspace past `MAX_SKILLS_PER_WORKSPACE`. -/// Uploads upsert, so names already present (`replacing`) don't count as new. -fn check_workspace_skill_capacity( - existing_total: i64, - replacing: i64, - upload_count: usize, -) -> Result<()> { - let new_count = upload_count as i64 - replacing; - if existing_total + new_count > MAX_SKILLS_PER_WORKSPACE as i64 { - return Err(Error::BadRequest(format!( - "workspace cannot store more than {MAX_SKILLS_PER_WORKSPACE} skills" - ))); - } - Ok(()) -} - -async fn list_skills( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, -) -> JsonResult> { - let mut tx = user_db.begin(&authed).await?; - let rows = sqlx::query!( - "SELECT name, description FROM ai_skill WHERE workspace_id = $1 ORDER BY name", - &w_id - ) - .fetch_all(&mut *tx) - .await?; - tx.commit().await?; - - Ok(Json( - rows.into_iter() - .map(|r| SkillListItem { name: r.name, description: r.description }) - .collect(), - )) -} - -async fn get_skill( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, name)): Path<(String, String)>, -) -> JsonResult { - let mut tx = user_db.begin(&authed).await?; - let row = sqlx::query!( - "SELECT name, description, instructions FROM ai_skill WHERE workspace_id = $1 AND name = $2", - &w_id, - &name - ) - .fetch_optional(&mut *tx) - .await?; - tx.commit().await?; - - row.map(|r| { - Json(Skill { name: r.name, description: r.description, instructions: r.instructions }) - }) - .ok_or_else(|| Error::NotFound(format!("no skill named {name:?} in workspace {w_id}"))) -} - -/// Bulk upsert the uploaded skills by name. Existing skills not in the payload -/// are left untouched — removal goes through `delete_skill`. -async fn upload_skills( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - Json(payload): Json, -) -> Result { - require_admin(authed.is_admin, &authed.username)?; - - if payload.skills.is_empty() { - return Err(Error::BadRequest("no skills to upload".to_string())); - } - if payload.skills.len() > MAX_SKILLS_PER_UPLOAD { - return Err(Error::BadRequest(format!( - "cannot upload more than {MAX_SKILLS_PER_UPLOAD} skills at a time" - ))); - } - for skill in &payload.skills { - validate_skill(skill)?; - } - let names = collect_upload_names(&payload.skills)?; - - let mut tx = db.begin().await?; - let counts = sqlx::query!( - r#"SELECT - COUNT(*)::bigint AS "total!", - COUNT(*) FILTER (WHERE name = ANY($2::text[]))::bigint AS "replacing!" - FROM ai_skill - WHERE workspace_id = $1"#, - &w_id, - &names - ) - .fetch_one(&mut *tx) - .await?; - check_workspace_skill_capacity(counts.total, counts.replacing, names.len())?; - - for (skill, name) in payload.skills.iter().zip(names.iter()) { - sqlx::query!( - r#"INSERT INTO ai_skill (workspace_id, name, description, instructions, edited_at, edited_by) - VALUES ($1, $2, $3, $4, now(), $5) - ON CONFLICT (workspace_id, name) DO UPDATE - SET description = EXCLUDED.description, - instructions = EXCLUDED.instructions, - edited_at = now(), - edited_by = EXCLUDED.edited_by"#, - &w_id, - name, - skill.description, - skill.instructions, - &authed.username, - ) - .execute(&mut *tx) - .await?; - } - - let audit_resource = names.join(","); - audit_log( - &mut *tx, - &authed, - "ai_skills.upload", - ActionKind::Update, - &w_id, - Some(&audit_resource), - Some([("skill_count", &names.len().to_string()[..])].into()), - ) - .await?; - tx.commit().await?; - - Ok(format!( - "Uploaded {} skill(s) to workspace {}", - payload.skills.len(), - &w_id - )) -} - -async fn delete_skill( - authed: ApiAuthed, - Extension(db): Extension, - Path((w_id, name)): Path<(String, String)>, -) -> Result { - require_admin(authed.is_admin, &authed.username)?; - - let mut tx = db.begin().await?; - let deleted = sqlx::query_scalar!( - "DELETE FROM ai_skill WHERE workspace_id = $1 AND name = $2 RETURNING name", - &w_id, - &name - ) - .fetch_optional(&mut *tx) - .await?; - - if deleted.is_none() { - tx.commit().await?; - return Err(Error::NotFound(format!( - "no skill named {name:?} in workspace {w_id}" - ))); - } - - audit_log( - &mut *tx, - &authed, - "ai_skills.delete", - ActionKind::Delete, - &w_id, - Some(&name), - None, - ) - .await?; - tx.commit().await?; - - Ok(format!("Deleted skill {name} from workspace {w_id}")) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn skill() -> SkillUpload { - SkillUpload { - name: "test-skill".to_string(), - description: "Useful for tests".to_string(), - instructions: "# Test\n\nDo the thing.".to_string(), - } - } - - #[test] - fn validate_skill_rejects_oversized_description() { - let mut skill = skill(); - skill.description = "x".repeat(MAX_SKILL_DESCRIPTION_CHARS + 1); - - assert!(matches!(validate_skill(&skill), Err(Error::BadRequest(_)))); - } - - #[test] - fn validate_skill_rejects_oversized_instructions() { - let mut skill = skill(); - skill.instructions = "x".repeat(MAX_SKILL_INSTRUCTIONS_BYTES + 1); - - assert!(matches!(validate_skill(&skill), Err(Error::BadRequest(_)))); - } - - #[test] - fn validate_skill_rejects_oversized_name() { - let mut skill = skill(); - skill.name = "a".repeat(MAX_SKILL_NAME_CHARS + 1); - - assert!(matches!(validate_skill(&skill), Err(Error::BadRequest(_)))); - } - - #[test] - fn validate_skill_rejects_non_slug_name() { - // Uppercase, underscore, space and punctuation are all outside the - // Claude SKILL.md `[a-z0-9-]` name charset. - for bad in ["My-Skill", "my_skill", "my skill", "skill!"] { - let mut skill = skill(); - skill.name = bad.to_string(); - - assert!( - matches!(validate_skill(&skill), Err(Error::BadRequest(_))), - "{bad:?} should be rejected" - ); - } - } - - #[test] - fn validate_skill_counts_description_in_characters() { - // 1024 two-byte chars exceed the byte limit but sit exactly on the - // character limit, so they must be accepted. - let mut skill = skill(); - skill.description = "é".repeat(MAX_SKILL_DESCRIPTION_CHARS); - - assert!(validate_skill(&skill).is_ok()); - } - - #[test] - fn workspace_capacity_allows_replacement_at_cap() { - // Already at the cap, but the upload only replaces an existing skill. - let at_cap = MAX_SKILLS_PER_WORKSPACE as i64; - assert!(check_workspace_skill_capacity(at_cap, 1, 1).is_ok()); - } - - #[test] - fn workspace_capacity_rejects_new_skill_over_cap() { - let at_cap = MAX_SKILLS_PER_WORKSPACE as i64; - assert!(matches!( - check_workspace_skill_capacity(at_cap, 0, 1), - Err(Error::BadRequest(_)) - )); - } - - #[test] - fn collect_upload_names_trims_and_collects() { - let names = collect_upload_names(&[skill()]).unwrap(); - assert_eq!(names, vec!["test-skill".to_string()]); - } - - #[test] - fn collect_upload_names_rejects_duplicates() { - // Names are compared after trimming, so whitespace can't smuggle a dup in. - let dup = SkillUpload { name: " test-skill ".to_string(), ..skill() }; - assert!(matches!( - collect_upload_names(&[skill(), dup]), - Err(Error::BadRequest(_)) - )); - } -} diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 28bbca9753..3e3e3277fd 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -491,7 +491,8 @@ async fn list_apps( FROM draft d \ LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email \ LEFT JOIN password p ON p.email = d.email AND p.super_admin = true \ - WHERE d.workspace_id = app.workspace_id AND d.path = app.path AND d.typ IN ('app', 'raw_app')) as draft_users", + WHERE d.workspace_id = app.workspace_id AND d.path = app.path AND d.typ IN ('app', 'raw_app') \ + AND (d.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL)) as draft_users", "folder_labels(app.workspace_id, app.path) as inherited_labels", ]) .left() diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index 04d21c7b53..d781e229d2 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -102,6 +102,12 @@ lazy_static::lazy_static! { (20260727151319, include_str!( "../../migrations/20260727151319_draft_only_listing_indexes.up.sql" ).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY")), + (20260826202939, include_str!( + "../../migrations/20260826202939_queue_suspended_resume_at_index.up.sql" + ).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY").replace("DROP INDEX", "DROP INDEX CONCURRENTLY")), + (20260826214706, include_str!( + "../../migrations/20260826214706_queue_suspended_drop_legacy_index.up.sql" + ).replace("DROP INDEX", "DROP INDEX CONCURRENTLY")), ].into_iter().collect(); } @@ -228,6 +234,8 @@ impl Migrate for CustomMigrator { // CONCURRENTLY operations cannot run inside a transaction block // or a multi-statement query (PostgreSQL requires top-level execution). // Split into individual statements and execute each separately. + // The split is naive, so a `;` anywhere in an overridden migration — + // inside a comment or a string literal included — splits mid-statement. for stmt in migration_sql.split(';') { let stmt = stmt.trim(); if !stmt.is_empty() diff --git a/backend/windmill-api/src/drafts.rs b/backend/windmill-api/src/drafts.rs index 864b99a49d..42d6a47930 100644 --- a/backend/windmill-api/src/drafts.rs +++ b/backend/windmill-api/src/drafts.rs @@ -215,6 +215,9 @@ fn list_drafts_query(all_users: bool) -> String { // row: fall back to their instance-derived username (`password.username`), or // their email when derivation is disabled (`password.username` is NULL). This // keeps the raw email out of the payload whenever a derived username exists. + // A null username means the legacy row downstream, so an owner that resolves to + // no name at all — an external JWT's subject has neither row — is dropped rather + // than surfaced as a second legacy entry. let draft_users = r#"CASE WHEN d.typ::text IN ('script', 'flow', 'app', 'raw_app') THEN ( SELECT json_agg(json_build_object('username', COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN du.email END)) ORDER BY COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN du.email END) NULLS LAST) @@ -222,6 +225,7 @@ fn list_drafts_query(all_users: bool) -> String { LEFT JOIN usr u ON u.workspace_id = du.workspace_id AND u.email = du.email LEFT JOIN password p ON p.email = du.email AND p.super_admin = true WHERE du.workspace_id = d.workspace_id AND du.path = d.path AND du.typ = d.typ + AND (du.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL) ) ELSE NULL END"#; // Default lists the user's own drafts AND the legacy NULL-email rows; with // `all_users` the filter is dropped to list every workspace draft. diff --git a/backend/windmill-api/src/hub_publish.rs b/backend/windmill-api/src/hub_publish.rs index 033c6ebce5..ccb530ce20 100644 --- a/backend/windmill-api/src/hub_publish.rs +++ b/backend/windmill-api/src/hub_publish.rs @@ -42,6 +42,11 @@ pub fn workspaced_service() -> Router { .route("/migrations", post(publish_migrations)) .route("/projects/{slug}/export", get(get_project_export)) .route("/projects/{slug}/submit", post(submit_project)) + .route("/projects/{slug}/withdraw", post(withdraw_project)) + .route( + "/projects/{slug}/discard_update", + post(discard_project_update), + ) .route("/project", get(get_project_by_source)) } @@ -554,9 +559,35 @@ async fn submit_project( .await } -// The Hub has no auth of its own: it validates bearer tokens by calling this -// instance's /api/users/whoami. Forwarding the caller's own token logs them in -// on the Hub as themselves (account auto-created on first use). +// Take a submission back out of review, keeping what was pushed for it. +async fn withdraw_project( + ctx: HubPublishCtx, + Path((_workspace, slug)): Path<(String, ProjectSlug)>, +) -> Result { + ctx.post( + &format!("/projects/{}/withdraw", slug), + &serde_json::json!({}), + ) + .await +} + +// Throw away the pending update to an already-published project. The published +// version is untouched — it never saw the update. +async fn discard_project_update( + ctx: HubPublishCtx, + Path((_workspace, slug)): Path<(String, ProjectSlug)>, +) -> Result { + ctx.post( + &format!("/projects/{}/discard_update", slug), + &serde_json::json!({}), + ) + .await +} + +// The Hub has no auth of its own: it validates bearer tokens by calling +// /api/users/whoami — on app.windmill.dev for the public Hub, on the paired +// instance for a private one. Forwarding the caller's own token logs them in on +// the Hub as themselves (account auto-created on first use). async fn get_from_hub( path: &str, source_id: &str, diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index a30cffbfbf..53044ee020 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -18,6 +18,7 @@ use quick_cache::sync::Cache; use serde_json::value::RawValue; use serde_json::Value; use sha2::{Digest, Sha256}; +use std::borrow::Cow; use std::collections::HashMap; use std::str::FromStr; use std::sync::Arc; @@ -108,6 +109,7 @@ use windmill_common::{ flows::{add_virtual_items_if_necessary, resolve_maybe_value, FlowValue}, jobs::{script_path_to_payload, CompletedJob, JobKind, JobPayload, QueuedJob, RawCode}, oauth2::HmacSha256, + query_builders, scripts::{ScriptHash, ScriptLang}, users::username_to_permissioned_as, utils::{not_found_if_none, now_from_db, paginate, require_admin, Pagination, StripPath}, @@ -8131,6 +8133,79 @@ pub async fn run_wait_result_flow_by_version( .await } +/// Whether request-supplied SQL from an operator may run. Operators can only run deployed +/// code, so a request their job token (`WM_TOKEN`) authenticates comes from code a +/// non-operator authored. The job must still be running, and the request must have the +/// shape `wmill.datatable()` sends (PostgreSQL against a `datatable://` database), so a +/// WM_TOKEN that leaked into job logs cannot be replayed to reach another target while the +/// job lives, in particular DuckDB, which runs in-process in the worker. +/// +/// What it does permit is any statement against the workspace's data tables, writes and DDL +/// included: the helper's body is an unrestricted SQL template and data tables carry no +/// per-user ACL. Narrowing that is a separate decision from this exemption. +/// +/// The database argument is only half the target: the executor honors a `-- database` +/// directive in the SQL over it, and `-- s3` redirects the result set, so both are refused. +/// Check them against the code the executor runs rather than the request's `content`, which +/// is not the same string once a `WM_INTERNAL_DB` marker expands. +async fn operator_may_run_datatable_query( + db: &DB, + w_id: &str, + job_id: Option, + language: Option<&ScriptLang>, + content: &str, + args: Option<&HashMap>>, +) -> error::Result { + let Some(job_id) = job_id else { + return Ok(false); + }; + if language != Some(&ScriptLang::Postgresql) { + return Ok(false); + } + // Parse the directives out of the code the executor actually runs: it expands a + // `WM_INTERNAL_DB` marker first, and a directive can be embedded in the expansion. + // An expansion that overrides the language would run something other than the SQL the + // language check above cleared, so it is refused along with a malformed marker. + let executed = + match query_builders::try_expand_internal_db_query(content, &ScriptLang::Postgresql) { + Some(Ok(expanded)) if expanded.language_override.is_none() => Cow::Owned(expanded.code), + Some(_) => return Ok(false), + None => Cow::Borrowed(content), + }; + if windmill_parser_sql::parse_db_resource(&executed).is_some() + || !matches!(windmill_parser_sql::parse_s3_mode(&executed), Ok(None)) + { + return Ok(false); + } + let targets_datatable = args + .and_then(|args| args.get("database")) + .and_then(|database| serde_json::from_str::(database.get()).ok()) + .is_some_and(|database| database.starts_with("datatable://")); + if !targets_datatable { + return Ok(false); + } + Ok(sqlx::query_scalar!( + "SELECT running AS \"running!\" FROM v2_job_queue WHERE id = $1 AND workspace_id = $2", + job_id, + w_id + ) + .fetch_optional(db) + .await? + .unwrap_or(false)) +} + +/// The refusal an operator gets from a preview route. Inside a job the caller never ran a +/// preview themselves, so name the one thing the job's token may do. +fn operator_preview_refusal(job_id: Option) -> error::Error { + let reason = if job_id.is_some() { + "Operators cannot run preview jobs for security reasons: from inside a job, an \ + operator may only run a wmill.datatable() query while that job is running" + } else { + "Operators cannot run preview jobs for security reasons" + }; + error::Error::NotAuthorized(reason.to_string()) +} + async fn run_preview_script( authed: ApiAuthed, Extension(db): Extension, @@ -8142,9 +8217,20 @@ async fn run_preview_script( #[cfg(feature = "enterprise")] check_license_key_valid().await?; if authed.is_operator { - return Err(error::Error::NotAuthorized( - "Operators cannot run preview jobs for security reasons".to_string(), - )); + // A deferred run would outlive the running job the exemption keys off. + if run_query.get_scheduled_for(&db).await?.is_some() + || !operator_may_run_datatable_query( + &db, + &w_id, + authed.job_id, + preview.language.as_ref(), + preview.content.as_deref().unwrap_or_default(), + preview.args.as_ref(), + ) + .await? + { + return Err(operator_preview_refusal(authed.job_id)); + } } // Preview runs arbitrary, request-supplied code. require_path_read_access_for_preview // only checks folder/namespace *read* access (and is a no-op when path is null), so a @@ -8239,13 +8325,21 @@ async fn run_inline_preview_script( Path(w_id): Path, Json(preview): Json, ) -> error::Result { - // Same arbitrary-code class as run_preview_script: operators are blocked from - // running request-supplied code, and a narrowly-scoped token must not escape - // its scope through inline preview. - if authed.is_operator { - return Err(error::Error::NotAuthorized( - "Operators cannot run preview jobs for security reasons".to_string(), - )); + // Same arbitrary-code class as run_preview_script, and every worker and standalone + // server exposes this route, so an operator is refused on the same terms. A + // narrowly-scoped token must not escape its scope through inline preview either. + if authed.is_operator + && !operator_may_run_datatable_query( + &db, + &w_id, + job_id, + Some(&preview.language), + &preview.content, + preview.args.as_ref(), + ) + .await? + { + return Err(operator_preview_refusal(job_id)); } check_scopes(&authed, || format!("jobs:run"))?; if let Some(job_id) = job_id { diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index de49949043..054af0f2fa 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -66,7 +66,9 @@ use crate::scim_oss::has_scim_token; use windmill_common::error::AppError; mod ai; -mod ai_skills; +#[cfg(feature = "private")] +mod ai_free_tier_ee; +mod ai_free_tier_oss; mod apps; mod apps_raw_bundle; pub use apps::invalidate_app_policy_cache; @@ -450,15 +452,15 @@ pub async fn run_server( // unless they are allowed — hence a separate layer rather than widening the // one every other route shares. (`Mcp-Param-*` is only sent for tool inputs // annotated with `x-mcp-header`, which no tool here declares.) + // + // The request's own header list is mirrored rather than enumerated: a browser + // MCP client may send any custom name for a preprocessor to read, and no fixed + // list could cover them. Nothing is granted by echoing it: the origin is + // `Any`, so browsers never attach credentials, and the endpoint authenticates + // each request on its own. let mcp_cors = CorsLayer::new() .allow_methods([http::Method::GET, http::Method::POST, http::Method::DELETE]) - .allow_headers([ - http::header::CONTENT_TYPE, - http::header::AUTHORIZATION, - http::HeaderName::from_static("mcp-protocol-version"), - http::HeaderName::from_static("mcp-method"), - http::HeaderName::from_static("mcp-name"), - ]) + .allow_headers(tower_http::cors::AllowHeaders::mirror_request()) // The 401 challenge is how a client discovers where to authorize (RFC 9728), // and it is not a safelisted response header, so without this a browser // client sees an empty one and has no way to begin the OAuth flow. @@ -710,7 +712,6 @@ pub async fn run_server( Router::new() }) .nest("/ai", ai::workspaced_service()) - .nest("/ai_skills", ai_skills::workspaced_service()) .nest("/npm_proxy", windmill_api_npm_proxy::workspaced_service()) .nest( "/path_autocomplete", diff --git a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs index 568ad854f9..aa2415f664 100644 --- a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs +++ b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs @@ -878,7 +878,7 @@ is, a different one moves it there and archives the old path"), EndpointTool { name: Cow::Borrowed("runScriptByPath"), description: Cow::Borrowed("run script by path"), - instructions: Cow::Borrowed("You should first use getScriptByPath to retrieve the script's schema and understand what arguments are expected."), + instructions: Cow::Borrowed("You should first use getScriptByPath to retrieve the script's schema and understand what arguments are expected. A resource argument (schema format `resource-`) is the bare string `$res:` as that whole argument value -- not a wrapper object, not a plain path. A variable is `$var:`."), path: Cow::Borrowed("/w/{workspace}/jobs/run/p/{path}"), method: Cow::Borrowed("POST"), path_params_schema: Some(serde_json::json!({ @@ -1419,7 +1419,7 @@ is, a different one moves it there and archives the old path"), EndpointTool { name: Cow::Borrowed("runFlowByPath"), description: Cow::Borrowed("run flow by path"), - instructions: Cow::Borrowed("You should first use getFlowByPath to retrieve the flow's schema and understand what arguments are expected."), + instructions: Cow::Borrowed("You should first use getFlowByPath to retrieve the flow's schema and understand what arguments are expected. A resource argument (schema format `resource-`) is the bare string `$res:` as that whole argument value -- not a wrapper object, not a plain path. A variable is `$var:`."), path: Cow::Borrowed("/w/{workspace}/jobs/run/f/{path}"), method: Cow::Borrowed("POST"), path_params_schema: Some(serde_json::json!({ diff --git a/backend/windmill-api/src/mcp/core.rs b/backend/windmill-api/src/mcp/core.rs index 2b4350bcfe..480e3c0841 100644 --- a/backend/windmill-api/src/mcp/core.rs +++ b/backend/windmill-api/src/mcp/core.rs @@ -12,7 +12,9 @@ use windmill_mcp::common::transform::transform_property_keys; use windmill_mcp::common::types::{ FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo, WorkspaceInfo, }; -use windmill_mcp::server::{BackendResult, EndpointTool, ErrorData, McpBackend, PathFilter}; +use windmill_mcp::server::{ + BackendResult, EndpointTool, ErrorData, McpBackend, McpRequest, PathFilter, +}; use crate::auth::AuthCache; use crate::db::ApiAuthed; @@ -214,8 +216,11 @@ impl McpBackend for WindmillBackend { workspace_id: &str, path: &str, args: Value, + request: &McpRequest<'_>, ) -> BackendResult { - let push_args = prepare_push_args(args); + let push_args = prepare_push_args(&self.db, workspace_id, path, false, args, request) + .await + .map_err(|e| ErrorData::internal_error(e.to_string(), None))?; let result = run_wait_result_script_by_path_internal( self.db.clone(), @@ -238,8 +243,11 @@ impl McpBackend for WindmillBackend { workspace_id: &str, path: &str, args: Value, + request: &McpRequest<'_>, ) -> BackendResult { - let push_args = prepare_push_args(args); + let push_args = prepare_push_args(&self.db, workspace_id, path, true, args, request) + .await + .map_err(|e| ErrorData::internal_error(e.to_string(), None))?; let result = run_wait_result_flow_by_path_internal( self.db.clone(), diff --git a/backend/windmill-api/src/mcp/utils.rs b/backend/windmill-api/src/mcp/utils.rs index 94271bb662..65c8069b85 100644 --- a/backend/windmill-api/src/mcp/utils.rs +++ b/backend/windmill-api/src/mcp/utils.rs @@ -11,15 +11,19 @@ use serde_json::Value; use sql_builder::prelude::*; use windmill_common::auth::create_jwt_token; use windmill_common::db::{Authed, UserDB}; +use windmill_common::error::Error; use windmill_common::scripts::{get_full_hub_script_by_path, Schema}; +use windmill_common::triggers::{RunnableFormat, RunnableFormatVersion, TriggerKind}; use windmill_common::utils::{query_elems_from_hub, StripPath}; use windmill_common::worker::to_raw_value; use windmill_common::{DB, HUB_BASE_URL}; use windmill_mcp::server::{ - non_empty_body_fields, BackendResult, EndpointTool, ErrorData, PathFilter, + non_empty_body_fields, BackendResult, EndpointTool, ErrorData, McpRequest, PathFilter, }; use windmill_mcp::{HubResponse, HubScriptInfo, ItemSchema, ResourceInfo, ResourceType}; +use windmill_trigger::trigger_helpers::{get_runnable_format, RunnableId}; +use crate::args::build_headers; use crate::db::ApiAuthed; use crate::HTTP_CLIENT; @@ -641,7 +645,7 @@ fn selects_endpoint_tool(caller_scopes: &[String], tool: &str) -> bool { .is_ok_and(|config| config.endpoints.iter().any(|e| e == tool)) } -/// Create HTTP request with authentication +/// Create HTTP request with authentication. pub async fn create_http_request( method: &str, url: &str, @@ -702,17 +706,113 @@ pub async fn create_http_request( .map_err(|e| ErrorData::internal_error(format!("Failed to execute request: {}", e), None)) } -/// Convert a JSON Value into PushArgsOwned for job execution -pub fn prepare_push_args(args: Value) -> windmill_queue::PushArgsOwned { +/// The `kind` an MCP-invoked runnable sees on its preprocessor event, alongside +/// `webhook`, `http` and the trigger kinds. +const MCP_TRIGGER_KEY: &str = "mcp"; + +/// A preprocessor's view of the MCP request that ran it. Mirrors the HTTP +/// trigger event: `body` is what the model sent, everything else describes the +/// call itself. +#[derive(serde::Serialize)] +struct McpPreprocessorEvent<'a> { + kind: &'a str, + body: Box, + headers: HashMap>, + tool_name: &'a str, +} + +/// Headers withheld from a preprocessor because they authenticate the connection. +/// +/// Not a security boundary: a webhook preprocessor receives all three. Withheld +/// because nothing needs them yet, and releasing one later is additive while +/// withdrawing one after runnables read it is not. +const WITHHELD_FROM_PREPROCESSOR: &[&str] = &["authorization", "cookie", "proxy-authorization"]; + +/// Every header a preprocessor may see. +fn preprocessor_headers( + headers: &http::HeaderMap, +) -> HashMap> { + let mut selected = build_headers(headers, None, true); + selected.retain(|name, _| { + !WITHHELD_FROM_PREPROCESSOR + .iter() + .any(|withheld| withheld.eq_ignore_ascii_case(name)) + }); + selected +} + +/// Build the job arguments for a script or flow run as an MCP tool. +/// +/// Shaped by the runnable's own format: a preprocessor receives the request as +/// an event, and a runnable without one receives only what the model sent. +pub async fn prepare_push_args( + db: &DB, + w_id: &str, + path: &str, + is_flow: bool, + args: Value, + request: &McpRequest<'_>, +) -> Result { + let mut main_args = HashMap::new(); if let Value::Object(map) = args { - let mut args_hash = HashMap::new(); for (k, v) in map { - args_hash.insert(k, to_raw_value(&v)); + main_args.insert(k, to_raw_value(&v)); } - windmill_queue::PushArgsOwned { extra: None, args: args_hash } - } else { - windmill_queue::PushArgsOwned::default() } + + let runnable_id = if is_flow { + RunnableId::from_flow_path(path) + } else { + // Resolves a `hub/` path to the hub script on its own. + RunnableId::from_script_path(path) + }; + + // MCP is not one of the `TRIGGER_KIND` enum values and does not need to be: + // the per-kind arms of the no-preprocessor heuristic are payload-shape + // special cases for message triggers, and `Webhook` reaches the same generic + // arm MCP wants while sharing that kind's format cache. + let runnable_format = get_runnable_format(runnable_id, w_id, db, &TriggerKind::Webhook).await?; + + Ok(match runnable_format { + // Without a preprocessor there is nowhere for a header to go that the + // model does not also write: its arguments *are* the runnable's + // parameters, so a header bound to one of them would be a value the model + // could set. The request is reachable through a preprocessor, where it + // arrives in a key of the event the model never fills. + RunnableFormat { has_preprocessor: false, .. } => { + windmill_queue::PushArgsOwned { args: main_args, extra: None } + } + RunnableFormat { has_preprocessor: true, version } => { + let headers = preprocessor_headers(request.headers); + match version { + RunnableFormatVersion::V2 => { + let event = McpPreprocessorEvent { + kind: MCP_TRIGGER_KEY, + body: to_raw_value(&main_args), + headers, + tool_name: request.tool_name, + }; + windmill_queue::PushArgsOwned { + args: HashMap::from([("event".to_string(), to_raw_value(&event))]), + extra: None, + } + } + RunnableFormatVersion::V1 => windmill_queue::PushArgsOwned { + args: main_args, + extra: Some(HashMap::from([( + "wm_trigger".to_string(), + to_raw_value(&serde_json::json!({ + "kind": MCP_TRIGGER_KEY, + MCP_TRIGGER_KEY: { + "headers": headers, + "tool_name": request.tool_name, + } + })), + )])), + }, + } + } + }) } /// Parse an HTTP response body into a JSON Value diff --git a/backend/windmill-api/src/offboarding.rs b/backend/windmill-api/src/offboarding.rs index a69441ea75..2998faa7f3 100644 --- a/backend/windmill-api/src/offboarding.rs +++ b/backend/windmill-api/src/offboarding.rs @@ -618,6 +618,7 @@ pub(crate) async fn offboard_global_user( sqlx::query!("DELETE FROM password WHERE email = $1", &email) .execute(&mut *tx) .await?; + windmill_common::user_drafts::delete_drafts_of_email(&mut *tx, &email).await?; sqlx::query!("DELETE FROM workspace_invite WHERE email = $1", &email) .execute(&mut *tx) .await?; diff --git a/backend/windmill-api/src/runnables.rs b/backend/windmill-api/src/runnables.rs index c145252251..ea6d0c5bba 100644 --- a/backend/windmill-api/src/runnables.rs +++ b/backend/windmill-api/src/runnables.rs @@ -261,7 +261,8 @@ fn branch_sqls() -> Branches { FROM draft d \ LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email \ LEFT JOIN password p ON p.email = d.email AND p.super_admin = true \ - WHERE d.workspace_id = o.workspace_id AND d.path = o.path AND {typ_pred}) as draft_users" + WHERE d.workspace_id = o.workspace_id AND d.path = o.path AND {typ_pred} \ + AND (d.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL)) as draft_users" ) }; diff --git a/backend/windmill-api/src/service_logs.rs b/backend/windmill-api/src/service_logs.rs index 99dc7c41a8..a54f275f46 100644 --- a/backend/windmill-api/src/service_logs.rs +++ b/backend/windmill-api/src/service_logs.rs @@ -88,6 +88,66 @@ async fn list_files( Ok(Json(rows)) } +/// Rebuild one source log file from the columnar store. +/// +/// Not the original bytes: the store holds a line's fields rather than its text, +/// so the JSON is re-serialized here and key order and whitespace are this +/// writer's. Everything a reader can see survives — the drawer this feeds +/// renders a prettified view of each line either way, and a line that was never +/// JSON comes back exactly as it was written. +#[cfg(all(feature = "tantivy", feature = "private"))] +async fn get_log_file_from_store( + db: &DB, + store: &windmill_indexer::service_logs_store_ee::Store, + path: &str, +) -> windmill_common::error::Result { + let (hostname, file_name) = path + .split_once('/') + .ok_or_else(|| Error::BadRequest("Invalid path".to_string()))?; + + // The store is partitioned by day and mode, neither of which the path + // carries. `log_file` names both, and its primary key starts with hostname. + let file = sqlx::query!( + // `mode!` because the column is NOT NULL and only the cast makes sqlx + // think otherwise; a silent default would look up a `mode=` partition + // that matches nothing and read as a missing file. + "SELECT mode::text AS \"mode!\", log_ts FROM log_file WHERE hostname = $1 AND file_path = $2 ORDER BY log_ts DESC LIMIT 1", + hostname, + file_name + ) + .fetch_optional(db) + .await? + .ok_or_else(|| Error::NotFound(format!("File {path} not found")))?; + + // A row registered by this version carries the minute in the file's own name, + // so the two agree and the second is redundant. One written before the + // uploader derived `log_ts` from the name carries a wall clock instead, and + // those outlive an upgrade by the retention period — which is also what makes + // the `ORDER BY` above worth having. The name is authoritative, so both go. + let mut known_ts = vec![chrono::DateTime::from_naive_utc_and_offset( + file.log_ts, + chrono::Utc, + )]; + if let Some(named) = file_name.rsplit('.').next().and_then(|s| { + chrono::NaiveDateTime::parse_from_str(s, windmill_common::tracing_init::LOG_TIMESTAMP_FMT) + .ok() + }) { + known_ts.push(chrono::DateTime::from_naive_utc_and_offset( + named, + chrono::Utc, + )); + } + + let text = windmill_indexer::service_logs_store_ee::read_log_file( + store, &file.mode, hostname, file_name, &known_ts, + ) + .await + .map_err(|e| Error::internal_err(format!("Error reading the service log store: {e}")))? + .ok_or_else(|| Error::NotFound(format!("File {path} not found")))?; + + Ok(content_plain(Body::from(text))) +} + async fn get_log_file( authed: ApiAuthed, Extension(db): Extension, @@ -104,27 +164,30 @@ async fn get_log_file( let s3_client = windmill_object_store::get_object_store().await; #[cfg(feature = "parquet")] if let Some(s3_client) = s3_client { - let path = format!("{}{}", windmill_common::tracing_init::LOGS_SERVICE, path); - let file = s3_client + use windmill_object_store::object_store_reexports::ObjectStoreError; + + // The raw file, for as long as it is there. It outlives its ingestion by + // one indexer pass at most, so this covers the most recent minutes of a + // host's logs byte for byte; everything older is rebuilt from the store. + let object_path = format!("{}{}", windmill_common::tracing_init::LOGS_SERVICE, path); + match s3_client .get(&windmill_object_store::object_store_reexports::Path::from( - path, + object_path, )) - .await; - match file { - Ok(file) => { - let bytes = file.bytes().await; - match bytes { - Ok(bytes) => { - return Ok(content_plain(Body::from(bytes::Bytes::from(bytes)))); - } - Err(e) => { - return Err(Error::internal_err(format!( - "Error pulling the bytes: {}", - e - ))); - } + .await + { + Ok(file) => match file.bytes().await { + Ok(bytes) => { + return Ok(content_plain(Body::from(bytes::Bytes::from(bytes)))); } - } + Err(e) => { + return Err(Error::internal_err(format!( + "Error pulling the bytes: {}", + e + ))); + } + }, + Err(ObjectStoreError::NotFound { .. }) => {} Err(e) => { return Err(Error::internal_err(format!( "Error fetching the file: {}", @@ -132,6 +195,11 @@ async fn get_log_file( ))); } } + + #[cfg(all(feature = "tantivy", feature = "private"))] + return get_log_file_from_store(&db, &s3_client, &path).await; + #[cfg(not(all(feature = "tantivy", feature = "private")))] + return Err(Error::NotFound(format!("File {path} not found"))); } let full_path = format!("{}{}", *TMP_WINDMILL_LOGS_SERVICE, path); // SECURITY (defense in depth): refuse to read through a symlink so a planted diff --git a/backend/windmill-api/src/token.rs b/backend/windmill-api/src/token.rs index 2268a63894..a1a9206196 100644 --- a/backend/windmill-api/src/token.rs +++ b/backend/windmill-api/src/token.rs @@ -99,7 +99,6 @@ fn build_standard_scope_domains() -> Vec { ("configs", "Configs", "Configuration management", false), ("oauth", "OAuth", "OAuth management", false), ("ai", "AI", "AI feature management", false), - ("ai_skills", "AI Skills", "AI skill management", false), ( "ai_evals", "AI Evals", diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 01eb32b7f6..da8532e9d1 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -18,7 +18,6 @@ use crate::teams_oss::{ connect_teams, edit_teams_command, run_teams_message_test_job, workspaces_list_available_teams_channels, workspaces_list_available_teams_ids, }; - use axum::{ extract::{Extension, Path}, routing::{get, post}, @@ -147,19 +146,34 @@ async fn edit_copilot_config( .await?; let workspace_has_config = ai_config.has_providers(); + let copilot_disabled = ai_config.copilot_disabled; let instance_ai_config = sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'ai_config'") .fetch_optional(&db) .await?; let settings_state = build_copilot_settings_state(workspace_has_config, instance_ai_config.as_ref()); - let effective_ai_config = if workspace_has_config { + // A provider-less instance config (e.g. `{}`) is unconfigured, same as build_copilot_settings_state + // treats it — so it must not shadow the free-tier fallback here either. + let instance_config_with_providers = instance_ai_config + .as_ref() + .and_then(|v| serde_json::from_value::(v.clone()).ok()) + .filter(|c| c.has_providers()); + let mut effective_ai_config = if workspace_has_config { ai_config - } else if let Some(instance_ai_config) = instance_ai_config { - serde_json::from_value::(instance_ai_config).unwrap_or_default() + } else if let Some(instance_config) = instance_config_with_providers { + instance_config + } else if let Some(free_config) = + crate::ai_free_tier_oss::free_tier_copilot_config(&db, &authed.email).await? + { + // Same fallback as get_copilot_info: with nothing configured, surface Windmill's free + // tier (EE-only) so clearing a workspace provider activates it immediately, instead of + // returning an empty config that disables AI until the next page reload re-fetches it. + free_config } else { AIConfig::default() }; + effective_ai_config.copilot_disabled = copilot_disabled; Ok(Json(EditCopilotConfigResponse { effective_ai_config, @@ -179,6 +193,7 @@ struct EditCopilotConfigResponse { } async fn get_copilot_info( + authed: ApiAuthed, Extension(db): Extension, Path(w_id): Path, ) -> JsonResult { @@ -194,19 +209,34 @@ async fn get_copilot_info( )) })?; - if let Some(workspace_ai_config) = workspace_ai_config.filter(|c| c.0.has_providers()) { - Ok(Json(workspace_ai_config.0)) - } else if let Some(instance_config) = + let copilot_disabled = workspace_ai_config + .as_ref() + .is_some_and(|c| c.0.copilot_disabled); + let instance_config = sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'ai_config'") .fetch_optional(&db) .await? - { - Ok(Json( - serde_json::from_value::(instance_config).unwrap_or_default(), - )) - } else { - Ok(Json(AIConfig::default())) - } + .and_then(|v| serde_json::from_value::(v).ok()) + // A provider-less instance config (e.g. `{}`) is unconfigured; don't let it shadow the + // free-tier fallback, matching the proxy and edit_copilot_config paths. + .filter(|c| c.has_providers()); + let mut effective = + if let Some(workspace_ai_config) = workspace_ai_config.filter(|c| c.0.has_providers()) { + workspace_ai_config.0 + } else if let Some(instance_config) = instance_config { + instance_config + } else if let Some(free_config) = + crate::ai_free_tier_oss::free_tier_copilot_config(&db, &authed.email).await? + { + // Nothing configured: fall back to Windmill's free tier (EE-only). The config + // carries a `free_tier` marker even once the user's grant is spent — with no + // providers, but telling the client *why* AI is off. + free_config + } else { + AIConfig::default() + }; + effective.copilot_disabled = copilot_disabled; + Ok(Json(effective)) } #[cfg(feature = "enterprise")] @@ -216,7 +246,14 @@ pub async fn get_critical_alerts( authed: ApiAuthed, Query(params): Query, ) -> JsonResult { - require_admin_or_devops(authed.is_admin, &authed.username, &authed.email, authed.job_id.is_some(), &db).await?; + require_admin_or_devops( + authed.is_admin, + &authed.username, + &authed.email, + authed.job_id.is_some(), + &db, + ) + .await?; crate::utils::get_critical_alerts(db, params, Some(w_id)).await } @@ -232,7 +269,14 @@ pub async fn acknowledge_critical_alert( Path((w_id, id)): Path<(String, i32)>, authed: ApiAuthed, ) -> Result { - require_admin_or_devops(authed.is_admin, &authed.username, &authed.email, authed.job_id.is_some(), &db).await?; + require_admin_or_devops( + authed.is_admin, + &authed.username, + &authed.email, + authed.job_id.is_some(), + &db, + ) + .await?; crate::utils::acknowledge_critical_alert(db, Some(w_id), id).await } diff --git a/backend/windmill-common/src/db_params.rs b/backend/windmill-common/src/db_params.rs deleted file mode 100644 index d52700376d..0000000000 --- a/backend/windmill-common/src/db_params.rs +++ /dev/null @@ -1,45 +0,0 @@ -use anyhow::Result; - -/// Parsed database connection parameters, shared across DB auth providers (IAM RDS, Entra ID, etc.) -#[derive(Debug, Clone)] -pub struct DatabaseParams { - pub hostname: String, - pub port: u64, - pub username: String, - pub database: String, -} - -/// Extract database connection parameters from a PostgreSQL URL -pub fn extract_database_params(database_url: &str) -> Result { - let url = url::Url::parse(database_url) - .map_err(|e| anyhow::anyhow!("Failed to parse database URL: {}", e))?; - - let hostname = url - .host_str() - .ok_or_else(|| anyhow::anyhow!("Database URL missing hostname"))? - .to_string(); - - let port = url.port().unwrap_or(5432) as u64; - - let username = if url.username().is_empty() { - return Err(anyhow::anyhow!("Database URL missing username")); - } else { - urlencoding::decode(url.username())?.to_string() - }; - - let database = url - .path() - .trim_start_matches('/') - .split('/') - .next() - .filter(|s| !s.is_empty()) - .ok_or_else(|| anyhow::anyhow!("Database URL missing database name"))? - .to_string(); - - Ok(DatabaseParams { - hostname, - port, - username, - database: urlencoding::decode(&database)?.to_string(), - }) -} diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index b539c1c6b3..5c4b4bddab 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -15,6 +15,7 @@ pub const OAUTH_SETTING: &str = "oauths"; pub const AI_CONFIG_SETTING: &str = "ai_config"; pub const RETENTION_PERIOD_SECS_SETTING: &str = "retention_period_secs"; pub const RETENTION_PERIOD_SECS_OVERRIDES_SETTING: &str = "retention_period_secs_overrides"; +pub const SERVICE_LOG_RETENTION_SECS_SETTING: &str = "service_log_retention_secs"; /// Upper bound on how many per-workspace retention overrides may be configured. The periodic monitor /// sweeps each override workspace in its own transaction every pass, so this keeps a pass bounded /// (and the feature is a targeted escape hatch for a handful of special workspaces, not a bulk knob). @@ -107,6 +108,7 @@ pub const JWT_SECRET_SETTING: &str = "jwt_secret"; pub const EMAIL_DOMAIN_SETTING: &str = "email_domain"; pub const OTEL_SETTING: &str = "otel"; pub const OTEL_TRACING_PROXY_SETTING: &str = "otel_tracing_proxy"; +pub const OTEL_TRACES_RETENTION_SECS_SETTING: &str = "otel_traces_retention_secs"; pub const APP_WORKSPACED_ROUTE_SETTING: &str = "app_workspaced_route"; pub const HTTP_ROUTE_WORKSPACED_ROUTE_SETTING: &str = "http_route_workspaced_route"; pub const SECRET_BACKEND_SETTING: &str = "secret_backend"; diff --git a/backend/windmill-common/src/indexer.rs b/backend/windmill-common/src/indexer.rs index 7b1ed407ab..559eba3bf9 100644 --- a/backend/windmill-common/src/indexer.rs +++ b/backend/windmill-common/src/indexer.rs @@ -94,6 +94,21 @@ pub async fn load_indexer_config(db: &DB) -> error::Result i64 { + let retention = crate::service_log_retention_secs(); + if max_index_time_window_secs > 0 { + std::cmp::min(max_index_time_window_secs, retention) + } else { + retention + } +} + pub fn get_env_var(env_var: &str) -> Option { match std::env::var(env_var).map(|x| x.parse()) { Ok(Ok(i)) => Some(i), @@ -136,3 +151,60 @@ pub fn get_indexer_rates_from_env() -> TantivyIndexerSettings { settings } + +#[cfg(test)] +mod tests { + use super::*; + + // One test rather than several: both halves share the process-wide retention, and the + // setter half writes it, which parallel tests would race. + #[test] + fn retention_rejects_unusable_values_and_the_index_window_clamps_to_it() { + use crate::{ + service_log_retention_secs, set_service_log_retention_secs, + DEFAULT_SERVICE_LOG_RETENTION_SECS, + }; + + // See `set_service_log_retention_secs` for why the two unusable directions land apart: + // too large keeps the intent by capping, non-positive cannot and falls back. + let rejected: Vec = [0, -1, i64::MIN] + .iter() + .map(|v| { + set_service_log_retention_secs(*v); + service_log_retention_secs() + }) + .collect(); + let capped: Vec = [i64::MAX, 60 * 60 * 24 * 365 * 101] + .iter() + .map(|v| { + set_service_log_retention_secs(*v); + service_log_retention_secs() + }) + .collect(); + + set_service_log_retention_secs(60 * 60 * 24 * 3); + let retention = service_log_retention_secs(); + let windows = [ + // `0` disables the extra shrinking rather than lifting the ceiling — the trap that + // makes an unset setting look unbounded. + service_log_index_window_secs(0), + // Retention is the ceiling: the index cannot reach lines whose `log_file` row is gone. + service_log_index_window_secs(retention * 2), + service_log_index_window_secs(60), + ]; + set_service_log_retention_secs(DEFAULT_SERVICE_LOG_RETENTION_SECS); + + assert_eq!( + rejected, + vec![DEFAULT_SERVICE_LOG_RETENTION_SECS; 3], + "a value that would expire everything must fall back to the default" + ); + assert_eq!( + capped, + vec![60 * 60 * 24 * 365 * 100; 2], + "an oversized value must cap, not shorten retention to the default" + ); + assert_eq!(retention, 60 * 60 * 24 * 3); + assert_eq!(windows, [retention, retention, 60]); + } +} diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 89db0ee769..b4128f80d2 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -42,7 +42,6 @@ pub mod db; mod db_entra_ee; #[cfg(all(feature = "enterprise", feature = "private"))] mod db_iam_ee; -pub mod db_params; pub mod dbt_manifest; pub mod deploy_origin; #[cfg(feature = "private")] @@ -148,9 +147,86 @@ pub const DEFAULT_MAX_CONNECTIONS_INDEXER: u32 = 5; pub const DEFAULT_HUB_BASE_URL: &str = "https://hub.windmill.dev"; pub const PRIVATE_HUB_MIN_VERSION: i32 = 10_000_000; -pub const SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 14; // 2 weeks retention period for logs +pub const DEFAULT_SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 14; // 2 weeks retention period for logs +pub const DEFAULT_OTEL_TRACES_RETENTION_SECS: i64 = 60 * 60 * 24 * 7; // 1 week retention period for HTTP request spans pub const WM_DEPLOYERS_GROUP: &str = "wm_deployers"; +/// A century. Every consumer has to survive `now - retention`, and the ceilings are much lower +/// than an `i64`: `DateTime` subtraction panics past year 262143, and the `( s)::interval` +/// the cleanup queries build overflows Postgres' microsecond field. +const MAX_RETENTION_SECS: i64 = 60 * 60 * 24 * 365 * 100; + +/// Clamp a configured retention window, in seconds, to one a cutoff can be built from. +/// +/// Shared by the retention windows that have no "keep forever" spelling, so that an unusable +/// value can never reach a cutoff. The two unusable directions are not the same mistake and must +/// not share a landing point: too large still says "keep these for a very long time", so it is +/// capped and the intent survives, whereas falling back would delete data the operator meant to +/// keep. A non-positive value has no such reading — every cutoff is `now - retention`, so it +/// lands at or after `now` and the next sweep expires the entire history. `0` is both what an +/// operator types by analogy with job retention, where it does mean keep forever, and what the +/// settings UI writes into a field that was merely focused, so it falls back to the default. +fn clamp_retention_secs(configured: i64, default: i64, what: &str) -> i64 { + if configured > MAX_RETENTION_SECS { + tracing::warn!( + "{what} retention of {configured}s exceeds the maximum of {MAX_RETENTION_SECS}s, \ + capping it there" + ); + MAX_RETENTION_SECS + } else if configured >= 1 { + configured + } else { + tracing::warn!( + "{what} retention of {configured}s would expire the entire history, \ + falling back to the default of {default}s" + ); + default + } +} + +/// Apply a configured service log retention, in seconds. +/// +/// The only way into [`SERVICE_LOG_RETENTION_SECS`]. Expiry reaches every copy of a log line: +/// the row, the file on disk, and the object-storage object. +pub fn set_service_log_retention_secs(configured: i64) { + let effective = clamp_retention_secs( + configured, + DEFAULT_SERVICE_LOG_RETENTION_SECS, + "service log", + ); + SERVICE_LOG_RETENTION_SECS.store(effective, std::sync::atomic::Ordering::Relaxed); +} + +/// Apply a configured OTEL trace retention, in seconds. +/// +/// The only way into [`OTEL_TRACES_RETENTION_SECS`]. +pub fn set_otel_traces_retention_secs(configured: i64) { + let effective = clamp_retention_secs( + configured, + DEFAULT_OTEL_TRACES_RETENTION_SECS, + "otel traces", + ); + OTEL_TRACES_RETENTION_SECS.store(effective, std::sync::atomic::Ordering::Relaxed); +} + +/// How long an HTTP request tracing span stays in `otel_traces`, in seconds. +/// +/// Spans are keyed by the job they were captured for and read back by the job detail view, so +/// this is the outer bound on how far back that view can show a job's HTTP requests. It is +/// independent of job retention: a span can outlive its job, or be swept while the job remains. +pub fn otel_traces_retention_secs() -> i64 { + OTEL_TRACES_RETENTION_SECS.load(std::sync::atomic::Ordering::Relaxed) +} + +/// How long a service log line stays retrievable, in seconds. +/// +/// The outer bound on everything service-log: the `log_file` rows, the raw files in object +/// storage, the columnar store queried by retrieval, and — through +/// [`indexer::service_log_index_window_secs`] — the search index. +pub fn service_log_retention_secs() -> i64 { + SERVICE_LOG_RETENTION_SECS.load(std::sync::atomic::Ordering::Relaxed) +} + /// Canonical form of a base URL, used as one of the inputs to the offline-license /// instance hash (`compute_instance_hash`). /// @@ -376,6 +452,14 @@ lazy_static::lazy_static! { /// workspace configured before its override could be read. pub static ref JOB_RETENTION_SECS_OVERRIDES_LOADED: AtomicBool = AtomicBool::new(false); pub static ref AUDIT_LOG_RETENTION_DAYS: AtomicI64 = AtomicI64::new(0); + /// Private on purpose: [`set_service_log_retention_secs`] is the only writer, so a value that + /// would expire every service log cannot reach a cutoff. Read it with + /// [`service_log_retention_secs`]. + static ref SERVICE_LOG_RETENTION_SECS: AtomicI64 = AtomicI64::new(DEFAULT_SERVICE_LOG_RETENTION_SECS); + /// Private on purpose, same as [`SERVICE_LOG_RETENTION_SECS`]: + /// [`set_otel_traces_retention_secs`] is the only writer, [`otel_traces_retention_secs`] the + /// only reader. + static ref OTEL_TRACES_RETENTION_SECS: AtomicI64 = AtomicI64::new(DEFAULT_OTEL_TRACES_RETENTION_SECS); pub static ref MONITOR_LOGS_ON_OBJECT_STORE: AtomicBool = AtomicBool::new(false); @@ -1479,6 +1563,17 @@ pub async fn create_custom_instance_database( Ok(()) } +/// Connection options parsed from a database URL. +/// +/// The only place a database URL becomes `PgConnectOptions`. Providers that mint the password +/// themselves override it on these and keep the rest: options assembled field by field instead +/// would drop every query parameter, `sslmode` and `sslrootcert` above all, leaving the +/// connection on sqlx's default TLS policy rather than the operator's. +pub fn base_connect_options(database_url: &str) -> Result { + sqlx::postgres::PgConnectOptions::from_str(database_url) + .map_err(|e| Error::InternalErr(format!("Failed to parse database URL: {}", e))) +} + #[derive(Clone)] pub enum DatabaseUrl { #[cfg(all(feature = "enterprise", feature = "private"))] @@ -1509,8 +1604,8 @@ impl DatabaseUrl { } /// Get PgConnectOptions for this database URL. - /// For token-based auth (IAM RDS, Entra ID), this returns options built directly from the - /// token to avoid double-encoding issues with temporary credentials. + /// For token-based auth (IAM RDS, Entra ID), this returns options carrying the current + /// token, set on the builder to avoid double-encoding temporary credentials. /// For static URLs, this parses the URL string. pub async fn connect_options(&self) -> Result { match self { @@ -1524,8 +1619,7 @@ impl DatabaseUrl { let guard = entra_url.read().await; Ok(guard.connect_options()) } - DatabaseUrl::Static(url) => sqlx::postgres::PgConnectOptions::from_str(url) - .map_err(|e| Error::InternalErr(format!("Failed to parse database URL: {}", e))), + DatabaseUrl::Static(url) => base_connect_options(url), } } diff --git a/backend/windmill-common/src/tracing_init.rs b/backend/windmill-common/src/tracing_init.rs index 5091c7e16d..b456e370fc 100644 --- a/backend/windmill-common/src/tracing_init.rs +++ b/backend/windmill-common/src/tracing_init.rs @@ -178,56 +178,40 @@ pub fn initialize_tracing( .with(logs_bridge.with_filter(otel_logs_filter)) .with(opentelemetry_filtered); - match *JSON_FMT { - true => { - // Stdout layer with its own filter - let stdout_layer = json_layer() - .with_writer(std::io::stdout) - .flatten_event(true) - .with_filter(stdout_env_filter) - .with_filter(create_targets_filter(default_env_filter)); + // The service log files are written to be indexed, not tailed, so they always carry the + // JSON format: it is what preserves level, target and the current span as fields rather + // than as text the index would have to recover by regex. JSON_FMT governs stdout only. + let file_layer = json_layer() + .with_writer(log_file_writer) + .flatten_event(true) + .with_filter(file_env_filter) + .with_filter(create_targets_filter(default_env_filter)); - // File layer with its own filter - let file_layer = json_layer() - .with_writer(log_file_writer) - .flatten_event(true) - .with_filter(file_env_filter) - .with_filter(create_targets_filter(default_env_filter)); + // Boxed so both arms have one type: the file layer is a single value and could not + // otherwise be typed against two different subscriber stacks. + let stdout_layer = match *JSON_FMT { + true => json_layer() + .with_writer(std::io::stdout) + .flatten_event(true) + .with_filter(stdout_env_filter) + .with_filter(create_targets_filter(default_env_filter)) + .boxed(), + false => compact_layer() + .with_writer(std::io::stdout) + .with_ansi(style.to_lowercase() != "never") + .with_file(true) + .with_line_number(true) + .with_target(false) + .with_filter(stdout_env_filter) + .with_filter(create_targets_filter(default_env_filter)) + .boxed(), + }; - base_layer - .with(stdout_layer) - .with(file_layer) - .with(CountingLayer::new()) - .init() - } - false => { - // Stdout layer with its own filter - let stdout_layer = compact_layer() - .with_writer(std::io::stdout) - .with_ansi(style.to_lowercase() != "never") - .with_file(true) - .with_line_number(true) - .with_target(false) - .with_filter(stdout_env_filter) - .with_filter(create_targets_filter(default_env_filter)); - - // File layer with its own filter - let file_layer = compact_layer() - .with_writer(log_file_writer) - .with_ansi(false) // No ANSI codes in log files - .with_file(true) - .with_line_number(true) - .with_target(false) - .with_filter(file_env_filter) - .with_filter(create_targets_filter(default_env_filter)); - - base_layer - .with(stdout_layer) - .with(file_layer) - .with(CountingLayer::new()) - .init() - } - } + base_layer + .with(stdout_layer) + .with(file_layer) + .with(CountingLayer::new()) + .init(); (_guard, meter_provider) } diff --git a/backend/windmill-common/src/user_drafts.rs b/backend/windmill-common/src/user_drafts.rs index db14aa438a..0ee6c518a5 100644 --- a/backend/windmill-common/src/user_drafts.rs +++ b/backend/windmill-common/src/user_drafts.rs @@ -256,7 +256,9 @@ async fn fetch_other_drafts_users( // row: fall back to their instance-derived username (`password.username`), or // their email when derivation is disabled. Else a real teammate's draft renders // as a phantom "Legacy draft". The genuine NULL-email legacy row keeps - // `username = None` (no `usr`/`password` match and `d.email` is NULL). + // `username = None` (no `usr`/`password` match and `d.email` is NULL), which is + // why an owner that resolves to no name at all — an external JWT's subject has + // neither row — is dropped instead: `None` is taken to mean "legacy" downstream. let rows = sqlx::query_as!( OtherDraftUser, r#"SELECT COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN d.email END) as "username?", @@ -272,6 +274,7 @@ async fn fetch_other_drafts_users( AND d.path = $2 AND d.typ = $3 AND (d.email IS NULL OR d.email <> $4) + AND (d.email IS NULL OR u.username IS NOT NULL OR p.email IS NOT NULL) ORDER BY d.email NULLS LAST"#, w_id, path, @@ -437,6 +440,67 @@ pub async fn overlay_or_draft_only( } } +/// Delete the drafts an address owns, across every workspace. +/// +/// `draft.email` carries no foreign key to `password`: a draft's owner is any principal the +/// instance authenticates, and an external JWT's subject never has a `password` row. Deleting an +/// account is therefore what has to delete its drafts — a delete path that skips this leaves them +/// behind forever, addressed to someone who no longer exists. Call it in the same transaction as +/// the account removal. +/// +/// No authorization of its own: it acts instance-wide on whatever address it is handed, so the +/// caller must already have authorized removing that account (superadmin, the account's own +/// holder, or SCIM). +pub async fn delete_drafts_of_email<'c>( + executor: impl sqlx::PgExecutor<'c>, + email: &str, +) -> Result<()> { + sqlx::query!("DELETE FROM draft WHERE email = $1", email) + .execute(executor) + .await?; + Ok(()) +} + +/// Move the drafts an address owns onto its new address, for the same reason +/// [`delete_drafts_of_email`] exists: no foreign key follows the rename, so drafts left behind are +/// stranded on an address that no longer authenticates. Same authorization contract, for a rename. +/// +/// The two addresses may each already hold a draft of the same item, since the destination can +/// belong to a principal with no account and so is not covered by the caller's "address is free" +/// check. `draft_pkey_with_user` admits only one, so the moving account's wins — which is also why +/// a rename onto the same address returns early: every row would collide with itself and be +/// cleared. Callers need not compare first (an IdP re-sending an unchanged `userName` does not). +pub async fn rename_drafts_of_email( + conn: &mut sqlx::PgConnection, + old_email: &str, + new_email: &str, +) -> Result<()> { + if old_email == new_email { + return Ok(()); + } + sqlx::query!( + "DELETE FROM draft dest + WHERE dest.email = $1 + AND EXISTS (SELECT 1 FROM draft src + WHERE src.email = $2 + AND src.workspace_id = dest.workspace_id + AND src.path = dest.path + AND src.typ = dest.typ)", + new_email, + old_email + ) + .execute(&mut *conn) + .await?; + sqlx::query!( + "UPDATE draft SET email = $1 WHERE email = $2", + new_email, + old_email + ) + .execute(&mut *conn) + .await?; + Ok(()) +} + /// Delete EVERY user's draft (and the legacy NULL-email row) at a path+kind. /// Use when the item is DELETED outright: it's gone for everyone, so leaving /// teammates' drafts behind would orphan them forever. Discarding one's OWN diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 7945e73b45..84284759c6 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -734,11 +734,17 @@ fn format_pull_query(peek: String) -> String { r } +// The `CASE` is `suspend <= 0 OR suspend_until <= now()` written as one indexable +// expression, equivalent only under the `suspend_until IS NOT NULL` guard. It must stay in +// sync with `queue_suspended_v2` (migration 20260826202939): if it no longer matches, the +// test silently reverts to a heap filter over every suspended row on every worker poll. pub fn make_suspended_pull_query(tags: &[String]) -> String { format_pull_query(format!( "SELECT id FROM v2_job_queue - WHERE suspend_until IS NOT NULL AND (suspend <= 0 OR suspend_until <= now()) AND tag IN ({}) + WHERE suspend_until IS NOT NULL + AND (CASE WHEN suspend <= 0 THEN '-infinity'::timestamptz ELSE suspend_until END) <= now() + AND tag IN ({}) ORDER BY priority DESC NULLS LAST, created_at FOR UPDATE SKIP LOCKED LIMIT 1", diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 5f558d325a..697fc69e44 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -175,7 +175,7 @@ pub enum ObjectType { DatatableMigration, } -pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28911/sync-script-to-git-repo-windmill"; +pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28931/sync-script-to-git-repo-windmill"; /// Hub script that applies a repository's state back into a workspace /// (the repo → Windmill / "pull" direction). Same script the UI runs from @@ -183,7 +183,7 @@ pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28911/sync-script-to-git-repo /// ignores the slug, so the slug is kept free of characters that would be /// percent-encoded into the run URL (a `:` becomes `%3A`, which some hardened /// reverse proxies reject as double-encoding when the client re-encodes it). -pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/28910/git-sync-init-repository-windmill"; +pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/28930/git-sync-init-repository-windmill"; /// Prefix used to identify fork workspaces. A workspace whose id starts with this string is a /// fork of another workspace. @@ -759,15 +759,24 @@ pub async fn count_workspace_forks(db: &crate::DB, root: &str) -> Result { Ok(count) } -/// Approximate paid seats of a workspace as `ceil(developers + operators/2)`, excluding disabled and -/// service-account members. Reuses billing's author/operator weighting, but counts provisioned -/// members rather than the active-user population billing meters, so it only ever loosens the fork -/// cap (never blocks a paid seat) — good enough for a soft guardrail. +/// The billable members of a workspace and the seats they add up to. +#[derive(Clone, Debug, Serialize)] +pub struct BillableSeats { + pub developers: i64, + pub operators: i64, + pub seats: i64, +} + +/// Billable members of `w_id` and the seats they cost, as `ceil(developers + operators/2)`. Service +/// accounts cannot log in and do not take a seat; a disabled member is not billed either. +/// +/// The workspace is invoiced by a job outside this codebase that counts the same rows with its own +/// SQL. The two must be changed together: this rule disagreeing with that one is what bills a +/// workspace for seats the product never credits it for. /// /// Unauthenticated metering helper: reads member counts for any `w_id`, so callers must already be /// authorized for that workspace (or run in trusted server-side code). -#[cfg(feature = "cloud")] -pub async fn count_paid_seats(db: &crate::DB, w_id: &str) -> Result { +pub async fn billable_seats(db: &crate::DB, w_id: &str) -> Result { let row = sqlx::query!( r#"SELECT COUNT(*) FILTER (WHERE NOT operator AND NOT disabled AND NOT is_service_account) AS "developers!", @@ -777,8 +786,18 @@ pub async fn count_paid_seats(db: &crate::DB, w_id: &str) -> Result { ) .fetch_one(db) .await - .map_err(|e| Error::internal_err(format!("counting paid seats of {w_id}: {e:#}")))?; - Ok(((row.developers as f64) + 0.5 * (row.operators as f64)).ceil() as i64) + .map_err(|e| Error::internal_err(format!("counting billable seats of {w_id}: {e:#}")))?; + Ok(BillableSeats { + developers: row.developers, + operators: row.operators, + seats: ((row.developers as f64) + 0.5 * (row.operators as f64)).ceil() as i64, + }) +} + +/// Seats only, for the fork cap. See [`billable_seats`]. +#[cfg(feature = "cloud")] +pub async fn count_paid_seats(db: &crate::DB, w_id: &str) -> Result { + Ok(billable_seats(db, w_id).await?.seats) } #[cfg(feature = "cloud")] @@ -2193,6 +2212,32 @@ pub fn lfs_entry_storage_ref(entry: &serde_json::Value) -> Option { Some(format!("{typ}:{path}")) } +pub const FILESYSTEM_STORAGE_DEV_ONLY_MSG: &str = + "Filesystem storage is only available in development builds of Windmill: it points the \ + workspace at a directory on the server's own disk rather than at a resource. Use an S3, \ + Azure Blob or Google Cloud Storage backend instead."; + +/// A filesystem workspace storage names a directory on the server's own disk, so it hands whoever +/// configures it — a workspace admin, or any member who can write a `filesystem` resource — +/// whatever the server process can reach, and it only resolves when server and workers share that +/// disk. It is there so local development can skip MinIO, hence debug builds only. Instance object +/// storage on local disk is a separate, superadmin-only setting and stays allowed everywhere. +pub fn filesystem_storage_allowed() -> bool { + cfg!(debug_assertions) +} + +/// Guards every site that builds an `ObjectStoreResource::Filesystem`, so nothing downstream can +/// reach a local-disk store: a stored config outlives the build that accepted it, and the resource +/// route never passes through the workspace-storage settings at all. +pub fn ensure_filesystem_storage_allowed() -> Result<()> { + if !filesystem_storage_allowed() { + return Err(Error::BadRequest( + FILESYSTEM_STORAGE_DEV_ONLY_MSG.to_string(), + )); + } + Ok(()) +} + /// Resolve a `$res:`/`$var:` reference tree to its concrete value (recursively, secrets /// decrypted). No permission checks — trusted server-side callers only; never echo the result /// to a user. diff --git a/backend/windmill-common/tests/billing_workspace.rs b/backend/windmill-common/tests/billing_workspace.rs index 1b0186a5cf..3025061249 100644 --- a/backend/windmill-common/tests/billing_workspace.rs +++ b/backend/windmill-common/tests/billing_workspace.rs @@ -3,7 +3,7 @@ use sqlx::{Pool, Postgres}; use windmill_common::workspaces::{ - count_paid_seats, count_workspace_forks, fork_chain_depth, fork_subtree_height, + billable_seats, count_paid_seats, count_workspace_forks, fork_chain_depth, fork_subtree_height, get_billing_workspace_id, invalidate_billing_workspace_cache, list_fork_descendants, }; @@ -106,11 +106,17 @@ async fn paid_seats_and_fork_count(db: Pool) { insert_member(&db, "seat-root", "dev2@w.dev", false, false, false).await; insert_member(&db, "seat-root", "op1@w.dev", true, false, false).await; insert_member(&db, "seat-root", "op2@w.dev", true, false, false).await; - // These must NOT count towards seats. + // These must NOT count towards seats. The service account is a non-operator, so counting it + // would inflate the developer tally the invoice line is written from, not the operator one. insert_member(&db, "seat-root", "disabled@w.dev", false, true, false).await; insert_member(&db, "seat-root", "svc@w.dev", false, false, true).await; assert_eq!(count_paid_seats(&db, "seat-root").await.unwrap(), 3); + let breakdown = billable_seats(&db, "seat-root").await.unwrap(); + assert_eq!( + (breakdown.developers, breakdown.operators, breakdown.seats), + (2, 2, 3) + ); insert_ws(&db, "seat-fork1", Some("seat-root"), false).await; insert_ws(&db, "seat-fork2", Some("seat-root"), false).await; diff --git a/backend/windmill-common/tests/user_drafts_rename.rs b/backend/windmill-common/tests/user_drafts_rename.rs new file mode 100644 index 0000000000..35d2b53858 --- /dev/null +++ b/backend/windmill-common/tests/user_drafts_rename.rs @@ -0,0 +1,28 @@ +use sqlx::{Pool, Postgres}; +use windmill_common::user_drafts::rename_drafts_of_email; + +/// A rename onto the same address has to be a no-op: the helper clears a draft the destination +/// already holds at the same item, and every row would be its own destination. SCIM PATCH sends +/// `userName` unconditionally, so an IdP re-sending an unchanged one reaches this. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn renaming_onto_the_same_address_keeps_the_drafts(db: Pool) { + sqlx::query( + "INSERT INTO draft(workspace_id, path, typ, value, email) \ + VALUES ('test-workspace', 'u/test-user/s', 'script', '{}'::json, 'test@windmill.dev')", + ) + .execute(&db) + .await + .expect("failed to seed draft"); + + let mut conn = db.acquire().await.unwrap(); + rename_drafts_of_email(&mut conn, "test@windmill.dev", "test@windmill.dev") + .await + .unwrap(); + + let kept: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM draft WHERE email = 'test@windmill.dev'") + .fetch_one(&db) + .await + .unwrap(); + assert_eq!(kept, 1); +} diff --git a/backend/windmill-dep-map/Cargo.toml b/backend/windmill-dep-map/Cargo.toml index 28b8552e8e..a93376d167 100644 --- a/backend/windmill-dep-map/Cargo.toml +++ b/backend/windmill-dep-map/Cargo.toml @@ -26,4 +26,5 @@ tracing.workspace = true lazy_static.workspace = true chrono.workspace = true itertools.workspace = true +futures.workspace = true uuid.workspace = true diff --git a/backend/windmill-dep-map/src/lib.rs b/backend/windmill-dep-map/src/lib.rs index ee6a3c3fd4..650e0e5f99 100644 --- a/backend/windmill-dep-map/src/lib.rs +++ b/backend/windmill-dep-map/src/lib.rs @@ -1,6 +1,7 @@ pub mod ci_tests; #[cfg(feature = "private")] pub mod ci_tests_ee; +pub mod lock_hash; pub mod scoped_dependency_map; pub mod trigger_dependents; pub mod workspace_dependencies; diff --git a/backend/windmill-dep-map/src/lock_hash.rs b/backend/windmill-dep-map/src/lock_hash.rs new file mode 100644 index 0000000000..50bd18a8ee --- /dev/null +++ b/backend/windmill-dep-map/src/lock_hash.rs @@ -0,0 +1,79 @@ +use std::collections::HashMap; + +use futures::TryStreamExt; +use sqlx::{Postgres, Transaction}; +use windmill_common::error::Result; +use windmill_common::scripts::hash_script; + +/// Records what the lock now at each path hashes to, which is one half of the comparison a relock +/// skip makes against what each importer resolved against. +/// +/// Writes any path in `w_id` and checks nothing: callers are responsible for having established +/// the caller's access to that workspace. A path repeated in `entries` keeps its last hash. +/// +/// Callers that write the lock itself in the same statement fold the upsert into that statement +/// instead; this is for the ones with nothing to fold it into. +pub async fn record_lock_hashes( + tx: &mut Transaction<'_, Postgres>, + w_id: &str, + entries: &[(String, i64)], +) -> Result<()> { + // Postgres rejects a whole statement that resolves a conflict on one key twice, so a path + // given more than once keeps its last hash, as it would if the two were written in order. + let mut deduped: HashMap<&str, i64> = HashMap::with_capacity(entries.len()); + for (path, hash) in entries { + deduped.insert(path.as_str(), *hash); + } + if deduped.is_empty() { + return Ok(()); + } + let (paths, hashes): (Vec, Vec) = deduped + .into_iter() + .map(|(path, hash)| (path.to_string(), hash)) + .unzip(); + // Recording a hash a path already has would still cut a row version, and the no-op push this + // is reached from is the mode a git-sync of an unchanged workspace runs in. + sqlx::query!( + "INSERT INTO lock_hash (workspace_id, path, lockfile_hash) + SELECT $1, * FROM UNNEST($2::text[], $3::bigint[]) + ON CONFLICT (workspace_id, path) DO UPDATE SET lockfile_hash = EXCLUDED.lockfile_hash + WHERE lock_hash.lockfile_hash IS DISTINCT FROM EXCLUDED.lockfile_hash", + w_id, + &paths[..], + &hashes[..] + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + +/// Records the hash of every live lock in `w_id`, for a workspace whose scripts arrived without +/// going through a deploy — a clone, which copies their locks verbatim and so would otherwise hold +/// none of the hashes describing them. +/// +/// Carries the same caller obligation as [`record_lock_hashes`]. +/// +/// `script.lock` is unbounded and a workspace holds one per script, so the rows are streamed and +/// each lock is hashed and dropped before the next arrives; only the hashes accumulate. +pub async fn record_lock_hashes_for_workspace( + tx: &mut Transaction<'_, Postgres>, + w_id: &str, +) -> Result<()> { + let mut entries: Vec<(String, i64)> = Vec::new(); + { + let mut rows = sqlx::query!( + "SELECT DISTINCT ON (path) path, lock FROM script + WHERE workspace_id = $1 AND NOT archived AND NOT deleted AND lock IS NOT NULL + ORDER BY path, created_at DESC", + w_id + ) + .fetch(&mut **tx); + + while let Some(row) = rows.try_next().await? { + if let Some(lock) = row.lock { + entries.push((row.path, hash_script(&lock))); + } + } + } + record_lock_hashes(tx, w_id, &entries).await +} diff --git a/backend/windmill-indexer/Cargo.toml b/backend/windmill-indexer/Cargo.toml index acf90ab57a..09b2bbe5b1 100644 --- a/backend/windmill-indexer/Cargo.toml +++ b/backend/windmill-indexer/Cargo.toml @@ -10,7 +10,13 @@ path = "src/lib.rs" [features] default = [] -parquet = ["windmill-common/parquet", "windmill-object-store/parquet"] +parquet = [ + "windmill-common/parquet", + "windmill-object-store/parquet", + "dep:datafusion", + "dep:object_store", + "dep:url", +] private = ["windmill-common/private"] enterprise = ["windmill-common/enterprise", "windmill-object-store/enterprise"] @@ -33,3 +39,6 @@ astral-tokio-tar.workspace = true lazy_static.workspace = true const_format.workspace = true flume.workspace = true +datafusion = { workspace = true, optional = true } +object_store = { workspace = true, optional = true } +url = { workspace = true, optional = true } diff --git a/backend/windmill-indexer/src/lib.rs b/backend/windmill-indexer/src/lib.rs index 6c13b551d1..f107ea911d 100644 --- a/backend/windmill-indexer/src/lib.rs +++ b/backend/windmill-indexer/src/lib.rs @@ -7,3 +7,5 @@ pub mod indexer_oss; #[cfg(feature = "private")] pub mod service_logs_ee; pub mod service_logs_oss; +#[cfg(all(feature = "private", feature = "parquet"))] +pub mod service_logs_store_ee; diff --git a/backend/windmill-mcp/src/common/schema.rs b/backend/windmill-mcp/src/common/schema.rs index 3f88f7e781..d49fa468ae 100644 --- a/backend/windmill-mcp/src/common/schema.rs +++ b/backend/windmill-mcp/src/common/schema.rs @@ -101,7 +101,7 @@ fn apply_resource_enrichment( let resources_count = resource_cache.len(); let description = match resource_type { Some(rt) => format!( - "This is a resource named `{}` with the following description: `{}`.\\nThe path of the resource should be used to specify the resource.\\n{}", + "This is a resource named `{}` with the following description: `{}`.\nPass it as the bare string `$res:` — the whole value of this argument, never an object wrapper like {{\"$res\": \"\"}} and never a plain path.\n{}", rt.name, rt.description.as_deref().unwrap_or("No description"), if resources_count == 0 { @@ -138,7 +138,7 @@ fn apply_resource_enrichment( ) }) .collect::>() - .join("\\n"); + .join("\n"); let prior_description = prop_map .get("description") .and_then(Value::as_str) @@ -147,7 +147,7 @@ fn apply_resource_enrichment( prop_map.insert( "description".to_string(), Value::String(format!( - "{}\\nHere are the available resources, in the format title:path. Title can be empty. Path should be used to specify the resource:\\n{}", + "{}\nHere are the available resources, one per line as `title: $res:path`. The title is only a label; pass the `$res:path` part verbatim as this argument's value:\n{}", prior_description, resources_description )), ); @@ -804,6 +804,10 @@ mod tests { let desc = node["description"].as_str().unwrap(); assert!(desc.contains("c_aws_account")); assert!(desc.contains("$res:f/platform/aws_dev")); + // MCP clients render this description verbatim, so the separators must be + // real newlines rather than the two-character escape. + assert!(desc.contains('\n')); + assert!(!desc.contains("\\n")); } #[test] diff --git a/backend/windmill-mcp/src/server/backend.rs b/backend/windmill-mcp/src/server/backend.rs index e42c0dacfc..7ca1eb08a1 100644 --- a/backend/windmill-mcp/src/server/backend.rs +++ b/backend/windmill-mcp/src/server/backend.rs @@ -16,6 +16,14 @@ use crate::server::endpoints::EndpointTool; /// Result type for backend operations using rmcp's ErrorData directly pub type BackendResult = Result; +/// What the backend needs about the HTTP request a tool call arrived on, in order +/// to hand a runnable the headers of the call that triggered it. +pub struct McpRequest<'a> { + pub headers: &'a http::HeaderMap, + /// The MCP tool name the caller invoked, reported to preprocessors. + pub tool_name: &'a str, +} + /// How a script/flow listing is narrowed by path at the SQL layer, *before* the /// `ITEMS_FETCH_MAX_LIMIT` cap applies. /// @@ -157,6 +165,7 @@ pub trait McpBackend: Send + Sync + Clone + 'static { workspace_id: &str, path: &str, args: Value, + request: &McpRequest<'_>, ) -> BackendResult; /// Run a flow and wait for result @@ -166,6 +175,7 @@ pub trait McpBackend: Send + Sync + Clone + 'static { workspace_id: &str, path: &str, args: Value, + request: &McpRequest<'_>, ) -> BackendResult; /// Call an endpoint tool (generated API endpoint) diff --git a/backend/windmill-mcp/src/server/mod.rs b/backend/windmill-mcp/src/server/mod.rs index b6fb7a5b0a..b97e374e98 100644 --- a/backend/windmill-mcp/src/server/mod.rs +++ b/backend/windmill-mcp/src/server/mod.rs @@ -12,7 +12,7 @@ pub mod tools; // Re-export main types pub use crate::common::types::{McpToken, MultiWorkspaceMcp, WorkspaceInfo}; -pub use backend::{BackendResult, McpAuth, McpBackend, PathFilter}; +pub use backend::{BackendResult, McpAuth, McpBackend, McpRequest, PathFilter}; pub use endpoints::{ endpoint_tool_to_mcp_tool, endpoint_tool_to_mcp_tool_multi, is_endpoint_read_only, list_workspaces_tool, non_empty_body_fields, EndpointTool, diff --git a/backend/windmill-mcp/src/server/runner.rs b/backend/windmill-mcp/src/server/runner.rs index f407504c4b..e27eaa8470 100644 --- a/backend/windmill-mcp/src/server/runner.rs +++ b/backend/windmill-mcp/src/server/runner.rs @@ -9,8 +9,10 @@ use crate::common::transform::{ extract_hub_version_id_from_hashed, extract_path_prefix_from_hashed, parse_tool_prefix, reverse_transform, reverse_transform_key, }; -use crate::common::types::{McpToken, MultiWorkspaceMcp, ResourceInfo, ToolableItem, WorkspaceId}; -use crate::server::backend::{McpAuth, McpBackend, PathFilter}; +use crate::common::types::{ + McpToken, MultiWorkspaceMcp, ResourceInfo, SchemaType, ToolableItem, WorkspaceId, +}; +use crate::server::backend::{McpAuth, McpBackend, McpRequest, PathFilter}; use crate::server::endpoints::{ endpoint_tool_to_mcp_tool, endpoint_tool_to_mcp_tool_multi, list_workspaces_tool, EndpointTool, }; @@ -101,16 +103,24 @@ enum McpMode { Multi(String), } +/// Everything a request carries besides its MCP payload. +struct McpContext { + auth: A, + mode: McpMode, + headers: http::HeaderMap, +} + impl Runner { /// Create a new Runner with the given backend pub fn new(backend: B) -> Self { Self { backend: Arc::new(backend) } } - /// Extract authentication and the workspace mode from request context + /// Extract authentication, the workspace mode and the HTTP request itself + /// from the request context fn extract_context( context: &RequestContext, - ) -> Result<(B::Auth, McpMode), ErrorData> { + ) -> Result, ErrorData> { let http_parts = context.extensions.get::().ok_or_else(|| { tracing::error!("http::request::Parts not found"); ErrorData::internal_error("http::request::Parts not found", None) @@ -148,7 +158,7 @@ impl Runner { McpMode::Single(workspace_id) }; - Ok((auth.clone(), mode)) + Ok(McpContext { auth: auth.clone(), mode, headers: http_parts.headers.clone() }) } } @@ -391,6 +401,18 @@ fn authorize_endpoint_call( Ok(()) } +/// Map the model's argument keys back to the runnable's original parameter names. +fn transform_call_args(args: Value, item_schema: &Option) -> Value { + let Value::Object(map) = args else { + return args; + }; + let mut args_hash = HashMap::new(); + for (k, v) in map { + args_hash.insert(reverse_transform_key(&k, item_schema), v); + } + Value::Object(args_hash.into_iter().collect()) +} + fn find_matching_path(candidates: Vec, request_name: &str) -> Option { candidates .into_iter() @@ -427,7 +449,7 @@ impl ServerHandler for Runner { _request: Option, context: RequestContext, ) -> Result { - let (auth, mode) = Self::extract_context(&context)?; + let McpContext { auth, mode, .. } = Self::extract_context(&context)?; // Parse MCP scopes to determine what to expose let scopes = auth.scopes().unwrap_or(&[]); @@ -455,7 +477,7 @@ impl ServerHandler for Runner { request: CallToolRequestParams, context: RequestContext, ) -> Result { - let (auth, mode) = Self::extract_context(&context)?; + let McpContext { auth, mode, headers } = Self::extract_context(&context)?; // Parse MCP scopes for authorization let scopes = auth.scopes().unwrap_or(&[]); @@ -464,6 +486,7 @@ impl ServerHandler for Runner { let read_only = auth.read_only(); let args = request.arguments.map(Value::Object).unwrap_or(Value::Null); + let mcp_request = McpRequest { headers: &headers, tool_name: request.name.as_ref() }; // Every tool here runs to completion in one round trip: none of them ask the // client for input, so the MRTR variants of `CallToolResponse` are never built. @@ -474,14 +497,22 @@ impl ServerHandler for Runner { &workspace_id, &scope_config, read_only, - request.name, + request.name.clone(), args, + &mcp_request, ) .await } McpMode::Multi(token) => { - self.call_tool_multi(&auth, &token, &scope_config, read_only, request.name, args) - .await + self.call_tool_multi( + &auth, + &token, + &scope_config, + read_only, + request.name.clone(), + args, + ) + .await } }?; Ok(result.into()) @@ -665,6 +696,7 @@ impl Runner { read_only: bool, name: std::borrow::Cow<'static, str>, args: Value, + request: &McpRequest<'_>, ) -> Result { // Check if this is an endpoint tool let endpoint_tools = self.backend.all_endpoint_tools(); @@ -777,17 +809,7 @@ impl Runner { .map_err(|e| ErrorData::internal_error(e.message, None))? }; - // Transform arguments back to original key names - let transformed_args = if let Value::Object(map) = args { - let mut args_hash = HashMap::new(); - for (k, v) in map { - let original_key = reverse_transform_key(&k, &item_schema); - args_hash.insert(original_key, v); - } - Value::Object(args_hash.into_iter().collect()) - } else { - args - }; + let transformed_args = transform_call_args(args, &item_schema); let script_or_flow_path = if is_hub { format!("hub/{}", path) @@ -798,11 +820,23 @@ impl Runner { // Execute script or flow let result = if tool_type == "script" { self.backend - .run_script(auth, workspace_id, &script_or_flow_path, transformed_args) + .run_script( + auth, + workspace_id, + &script_or_flow_path, + transformed_args, + request, + ) .await } else { self.backend - .run_flow(auth, workspace_id, &script_or_flow_path, transformed_args) + .run_flow( + auth, + workspace_id, + &script_or_flow_path, + transformed_args, + request, + ) .await }; diff --git a/backend/windmill-object-store/src/lib.rs b/backend/windmill-object-store/src/lib.rs index 0f919f1048..cc588d485f 100644 --- a/backend/windmill-object-store/src/lib.rs +++ b/backend/windmill-object-store/src/lib.rs @@ -1171,6 +1171,7 @@ pub fn lfs_to_object_store_resource( Ok(ObjectStoreResource::Gcs(gcs_resource)) } LargeFileStorage::FilesystemStorage(fs) => { + windmill_common::workspaces::ensure_filesystem_storage_allowed()?; Ok(ObjectStoreResource::Filesystem(FilesystemSettings { root_path: fs.root_path.clone(), })) diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index a1bd8034f0..c7c119a5a7 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -4671,6 +4671,12 @@ pub async fn check_debouncing_within_limits( } } +/// Whether the tag's queue name is computed from the job's arguments, so that a caller holding +/// arguments it could not build knows the tag cannot be built either. +pub fn tag_reads_args(tag: &str) -> bool { + RE_ARG_TAG.is_match(tag) +} + pub fn interpolate_args(x: String, args: &PushArgs, workspace_id: &str) -> String { // Save this value to avoid parsing twice let workspaced = x.as_str().replace("$workspace", workspace_id).to_string(); diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index b107d34be4..f9bfe2f4df 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -130,6 +130,12 @@ pub struct EditResourceType { pub schema: Option, pub description: Option, pub is_fileset: Option, + /// Doubly optional so an edit can distinguish the two things a plain + /// `Option` conflates: an absent field leaves the extension alone, while an + /// explicit `null` clears it. A hub pull relies on both — a type that stops + /// being a file type has to stop being one locally too. + #[serde(default, deserialize_with = "windmill_common::more_serde::double_option")] + pub format_extension: Option>, } #[derive(FromRow, Serialize, Deserialize)] @@ -2803,10 +2809,42 @@ async fn update_resource_type( if let Some(is_fileset) = ns.is_fileset { sqlb.set("is_fileset", if is_fileset { "TRUE" } else { "FALSE" }); } + if let Some(format_extension) = ns.format_extension.clone() { + match format_extension { + Some(ext) => sqlb.set_str("format_extension", ext), + None => sqlb.set("format_extension", "NULL"), + }; + } sqlb.set_str("edited_at", "now()"); let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let mut tx = user_db.begin(&authed).await?; + // Creation refuses the pair outright, so an edit must too — otherwise the same + // impossible type (a set of files that is also one file) is reachable by setting + // either half on an existing row. Whichever half the request omits is read from + // the row being edited, inside this transaction and with the row locked: read + // outside it, two concurrent edits each supplying one half would both pass. + let current = sqlx::query!( + "SELECT is_fileset, format_extension FROM resource_type + WHERE name = $1 AND workspace_id = $2 FOR UPDATE", + &name, + &w_id + ) + .fetch_optional(&mut *tx) + .await?; + let effective_is_fileset = ns + .is_fileset + .unwrap_or_else(|| current.as_ref().map(|c| c.is_fileset).unwrap_or(false)); + let effective_format_extension = match &ns.format_extension { + Some(value) => value.clone(), + None => current.and_then(|c| c.format_extension), + }; + if effective_is_fileset && effective_format_extension.is_some() { + return Err(Error::BadRequest( + "A fileset resource type cannot have a format_extension".to_string(), + )); + } + sqlx::query(&sql).execute(&mut *tx).await?; audit_log( &mut *tx, diff --git a/backend/windmill-trigger-http/src/handler.rs b/backend/windmill-trigger-http/src/handler.rs index 14e1b9a4d2..68986bb6f9 100644 --- a/backend/windmill-trigger-http/src/handler.rs +++ b/backend/windmill-trigger-http/src/handler.rs @@ -268,6 +268,15 @@ pub async fn create_many_http_triggers( format!("http_triggers:write:{}", &new_http_trigger.base.path) })?; + // This route inserts directly, bypassing the shared create handler. + // `error_wrapper` would turn the rejection into a 500. + new_http_trigger.error_handling.validate().map_err(|err| { + Error::BadRequest(format!( + "Error occurred for HTTP route at route path: {}, error: {}", + new_http_trigger.config.route_path, err + )) + })?; + handler .validate_new(&db, &w_id, &new_http_trigger.config) .await @@ -570,8 +579,8 @@ impl TriggerCrud for HttpTrigger { route_path, &route_path_key, Some(effective_workspaced), - trigger.config.wrap_body, - trigger.config.raw_string, + trigger.config.wrap_body.unwrap_or(false), + trigger.config.raw_string.unwrap_or(false), trigger.config.authentication_resource_path, trigger.base.script_path, trigger.base.path, @@ -626,8 +635,8 @@ impl TriggerCrud for HttpTrigger { workspace_id = $20 AND path = $21 "#, - trigger.config.wrap_body, - trigger.config.raw_string, + trigger.config.wrap_body.unwrap_or(false), + trigger.config.raw_string.unwrap_or(false), trigger.config.authentication_resource_path, trigger.base.script_path, trigger.base.path, diff --git a/backend/windmill-trigger/src/handler.rs b/backend/windmill-trigger/src/handler.rs index 28898489f4..8ef7c1ee39 100644 --- a/backend/windmill-trigger/src/handler.rs +++ b/backend/windmill-trigger/src/handler.rs @@ -554,6 +554,8 @@ async fn create_trigger( ))); } + new_trigger.error_handling.validate()?; + handler .validate_new(&db, &workspace_id, &new_trigger.config) .await?; @@ -815,6 +817,8 @@ async fn update_trigger( ) })?; + edit_trigger.error_handling.validate()?; + handler .validate_edit(&db, &workspace_id, &edit_trigger.config, path) .await?; diff --git a/backend/windmill-trigger/src/types.rs b/backend/windmill-trigger/src/types.rs index dcf9a136be..2c1b854396 100644 --- a/backend/windmill-trigger/src/types.rs +++ b/backend/windmill-trigger/src/types.rs @@ -84,6 +84,31 @@ pub struct TriggerErrorHandling { pub retry: Option>, } +impl TriggerErrorHandling { + /// Schedule and workspace error handlers encode script-vs-flow as a + /// `script/`/`flow/` prefix; a trigger's handler is always a script, so a + /// prefixed path here would be looked up verbatim as a script name and fail + /// only once the trigger errors, which is when the handler is needed. + pub fn validate(&self) -> windmill_common::error::Result<()> { + let Some(path) = self.error_handler_path.as_deref() else { + return Ok(()); + }; + if let Some(bare) = path.strip_prefix("script/") { + return Err(windmill_common::error::Error::BadRequest(format!( + "error_handler_path is a plain script path, not the prefixed form a schedule \ + error handler takes: got '{path}', use '{bare}'" + ))); + } + if path.starts_with("flow/") { + return Err(windmill_common::error::Error::BadRequest(format!( + "error_handler_path must be a script: a trigger error handler cannot be a flow \ + (got '{path}')" + ))); + } + Ok(()) + } +} + #[derive(Serialize, Deserialize, Clone)] pub struct Trigger where @@ -253,6 +278,43 @@ mod tests { use super::*; use serde_json::json; + // --- TriggerErrorHandling::validate --- + + fn error_handling(path: Option<&str>) -> TriggerErrorHandling { + TriggerErrorHandling { + error_handler_path: path.map(str::to_string), + error_handler_args: None, + retry: None, + } + } + + #[test] + fn test_error_handler_path_accepts_bare_and_hub_paths() { + for path in [None, Some("f/team/handler"), Some("u/admin/handler")] { + assert!(error_handling(path).validate().is_ok(), "{path:?}"); + } + assert!(error_handling(Some("hub/13953/windmill/handler")) + .validate() + .is_ok()); + } + + #[test] + fn test_error_handler_path_rejects_prefixed_paths() { + // The rejection names the bare path to use, so the caller can fix it + // without knowing which of the two conventions a trigger follows. + let err = error_handling(Some("script/f/team/handler")) + .validate() + .unwrap_err() + .to_string(); + assert!(err.contains("f/team/handler"), "{err}"); + + let err = error_handling(Some("flow/f/team/handler")) + .validate() + .unwrap_err() + .to_string(); + assert!(err.contains("cannot be a flow"), "{err}"); + } + // --- TriggerMode serde --- #[test] diff --git a/backend/windmill-types/src/jobs.rs b/backend/windmill-types/src/jobs.rs index db20bc1251..5190efe380 100644 --- a/backend/windmill-types/src/jobs.rs +++ b/backend/windmill-types/src/jobs.rs @@ -632,6 +632,28 @@ pub enum JobPayload { }, } +impl JobPayload { + /// Whether the payload itself declares a dedicated worker, in which case `push` replaces + /// whatever tag it is handed and the caller's tag never reaches the queue. + /// + /// This reads what the payload carries, not what `push` will conclude: a `SingleStepFlow` + /// loads the flag from the script row at push time and reports `false` here. That only + /// matters to a caller reasoning about the tag, and for those the answer is the same either + /// way, since `push` replaces the tag in exactly the case this misses. + pub fn is_dedicated_worker(&self) -> bool { + let dedicated_worker = match self { + JobPayload::ScriptHash { dedicated_worker, .. } + | JobPayload::FlowScript { dedicated_worker, .. } + | JobPayload::Dependencies { dedicated_worker, .. } + | JobPayload::FlowDependencies { dedicated_worker, .. } + | JobPayload::Flow { dedicated_worker, .. } => dedicated_worker, + JobPayload::Code(raw) => &raw.dedicated_worker, + _ => &None, + }; + dedicated_worker.is_some_and(|x| x) + } +} + #[derive(Clone, Serialize, Deserialize, Debug)] pub struct SkipHandler { pub path: String, diff --git a/backend/windmill-types/src/more_serde.rs b/backend/windmill-types/src/more_serde.rs index d4b648d2d5..f9cbe739c5 100644 --- a/backend/windmill-types/src/more_serde.rs +++ b/backend/windmill-types/src/more_serde.rs @@ -38,6 +38,25 @@ pub fn is_default(t: &T) -> bool { &T::default() == t } +pub fn maybe_number<'de, T, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, + T: FromStr + serde::Deserialize<'de>, + ::Err: Display, +{ + #[derive(Deserialize)] + #[serde(untagged)] + enum NumericOrString { + String(String), + RawT(T), + } + + match NumericOrString::::deserialize(deserializer)? { + NumericOrString::String(s) => T::from_str(&s).map_err(serde::de::Error::custom), + NumericOrString::RawT(i) => Ok(i), + } +} + pub fn maybe_number_opt<'de, T, D>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de>, @@ -66,3 +85,60 @@ where NumericOrNull::Null => Ok(None), } } + +/// Deserializer for a doubly-optional field, so a struct can tell an absent key +/// (`None`) from an explicit `null` (`Some(None)`). +/// +/// Plain serde collapses both into the outer `None`, which makes the distinction +/// unusable exactly where it matters: a payload that omits a field means "leave it +/// alone", while one that sends `null` means "clear it". +/// +/// ```ignore +/// #[serde(default, deserialize_with = "double_option", skip_serializing_if = "Option::is_none")] +/// pub field: Option>, +/// ``` +pub fn double_option<'de, T, D>(deserializer: D) -> Result>, D::Error> +where + T: serde::Deserialize<'de>, + D: serde::Deserializer<'de>, +{ + serde::Deserialize::deserialize(deserializer).map(Some) +} + +#[cfg(test)] +mod tests { + use serde::Deserialize; + + #[derive(Deserialize)] + struct WithMaybeNumber { + #[serde(deserialize_with = "super::maybe_number")] + n: i64, + } + + #[test] + fn maybe_number_accepts_number() { + let v: WithMaybeNumber = serde_json::from_value(serde_json::json!({ "n": 12345 })).unwrap(); + assert_eq!(v.n, 12345); + } + + #[test] + fn maybe_number_accepts_string() { + let v: WithMaybeNumber = + serde_json::from_value(serde_json::json!({ "n": "12345" })).unwrap(); + assert_eq!(v.n, 12345); + } + + #[test] + fn maybe_number_rejects_non_numeric_string() { + assert!( + serde_json::from_value::(serde_json::json!({ "n": "abc" })).is_err() + ); + } + + #[test] + fn maybe_number_rejects_null() { + assert!( + serde_json::from_value::(serde_json::json!({ "n": null })).is_err() + ); + } +} diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 1fb566e57f..77221edf22 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -1256,6 +1256,7 @@ pub async fn prebundle_bun_script( token: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, temp_script_refs: &Option>, + modules: Option<&HashMap>, ) -> Result<()> { let (local_path, remote_path) = compute_bundle_local_and_remote_path( inner_content, @@ -1264,6 +1265,7 @@ pub async fn prebundle_bun_script( db, w_id, temp_script_refs, + modules, ) .await; if exists_in_cache(&local_path, &remote_path).await { @@ -1442,6 +1444,7 @@ pub async fn compute_bundle_local_and_remote_path( db: Option<&DB>, w_id: &str, temp_script_refs: &Option>, + modules: Option<&HashMap>, ) -> (String, String) { let mut input_src = format!("{inner_content}{lock}",); @@ -1470,7 +1473,10 @@ pub async fn compute_bundle_local_and_remote_path( let ws_suffix = crate::workspace_registry_cache_suffix(w_id).await; input_src.push_str(&ws_suffix); - let hash = windmill_common::utils::calculate_hash(&input_src); + + // The loader resolves relative imports against the module files in the job dir, so + // their content is inlined into the bundle this name covers. + let hash = crate::worker::artifact_cache_name(input_src, modules); let local_path = format!("{}/{hash}", *BUN_BUNDLE_CACHE_DIR); #[cfg(windows)] @@ -1569,6 +1575,7 @@ pub async fn handle_bun_job( Some(db), &job.workspace_id, &temp_script_refs, + modules.as_ref(), ) .await } @@ -2962,9 +2969,8 @@ pub async fn handle_wac_v2_output( version: flow_info.version, labels: flow_info.labels.clone(), }; - let on_behalf_of = flow_info - .on_behalf_of(&job.workspace_id, db) - .await?; + let on_behalf_of = + flow_info.on_behalf_of(&job.workspace_id, db).await?; let step_args: HashMap> = step .args .iter() @@ -4382,4 +4388,49 @@ export function main(x: number) { return x; }"#; assert!(wrapper.contains(r#"line.startsWith("exec_preprocess:")"#)); assert!(wrapper.contains(r#"line.startsWith("exec:")"#)); } + + /// The bundle cache is global and content-keyed, so a key that ignores the inline + /// modules hands one workspace's bundle — attacker helper code and all — to the next + /// job whose main content and lockfile happen to match. + #[tokio::test] + async fn bundle_cache_key_separates_inline_module_content() { + use windmill_common::scripts::ScriptModule; + + async fn key_for(modules: Option<&HashMap>) -> String { + compute_bundle_local_and_remote_path( + "import { h } from './helper.ts';\nexport async function main() { return h(); }", + "{}\n//bun.lock\n", + "u/alice/script", + None, + "w1", + &None, + modules, + ) + .await + .1 + } + fn modules(content: &str) -> HashMap { + HashMap::from([( + "helper.ts".to_string(), + ScriptModule { + content: content.to_string(), + language: ScriptLang::Bun, + lock: None, + }, + )]) + } + + let attacker = key_for(Some(&modules("export const h = () => 'attacker'"))).await; + let victim = key_for(Some(&modules("export const h = () => 'victim'"))).await; + assert_ne!(attacker, victim); + assert_eq!( + attacker, + key_for(Some(&modules("export const h = () => 'attacker'"))).await, + "same modules must still share a cache slot" + ); + + // An absent map and an empty one are the same script, so they share a slot. + assert_eq!(key_for(None).await, key_for(Some(&HashMap::new())).await); + assert_ne!(key_for(None).await, attacker); + } } diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 6ee22fd882..f6391eab07 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -1640,6 +1640,7 @@ pub(crate) async fn get_workspace_s3_resource_path( ) } Some(LargeFileStorage::FilesystemStorage(fs)) => { + windmill_common::workspaces::ensure_filesystem_storage_allowed()?; return Ok(Some( windmill_object_store::ObjectStoreResource::Filesystem( windmill_object_store::FilesystemSettings { root_path: fs.root_path.clone() }, diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs index c95107d504..d393171f7f 100644 --- a/backend/windmill-worker/src/csharp_executor.rs +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -13,7 +13,7 @@ use itertools::Itertools; #[cfg(feature = "csharp")] use tokio::{fs::File, io::AsyncReadExt, process::Command}; #[cfg(feature = "csharp")] -use windmill_common::{utils::calculate_hash, worker::write_file}; +use windmill_common::worker::write_file; #[cfg(feature = "csharp")] use crate::global_cache::save_cache; @@ -72,13 +72,21 @@ const CSHARP_OBJECT_STORE_PREFIX: &str = /// Cache key of a C# build. The run path and the deploy-time prebuild must derive it the /// same way or the prebuilt binary is never found and gets rebuilt on first run. #[cfg(feature = "csharp")] -async fn csharp_cache_key(code: &str, requirements_o: Option<&str>, w_id: &str) -> String { - let mut hash = calculate_hash(&format!( +async fn csharp_cache_key( + code: &str, + requirements_o: Option<&str>, + w_id: &str, + modules: Option<&HashMap>, +) -> String { + // The SDK project globs every `.cs` under the job dir, so companion modules are + // compiled into the binary this key names and have to be part of it. + let base = format!( "{}{}{}", code, requirements_o.unwrap_or(""), DOTNET_TARGET_FRAMEWORK.as_str() - )); + ); + let mut hash = crate::worker::artifact_cache_name(base, modules); hash.push_str(&crate::workspace_registry_cache_suffix(w_id).await); hash } @@ -487,10 +495,11 @@ pub async fn prebuild_csharp_binary( worker_name: &str, base_internal_url: &str, occupancy_metrics: &mut OccupancyMetrics, + modules: Option<&HashMap>, ) -> error::Result> { check_executor_binary_exists("dotnet", DOTNET_PATH.as_str(), "C#")?; - let hash = csharp_cache_key(code, Some(lock), &job.workspace_id).await; + let hash = csharp_cache_key(code, Some(lock), &job.workspace_id, modules).await; let remote_path = format!("{CSHARP_OBJECT_STORE_PREFIX}{hash}"); if crate::global_cache::exists_in_object_store(&remote_path).await { return Ok(None); @@ -549,6 +558,7 @@ pub async fn handle_csharp_job( _worker_name: &str, _envs: HashMap, _occupancy_metrics: &mut OccupancyMetrics, + _modules: Option<&HashMap>, ) -> Result, Error> { Err(anyhow!("C# is not available because the feature is not enabled").into()) } @@ -568,6 +578,7 @@ pub async fn handle_csharp_job( worker_name: &str, envs: HashMap, occupancy_metrics: &mut OccupancyMetrics, + modules: Option<&HashMap>, ) -> Result, Error> { check_executor_binary_exists("dotnet", DOTNET_PATH.as_str(), "C#")?; @@ -575,6 +586,7 @@ pub async fn handle_csharp_job( inner_content, requirements_o.map(|x| x.as_str()), &job.workspace_id, + modules, ) .await; let bin_path = format!("{}/{hash}", *CSHARP_CACHE_DIR); diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index 1d2009c5c0..9fe6d4a616 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -230,8 +230,15 @@ fn go_runtime_int32(v: &str) -> Option { /// Cache key of a Go build. The run path and the deploy-time prebuild must derive it the /// same way or the prebuilt binary is never found and gets rebuilt on first run. -fn go_cache_key(code: &str, maybe_lock: &MaybeLock) -> String { - calculate_hash(&format!("{}{:?}v2", code, maybe_lock)) +fn go_cache_key( + code: &str, + maybe_lock: &MaybeLock, + modules: Option<&HashMap>, +) -> String { + // A module whose path starts with `go/` lands inside the module dir this builds, so + // its content ends up in the binary the key names. + let base = format!("{}{:?}v2", code, maybe_lock); + crate::worker::artifact_cache_name(base, modules) } /// Install the deps, generate the entrypoint wrapper, `go build`, and push the binary to @@ -459,9 +466,10 @@ pub async fn prebuild_go_binary( worker_name: &str, base_internal_url: &str, occupancy_metrics: &mut OccupancyMetrics, + modules: Option<&HashMap>, ) -> Result, Error> { let maybe_lock = MaybeLock::Resolved { lock: lock.to_string() }; - let hash = go_cache_key(code, &maybe_lock); + let hash = go_cache_key(code, &maybe_lock, modules); let remote_path = format!("{GO_OBJECT_STORE_PREFIX}{hash}"); if crate::global_cache::exists_in_object_store(&remote_path).await { return Ok(None); @@ -512,6 +520,7 @@ pub async fn handle_go_job( envs: HashMap, occupation_metrics: &mut OccupancyMetrics, maybe_lock: MaybeLock, + modules: Option<&HashMap>, ) -> Result, Error> { //go does not like executing modules at temp root let job_dir = &format!("{job_dir}/go"); @@ -520,7 +529,7 @@ pub async fn handle_go_job( .create(&job_dir) .expect("could not create go job dir"); - let hash = go_cache_key(inner_content, &maybe_lock); + let hash = go_cache_key(inner_content, &maybe_lock, modules); let bin_path = format!("{}/{hash}", *GO_BIN_CACHE_DIR); let remote_path = format!("{GO_OBJECT_STORE_PREFIX}{hash}"); let (cache, cache_logs) = crate::global_cache::load_cache(&bin_path, &remote_path, false).await; diff --git a/backend/windmill-worker/src/java_executor.rs b/backend/windmill-worker/src/java_executor.rs index bb0518f22d..2e47fced18 100644 --- a/backend/windmill-worker/src/java_executor.rs +++ b/backend/windmill-worker/src/java_executor.rs @@ -66,6 +66,7 @@ pub(crate) struct JobHandlerInput<'a> { pub requirements_o: Option<&'a String>, pub shared_mount: &'a str, pub worker_name: &'a str, + pub modules: Option<&'a HashMap>, } pub async fn handle_java_job<'a>(mut args: JobHandlerInput<'a>) -> Result, Error> { @@ -612,25 +613,33 @@ async fn compile<'a>( inner_content, requirements_o, parent_runnable_path, + modules, .. }: &mut JobHandlerInput<'a>, classpath: &'a str, // plugins: Vec<&'a str>, ) -> Result<(), Error> { - fn compute_hash(code: &str, requirements_o: Option<&String>) -> String { - calculate_hash(&format!( + // The cached artifact is the whole `target/` dir, and companion modules are written + // into the job dir before this runs, so their content can land in it. + fn compute_hash( + code: &str, + requirements_o: Option<&String>, + modules: Option<&HashMap>, + ) -> String { + let base = format!( "{}{}", code, requirements_o .as_ref() .map(|x| x.to_string()) .unwrap_or_default() - )) + ); + crate::worker::artifact_cache_name(base, modules) } let reserved_variables = get_reserved_variables(job, &client.token, conn, parent_runnable_path.clone()).await?; let ws_suffix = crate::workspace_registry_cache_suffix(&job.workspace_id).await; - let mut hash = compute_hash(inner_content, *requirements_o); + let mut hash = compute_hash(inner_content, *requirements_o, *modules); hash.push_str(&ws_suffix); let bin_path = format!("{}/{hash}", *JAVA_CACHE_DIR); let remote_path = format!("java_jar/{hash}"); diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index b9d53a801b..dff7d58c2d 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -14,7 +14,6 @@ use tokio::{ }; use windmill_common::{ error::{self, Error}, - utils::calculate_hash, worker::{write_file, Connection}, }; use windmill_queue::MiniPulledJob; @@ -602,8 +601,13 @@ pub async fn build_rust_crate( /// Cache key of a Rust build. The run path and the deploy-time prebuild must derive it /// the same way or the prebuilt binary is never found and gets rebuilt on first run. -async fn rust_cache_key(code: &str, requirements_o: Option<&String>, w_id: &str) -> String { - let mut hash = compute_rust_hash(code, requirements_o); +async fn rust_cache_key( + code: &str, + requirements_o: Option<&String>, + w_id: &str, + modules: Option<&HashMap>, +) -> String { + let mut hash = compute_rust_hash(code, requirements_o, modules); hash.push_str(&crate::workspace_registry_cache_suffix(w_id).await); hash } @@ -621,11 +625,12 @@ pub async fn prebuild_rust_binary( worker_name: &str, base_internal_url: &str, occupancy_metrics: &mut OccupancyMetrics, + modules: Option<&HashMap>, ) -> error::Result> { ensure_rust_runtime_dirs(); check_executor_binary_exists("cargo", CARGO_PATH.as_str(), "rust")?; - let hash = rust_cache_key(code, Some(&lock.to_string()), &job.workspace_id).await; + let hash = rust_cache_key(code, Some(&lock.to_string()), &job.workspace_id, modules).await; let remote_path = format!("{RUST_OBJECT_STORE_PREFIX}{hash}"); if crate::global_cache::exists_in_object_store(&remote_path).await { return Ok(None); @@ -652,15 +657,22 @@ pub async fn prebuild_rust_binary( Ok(Some(logs)) } -pub fn compute_rust_hash(code: &str, requirements_o: Option<&String>) -> String { - calculate_hash(&format!( +pub fn compute_rust_hash( + code: &str, + requirements_o: Option<&String>, + // Companion modules are written into the crate dir and compiled into the binary this + // key names, so leaving them out shares one script's binary with another. + modules: Option<&HashMap>, +) -> String { + let base = format!( "{}{}", code, requirements_o .as_ref() .map(|x| x.to_string()) .unwrap_or_default() - )) + ); + crate::worker::artifact_cache_name(base, modules) } #[tracing::instrument(level = "trace", skip_all)] @@ -679,11 +691,12 @@ pub async fn handle_rust_job( worker_name: &str, envs: HashMap, occupancy_metrics: &mut OccupancyMetrics, + modules: Option<&HashMap>, ) -> Result, Error> { ensure_rust_runtime_dirs(); check_executor_binary_exists("cargo", CARGO_PATH.as_str(), "rust")?; - let hash = rust_cache_key(inner_content, requirements_o, &job.workspace_id).await; + let hash = rust_cache_key(inner_content, requirements_o, &job.workspace_id, modules).await; let bin_path = format!("{}/{hash}", *RUST_CACHE_DIR); let remote_path = format!("{RUST_OBJECT_STORE_PREFIX}{hash}"); diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 350f0c5a61..7e620439b0 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -950,6 +950,51 @@ pub async fn workspace_registry_cache_suffix(w_id: &str) -> String { } } +/// The name a build artifact is cached under, derived from `base` — the runnable's own +/// cache-key input — and its inline modules. +/// +/// `write_module_files` puts module content in the job dir where the build inlines it into +/// the artifact, so a name without it serves one runnable's modules to another whose main +/// content and lockfile match — across workspaces, the cache being global. +/// +/// Only the path and content may name the artifact, because they are all the build reads. +/// `ScriptModule::lock` especially must stay out: deploy regenerates it *after* the parent +/// has prebuilt, so naming it would strand every prebuilt artifact. +pub(crate) fn artifact_cache_name( + base: String, + modules: Option<&std::collections::HashMap>, +) -> String { + let Some(modules) = modules.filter(|m| !m.is_empty()) else { + // Byte-identical to the name a module-free runnable had before modules entered this, + // so its cached artifacts stay reachable. A pre-fix multi-file runnable also stored + // here, so one of those stays reachable too — accepted over invalidating every cache, + // and once this ships nothing can be stored here with module content in it again. + return windmill_common::utils::calculate_hash(&base); + }; + let mut entries: Vec<(&String, &ScriptModule)> = modules.iter().collect(); + entries.sort_by(|a, b| a.0.cmp(b.0)); + // `base` ends in caller-supplied bytes (a preview brings its own lockfile), so it is + // sealed to a fixed width before the module block is appended — raw, a crafted lockfile + // could spell out another runnable's block and reach its slot. + let mut keyed = format!( + "{}:modules:{}", + windmill_common::utils::calculate_hash(&base), + entries.len() + ); + for (path, module) in entries { + // Both length-prefixed, else `{"a": "bc"}` and `{"ab": "c"}` encode alike. + keyed.push_str(&format!( + ":{}:{path}:{}:{}", + path.len(), + module.content.len(), + module.content, + )); + } + // Its own namespace: `calculate_hash` emits hex, so however a module-free runnable + // crafts its content and lockfile it can never land on a module-bearing name. + format!("mod-{}", windmill_common::utils::calculate_hash(&keyed)) +} + pub fn is_sandboxing_enabled() -> bool { if !*DISABLE_NSJAIL { return true; @@ -3532,7 +3577,13 @@ pub async fn run_worker( job.kind, JobKind::Script | JobKind::Preview | JobKind::FlowScript ) { - if !dedicated_workers.is_empty() { + // A job carrying a pre-run error never runs its code: it only has to be + // pulled so `handle_queued_job` can fail it. Both hand-off paths below + // dispatch by path and return before that check, so a job sent down them + // would run with whatever arguments survived the failure. + let fails_before_running = job.pre_run_error.is_some(); + + if !dedicated_workers.is_empty() && !fails_before_running { let dedicated_worker_tx = job.runnable_path.as_ref().and_then(|path| { // For flow steps inside branches/loops, runnable_path includes // nesting segments (e.g. f/flow/branchone-0/a) but the dedicated @@ -3577,7 +3628,7 @@ pub async fn run_worker( NextJob::Http(_) => None, }; - if let Some(flow_runners) = flow_runners { + if let Some(flow_runners) = flow_runners.filter(|_| !fails_before_running) { let key_o = job.flow_step_id.as_ref().map(|x| x.to_string()); if let Some(key) = key_o { if let Some(flow_runner_tx) = flow_runners.runners.get(&key) { @@ -5453,7 +5504,9 @@ async fn handle_code_execution_job( None => job, }; - // For preview jobs, extract modules from args._MODULES if not already set + // Any job kind, not just previews: whatever is here is what gets written to the job dir + // and built in, so the agent-worker server precomputing a cache name has to resolve + // modules the same way (`windmill-api-agent-workers`, `get_code_and_lock`). let modules = modules_from_data.clone().or_else(|| { job.args.as_ref().and_then(|args| { args.get("_MODULES").and_then(|raw| { @@ -5646,11 +5699,106 @@ mod write_module_files_tests { use super::*; use std::collections::HashMap; use windmill_common::scripts::ScriptLang; + use windmill_common::utils::calculate_hash; fn module(content: &str) -> ScriptModule { ScriptModule { content: content.to_string(), language: ScriptLang::Python3, lock: None } } + /// Every language's artifact cache name funnels module content through this, so an + /// ambiguous encoding puts two different runnables back on one name. + #[test] + fn artifact_name_cannot_be_re_cut_into_another_module_map() { + fn name(entries: &[(&str, &str)]) -> String { + let map: HashMap = entries + .iter() + .map(|(p, c)| (p.to_string(), module(c))) + .collect(); + artifact_cache_name("base".to_string(), Some(&map)) + } + + // Naive `path + content` concatenation renders both of these as "abc". + assert_ne!(name(&[("a", "bc")]), name(&[("ab", "c")])); + // Splitting one module into two must not read back as the joined one. + assert_ne!(name(&[("a", "b"), ("c", "d")]), name(&[("ac", "bd")])); + // Iteration order of the map must not move the name. + assert_eq!( + name(&[("a", "1"), ("b", "2")]), + name(&[("b", "2"), ("a", "1")]) + ); + } + + /// A module-free runnable must keep the exact name it had before modules entered the + /// derivation, or upgrading strands every artifact already in the cache. + #[test] + fn artifact_name_is_unchanged_without_modules() { + assert_eq!( + artifact_cache_name("code+lock".to_string(), None), + calculate_hash("code+lock") + ); + assert_eq!( + artifact_cache_name("code+lock".to_string(), Some(&HashMap::new())), + calculate_hash("code+lock") + ); + } + + /// A preview brings its own source and lockfile, so a module-free runnable picks its + /// whole `base`. Module-bearing names live in their own namespace precisely so that no + /// crafted `base` can be made to land on one. + #[test] + fn a_module_free_runnable_cannot_forge_a_module_bearing_name() { + let modules = HashMap::from([("h.ts".to_string(), module("evil"))]); + let victim = artifact_cache_name("code+lock".to_string(), Some(&modules)); + + // `calculate_hash` emits hex, so the namespace is unreachable however `base` is + // chosen — including by feeding it the victim's own name. + assert!(victim.starts_with("mod-")); + assert_ne!(artifact_cache_name(victim.clone(), None), victim); + assert!(!artifact_cache_name("anything".to_string(), None).starts_with("mod-")); + } + + /// The `mod-` namespace separates module-free from module-bearing, and nothing separates + /// two module-bearing runnables — only the seal does. Unsealed, `base` is variable-width, + /// so the split between it and the module block is ambiguous and a preview (which brings + /// its own source *and* lockfile) can absorb part of another runnable's block. + #[test] + fn a_module_bearing_runnable_cannot_absorb_another_ones_block() { + let victim = artifact_cache_name( + "V".to_string(), + Some(&HashMap::from([( + "h.ts".to_string(), + module(":modules:1:1:a:1:b"), + )])), + ); + // Byte-identical to the victim's without the seal: the forger's `base` spells out the + // victim's leading block, leaving its own single module to supply the tail. + let forged = artifact_cache_name( + "V:modules:1:4:h.ts:18:".to_string(), + Some(&HashMap::from([("a".to_string(), module("b"))])), + ); + + assert_ne!(victim, forged); + } + + /// Deploy fills a module's lock in after the parent has prebuilt, so a name that moved + /// with it would leave every prebuilt artifact unreachable by the runs it was built for. + #[test] + fn artifact_name_ignores_the_lock_deploy_fills_in_later() { + let prebuild = artifact_cache_name( + "base".to_string(), + Some(&HashMap::from([("h.ts".to_string(), module("x"))])), + ); + + let mut locked = module("x"); + locked.lock = Some("{}\n//bun.lock\n".to_string()); + let after_deploy = artifact_cache_name( + "base".to_string(), + Some(&HashMap::from([("h.ts".to_string(), locked)])), + ); + + assert_eq!(prebuild, after_deploy); + } + #[test] fn contained_relative_path_rejects_traversal_and_absolute() { assert!(is_contained_relative_path("u/admin/pkg")); @@ -6380,6 +6528,7 @@ mount {{ envs, occupancy_metrics, maybe_lock, + modules.as_ref(), )) .await } @@ -6509,6 +6658,7 @@ mount {{ worker_name, envs, occupancy_metrics, + modules.as_ref(), )) .await } @@ -6567,6 +6717,7 @@ mount {{ worker_name, envs, occupancy_metrics, + modules.as_ref(), )) .await } @@ -6631,6 +6782,7 @@ mount {{ worker_name, envs, occupancy_metrics, + modules: modules.as_ref(), })) .await } diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index bd39afa8c7..34f46307cf 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -70,9 +70,9 @@ use windmill_queue::schedule::get_schedule_opt; use windmill_queue::{ add_completed_job, add_completed_job_error, append_logs, get_mini_pulled_job, insert_concurrency_key_capped, interpolate_args, - report_error_to_workspace_handler_or_critical_side_channel, try_schedule_next_job, CanceledBy, - FlowRunners, MiniCompletedJob, MiniPulledJob, PushArgs, PushIsolationLevel, SameWorkerPayload, - WrappedError, + report_error_to_workspace_handler_or_critical_side_channel, tag_reads_args, + try_schedule_next_job, CanceledBy, FlowRunners, MiniCompletedJob, MiniPulledJob, PushArgs, + PushIsolationLevel, SameWorkerPayload, WrappedError, }; use windmill_audit::audit_oss::audit_log; @@ -4403,6 +4403,23 @@ async fn push_next_flow_job( payload_tag.tag.as_deref(), ); + // `push_args` is empty once the input transforms failed, so a tag reading `$args[...]` + // interpolates to a queue nobody serves and the step sits there instead of reporting + // the error. Send it to the flow's tag, which a worker is provably serving right now. + // + // A step handed over by id, or one whose tag `push` replaces, never reaches a worker + // through its tag, so rewriting theirs would be noise. + let step_is_pulled_by_tag = !continue_on_same_worker + && !continue_with_runners + && !payload_tag.payload.is_dedicated_worker(); + let reroute_to_flow_tag = + err.is_some() && step_is_pulled_by_tag && tag.as_deref().is_some_and(tag_reads_args); + let tag = if reroute_to_flow_tag { + Some(flow_job.tag.clone()) + } else { + tag + }; + let (email, permissioned_as) = if let Some(on_behalf_of) = payload_tag.on_behalf_of.as_ref() { (&on_behalf_of.email, on_behalf_of.permissioned_as.clone()) @@ -4421,8 +4438,7 @@ async fn push_next_flow_job( .as_deref() .filter(|t| !t.is_empty() && *t != flow_job.tag.as_str()) { - let is_super_admin = - windmill_common::auth::is_super_admin_email(db, email).await?; + let is_super_admin = windmill_common::auth::is_super_admin_email(db, email).await?; check_tag_available_for_workspace_internal( db, &flow_job.workspace_id, @@ -6155,9 +6171,7 @@ pub async fn script_to_payload( .await? .prefetch_cached(&db) .await?; - let on_behalf_of = script_info - .on_behalf_of(&flow_job.workspace_id, db) - .await?; + let on_behalf_of = script_info.on_behalf_of(&flow_job.workspace_id, db).await?; let ScriptHashInfo { tag, cache_ttl, diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 5cfb9f08ce..57b859075a 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -199,6 +199,7 @@ async fn handle_build_binary_job( worker_name, base_internal_url, occupancy_metrics, + script_data.modules.as_ref(), ) .await? } @@ -214,6 +215,7 @@ async fn handle_build_binary_job( worker_name, base_internal_url, occupancy_metrics, + script_data.modules.as_ref(), ) .await? } @@ -235,6 +237,7 @@ async fn handle_build_binary_job( worker_name, base_internal_url, occupancy_metrics, + script_data.modules.as_ref(), ) .await? } @@ -3155,7 +3158,11 @@ async fn capture_dependency_job( ) .await? { - if !wd_exist { + // Nothing here writes the module files, so a bundle built now resolves a + // multi-file script's relative imports remotely rather than from its + // modules; caching that under a key naming them would serve the wrong + // code to every run. Leave it to the first run instead. + if !wd_exist && modules.map_or(true, |m| m.is_empty()) { crate::bun_executor::prebundle_bun_script( job_raw_code, &lock, @@ -3169,6 +3176,7 @@ async fn capture_dependency_job( &token, &mut Some(occupancy_metrics), temp_script_refs, + modules, ) .await?; } diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 55734e52e8..6a165901e4 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.796.0"; +export const VERSION = "v1.803.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/commands/app/bundle.ts b/cli/src/commands/app/bundle.ts index 0ebbdaa076..286e878cfb 100644 --- a/cli/src/commands/app/bundle.ts +++ b/cli/src/commands/app/bundle.ts @@ -191,7 +191,14 @@ function createSveltePlugin(appDir: string): any { // Convert Svelte syntax to JavaScript try { - const { js, warnings } = svelte.compile(source, { filename }); + // The raw-app editor's in-browser bundler compiles with + // `css: "injected"`, so this must too, or the same app renders + // styled there and unstyled once the CLI builds it: Svelte's default + // ("external") hands the +`, + "styles_entry.ts": `import Styled from './Styled.svelte'; +export default Styled; +`, + }); + + const js = await bundle("styles_entry.ts"); + + const scopeClass = js.match(/

= {}; +let remoteUnreadable = false; +let pushedFiles: Record | undefined; mock.module("../gen/services.gen.ts", () => ({ - getSharedUi: async (_args: { workspace: string }) => ({ files: remoteFiles }), + getSharedUi: async (_args: { workspace: string }) => { + if (remoteUnreadable) throw new Error("shared UI store unreadable"); + return { files: remoteFiles }; + }, + updateSharedUi: async (args: { + workspace: string; + requestBody: { files: Record }; + }) => { + pushedFiles = args.requestBody.files; + }, })); -const { diffSharedUi } = await import("../src/commands/shared_ui.ts"); +const { diffSharedUi, pushSharedUi } = await import( + "../src/commands/shared_ui.ts" +); describe("diffSharedUi", () => { const ws = "test-workspace"; @@ -25,6 +38,8 @@ describe("diffSharedUi", () => { beforeEach(() => { remoteFiles = {}; + remoteUnreadable = false; + pushedFiles = undefined; prevCwd = process.cwd(); tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "wm-shared-ui-")); process.chdir(tmpDir); @@ -89,4 +104,46 @@ describe("diffSharedUi", () => { const changes = await diffSharedUi(ws); expect(changes).toEqual([]); }); + + test("emits nothing under keepDeleted when the remote store is unreadable", async () => { + // pushSharedUi skips the push rather than clearing a store it can't read, + // so the preview must show that same nothing. + remoteUnreadable = true; + writeUi("theme.json", "{}"); + expect(await diffSharedUi(ws, true)).toEqual([]); + }); +}); + +describe("pushSharedUi with keepDeleted", () => { + const ws = "test-workspace"; + let tmpDir: string; + let prevCwd: string; + + beforeEach(() => { + remoteFiles = {}; + remoteUnreadable = false; + pushedFiles = undefined; + prevCwd = process.cwd(); + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "wm-shared-ui-push-")); + process.chdir(tmpDir); + }); + + afterEach(() => { + process.chdir(prevCwd); + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + test("carries remote-only files, including ui/__proto__, into the pushed map", async () => { + // JSON.parse, not a literal: `{__proto__: …}` sets the prototype instead of + // creating the own property the API response really has. + remoteFiles = JSON.parse('{"__proto__":"keep me","extra.json":"1"}'); + fs.mkdirSync(path.join(tmpDir, "ui"), { recursive: true }); + fs.writeFileSync(path.join(tmpDir, "ui", "theme.json"), "{}", "utf-8"); + + expect(await pushSharedUi(ws, true)).toBe(true); + // The store is written whole, so anything missing here is deleted. + expect(pushedFiles!["extra.json"]).toEqual("1"); + expect(Object.getOwnPropertyDescriptor(pushedFiles!, "__proto__")?.value) + .toEqual("keep me"); + }); }); diff --git a/cli/test/sync_pull_push.test.ts b/cli/test/sync_pull_push.test.ts index 343ed49ef7..24c4a282a3 100644 --- a/cli/test/sync_pull_push.test.ts +++ b/cli/test/sync_pull_push.test.ts @@ -2774,3 +2774,102 @@ kind: script }); }); }); + +describe("keep deleted", () => { + test("Integration: --keep-deleted keeps items absent from the other side", async () => { + await withTestBackend(async (backend, tempDir) => { + const uniqueId = Date.now(); + const scriptPath = `f/test/keep_deleted_${uniqueId}`; + + const resp = await backend.apiRequest!( + `/api/w/${backend.workspace}/scripts/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: scriptPath, + content: 'export async function main() { return "keep me"; }', + language: "bun", + summary: "Kept by --keep-deleted", + schema: { + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + properties: {}, + required: [], + }, + }), + } + ); + expect(resp.status).toBeLessThan(300); + await resp.text(); + + await writeWmillYaml(tempDir); + expect( + (await backend.runCLICommand(["sync", "pull", "--yes"], tempDir)).code + ).toEqual(0); + + const contentFile = `${scriptPath}.ts`; + const metadataFile = `${scriptPath}.script.yaml`; + expect(await listFilesRecursive(tempDir)).toContain(contentFile); + + // Push direction: the remote script survives losing its local files. + await rm(join(tempDir, contentFile)); + await rm(join(tempDir, metadataFile)); + expect( + ( + await backend.runCLICommand( + ["sync", "push", "--yes", "--keep-deleted"], + tempDir + ) + ).code + ).toEqual(0); + // A push deletion archives the script rather than removing the row, so + // `archived` — not the status code — is what says it survived. + const remote = await backend.apiRequest!( + `/api/w/${backend.workspace}/scripts/get/p/${scriptPath}` + ); + expect(remote.status).toEqual(200); + expect((await remote.json()).archived).not.toEqual(true); + + // Pull direction: a file with no remote counterpart survives the pull. + const localOnly = `f/test/local_only_${uniqueId}.ts`; + await writeFile( + join(tempDir, localOnly), + 'export async function main() { return "local only"; }', + "utf-8" + ); + expect( + ( + await backend.runCLICommand( + ["sync", "pull", "--yes", "--keep-deleted"], + tempDir + ) + ).code + ).toEqual(0); + const afterPull = await listFilesRecursive(tempDir); + expect(afterPull).toContain(localOnly); + // Adds still apply: the script deleted above is written back. + expect(afterPull).toContain(contentFile); + + // An empty changeset falls past the dry-run return, on to the shared-UI + // step — which writes to disk, so a dry run must skip it. + // Including the metadata and lock the pull's auto-fill generated for it. + for (const ext of [".ts", ".script.yaml", ".script.lock"]) { + await rm(join(tempDir, `f/test/local_only_${uniqueId}${ext}`), { + force: true, + }); + } + await mkdir(join(tempDir, "ui"), { recursive: true }); + await writeFile(join(tempDir, "ui", "custom.css"), "body{}", "utf-8"); + const dryRun = await backend.runCLICommand( + ["sync", "pull", "--dry-run"], + tempDir + ); + expect(dryRun.code).toEqual(0); + // Guards against a vacuous pass: a non-empty changeset would return at + // the dry-run check above and never reach the shared-UI step. + expect(dryRun.stdout + dryRun.stderr).toContain("0 changes to apply"); + expect(await listFilesRecursive(tempDir)).toContain("ui/custom.css"); + }); + }); +}); diff --git a/docker-compose.yml b/docker-compose.yml index a801a0ce7c..bd1e060f22 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,15 +8,28 @@ x-logging: &default-logging compress: "true" services: + ## UPGRADING FROM POSTGRES 16: db_data holds a cluster 18 cannot read, so the + ## container exits with an explanatory error rather than coming up blank. Migrating + ## means dumping the WHOLE cluster (pg_dumpall), never just the windmill database: + ## Windmill keeps datatable, DuckLake and wm_fork_* databases beside it and grants + ## its RLS policies to cluster-level roles, and a single-database dump loses both + ## silently. Full procedure, and why 16 is still a valid choice until Nov 2028: + ## https://www.windmill.dev/docs/advanced/self_host#upgrade-postgresql-to-18 db: deploy: # To use an external database, set replicas to 0 and set DATABASE_URL to the external database url in the .env file replicas: 1 - image: postgres:16 + image: postgres:18 shm_size: 1g restart: unless-stopped volumes: - - db_data:/var/lib/postgresql/data + # From 18 on the official image keeps the cluster in a major-version + # subdirectory (/var/lib/postgresql/18/docker), so the mount has to be the + # parent directory: that is what lets pg_upgrade see an old and a new + # cluster inside a single mount point. Mounting the pre-18 .../data path + # instead makes the image exit rather than start, which is what turns a + # stale 16 cluster into a loud failure instead of an empty instance. + - db_data:/var/lib/postgresql expose: - 5432 environment: diff --git a/docs/feature-telemetry.md b/docs/feature-telemetry.md index 57467e25ec..5d7f0b2c29 100644 --- a/docs/feature-telemetry.md +++ b/docs/feature-telemetry.md @@ -4,10 +4,10 @@ anonymous usage-stats payload. It answers "does anyone use this, and which variant do they pick" without any identifying data leaving the instance. -It currently carries 21 registered actions across nine features (`ai_session`, `ai_chat`, -`flow_editor`, `flow_run`, `flow_step`, `trigger`, `command_script`, `hub_script`, -`usage_meter`). Nearly all of the product is uninstrumented, so new user-facing work is the -opportunity to change that. +It currently carries 28 registered actions across fourteen features (`ai_session`, `ai_chat`, +`ai_fix`, `ai_agent`, `ai_agent_eval`, `flow_editor`, `flow_run`, `flow_step`, `run_form`, +`debugger`, `trigger`, `command_script`, `hub_script`, `usage_meter`). Nearly all of the +product is uninstrumented, so new user-facing work is the opportunity to change that. ## When to instrument diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 6d72964274..6fd6413d59 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.796.0", + "version": "1.803.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.796.0", + "version": "1.803.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { @@ -29,6 +29,7 @@ "@windmill-labs/svelte-dnd-action": "^0.9.44", "@xterm/addon-fit": "^0.10.0", "@xyflow/svelte": "^1.0.0", + "acorn": "^8.15.0", "ag-charts-community": "^9.0.1", "ag-charts-enterprise": "^9.0.1", "ag-grid-community": "^31.3.4", @@ -13195,9 +13196,9 @@ } }, "node_modules/svelte-check/node_modules/picomatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", - "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", "dev": true, "license": "MIT", "optional": true, diff --git a/frontend/package.json b/frontend/package.json index 1d984c8587..f8f952ede6 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.796.0", + "version": "1.803.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", @@ -105,6 +105,7 @@ "@windmill-labs/svelte-dnd-action": "^0.9.44", "@xterm/addon-fit": "^0.10.0", "@xyflow/svelte": "^1.0.0", + "acorn": "^8.15.0", "ag-charts-community": "^9.0.1", "ag-charts-enterprise": "^9.0.1", "ag-grid-community": "^31.3.4", diff --git a/frontend/src/lib/aiStore.test.ts b/frontend/src/lib/aiStore.test.ts index fed6147b62..be8980e93a 100644 --- a/frontend/src/lib/aiStore.test.ts +++ b/frontend/src/lib/aiStore.test.ts @@ -35,6 +35,24 @@ describe('setCopilotInfo legacy /thinking migration', () => { expect(info.aiModels.map((m) => m.model)).toEqual(['claude-sonnet-4-6']) }) + it('keeps the models but turns the assistant off when the workspace disabled it', () => { + setCopilotInfo({ + providers: { + anthropic: { + resource_path: 'u/admin/anthropic', + models: ['claude-sonnet-4-6'] + } + }, + copilot_disabled: true + }) + + const info = get(copilotInfo) + expect(info.enabled).toBe(false) + expect(info.workspaceDisabled).toBe(true) + // The providers still describe what AI agent steps can run on. + expect(info.aiModels.map((m) => m.model)).toEqual(['claude-sonnet-4-6']) + }) + it('defaults provider web search on unless explicitly disabled', () => { setCopilotInfo({ providers: { diff --git a/frontend/src/lib/aiStore.ts b/frontend/src/lib/aiStore.ts index 0955f68ee5..6ac98d69e8 100644 --- a/frontend/src/lib/aiStore.ts +++ b/frontend/src/lib/aiStore.ts @@ -7,6 +7,7 @@ import { type AIProviderModel, type AIProvider, type AIConfig, + type FreeTierInfo, type ModelPriceOverride } from './gen' import { @@ -40,6 +41,10 @@ export const copilotSessionModel = writable( export const copilotInfo = writable<{ enabled: boolean + // The workspace hid the assistant (`ai_config.copilot_disabled`). `enabled` is then false + // whatever the providers say, and the AI entry points that nudge "configure AI" when + // `enabled` is off render nothing at all instead. + workspaceDisabled: boolean codeCompletionModel?: AIProviderModel defaultModel?: AIProviderModel metadataModel?: AIProviderModel @@ -49,8 +54,13 @@ export const copilotInfo = writable<{ /** Negotiated rates per `provider:model`, overriding the built-in price table. */ modelPricing?: Record webSearchEnabledProviders?: Partial> + // Set only when the workspace has no AI provider of its own and is running on + // Windmill's free tier. `exhausted` means the grant is spent: there is no model, but + // that is a different state from "never configured" and the UI must say so. + freeTier?: FreeTierInfo }>({ enabled: false, + workspaceDisabled: false, codeCompletionModel: undefined, defaultModel: undefined, metadataModel: undefined, @@ -66,7 +76,7 @@ export const copilotInfo = writable<{ aiUserDisabled.subscribe((disabled) => { copilotInfo.update((info) => ({ ...info, - enabled: info.aiModels.length > 0 && !disabled + enabled: info.aiModels.length > 0 && !disabled && !info.workspaceDisabled })) }) @@ -121,9 +131,11 @@ export function setCopilotInfo(aiConfig: AIConfig) { return model }) + const workspaceDisabled = aiConfig.copilot_disabled === true copilotInfo.set({ - // Providers are configured; the per-user opt-out is the only thing that can gate it off. - enabled: !get(aiUserDisabled), + // Providers are configured; only the workspace or per-user opt-outs can gate it off. + enabled: !workspaceDisabled && !get(aiUserDisabled), + workspaceDisabled, // Strip the deprecated /thinking suffix from the configured model slots too, // otherwise a workspace whose default still carries it sends an invalid model id. codeCompletionModel: stripModelSuffix(aiConfig.code_completion_model), @@ -132,22 +144,27 @@ export function setCopilotInfo(aiConfig: AIConfig) { aiModels: aiModels, customPrompts: aiConfig.custom_prompts ?? {}, maxTokensPerModel: aiConfig.max_tokens_per_model ?? {}, + webSearchEnabledProviders, modelPricing: aiConfig.model_pricing ?? {}, - webSearchEnabledProviders + freeTier: aiConfig.free_tier }) } else { copilotSessionModel.set(undefined) copilotInfo.set({ enabled: false, + workspaceDisabled: aiConfig.copilot_disabled === true, codeCompletionModel: undefined, defaultModel: undefined, metadataModel: undefined, aiModels: [], customPrompts: {}, maxTokensPerModel: {}, + webSearchEnabledProviders: {}, modelPricing: {}, - webSearchEnabledProviders: {} + // An exhausted free grant lands here — no providers, but the reason AI is off + // is "you used it up", not "you never set it up". + freeTier: aiConfig.free_tier }) } } diff --git a/frontend/src/lib/components/AppConnectDrawer.svelte b/frontend/src/lib/components/AppConnectDrawer.svelte index 67c19b0fe7..8de484929e 100644 --- a/frontend/src/lib/components/AppConnectDrawer.svelte +++ b/frontend/src/lib/components/AppConnectDrawer.svelte @@ -6,7 +6,7 @@ import DrawerContent from './common/drawer/DrawerContent.svelte' import AppConnectInner from './AppConnectInner.svelte' - import DarkModeObserver from './DarkModeObserver.svelte' + import GoogleSigninButton from './GoogleSigninButton.svelte' import IconedResourceType from './IconedResourceType.svelte' import { addResourceTitle } from './resourceTypeDisplay' @@ -22,6 +22,10 @@ disableChatOffset = false }: Props = $props() + /** Set by `open(rt, fillPath)`, not by the parent: which resource this run fills is a + * property of the click, and a prop would go stale between two different rows. */ + let fillPath: string | undefined = $state(undefined) + let drawer: Drawer | undefined = $state() let resourceType = $state('') let step = $state(1) @@ -32,29 +36,46 @@ let appConnectInner: AppConnectInner | undefined = $state(undefined) let rtToLoad: string | undefined = $state('') - export async function open(rt?: string) { + /** `fill` connects into a resource that already exists, instead of creating one. */ + export async function open(rt?: string, fill?: string) { + fillPath = fill + handedOff = false rtToLoad = rt drawer?.openDrawer?.() } + /** + * Hand off to the inner component exactly once per opening. The reactive statement below + * re-runs both when `rtToLoad` changes and when `appConnectInner` binds — and it binds + * afresh on every opening, since the drawer destroys its content on close. A second + * `open()` runs `next()` a second time, which walks a drawer opened on a resource type + * straight past the Connect button and into `window.open`; a popup opened from a reactive + * effect rather than from the click is blocked, leaving "Finish connection in popup + * window" with no popup behind it. + * + * A flag rather than the last resource type: `open()` with no argument leaves `rtToLoad` + * undefined, which compares equal to the initial state and would skip the hand-off + * entirely — the resources page opens it that way. + */ + let handedOff = false function onRtToLoadChange(rtToLoad: string | undefined) { + if (handedOff) return + handedOff = true appConnectInner?.open(rtToLoad) } const dispatch = createEventDispatcher() - let darkMode: boolean = $state(false) run(() => { appConnectInner && onRtToLoadChange(rtToLoad) }) - - { step = 1 + handedOff = false dispatch('close') }} size="700px" @@ -83,22 +104,26 @@ on:refresh express={expressOAuthSetup} {workspace} + {fillPath} /> {#snippet actions()}
- {#if step > 1} - + + {#if step > 1 && !rtToLoad} + {/if} {#if isGoogleSignin} - + appConnectInner?.next()} /> {:else} -
{:else} -
+ +
{#if step > 2} - + {/if} - + {#if isGoogleSignin} + appConnect?.next()} /> + {:else} + + {/if}
{/if} @@ -64,6 +72,7 @@ bind:resourceType bind:disabled bind:manual + bind:isGoogleSignin on:error on:refresh /> diff --git a/frontend/src/lib/components/CenteredModal.svelte b/frontend/src/lib/components/CenteredModal.svelte index e3f8ea1604..3250b3a16a 100644 --- a/frontend/src/lib/components/CenteredModal.svelte +++ b/frontend/src/lib/components/CenteredModal.svelte @@ -6,6 +6,9 @@ interface Props { subtitle?: string | undefined + /** Rendered under the title, for a subtitle that needs markup (a link, say). + * Sits below `subtitle` when both are given. */ + subtitleSnippet?: import('svelte').Snippet title?: string large?: boolean centerVertically?: boolean @@ -16,6 +19,7 @@ let { subtitle = undefined, + subtitleSnippet = undefined, title = 'Windmill', large = false, centerVertically = true, @@ -60,6 +64,9 @@ {subtitle}

{/if} + {#if subtitleSnippet} +
{@render subtitleSnippet()}
+ {/if}
{#if children} diff --git a/frontend/src/lib/components/Dev.svelte b/frontend/src/lib/components/Dev.svelte index 54a68dd5a3..c7e83b8a80 100644 --- a/frontend/src/lib/components/Dev.svelte +++ b/frontend/src/lib/components/Dev.svelte @@ -1,6 +1,7 @@ @@ -223,7 +209,7 @@ Files
diff --git a/frontend/src/lib/components/FilterSearchbar.svelte b/frontend/src/lib/components/FilterSearchbar.svelte index 5078e0a83b..51026f10fd 100644 --- a/frontend/src/lib/components/FilterSearchbar.svelte +++ b/frontend/src/lib/components/FilterSearchbar.svelte @@ -9,6 +9,11 @@ type: 'string' | 'number' | 'boolean' allowMultiple?: boolean format?: 'json' + /** Boolean only: the value the filter holds while unset (defaults to false). + * Selecting a filter whose default is false sets it to true immediately rather + * than opening a true/false picker whose only useful choice is true. A default-true + * boolean still opens the picker, since choosing false is the meaningful action. */ + default?: boolean } | { type: 'date' @@ -121,16 +126,34 @@ // Create the filter instance object const filterInstance: { val: Partial> } = $state({ val: {} }) - // Sync URL params to filter instance on initialization and when URL changes + // Sync URL params to filter instance, reactively. Reading urlFilter[key] tracked + // means browser Back/Forward — which mutates useSearchParams' cells on popstate — + // flows into the instance (chips, kind toggle, results), not just the first render. + // The write happens untracked so it can't self-trigger, and the equality check plus + // the reverse effect's own guard keep the two directions from ping-ponging. for (const key of Object.keys(schemaRec)) { - let urlValue = urlFilter[key] - if (schemaRec[key].type === 'date' && typeof urlValue === 'string') { - const d = new Date(urlValue) - urlValue = isNaN(d.getTime()) ? null : d - } - if (urlValue !== undefined && urlValue !== null) { - ;(filterInstance.val as any)[key] = urlValue - } + $effect(() => { + let urlValue = urlFilter[key] + if (schemaRec[key].type === 'date' && typeof urlValue === 'string') { + const d = new Date(urlValue) + urlValue = isNaN(d.getTime()) ? null : d + } + untrack(() => { + const current = (filterInstance.val as any)[key] + const same = + urlValue instanceof Date && current instanceof Date + ? urlValue.getTime() === current.getTime() + : current === (urlValue ?? undefined) + if (same) return + if (urlValue !== undefined && urlValue !== null) { + ;(filterInstance.val as any)[key] = urlValue + } else if (current !== undefined) { + // Key dropped from the URL (Back to a state without it): clear it so a + // stale chip / filter doesn't linger against the navigated-to URL. + delete (filterInstance.val as any)[key] + } + }) + }) } // Sync filter instance changes back to URL params @@ -275,6 +298,17 @@ class?: string placeholder?: string autofocus?: boolean + // Applied as the id of the underlying editable, so a parent can focus it or recognise its + // key events by id (the searchbar is a contenteditable, not an ). + inputId?: string + // Free-text mode: while the input holds only free text (no specific filter tag is + // being edited and no non-default filter is set), suppress the suggestions dropdown + // so it behaves like a plain search box. This frees the arrow keys for the + // surrounding UI (e.g. a results list). The dropdown returns the moment a specific + // filter is present (e.g. `path: u/me/abc`). + hideDropdownOnFreeText?: boolean + // Notified whenever the dropdown's effective visibility changes + onDropdownVisibleChange?: (visible: boolean) => void } type SchemaT = FilterSchemaRec // TODO: Generic @@ -284,7 +318,10 @@ presets: _presets = [], class: className, placeholder = 'Filter...', - autofocus + autofocus, + hideDropdownOnFreeText = false, + onDropdownVisibleChange, + inputId }: Props = $props() let _value = new DebouncedTempValue( @@ -298,6 +335,24 @@ let currentTag: keyof SchemaT | undefined = $state() let currentTextSegment = $state({ text: '', start: 0, end: 0 }) let open = $state(false) + + // A specific filter is in play when a tag is being edited or any non-free-text filter + // is set. + let hasSpecificFilter = $derived( + !!currentTag || Object.keys(value).some((k) => k !== '_default_') + ) + // A plain search term is being typed (free text, no specific filter). + let hasFreeText = $derived(!!String(value['_default_'] ?? '').trim()) + // Effective dropdown visibility. Free-text mode suppresses the dropdown ONLY while the + // user is typing a plain search term: it still opens when the input is empty (so the + // available filters stay discoverable) and whenever a specific filter is set or being + // edited. That leaves the arrow keys for the surrounding list only during free-text search. + let dropdownVisible = $derived( + open && (!hideDropdownOnFreeText || hasSpecificFilter || !hasFreeText) + ) + $effect(() => { + onDropdownVisibleChange?.(dropdownVisible) + }) let inputElement: HTMLDivElement | undefined = $state() let highlightedIndex = $state(0) let taggedTextInput: TaggedTextInput | undefined = $state() @@ -347,9 +402,17 @@ key, filterSchema, onClick: () => { - // Replace the text segment with the new filter tag const before = asText.val.slice(0, currentTextSegment.start) const after = asText.val.slice(currentTextSegment.end) + if (schema[key].type === 'boolean' && schema[key].default !== true) { + // Set the only useful value and reparse to canonical text. The space is + // required: dropping the segment must not fuse the tags that flanked it. + asText.val = `${before} ${after}` + value[key] = true as any + asText.reparse() + return + } + // Replace the text segment with the new (empty) filter tag; the value picker opens. asText.val = `${before}${before && !before.endsWith(' ') ? ' ' : ''}${key}:\\\u00A0${after}`.trim() + '\u00A0' @@ -406,6 +469,28 @@ onClick: () => setValueForCurrentTag(false) } ] + } else if (filter.type === 'string' && filter.format !== 'json') { + // A plain string filter has no fixed options, but any presets targeting this tag + // (`:`) are exactly its useful values — surface them as suggestions so + // picking one is a click, matching the top-level preset row. Unescape the tagged + // syntax's `\ ` back to a real space for the stored value. + const prefix = `${String(currentTag)}:` + const suffix = String(value[currentTag!] ?? '') + .trim() + .toLowerCase() + return _presets + .filter((p) => p.value.startsWith(prefix) && !asText.val.includes(p.value)) + .map((p) => { + const raw = p.value.slice(prefix.length).replace(/^\\ /, '').replace(/\\ /g, ' ') + return { name: p.name, raw } + }) + .filter((p) => !suffix || p.raw.toLowerCase().includes(suffix)) + .map((p) => ({ + type: 'option' as const, + option: { value: p.raw, label: p.name }, + onClick: () => appendOrSetValueForCurrentTag(p.raw), + onNegativeClick: undefined + })) } } return [] @@ -514,7 +599,9 @@ } function handleKeyDown(e: KeyboardEvent) { - if (!open) return + // In free-text mode the dropdown is hidden; let arrow/enter keys pass through to + // the surrounding UI (e.g. list navigation) rather than steering a hidden menu. + if (!dropdownVisible) return if (e.key === 'Escape') { open = false return @@ -601,6 +688,7 @@ > (open = true)} + onKeyDown={(e) => { + // In free-text mode the searchbar coexists with a list that owns Arrow/Enter, so opening + // the dropdown on a bare navigation key would steal them from an empty box. Typing, click, + // or an already-open dropdown still open/keep it. Other searchbars keep opening on any key. + if ( + !hideDropdownOnFreeText || + !['ArrowDown', 'ArrowUp', 'ArrowLeft', 'ArrowRight', 'Enter', 'Escape', 'Tab'].includes( + e.key + ) + ) { + open = true + } + }} {autofocus} /> {#if asText.val} @@ -630,9 +730,10 @@
inputElement?.getBoundingClientRect() ?? new DOMRect()} - innerClass="!max-h-[30rem]" + innerClass="!max-h-[25rem]" strictWidth > @@ -747,6 +848,20 @@ class="border border-border-light rounded min-h-[4rem]" /> + {:else if filter.type === 'string'} + {#if menuItems.length} +
+ {#each menuItems as item, index} + {#if item.type === 'option' && item.option} + {@render menuItem({ + onClick: item.onClick, + label: item.option.label || item.option.value, + highlighted: index === highlightedIndex + })} + {/if} + {/each} +
+ {/if} {/if} {/snippet} diff --git a/frontend/src/lib/components/FlowPreviewContent.svelte b/frontend/src/lib/components/FlowPreviewContent.svelte index dc42f9828c..d895cb4e2e 100644 --- a/frontend/src/lib/components/FlowPreviewContent.svelte +++ b/frontend/src/lib/components/FlowPreviewContent.svelte @@ -34,6 +34,8 @@ import InputSelectedBadge from './schema/InputSelectedBadge.svelte' import Toggle from './Toggle.svelte' import JsonInputs from './JsonInputs.svelte' + import { argsToJsonPayload } from '$lib/schema' + import type { Schema } from '$lib/common' import FlowHistoryJobPicker from './FlowHistoryJobPicker.svelte' import type { DurationStatus, GraphModuleState } from './graph' import { getStepHistoryLoaderContext } from './stepHistoryLoader.svelte' @@ -264,8 +266,12 @@ previewArgs.val = input inputSelected = type preventEscape = true - jsonEditor?.setCode(JSON.stringify(previewArgs.val ?? {}, null, '\t')) } + // Deselecting restores the args the same way selecting replaced them, so both branches + // owe the editor an overwrite — it holds a payload for the input being left behind. + jsonEditor?.setCode( + argsToJsonPayload(flowStore.val.schema as Schema | undefined, previewArgs.val) + ) } export function refresh() { @@ -510,8 +516,7 @@ rightTooltip: 'Fill args from JSON' }} lightMode - on:change={(e) => { - jsonEditor?.setCode(JSON.stringify(previewArgs.val ?? {}, null, '\t')) + on:change={() => { refresh() }} /> @@ -526,6 +531,10 @@ previewArgs.val = e.detail } }} + initialCode={argsToJsonPayload( + flowStore.val.schema as Schema | undefined, + previewArgs.val + )} updateOnBlur={false} placeholder={`Write args as JSON.

Example:

{
  "foo": "12"
}`} /> diff --git a/frontend/src/lib/components/FolderEditor.svelte b/frontend/src/lib/components/FolderEditor.svelte index dede17b953..e8f37a625c 100644 --- a/frontend/src/lib/components/FolderEditor.svelte +++ b/frontend/src/lib/components/FolderEditor.svelte @@ -6,19 +6,23 @@ FolderService, UserService, GranularAclService, - GroupService + GroupService, + type User } from '$lib/gen' - import TableCustom from './TableCustom.svelte' + import DataTable from './table/DataTable.svelte' + import Head from './table/Head.svelte' + import Row from './table/Row.svelte' + import Cell from './table/Cell.svelte' import { DEMO_RESTRICTION_HINT, isDemoWorkspaceRestricted } from '$lib/cloud' - import { Alert, Button, Drawer, DrawerContent } from './common' + import { Alert, Button } from './common' import Skeleton from './common/skeleton/Skeleton.svelte' - import GroupEditor from './GroupEditor.svelte' + import GroupEditorDrawer from './GroupEditorDrawer.svelte' import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' - import { ArrowDown, ArrowUp, Eye, Plus, Trash } from 'lucide-svelte' + import { ArrowDown, ArrowUp, Eye, Pen, Plus, Trash } from 'lucide-svelte' import Label from './Label.svelte' import { sendUserToast } from '$lib/toast' - import { createEventDispatcher, untrack } from 'svelte' + import { onMount, tick, untrack } from 'svelte' import Select from './select/Select.svelte' import { safeSelectItems } from './select/utils.svelte' import TextInput from './text_input/TextInput.svelte' @@ -28,97 +32,251 @@ import CollapseLink from './CollapseLink.svelte' import LabelsInput from './LabelsInput.svelte' import Badge from './common/badge/Badge.svelte' + import InputError from './InputError.svelte' + import Popover from './meltComponents/Popover.svelte' + import { deepEqual } from 'fast-equals' + import { + folderPermissionDiff, + isFolderDraftDirty, + type FolderDraft, + type FolderRole + } from '$lib/folderDraft' + + const VALID_FOLDER_NAME = /^[a-zA-Z_0-9-]+$/ + + const ROLE_TOOLTIPS = { + viewer: + 'A viewer of a folder has read-only access to all the elements (scripts/flows/apps/schedules/resources/variables) inside the folder', + writer: + 'A writer of a folder has read AND write access to all the elements (scripts/flows/apps/schedules/resources/variables) inside the folder', + admin: + 'An admin of a folder has read AND write access to all the elements inside the folders and can manage the permissions as well as add new admins' + } + + const MEMBERS_EXPLAINER = + "A member is a user or group with a role on this folder. The role applies to every script, flow, app, resource, variable and schedule inside it: viewers can read them, writers can also edit them, and admins can additionally manage the folder's members." + + // Edits mutate `draft` only; `save()` is the sole writer to the backend, and `baseline` is + // what the folder held when it was loaded, so comparing the two gives both the dirty state + // and the permission calls to replay. Both live in `folderDraft.ts`, with tests. + type Role = FolderRole interface Props { + /** In `new` mode this is the name being typed, hence bindable. */ name: string + mode?: 'edit' | 'new' + /** Drives the parent drawer's Save button, which lives above this component. */ + onCanSaveChange?: (canSave: boolean) => void + /** Drives the parent drawer's discard confirmation on close. Unlike `canSave` + * this stays true for edits that cannot be saved yet (an invalid rule, a name + * already taken) — closing would still throw them away. */ + onUnsavedChange?: (unsaved: boolean) => void + /** False while Save would create rather than update, which an `edit` drawer reaches + * when the folder turns out not to exist. The drawer labels itself from this. */ + onExistsChange?: (exists: boolean) => void + /** Edit a folder of this workspace rather than the active one. The folder picker + * can be aimed elsewhere (the project import wizard picks a destination workspace + * before entering it), and the folder must be written where it was listed. */ + workspace?: string } - let { name }: Props = $props() - let can_write = $state(false) + let { + name = $bindable(), + mode = 'edit', + onCanSaveChange, + onUnsavedChange, + onExistsChange, + workspace + }: Props = $props() - type Role = 'viewer' | 'writer' | 'admin' - let folder: Folder | undefined - let perms: { owner_name: string; role: Role }[] | undefined = $state(undefined) - let usernames: string[] = $state([]) - let groups: string[] = $state([]) - let ownerItem: string = $state('') + const targetWorkspace = $derived(workspace ?? $workspaceStore ?? '') + const aimedElsewhere = $derived(!!workspace && workspace !== $workspaceStore) - let newGroup: Drawer | undefined = $state(undefined) - let viewGroup: Drawer | undefined = $state(undefined) - - async function loadUsernames(): Promise { - usernames = await UserService.listUsernames({ workspace: $workspaceStore! }) - } - - async function loadGroups(): Promise { - groups = await GroupService.listGroupNames({ workspace: $workspaceStore! }) - } - - async function load() { - loadUsernames() - loadGroups() - await loadFolder() - } - - async function addToFolder() { - await GranularAclService.addGranularAcls({ - workspace: $workspaceStore ?? '', - path: name, - kind: 'folder', - requestBody: { - owner: (ownerKind == 'user' ? 'u/' : 'g/') + ownerItem - } - }) - ownerItem = '' - loadFolder() - } - - let folderNotFound: boolean | undefined = $state(undefined) - - async function loadFolder(): Promise { - try { - folder = await FolderService.getFolder({ workspace: $workspaceStore!, name }) - summary = folder.summary ?? '' - labels = [...(folder.labels ?? [])] - defaultPermissionedAs = (folder.default_permissioned_as ?? []).map((r) => ({ ...r })) - can_write = - $userStore != undefined && - (folder?.owners.includes('u/' + $userStore.username) || - ($userStore.is_admin ?? false) || - ($userStore.is_super_admin ?? false) || - $userStore.pgroups.findIndex((x) => folder?.owners.includes(x)) != -1) - - perms = Array.from( - new Set( - Object.entries(folder?.extra_perms ?? {}) - .map((x) => x[0]) - .concat(folder?.owners ?? []) - ) - ).map((x) => { - return { - owner_name: x, - role: getRole(x) + // `$userStore` describes the workspace the app is *in*. Aimed at another one it answers + // the wrong question — a folder admin there would get read-only controls, and a + // non-member would get write ones — so resolve the membership of the workspace being + // edited. `whoami` returns group names unprefixed; `owners` holds them `g/`-prefixed. + let targetUser: User | undefined = $state(undefined) + const membership = $derived.by(() => { + if (!aimedElsewhere) { + return $userStore + ? { + username: $userStore.username, + is_admin: $userStore.is_admin ?? false, + is_super_admin: $userStore.is_super_admin ?? false, + pgroups: $userStore.pgroups ?? [], + groups: $userStore.groups ?? [] + } + : undefined + } + return targetUser + ? { + username: targetUser.username, + is_admin: targetUser.is_admin ?? false, + is_super_admin: targetUser.is_super_admin ?? false, + pgroups: (targetUser.groups ?? []).map((g) => 'g/' + g), + groups: targetUser.groups ?? [] } - }) - reloadHistory++ - } catch (e) { - folderNotFound = true + : undefined + }) + + async function loadTargetUser(): Promise { + if (!aimedElsewhere || !workspace) return + try { + targetUser = await UserService.whoami({ workspace }) + } catch { + // Not a member, or the call failed: no membership means read-only controls, + // which is the safe reading — the write would be refused anyway. + targetUser = undefined } } - // --- default_permissioned_as rules editor --- - let defaultPermissionedAs: FolderDefaultPermissionedAs = $state([]) + let can_write = $state(false) + let folder: Folder | undefined + let usernames: string[] = $state([]) + let groups: string[] = $state([]) + let folderNames: string[] = $state([]) + let ownerItem: string = $state('') + + let groupEditorDrawer: GroupEditorDrawer | undefined = $state(undefined) + let addMemberPopover: Popover | undefined = $state(undefined) + let nameInput: TextInput | undefined = $state(undefined) + + let baseline: FolderDraft | undefined = $state(undefined) + // Empty, not `emptyDraft()`: that one seeds the caller as an admin, which is true of a + // folder being created and a lie about one whose read failed. Every path that wants the + // seeded row calls `emptyDraft()` itself. + let draft: FolderDraft = $state({ + summary: '', + labels: [], + defaultPermissionedAs: [], + perms: [] + }) + let labelsInput: LabelsInput | undefined = $state() + let pendingLabel = $state('') + let folderNotFound: boolean | undefined = $state(undefined) + let loaded = $state(false) + + // A name typed in `new` mode, and one whose folder turned out not to exist, both + // end up at `createFolder` on save. + const isNew = $derived(mode === 'new' || folderNotFound === true) + + function emptyDraft(): FolderDraft { + return { + summary: '', + labels: [], + defaultPermissionedAs: [], + // The backend makes the creator an owner whatever we send, so the table + // shows that from the start rather than after the first reload. + perms: membership ? [{ owner_name: 'u/' + membership.username, role: 'admin' as Role }] : [] + } + } + + function setDraft(value: FolderDraft) { + baseline = structuredClone(value) + draft = structuredClone(value) + } + + async function loadUsernames(): Promise { + usernames = await UserService.listUsernames({ workspace: targetWorkspace }) + } + + async function loadGroups(): Promise { + groups = await GroupService.listGroupNames({ workspace: targetWorkspace }) + } + + async function loadFolderNames(): Promise { + folderNames = await FolderService.listFolderNames({ workspace: targetWorkspace }) + } + + /** Fills a picker or a validation list. The editor is usable before these land, so they + * run alongside the folder read — but a rejection has to be reported: unhandled, it + * leaves the list silently empty and duplicate names stop being caught. */ + function loadAside(load: () => Promise): void { + load().catch((e) => sendUserToast(e?.body ?? String(e), true)) + } + + async function load() { + loadAside(loadUsernames) + loadAside(loadGroups) + // Before the folder read: `can_write` is computed from this membership. + await loadTargetUser() + if (mode === 'new') { + loadAside(loadFolderNames) + can_write = true + setDraft(emptyDraft()) + loaded = true + } else { + await loadFolder() + } + } + + function grant(close: () => void) { + const owner = (ownerKind == 'user' ? 'u/' : 'g/') + ownerItem + if (!draft.perms.some((p) => p.owner_name === owner)) { + draft.perms.push({ owner_name: owner, role: newMemberRole }) + } + ownerItem = '' + close() + } + + /** `baselineOnly` re-reads the folder without touching the draft: after a save that + * committed some of its calls and then failed, the baseline must become what the server + * actually holds while the draft stays the user's intent — the applied changes then stop + * counting as dirty, and the ones still missing stay dirty and retryable. */ + async function loadFolder(opts?: { baselineOnly?: boolean }): Promise { + const apply = (value: FolderDraft) => + opts?.baselineOnly ? (baseline = structuredClone(value)) : setDraft(value) + try { + folder = await FolderService.getFolder({ workspace: targetWorkspace, name }) + folderNotFound = false + can_write = + membership != undefined && + (folder?.owners.includes('u/' + membership.username) || + membership.is_admin || + membership.is_super_admin || + membership.pgroups.findIndex((x) => folder?.owners.includes(x)) != -1) + + apply({ + summary: folder.summary ?? '', + labels: [...(folder.labels ?? [])], + defaultPermissionedAs: (folder.default_permissioned_as ?? []).map((r) => ({ ...r })), + perms: Array.from( + new Set( + Object.entries(folder?.extra_perms ?? {}) + .map((x) => x[0]) + .concat(folder?.owners ?? []) + ) + ).map((x) => ({ owner_name: x, role: getRole(x) })) + }) + reloadHistory++ + } catch (e) { + // Only a folder that is genuinely absent may replace the draft — it can be created + // from here, so the editor opens on an empty one rather than a dead end. Any other + // failure (network, 5xx) must leave the draft alone: overwriting it here would + // discard the user's edits and clear `unsaved` with them. + if (e?.status === 404) { + folderNotFound = true + can_write = true + apply(emptyDraft()) + } else { + sendUserToast(e?.body ?? String(e), true) + } + } finally { + loaded = true + } + } const restricted = $derived( - isDemoWorkspaceRestricted($workspaceStore, $userStore?.is_admin, $userStore?.is_super_admin) + isDemoWorkspaceRestricted(targetWorkspace, membership?.is_admin, membership?.is_super_admin) ) const canEditDefaults = $derived( can_write && !restricted && - ($userStore?.is_admin || - $userStore?.is_super_admin || - ($userStore?.groups ?? []).includes('wm_deployers')) + (membership?.is_admin || + membership?.is_super_admin || + (membership?.groups ?? []).includes('wm_deployers')) ) function isValidGlob(glob: string): boolean { @@ -135,58 +293,47 @@ return /^[ug]\/.+/.test(value) || value.includes('@') } - // Split a permissioned_as value like "u/alice" or "g/prod" into its kind and name. - function ruleKind(value: string): 'user' | 'group' { + // Split an owner value like "u/alice" or "g/prod" into its kind and name. + function ownerKindOf(value: string): 'user' | 'group' { return value.startsWith('g/') ? 'group' : 'user' } - function ruleName(value: string): string { + function ownerNameOf(value: string): string { if (value.startsWith('u/') || value.startsWith('g/')) return value.slice(2) return value } function setRulePermissionedAs(idx: number, kind: 'user' | 'group', name: string) { const prefix = kind === 'user' ? 'u/' : 'g/' - defaultPermissionedAs[idx].permissioned_as = prefix + name + draft.defaultPermissionedAs[idx].permissioned_as = prefix + name } + // Only blocks a save for someone who can see the rules. The backend accepts values this + // rejects (`u/` alone passes `validate_default_permissioned_as`), so a folder admin who + // is not a workspace admin could otherwise meet a permanently disabled Save with no rule + // on screen to explain it. const defaultRulesInvalid = $derived( - defaultPermissionedAs.some( - (r) => !isValidGlob(r.path_glob) || !isValidPermissionedAs(r.permissioned_as) - ) + canEditDefaults && + draft.defaultPermissionedAs.some( + (r) => !isValidGlob(r.path_glob) || !isValidPermissionedAs(r.permissioned_as) + ) ) function addDefaultRule() { - defaultPermissionedAs = [...defaultPermissionedAs, { path_glob: '**', permissioned_as: '' }] + draft.defaultPermissionedAs = [ + ...draft.defaultPermissionedAs, + { path_glob: '**', permissioned_as: '' } + ] } function removeDefaultRule(idx: number) { - defaultPermissionedAs = defaultPermissionedAs.filter((_, i) => i !== idx) + draft.defaultPermissionedAs = draft.defaultPermissionedAs.filter((_, i) => i !== idx) } function moveDefaultRule(idx: number, delta: -1 | 1) { - const next = [...defaultPermissionedAs] + const next = [...draft.defaultPermissionedAs] const target = idx + delta if (target < 0 || target >= next.length) return ;[next[idx], next[target]] = [next[target], next[idx]] - defaultPermissionedAs = next - } - - async function saveDefaultRules() { - if (defaultRulesInvalid) { - sendUserToast('Some rules have invalid globs or permissioned_as values', true) - return - } - try { - await FolderService.updateFolder({ - workspace: $workspaceStore ?? '', - name, - requestBody: { default_permissioned_as: defaultPermissionedAs } - }) - sendUserToast('Default permissioned_as rules updated') - dispatch('update') - loadFolder() - } catch (e) { - sendUserToast(e.body ?? String(e), true) - } + draft.defaultPermissionedAs = next } function getRole(x: string): Role { @@ -204,51 +351,208 @@ } let ownerKind: 'user' | 'group' = $state('user') - let groupCreated: string | undefined = $state(undefined) - let newGroupName: string = $state('') - let summary: string = $state('') - let labels: string[] | undefined = $state(undefined) + let newMemberRole: Role = $state('viewer') - async function saveLabels() { + // Set when the group editor is opened from the add-member form, so that saving returns + // there. Opened from a member row instead, that group is already a member and reopening + // the form on it would offer to add it twice. + let groupEditorInterruptedPicker = false + + function openGroupEditor(groupName: string, fromPicker: boolean) { + groupEditorInterruptedPicker = fromPicker + if (groupName) groupEditorDrawer?.initEdit(groupName) + else groupEditorDrawer?.initNew() + } + + async function onGroupSaved(groupName: string) { + // The group has to be in `groups` before the picker reopens, or the value set below + // has no matching item to show. try { - await FolderService.updateFolder({ - workspace: $workspaceStore ?? '', - name, - requestBody: { labels: labels ?? [] } - }) - sendUserToast('Folder labels updated') - dispatch('update') + await loadGroups() } catch (e) { - sendUserToast(e.body ?? String(e), true) - loadFolder() + sendUserToast(e?.body ?? String(e), true) + } + if (!groupEditorInterruptedPicker) return + // Editing a group was a detour from adding a member: come back to the form on that + // group so the interrupted job can be finished. + ownerKind = 'group' + ownerItem = groupName + addMemberPopover?.open() + } + + // Guarded on `mode`, not `isNew`: the name field is rendered only in `new` mode, so on the + // not-found branch there is no input to annotate and no name the user could correct. + const nameError = $derived( + mode !== 'new' + ? '' + : !name + ? '' + : !VALID_FOLDER_NAME.test(name) + ? 'Folder name can only contain alphanumeric characters, underscores, and hyphens' + : folderNames.includes(name) + ? 'A folder with this name already exists' + : '' + ) + + // `create_folder` folds the caller into `owners` with write whatever the payload says, so + // on create their own row is fixed: offering to demote or remove it would be a change the + // backend silently discards. + // An invalid rule disables Save, so the section holding it is held open rather than merely + // opened once: collapsing it would hide the only explanation for the disabled button. + let defaultRulesOpen = $state(false) + + function isFixedCreatorRow(owner: string): boolean { + return isNew && owner === 'u/' + membership?.username + } + + // The label input holds typed text until Enter or a blur, and that text is an edit like + // any other: it has to count as dirty here, or Save stays disabled when it is the only + // change and closing drops it without asking. `save()` flushes it into `draft.labels`. + const dirty = $derived(isFolderDraftDirty(draft, baseline) || pendingLabel !== '') + // A typed name is progress too, even before any other field is touched. + const unsaved = $derived(dirty || (mode === 'new' && !!name)) + + $effect(() => { + onCanSaveChange?.( + isNew + ? loaded && !!name && !nameError && !restricted && !defaultRulesInvalid + : can_write && dirty && !defaultRulesInvalid + ) + }) + + $effect(() => { + onUnsavedChange?.(unsaved) + }) + + $effect(() => { + onExistsChange?.(!isNew) + }) + + /** Replays the permission rows the user changed. `updateFolder` could write + * `owners`/`extra_perms` wholesale in the same call as the settings, but it only + * logs a single "update owners"/"update acl" entry, so the permission history + * would stop naming who was granted what. The diff itself is in `folderDraft.ts`. */ + async function applyPermissionChanges(next: FolderDraft['perms'], prev: FolderDraft['perms']) { + const workspace = targetWorkspace + const callerOwners = membership + ? ['u/' + membership.username, ...(membership.pgroups ?? [])] + : [] + for (const call of folderPermissionDiff(prev, next, callerOwners)) { + switch (call.kind) { + case 'grantAdmin': + await FolderService.addOwnerToFolder({ + workspace, + name, + requestBody: { owner: call.owner } + }) + break + case 'demoteAdmin': + await FolderService.removeOwnerToFolder({ + workspace, + name, + requestBody: { owner: call.owner, write: call.write } + }) + break + case 'setAcl': + await GranularAclService.addGranularAcls({ + workspace, + path: name, + kind: 'folder', + requestBody: { owner: call.owner, write: call.write } + }) + break + case 'remove': + // Sequential, and `removeowner` first: the write policy refuses it when the + // member being removed is the caller's last admin handle. Failing there leaves + // the folder untouched, where the other order strands a member with no grant + // but still in `owners`. + await FolderService.removeOwnerToFolder({ + workspace, + name, + requestBody: { owner: call.owner } + }) + await GranularAclService.removeGranularAcls({ + workspace, + path: name, + kind: 'folder', + requestBody: { owner: call.owner } + }) + break + } } } - async function addGroup() { - await GroupService.createGroup({ - workspace: $workspaceStore ?? '', - requestBody: { name: newGroupName } - }) - groupCreated = newGroupName - $userStore?.folders?.push(newGroupName) - loadGroups() - ownerItem = newGroupName + export async function save(): Promise<{ name: string; created: boolean } | undefined> { + // Clicking Save blurs the label input, which commits its text on a delay — after the + // snapshot below. Take the label first or it is dropped as the drawer closes. + labelsInput?.flushPendingLabel() + const next = $state.snapshot(draft) as FolderDraft + const prev = baseline as FolderDraft + // Captured before the write: an edit-branch save reloads, which clears `folderNotFound`. + const created = isNew + try { + if (created) { + await FolderService.createFolder({ + workspace: targetWorkspace, + requestBody: { + name, + summary: next.summary, + labels: next.labels, + default_permissioned_as: next.defaultPermissionedAs, + owners: next.perms.filter((p) => p.role === 'admin').map((p) => p.owner_name), + extra_perms: Object.fromEntries( + next.perms.map((p) => [p.owner_name, p.role !== 'viewer']) + ) + } + }) + sendUserToast(`Folder ${name} created`) + } else { + const requestBody: { + summary?: string + labels?: string[] + default_permissioned_as?: FolderDefaultPermissionedAs + } = {} + if (next.summary !== prev.summary) requestBody.summary = next.summary + if (!deepEqual(next.labels, prev.labels)) requestBody.labels = next.labels + if (!deepEqual(next.defaultPermissionedAs, prev.defaultPermissionedAs)) { + requestBody.default_permissioned_as = next.defaultPermissionedAs + } + if (Object.keys(requestBody).length > 0) { + await FolderService.updateFolder({ workspace: targetWorkspace, name, requestBody }) + } + await applyPermissionChanges(next.perms, prev.perms) + await loadFolder() + sendUserToast('Folder updated') + } + return { name, created } + } catch (e) { + sendUserToast(e.body ?? String(e), true) + // A failed create is not proof the folder is absent: `create_folder` commits before a + // git-sync step that can still fail the request. Only the name conflict says it was + // never written. Report rather than resolve — a folder found by name may be someone + // else's, and adopting it would send this draft's writes there. + const nameTaken = String(e?.body ?? '').includes('already exists') + if (created && !nameTaken) { + sendUserToast(`Folder ${name} may have been created anyway — reopen it to check`, true) + } + // Reconcile after any edit-path failure rather than tracking which calls landed: + // these handlers commit before a git-sync step that can still fail the request, so + // a rejection is not proof nothing was written. The baseline moves to what the + // server now holds and the draft stays, so a retry re-sends only what is missing. + if (!created) await loadFolder({ baselineOnly: true }) + return undefined + } } - const dispatch = createEventDispatcher() - - async function updateFolder() { - await FolderService.updateFolder({ - workspace: $workspaceStore ?? '', - name, - requestBody: { summary } - }) - sendUserToast('Folder summary updated') - dispatch('update') - loadFolder() - } + // The stores are read only to wait until they are populated, and the load runs once: this + // editor holds an unsaved draft, and the layout re-`set`s `$userStore` periodically — a + // second `load()` would overwrite the draft with the server's state and lose the edits + // silently, `unsaved` included. The drawer remounts this component per opening. + let loadStarted = false $effect.pre(() => { + if (loadStarted) return if ($workspaceStore && $userStore) { + loadStarted = true untrack(() => { load() }) @@ -256,47 +560,41 @@ }) let reloadHistory = $state(0) + + onMount(async () => { + if (mode !== 'new') return + // The editor is remounted per drawer opening, so mount is the moment the + // create form appears; the input only exists after the first render. + await tick() + nameInput?.focus() + }) - - { - newGroup?.closeDrawer() - groupCreated = undefined - }} - > - {#if !groupCreated} -
- - -
- {:else} - - {/if} - - - - - - - - +
- + {/if} + +
{#if can_write} - + (pendingLabel = v)} + /> {:else}
- {#each labels ?? [] as label (label)} + {#each draft.labels as label (label)} {label} {:else} No labels @@ -319,257 +622,261 @@
-
diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index 62bbe1ef93..ce5323fe2b 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -1069,8 +1069,10 @@
  • git sync repo count (sync vs promotion mode)
  • feature usage (counts of which product features are used, including AI provider and - model identifiers, the names of public hub scripts used, and the plan tier and quota - shown when the execution meter is opened, last 30 days)
  • feature adoption (counts of which flow, script, trigger and worker features your @@ -1121,8 +1123,10 @@
  • development instance status
  • feature usage (counts of which product features are used, including AI provider and - model identifiers, the names of public hub scripts used, and the plan tier and quota - shown when the execution meter is opened, last 30 days)
  • feature adoption (counts of which flow, script, trigger and worker features your @@ -1141,6 +1145,31 @@ description="Configure default timeouts and retention policies for job execution." link="https://www.windmill.dev/docs/advanced/instance_settings#jobs" /> + {:else if category == 'Service logs'} + + {#if !$values['object_store_cache_config']} +
    + + Instance object storage is not configured, so every server and worker keeps its log + files on its own disk. This page lists what each host wrote, but can only open the files + belonging to the replica serving the request — another host's are listed and not + readable — and a host's files go with it when it is replaced. Retention below still + governs the entries in the database and the files on disk. + +
    + {:else if !$enterpriseLicense} +
    + + Log files are uploaded to instance object storage, and the indexer that would ingest + them into the columnar store and delete each one afterwards is an enterprise feature. + Retention below expires the database entries and the local files; the uploaded copies + are only removed when Delete logs from s3 periodically is on under Object Storage. + +
    + {/if} {:else if category == 'Object Storage'} import SimpleEditor from '$lib/components/SimpleEditor.svelte' - import { createEventDispatcher } from 'svelte' + import { createEventDispatcher, untrack } from 'svelte' const dispatch = createEventDispatcher() @@ -8,18 +8,48 @@ updateOnBlur?: boolean placeholder?: string selected?: boolean + /** Content the editor opens with, and keeps following while the buffer is untouched — so a + * payload nobody has typed into tracks the schema instead of going stale. The first edit + * hands the buffer to the user and later changes stop overwriting it. */ + initialCode?: string } let { updateOnBlur = true, placeholder = 'Write a JSON payload. The input schema will be inferred.

    Example:

    {
      "foo": "12"
    }', - selected = false + selected = false, + initialCode = '' }: Props = $props() - let pendingJson = $state('') + let pendingJson = $state(untrack(() => initialCode)) + // The last content this component wrote, kept only to skip a reseed that would replace the + // buffer with what it already holds — `setValue` resets the cursor and the undo stack. + let seededCode = untrack(() => initialCode) + // Latched from Monaco's own change event, never from `pendingJson`: that trails the buffer by + // SimpleEditor's debounce, a window in which typed text still looks like the seeded payload + // and a reseed lands on top of it. + let userEdited = false let simpleEditor: SimpleEditor | undefined = $state(undefined) let focusTrap: HTMLElement | undefined = $state() + $effect(() => { + const next = initialCode + untrack(() => { + if (next !== seededCode && !userEdited) { + seed(next) + } + }) + }) + + // `SimpleEditor.setCode` cancels the change burst its own `setValue` opens, so reseeding + // never dispatches `select` — the payload reaches `args` only when the user edits it. + function seed(code: string) { + seededCode = code + userEdited = false + pendingJson = code + simpleEditor?.setCode(code) + } + function updatePayloadFromJson(jsonInput: string) { if (jsonInput === undefined || jsonInput === null || jsonInput.trim() === '') { dispatch('select', undefined) @@ -33,8 +63,10 @@ } } + /** Authoritative overwrite: replaces the buffer whether or not it has been typed into, and + * re-establishes it as the content to keep following. */ export function setCode(code: string) { - simpleEditor?.setCode(code) + seed(code) } export function resetSelected(dispatchEvent?: boolean) { @@ -59,6 +91,7 @@
    (userEdited = true)} on:focus={() => { if (updateOnBlur) { dispatch('focus') diff --git a/frontend/src/lib/components/LabelsInput.svelte b/frontend/src/lib/components/LabelsInput.svelte index 933fe38ea1..c7eda30d66 100644 --- a/frontend/src/lib/components/LabelsInput.svelte +++ b/frontend/src/lib/components/LabelsInput.svelte @@ -8,9 +8,22 @@ labels: string[] | undefined onchange?: () => void class?: string + /** Suggest the labels of this workspace rather than the active one, for an editor + * aimed elsewhere (the folder drawer opened from a cross-workspace picker). */ + workspace?: string + /** Text typed into the input but not yet added to `labels`. An editor with a Save + * button needs it: without it that text is invisible to the editor's dirty state, + * so it is silently dropped on close and cannot even enable Save on its own. */ + onPendingChange?: (pending: string) => void } - let { labels = $bindable(), onchange, class: clazz = '' }: Props = $props() + let { + labels = $bindable(), + onchange, + class: clazz = '', + workspace, + onPendingChange + }: Props = $props() let adding = $state(false) let inputValue = $state('') @@ -34,9 +47,13 @@ !(labels ?? []).includes(trimmedInput) ) + $effect(() => { + onPendingChange?.(adding ? trimmedInput : '') + }) + async function loadExistingLabels() { try { - const resp = await fetch(`/api/w/${$workspaceStore}/labels/list`) + const resp = await fetch(`/api/w/${workspace ?? $workspaceStore}/labels/list`) if (resp.ok) existingLabels = await resp.json() } catch {} } @@ -82,8 +99,15 @@ addLabel() // either "Create new" selected or free text } } else if (e.key === 'Escape') { + // Escape cancels the label, and nothing else. Left to bubble it also reaches + // whatever encloses us — a drawer or dialog closes on it, and one guarding on + // unsaved changes reads `pending` before this clears it, so it prompts to + // discard work this key just discarded. + e.preventDefault() + e.stopPropagation() inputValue = '' adding = false + onPendingChange?.('') } else if (e.key === 'ArrowDown') { e.preventDefault() const maxIdx = suggestions.length + (showCreateNew ? 1 : 0) - 1 @@ -100,6 +124,14 @@ if (adding) addLabel() }, 150) } + + /** Add whatever is typed but not yet committed, right now. Blur commits on a 150ms + * grace period, so a caller that reads `labels` in the same tick as the blur — a Save + * button, which blurs this input by being clicked — would miss the last label. + * `adding` is cleared here, so the pending timer then finds nothing to do. */ + export function flushPendingLabel(): void { + if (adding) addLabel() + }
    diff --git a/frontend/src/lib/components/Login.svelte b/frontend/src/lib/components/Login.svelte index d1dc3ec7a1..10f996a33f 100644 --- a/frontend/src/lib/components/Login.svelte +++ b/frontend/src/lib/components/Login.svelte @@ -329,7 +329,11 @@ } else { goto(resolvedRd ?? '/') } - } else if (resolvedRd?.startsWith('/user/workspaces')) { + // See (root)/+layout.svelte for why /projects/import skips the picker. + } else if ( + resolvedRd?.startsWith('/user/workspaces') || + resolvedRd?.startsWith(`${base}/projects/import`) + ) { goto(resolvedRd) } else if (resolvedRd == '/#user-settings') { goto(`/user/workspaces#user-settings`) diff --git a/frontend/src/lib/components/ModuleTest.svelte b/frontend/src/lib/components/ModuleTest.svelte index 2adba428ee..50e915a4cb 100644 --- a/frontend/src/lib/components/ModuleTest.svelte +++ b/frontend/src/lib/components/ModuleTest.svelte @@ -50,14 +50,19 @@ let jobProgressReset: () => void = () => {} let stepHistoryLoader = getStepHistoryLoaderContext() + // Every explicit run re-evaluates the args with errors surfaced. The reactive evaluations + // that follow each flow edit stay quiet, so without this a failing expression is silently + // `undefined` in what the run is built from. Manually edited args are preserved across the + // refresh by `initializeFromSchema`. export function runTestWithStepArgs() { - const args = stepsInputArgs.getStepArgs(mod.id) - runTest(args) - } - - export function loadArgsAndRunTest() { - stepsInputArgs?.updateStepArgs(mod.id, flowStateStore.val, flowStore?.val, previewArgs?.val) - runTestWithStepArgs() + stepsInputArgs?.updateStepArgs( + mod.id, + flowStateStore.val, + flowStore?.val, + previewArgs?.val, + true + ) + runTest(stepsInputArgs.getStepArgs(mod.id)) } // A step's timeout is an InputTransform. Only a static numeric value can be applied diff --git a/frontend/src/lib/components/ObjectStoreConfigSettings.svelte b/frontend/src/lib/components/ObjectStoreConfigSettings.svelte index c7f7065565..14658a4228 100644 --- a/frontend/src/lib/components/ObjectStoreConfigSettings.svelte +++ b/frontend/src/lib/components/ObjectStoreConfigSettings.svelte @@ -305,6 +305,7 @@ resourceType="s3_bucket" workspaceOverride="admins" buttonTextOverride="Test from a worker" + viaWorker />
    diff --git a/frontend/src/lib/components/ProjectContentBadges.svelte b/frontend/src/lib/components/ProjectContentBadges.svelte new file mode 100644 index 0000000000..dfb8e0eb90 --- /dev/null +++ b/frontend/src/lib/components/ProjectContentBadges.svelte @@ -0,0 +1,73 @@ + + + + +
    + {#each shown as c (c.label)} + + {c.count} + {c.label}{c.count === 1 ? '' : 's'} + + {/each} +
    diff --git a/frontend/src/lib/components/ResourceEditorDrawer.svelte b/frontend/src/lib/components/ResourceEditorDrawer.svelte index eec4c7579b..296ab693a8 100644 --- a/frontend/src/lib/components/ResourceEditorDrawer.svelte +++ b/frontend/src/lib/components/ResourceEditorDrawer.svelte @@ -22,8 +22,16 @@ let { workspace = undefined, disableChatOffset = false, - onRestored = undefined - }: { workspace?: string; disableChatOffset?: boolean; onRestored?: () => void } = $props() + onRestored = undefined, + onSaved = undefined + }: { + workspace?: string + disableChatOffset?: boolean + onRestored?: () => void + /** Fires after Save has written, for a caller showing state derived from the + * resource — `onRestored` only covers restoring an old version. */ + onSaved?: () => void + } = $props() let drawer: Drawer | undefined = $state() let historyDrawer: Drawer | undefined = $state() @@ -149,9 +157,14 @@ variant="accent" unifiedSize="md" startIcon={{ icon: Save }} - on:click={() => { - resourceEditor?.save() + on:click={async () => { + // Closed before the write is awaited, the way it always was: `save()` toasts its + // own failures and never rejects, so waiting would only add visible lag to every + // caller of this drawer. `onSaved` still fires after the write lands. + const saved = resourceEditor?.save() drawer?.closeDrawer() + await saved + onSaved?.() }} disabled={!canSave} > diff --git a/frontend/src/lib/components/RunForm.svelte b/frontend/src/lib/components/RunForm.svelte index 89eaf8f7ab..a371f3eea9 100644 --- a/frontend/src/lib/components/RunForm.svelte +++ b/frontend/src/lib/components/RunForm.svelte @@ -19,6 +19,7 @@ import { page } from '$app/state' import { replaceState } from '$app/navigation' import JsonInputs from '$lib/components/JsonInputs.svelte' + import { argsToJsonPayload } from '$lib/schema' import { triggerableByAI } from '$lib/actions/triggerableByAI.svelte' import InputSelectedBadge from './schema/InputSelectedBadge.svelte' import { untrack } from 'svelte' @@ -53,6 +54,8 @@ args = scriptArgs psCommonParams = commonParams reloadArgs++ + // `reloadArgs` only keys the form; the JSON editor reads its payload once, at mount. + syncJsonEditor() } export async function run(overrideScheduledForStr?: string | undefined | null) { @@ -199,8 +202,10 @@ return result } - export function setCode(code: string) { - jsonEditor?.setCode(code) + /** Rewrite the open JSON editor from the current args. Only for args replaced from outside + * the editor: entering the JSON view already starts from whatever `args` holds. */ + export function syncJsonEditor() { + jsonEditor?.setCode(argsToJsonPayload(runnable?.schema, args)) } $effect(() => { overrideTag @@ -320,6 +325,7 @@ args = enforceDisabledDefaults(e.detail) } }} + initialCode={argsToJsonPayload(runnable.schema, args)} updateOnBlur={false} placeholder={`Write args as JSON.

    Example:

    {
      "foo": "12"
    }`} /> diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index 7bc4774e9f..fdab141d03 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -122,6 +122,7 @@ import { updateDelegateToGitRepoConfig, insertAdditionalInventories } from '$lib/ansibleUtils' import { copilotInfo } from '$lib/aiStore' import JsonInputs from '$lib/components/JsonInputs.svelte' + import { argsToJsonPayload } from '$lib/schema' import Toggle from './Toggle.svelte' import { deepEqual } from 'fast-equals' import { usePreparedAssetSqlQueries } from '$lib/infer.svelte' @@ -310,6 +311,10 @@ let moduleTestState: Record; schema: Schema }> = $state({}) let testPanelArgs: Record = $state({}) let testPanelSchema: Schema = $state(emptySchema()) + // Bumped whenever the args under test are replaced from outside the arg panel. Both arg + // views key off it: without a bump the JSON editor keeps showing, and on the next + // keystroke commits, the payload it was seeded with for the previous args. + let argsRender = $state(0) // editorCode is what the editor shows; code always holds the main script content let editorCode: string = $state(code) // Sync editorCode when code changes externally (template reset, copilot, @@ -329,7 +334,13 @@ }) function switchToModule(modulePath: string) { - if (activeModuleTab !== null && modules && activeModuleTab !== modulePath) { + // Re-clicking the tab you are already on is a no-op. Re-running the body would reset this + // module's test state whenever its inference is still pending or has failed (the catch + // leaves `moduleTestState` unwritten), losing both the filled-in args and the arg views. + if (activeModuleTab === modulePath) { + return + } + if (activeModuleTab !== null && modules) { // Switching from another module: save its content and test state modules[activeModuleTab] = { ...modules[activeModuleTab], content: editorCode } moduleTestState[activeModuleTab] = { args: testPanelArgs, schema: testPanelSchema } @@ -345,13 +356,20 @@ } else { testPanelArgs = {} testPanelSchema = emptySchema() + // Inference lands after the bump below, so the editor opens on `{}` and the arg + // views follow the schema in once it arrives. Remounting them again on arrival + // instead would discard anything typed while it was in flight. inferModuleSchema() } + argsRender++ } } function switchToMain() { - if (activeModuleTab !== null && modules) { + if (activeModuleTab === null) { + return + } + if (modules) { // Save current module content and test state modules[activeModuleTab] = { ...modules[activeModuleTab], content: editorCode } moduleTestState[activeModuleTab] = { args: testPanelArgs, schema: testPanelSchema } @@ -360,6 +378,7 @@ editorCode = code lastSyncedCode = code editor?.setCode(editorCode) + argsRender++ } // Whether the open file is tested as a runnable of its own. A `__mod` helper @@ -854,6 +873,7 @@ export function setArgs(nargs: Record) { args = nargs + argsRender++ } export async function runTest(opts?: { cascade?: boolean; skipDdlGuard?: boolean }) { @@ -1654,15 +1674,30 @@ $effect(() => { !hasPreprocessor && (selectedTab = 'main') }) + // `main` and `preprocessor` describe the same args under different schemas; every other tab + // (`diagram`) runs against main's schema, so it collapses into `main` here. + let lastSchemaTab = untrack(() => (selectedTab === 'preprocessor' ? 'preprocessor' : 'main')) $effect(() => { // Only depend on selectedTab (preprocessor ↔ main toggle). // Code changes are handled by the editor on:change handler and // explicit inferSchema calls (initContent, onMount), so we read // `code` inside untrack to avoid a redundant double-inference race. - selectedTab && untrack(() => code && inferSchema(code)) + selectedTab && + untrack(() => { + const schemaTab = selectedTab === 'preprocessor' ? 'preprocessor' : 'main' + const switched = schemaTab !== lastSchemaTab + lastSchemaTab = schemaTab + if (!code) return + // Bump on the switch itself, not on the inference it starts: the other tab's schema + // only lands once that resolves, and remounting the arg views then would discard + // anything typed while it was in flight. An untouched editor follows the schema in. + if (switched) { + argsRender++ + } + inferSchema(code) + }) }) - let argsRender = $state(0) export async function updateArgs(newArgs: Record) { if (Object.keys(newArgs).length > 0) { args = { ...newArgs } @@ -2299,19 +2334,24 @@ style="height: {!schemaHeight || schemaHeight < 600 ? 600 : schemaHeight}px" data-schema-picker > - { - if (e.detail) { - if (onModuleArgs) { - testPanelArgs = e.detail - } else { - args = e.detail + {#key argsRender} + { + if (e.detail) { + if (onModuleArgs) { + testPanelArgs = e.detail + } else { + args = e.detail + } } - } - }} - updateOnBlur={false} - placeholder={`Write args as JSON.

    Example:

    {
      "foo": "12"
    }`} - /> + }} + initialCode={onModuleArgs + ? argsToJsonPayload(testPanelSchema, testPanelArgs) + : argsToJsonPayload(schema, args)} + updateOnBlur={false} + placeholder={`Write args as JSON.

    Example:

    {
      "foo": "12"
    }`} + /> + {/key}
    {:else}
    diff --git a/frontend/src/lib/components/ServiceLogsInner.svelte b/frontend/src/lib/components/ServiceLogsInner.svelte index bad18661b6..16bdd7d2ad 100644 --- a/frontend/src/lib/components/ServiceLogsInner.svelte +++ b/frontend/src/lib/components/ServiceLogsInner.svelte @@ -2,7 +2,7 @@ import { createBubbler, preventDefault } from 'svelte/legacy' const bubble = createBubbler() - import { IndexSearchService, ServiceLogsService } from '$lib/gen' + import { IndexSearchService, ServiceLogsService, type LogSearchHit } from '$lib/gen' import TimeframeSelect, { serviceLogsTimeframes, @@ -253,39 +253,50 @@ try { let res = '' log.split('\n').forEach((line) => { + // A file can hold both formats: the ones written before the layer + // switched to JSON, and panics or subprocess output that was never + // JSON to begin with. Those lines pass through as they are rather + // than being dropped, which would render the file blank. + let obj: any = undefined if (line.startsWith('{') && line.endsWith('}')) { - let obj = JSON.parse(line) - if (typeof obj == 'object') { - let nl = '' - if (obj['timestamp']) { - nl += obj['timestamp'] + ' ' - } - if (obj['level']) { - let lvl = obj['level'] - if (lvl == 'ERROR') { - nl += '\x1b[31mERROR\x1b[0m ' - } else if (lvl == 'INFO') { - nl += '\x1b[32mINFO\x1b[0m ' - } else { - nl += obj['level'] + ' ' - } - } - if (obj['message']) { - nl += obj['message'] + ' ' - } - delete obj['timestamp'] - delete obj['level'] - delete obj['message'] - Object.keys(obj).forEach((key) => { - nl += - key + - '=' + - (typeof obj[key] == 'object' ? JSON.stringify(obj[key]) : obj[key]) + - ' ' - }) - res += nl + '\n' + try { + obj = JSON.parse(line) + } catch { + obj = undefined } } + if (obj === null || typeof obj !== 'object') { + res += line + '\n' + } else { + let nl = '' + if (obj['timestamp']) { + nl += obj['timestamp'] + ' ' + } + if (obj['level']) { + let lvl = obj['level'] + if (lvl == 'ERROR') { + nl += '\x1b[31mERROR\x1b[0m ' + } else if (lvl == 'INFO') { + nl += '\x1b[32mINFO\x1b[0m ' + } else { + nl += obj['level'] + ' ' + } + } + if (obj['message']) { + nl += obj['message'] + ' ' + } + delete obj['timestamp'] + delete obj['level'] + delete obj['message'] + Object.keys(obj).forEach((key) => { + nl += + key + + '=' + + (typeof obj[key] == 'object' ? JSON.stringify(obj[key]) : obj[key]) + + ' ' + }) + res += nl + '\n' + } }) return res @@ -294,6 +305,23 @@ } } + // A hit is one log line with its fields already separated, so rendering it is + // formatting rather than parsing — there is no JSON to prettify and no + // snippet to highlight. + function renderHit(hit: LogSearchHit): string { + const level = + hit.level === 'ERROR' + ? '\x1b[31mERROR\x1b[0m' + : hit.level === 'WARN' + ? '\x1b[33mWARN\x1b[0m' + : hit.level === 'INFO' + ? '\x1b[32mINFO\x1b[0m' + : hit.level + return [hit.ts, level, hit.message, hit.target ? `target=${hit.target}` : ''] + .filter(Boolean) + .join(' ') + } + let logs: any = $state() let debounceTimeout: number | undefined = undefined @@ -399,7 +427,9 @@ ) { const res = await ServiceLogsService.getLogFile({ path: `${hostname}/${path}` }) - content = processLogWithJsonFmt(ansi_up.ansi_to_html(res), jsonFmt) + // Prettify first: it emits its own ANSI for the level, which converting + // beforehand would leave in the output as literal escapes. + content = ansi_up.ansi_to_html(processLogWithJsonFmt(res, jsonFmt)) hitLineNumber = lineNumber logDrawerOpen = true @@ -687,23 +717,19 @@
    {:else if logs != undefined}
    - {#each logs.hits as { snippet_fragment, snippet_highlighted, document }} + + {#each logs.hits ?? [] as hit, i (`${i}:${hit.file_path}:${hit.line_no}`)} { - let logLineNumber = document.line_number[0] - let logFile = document.file_name[0] - let host = document.host[0] - let jsonFmt = document.json_fmt[0] - seeLogContext(logLineNumber, logFile, host, jsonFmt) - }} + content={renderHit(hit)} + highlighted={[]} + onClick={() => seeLogContext(hit.line_no, hit.file_path, hit.host, true)} /> {/each} - {#if logs.hits.length === 0} + {#if (logs.hits ?? []).length === 0}
    No logs
    {/if} - {#if logs.hits.length === 1000} + {#if (logs.hits ?? []).length === 1000}
    Older matches were truncated from this search, try refining your filters to get more precise results. diff --git a/frontend/src/lib/components/ShareModal.svelte b/frontend/src/lib/components/ShareModal.svelte index 29da96cf98..471ed74099 100644 --- a/frontend/src/lib/components/ShareModal.svelte +++ b/frontend/src/lib/components/ShareModal.svelte @@ -252,7 +252,7 @@ {/if}
    Extra permissions ({acls?.length ?? 0})Extra members ({acls?.length ?? 0}) {#if linkedVarPaths.length > 0}
    @@ -299,7 +299,7 @@ size="lg" variant="accent" disabled={!newOwner} - on:click={() => addAcl(newOwner, write)}>Add permission addAcl(newOwner, write)}>Add member
    {/if} @@ -307,7 +307,7 @@ {#snippet headerRow()} - owner + member diff --git a/frontend/src/lib/components/SimpleEditor.svelte b/frontend/src/lib/components/SimpleEditor.svelte index 2764b130e2..65713e19f7 100644 --- a/frontend/src/lib/components/SimpleEditor.svelte +++ b/frontend/src/lib/components/SimpleEditor.svelte @@ -64,6 +64,9 @@ const CHANGE_TIMEOUT = 200 let changeTimeoutId: number | undefined = undefined + // Monaco fires onDidChangeModelContent synchronously from within `setValue`, so without + // this an authoritative overwrite reads as a user edit on the `input` event. + let applyingCode = false let divEl: HTMLDivElement | null = null let editor = $state(null) @@ -74,6 +77,10 @@ let width = $state(0) let initialized = $state(false) let placeholderVisible = $state(false) + // Monaco's content origin. The placeholder is a plain overlay on the editor container, so + // without these it sits over the line-number gutter and off the line-1 baseline. + let contentLeft = $state(0) + let contentLineHeight = $state(0) let mounted = $state(false) let valueAfterDispose: string | undefined = undefined @@ -179,7 +186,12 @@ if (ncode != code) { code = ncode } - editor?.setValue(ncode) + applyingCode = true + try { + editor?.setValue(ncode) + } finally { + applyingCode = false + } // setValue emits a change event of its own; drop the burst it opens so an edit // made right after an authoritative overwrite still counts as a leading change. cancelPendingChanges() @@ -454,6 +466,12 @@ changeTimeoutId = undefined updateCode() }, CHANGE_TIMEOUT) + // `change` trails the buffer by CHANGE_TIMEOUT, too late for a consumer that has to + // know the moment the buffer stopped being the one it wrote. `input` says only that, + // carrying no value: read `getCode()` for what is on screen. + if (!applyingCode) { + dispatch('input') + } if (leading) { updateCode() } @@ -533,6 +551,13 @@ } if (placeholder) { + const syncPlaceholderOrigin = () => { + if (!editor) return + contentLeft = editor.getLayoutInfo().contentLeft + contentLineHeight = editor.getOption(meditor.EditorOption.lineHeight) + } + syncPlaceholderOrigin() + editor.onDidLayoutChange(syncPlaceholderOrigin) editor.onDidChangeModelContent(() => { if (!editor) return const value = editor.getValue() @@ -755,9 +780,10 @@ {#if placeholder}
    {@html placeholder}
    diff --git a/frontend/src/lib/components/TaggedTextInput.svelte b/frontend/src/lib/components/TaggedTextInput.svelte index 23bbc65e6b..9f19195b85 100644 --- a/frontend/src/lib/components/TaggedTextInput.svelte +++ b/frontend/src/lib/components/TaggedTextInput.svelte @@ -8,6 +8,7 @@ onTextSegmentAtCursorChange, onKeyDown, autofocus, + id, class: className = '' }: { tags: { regex: RegExp; id: string; onClear?: () => void }[] @@ -18,6 +19,7 @@ onTextSegmentAtCursorChange?: (segment: { text: string; start: number; end: number }) => void onKeyDown?: (e: KeyboardEvent) => void autofocus?: boolean + id?: string class?: string } = $props() @@ -337,7 +339,13 @@ function handleKeyDown(e: KeyboardEvent) { onKeyDown?.(e) - if (e.key === 'ArrowDown' || e.key === 'ArrowUp' || e.key === 'Enter') return + // Single-line filter input: block Enter's default newline insertion. Surrounding handlers + // (suggestion select, list open) still run on bubble; only the contenteditable break is gone. + if (e.key === 'Enter') { + e.preventDefault() + return + } + if (e.key === 'ArrowDown' || e.key === 'ArrowUp') return const cursorPos = getCursorPosition() const text = getTextContent() @@ -509,6 +517,7 @@
    - import { type CompletedJob, JobService, type Preview } from '$lib/gen' + import { + type CompletedJob, + JobService, + type Preview, + ResourceService, + SettingService, + UserService, + VariableService + } from '$lib/gen' import { Database, Loader2 } from 'lucide-svelte' import Button from './common/button/Button.svelte' @@ -13,15 +21,57 @@ resourceType: string | undefined args?: Record | any buttonTextOverride?: string | undefined + // Object-storage types only: probe from a preview job (proves a worker reaches the API) + // instead of the browser. The job gets a short-lived token minted for the caller, since a + // job token is never a super admin, and that token is readable in the job's stored args + // until revoked: only use it where the workspace's job readers may hold the caller's rights. + viaWorker?: boolean } let { workspaceOverride = undefined, resourceType, args = {}, - buttonTextOverride = undefined + buttonTextOverride = undefined, + viaWorker = false }: Props = $props() + // Object-storage resource types share one probe, the API's own connectivity test, which runs + // on the API server with the caller's privileges. Each type maps its resource to the + // ObjectSettings body that route expects. + const objectStorageBody: { [key: string]: (args: any) => Record } = { + s3: (s3) => ({ + type: 'S3', + region: s3.region, + bucket: s3.bucket, + endpoint: s3.endPoint, + port: s3.port, + allow_http: !s3.useSSL, + access_key: s3.accessKey, + secret_key: s3.secretKey, + path_style: s3.pathStyle + }), + azure_blob: (s3) => ({ type: 'Azure', ...s3 }), + s3_bucket: (bucket) => bucket + } + + const OBJECT_STORAGE_TEST_SCRIPT = ` +export async function main(bucket: any, api_token: string) { + const res = await fetch(process.env.BASE_URL + '/api/settings/test_object_storage_config', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer ' + api_token, + }, + body: JSON.stringify(bucket), + }) + if (!res.ok) { + throw new Error(await res.text()) + } + return await res.text() +} +` + const scripts: { [key: string]: { code: string @@ -68,71 +118,18 @@ argName: 'database' }, s3: { - code: ` -import * as wmill from "windmill-client" - -type S3 = object - -export async function main(s3: S3) { - return fetch(process.env["BASE_URL"] + '/api/settings/test_object_storage_config', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer ' + process.env["WM_TOKEN"], - }, - body: JSON.stringify({ - type: "S3", - region: s3.region, - bucket: s3.bucket, - endpoint: s3.endPoint, - port: s3.port, - allow_http: !s3.useSSL, - access_key: s3.accessKey, - secret_key: s3.secretKey, - path_style: s3.pathStyle, - }), - }).then(async (res) => { - if (!res.ok) { - throw new Error(await res.text()) - } - return res.text() - }) -} -`, + code: OBJECT_STORAGE_TEST_SCRIPT, lang: 'bun', - argName: 's3', + argName: 'bucket', tooltip: - 'The storage operations of this test run on the Windmill server (the API process), not on the worker. If no access key/secret key is set, the ambient AWS credentials of the server (environment variables, instance role) are used — scripts using this resource directly through an S3 SDK resolve credentials on the worker instead, so results may differ.' + 'The storage operations of this test run on the Windmill server (the API process) with your permissions, not on the worker. Non-super-admins can only test public endpoints with an explicit access key and secret key; super admins can also test private endpoints and rely on the ambient AWS credentials of the server (environment variables, instance role). Scripts using this resource directly through an S3 SDK resolve credentials on the worker instead, so results may differ.' }, azure_blob: { - code: ` -import * as wmill from "windmill-client" - -type S3 = object - -export async function main(s3: S3) { - return fetch(process.env["BASE_URL"] + '/api/settings/test_object_storage_config', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer ' + process.env["WM_TOKEN"], - }, - body: JSON.stringify({ - type: "Azure", - ...s3 - }), - }).then(async (res) => { - if (!res.ok) { - throw new Error(await res.text()) - } - return res.text() - }) -} -`, + code: OBJECT_STORAGE_TEST_SCRIPT, lang: 'bun', - argName: 's3', + argName: 'bucket', tooltip: - 'The storage operations of this test run on the Windmill server (the API process), not on the worker.' + 'The storage operations of this test run on the Windmill server (the API process) with your permissions, not on the worker. Non-super-admins can only test public endpoints with an explicit access key.' }, graphql: { code: '{ __typename }', @@ -158,61 +155,163 @@ export async function main(s3: S3) { } }, s3_bucket: { - code: ` - -const process = require('process'); - -export async function main(bucket: any) { - const req = await fetch(process.env.BASE_URL + '/api/settings/test_object_storage_config', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer ' + process.env.WM_TOKEN, - }, - body: JSON.stringify(bucket), - }); - if (!req.ok) { - throw new Error(await req.text()); - } - return await req.text(); -} -`, + code: OBJECT_STORAGE_TEST_SCRIPT, lang: 'bun', argName: 'bucket', tooltip: - "The storage operations of this test run on the Windmill server (the API process). If no credentials are configured, the server's ambient credentials for the configured provider (environment variables, instance role) are used." + "The storage operations of this test run on the Windmill server (the API process) with your permissions. Non-super-admins can only test public endpoints with explicit credentials; super admins can also test private endpoints and rely on the server's ambient credentials for the configured provider (environment variables, instance role)." } } let loading = $state(false) + + // The token is revoked as soon as the job settles; the expiry only covers a browser that + // goes away mid-test. It is readable in the job's stored args until then, hence the short life. + const API_TOKEN_TTL_MS = 60_000 + // Tokens are addressed by their first 10 characters (TOKEN_PREFIX_LEN on the backend). + const API_TOKEN_PREFIX_LEN = 10 + + async function mintApiToken(): Promise { + return await UserService.createToken({ + requestBody: { + label: `test connection: ${resourceType}`, + expiration: new Date(Date.now() + API_TOKEN_TTL_MS).toISOString(), + scopes: ['settings:write'] + } + }) + } + + async function revokeApiToken(token: string | undefined) { + if (!token) return + try { + await UserService.deleteToken({ tokenPrefix: token.slice(0, API_TOKEN_PREFIX_LEN) }) + } catch (err) { + console.error(err) + } + } + + // A preview job gets its arguments interpolated on the worker: `$var:`, `$jsonvar:` and + // `$res:` references are replaced with the job's privileges before the script runs. The + // browser path has to do the same with the caller's session, or a secret stored as a linked + // variable (what the "Add resource" drawer saves) is sent verbatim as the credential. + async function resolveReferences(value: any, workspace: string): Promise { + if (typeof value === 'string') { + if (value.startsWith('$var:')) { + return await VariableService.getVariableValue({ + workspace, + path: value.slice('$var:'.length) + }) + } + if (value.startsWith('$jsonvar:')) { + return JSON.parse( + await VariableService.getVariableValue({ + workspace, + path: value.slice('$jsonvar:'.length) + }) + ) + } + if (value.startsWith('$res:')) { + return await ResourceService.getResourceValueInterpolated({ + workspace, + path: value.slice('$res:'.length) + }) + } + return value + } + if (Array.isArray(value)) { + return await Promise.all(value.map((v) => resolveReferences(v, workspace))) + } + if (value && typeof value === 'object') { + const resolved: Record = {} + for (const [key, v] of Object.entries(value)) { + resolved[key] = await resolveReferences(v, workspace) + } + return resolved + } + return value + } + + // The route bounds the probe only for non-super-admins; a super admin's probe against an + // endpoint that accepts the connection and never answers would otherwise spin here forever. + const BROWSER_TEST_TIMEOUT_MS = 15_000 + + async function testObjectStorageFromBrowser(body: Record, workspace: string) { + let timer: ReturnType | undefined = undefined + try { + const request = SettingService.testObjectStorageConfig({ + requestBody: await resolveReferences(body, workspace) + }) + await Promise.race([ + request, + new Promise((_, reject) => { + timer = setTimeout(() => { + request.cancel() + reject( + new Error( + `no answer from the storage endpoint after ${BROWSER_TEST_TIMEOUT_MS / 1000}s` + ) + ) + }, BROWSER_TEST_TIMEOUT_MS) + }) + ]) + sendUserToast('Connection successful', false) + } catch (err: any) { + sendUserToast('Connection error: ' + (err?.body ?? err?.message ?? err), true) + } finally { + clearTimeout(timer) + loading = false + } + } + async function testConnection() { if (!resourceType) return loading = true const resourceScript = scripts[resourceType] + const workspace = workspaceOverride ?? $workspaceStore! + const objectStorageArgs: Record | undefined = + resourceType in objectStorageBody ? objectStorageBody[resourceType](args) : undefined - const job = await JobService.runScriptPreview({ - workspace: workspaceOverride ?? $workspaceStore!, - requestBody: { - path: `testConnection: ${resourceType}`, - language: resourceScript.lang as Preview['language'], - content: resourceScript.code, - args: { - [resourceScript.argName]: args - } + if (objectStorageArgs && !viaWorker) { + await testObjectStorageFromBrowser(objectStorageArgs, workspace) + return + } + + let apiToken: string | undefined = undefined + let job: string + try { + if (objectStorageArgs) { + apiToken = await mintApiToken() } - }) + job = await JobService.runScriptPreview({ + workspace, + requestBody: { + path: `testConnection: ${resourceType}`, + language: resourceScript.lang as Preview['language'], + content: resourceScript.code, + args: objectStorageArgs + ? { bucket: objectStorageArgs, api_token: apiToken } + : { [resourceScript.argName]: args } + } + }) + } catch (err: any) { + loading = false + await revokeApiToken(apiToken) + sendUserToast('Connection error: ' + (err?.body ?? err?.message ?? err), true) + return + } tryEvery({ tryCode: async () => { let testResult = await JobService.getCompletedJob({ - workspace: workspaceOverride ?? $workspaceStore!, + workspace, id: job }) if (resourceScript.additionalCheck) { testResult = resourceScript.additionalCheck(testResult) } loading = false + revokeApiToken(apiToken) sendUserToast( testResult.success ? 'Connection successful' @@ -222,13 +321,14 @@ export async function main(bucket: any) { }, timeoutCode: async () => { loading = false + revokeApiToken(apiToken) sendUserToast( 'Connection did not resolve after 5s or job did not start. Do you have native workers or a worker group listening to the proper tag available?', true ) try { await JobService.cancelQueuedJob({ - workspace: workspaceOverride ?? $workspaceStore!, + workspace, id: job, requestBody: { reason: diff --git a/frontend/src/lib/components/aiEvals/AgentEvalModal.svelte b/frontend/src/lib/components/aiEvals/AgentEvalModal.svelte index ff014f403a..61d88819e7 100644 --- a/frontend/src/lib/components/aiEvals/AgentEvalModal.svelte +++ b/frontend/src/lib/components/aiEvals/AgentEvalModal.svelte @@ -27,18 +27,14 @@ let trail = $derived( location ? [{ label: TITLE, onclick: location.back }, { label: location.label }] : undefined ) - let description = $derived( - location - ? undefined - : 'Each run answers a dataset of cases with this agent and scores the answers, so runs can be compared.' - ) - import { createGrid, type GridApi } from 'ag-grid-community' - import 'ag-grid-community/styles/ag-grid.css' - import 'ag-grid-community/styles/ag-theme-alpine.css' - import '$lib/components/apps/components/display/table/theme/windmill-theme.css' - import { transformColumnDefs } from '$lib/components/apps/components/display/table/utils' - import { multilineCellColDef } from '$lib/components/apps/components/display/table/multilineCellEditor' - import DarkModeObserver from '$lib/components/DarkModeObserver.svelte' + import DataTable from '$lib/components/table/DataTable.svelte' + import Head from '$lib/components/table/Head.svelte' + import Cell from '$lib/components/table/Cell.svelte' + import Row from '$lib/components/table/Row.svelte' + import EditableTextarea from '$lib/components/common/EditableTextarea.svelte' + import { Button, EmptyState } from '$lib/components/common' + import { ListPlus, Plus, Trash2 } from 'lucide-svelte' import { untrack } from 'svelte' import type { CaseDraft } from './evalUtils' let { cases = $bindable(), onRemove, - locked = false, - onEditingChange + onAdd, + focusCaseId = undefined, + locked = false }: { /** The drawer's working copy. Edits land here as they are made; the drawer writes them. */ cases: CaseDraft[] /** Asked before a row goes, since a stored case has runs that executed it. */ onRemove: (c: CaseDraft) => void + /** Adds a case. The empty table offers this itself, where the first row would be. */ + onAdd: () => void + /** A case to open for typing as soon as it appears — the one just added, so a new row is + * ready to be filled in rather than waiting to be clicked. */ + focusCaseId?: string /** The cases are being written: an edit made now is one the request already left behind. */ locked?: boolean - /** A cell opened or closed. Reported up so Save can be pressed for an edit the list does not - * hold yet: the press is what commits the cell, so it has to reach a live button. */ - onEditingChange?: (editing: boolean) => void } = $props() - type Row = { id: string; question: string; expected: string } + /** A dataset is capped at 1000 cases and every row here mounts two editors and a button, so the + * whole set at once is thousands of components for a table you read a screenful of. ag-grid + * virtualised its rows; `DataTable` paginates instead, which is the primitive already here. */ + const CASES_PER_PAGE = 25 + let currentPage = $state(1) + /** Bound to `DataTable`'s own per-page selector, so changing it there re-slices here rather + * than only relabelling the footer. */ + let perPage = $state(CASES_PER_PAGE) + let lastPage = $derived(Math.max(1, Math.ceil(cases.length / perPage))) + let pageCases = $derived(cases.slice((currentPage - 1) * perPage, currentPage * perPage)) + // A case removed from the last page leaves it empty; step back rather than show nothing. + $effect(() => { + if (currentPage > lastPage) currentPage = lastPage + }) + /** `expected` is whatever the case holds: a bare string, or a value shown as JSON. Read back + * the same way, so a case that held an object keeps holding one. */ function expectedToText(value: unknown): string { if (value == undefined) return '' return typeof value === 'string' ? value : JSON.stringify(value, null, 2) } - function toRow(c: CaseDraft): Row { - return { - id: c.id ?? '', - question: c.input?.user_message ?? '', - expected: expectedToText(c.expected) - } - } - - let api: GridApi | undefined = $state() - let eGui: HTMLDivElement | undefined = $state() - let darkMode = $state(false) - - const defaultColDef = { - flex: 1, - minWidth: 120, - ...multilineCellColDef - } - - $effect(() => eGui && untrack(() => mountGrid())) - function mountGrid() { - if (!eGui || api) return - createGrid(eGui, { - rowData: untrack(() => cases.map(toRow)), - columnDefs: columnDefs(), - defaultColDef: { ...defaultColDef, editable: untrack(() => !locked) }, - onCellValueChanged: (e) => { - const target = cases.find((c) => c.id === (e.data as Row).id) - if (!target) return - if (e.colDef.field === 'question') { - target.input = { ...target.input, user_message: e.newValue ?? '' } - } else if (e.colDef.field === 'expected') { - setExpected(target, e.newValue ?? '') - } - }, - onCellEditingStarted: () => onEditingChange?.(true), - onCellEditingStopped: () => onEditingChange?.(false), - suppressColumnMoveAnimation: true, - suppressDragLeaveHidesColumns: true, - onGridReady: (e) => (api = e.api) + /** The question editors, by case id, so a newly added row can be opened. */ + let questionEditors = $state>({}) + $effect(() => { + const id = focusCaseId + if (!id) return + untrack(() => { + // Onto the page the case is on before reaching for its editor: paginated, a case added to + // a full page is not rendered yet, and there would be nothing to open. + const at = cases.findIndex((c) => c.id === id) + if (at < 0) return + currentPage = Math.floor(at / perPage) + 1 + // After the row has been rendered and registered itself. + requestAnimationFrame(() => questionEditors[id]?.edit()) }) - } + }) - /** Text that parses as JSON is stored as JSON, so a structured answer can be written by hand. */ function setExpected(c: CaseDraft, text: string) { + // Cleared means the case has no expected answer, which is not the same as expecting the + // empty string: a scorer reads `undefined` as "nothing to measure here" and leaves the case + // out of its mean, where `''` scores it a hard zero. const trimmed = text.trim() if (!trimmed) { c.expected = undefined @@ -88,45 +82,93 @@ c.expected = text } } - - function columnDefs() { - return transformColumnDefs({ - columnDefs: [ - { field: 'question', headerName: 'Question', flex: 3 }, - { field: 'expected', headerName: 'Expected', flex: 2 } - ] as any, - onDelete: (values) => { - if (locked) return - const target = cases.find((c) => c.id === (values as Row).id) - if (target) onRemove(target) - } - }) - } - - // Keyed on which cases are in the list rather than on what is in them: pushing rows back on - // each keystroke would reset the cell being typed in. - let rowKey = $derived(cases.map((c) => c.id).join(',')) - $effect(() => { - rowKey - untrack(() => api?.updateGridOptions({ rowData: cases.map(toRow) })) - }) - - $effect(() => { - const editable = !locked - untrack(() => api?.updateGridOptions({ defaultColDef: { ...defaultColDef, editable } })) - }) - - /** Commit whatever cell is open into `cases`, and wait for it to land there: the drawer reads - * the list to save it, and a cell still being typed in is an edit that press is saving. */ - export async function flush() { - api?.stopEditing() - // The commit reaches `cases` through the grid's own event queue, a macrotask away: reading in - // the same turn, or after a microtask, reads the list as it was before the cell was typed in. - await new Promise((resolve) => setTimeout(resolve, 0)) - } - - -
    +{#if cases.length === 0} + +{:else} + perPage} + bind:currentPage + bind:perPage + rowCount={cases.length} + hasMore={currentPage < lastPage} + showPrev={currentPage > 1} + on:next={() => (currentPage = Math.min(currentPage + 1, lastPage))} + on:previous={() => (currentPage = Math.max(currentPage - 1, 1))} + > + + + + + + + + Question + Expected + + + + + {#each pageCases as c (c.id)} + + + + (c.input = { ...(c.input ?? {}), user_message: v })} + /> + + + setExpected(c, v)} + /> + + +
    -
    + +
    Cases {workingCases.length} @@ -369,13 +372,13 @@ Add a case
    -
    +
    (casesEditing = v)} />
    @@ -397,7 +400,7 @@ unifiedSize="md" variant="accent" loading={saving} - disabled={writing || !path || !!pathError || (nothingToSave && !casesEditing)} + disabled={writing || !path || !!pathError || nothingToSave} onclick={saveDataset} > Save @@ -408,7 +411,8 @@ variant="accent" startIcon={{ icon: Plus }} loading={creating} - disabled={creating || !path || !!pathError} + disabled={creating || !path || !!pathError || noCases} + title={noCasesTitle} onclick={createDataset} > Create dataset diff --git a/frontend/src/lib/components/aiEvals/EvalRunDialog.svelte b/frontend/src/lib/components/aiEvals/EvalRunDialog.svelte index d4d1714fd0..63fe366628 100644 --- a/frontend/src/lib/components/aiEvals/EvalRunDialog.svelte +++ b/frontend/src/lib/components/aiEvals/EvalRunDialog.svelte @@ -47,16 +47,6 @@ let hasDraft = $derived(editedConfig !== undefined) let dataset = $state(undefined) let hoveringDataset = $state(false) - /** Set while the dialog stands aside for the dataset drawer, holding the pane's dataset as it - * was on the way out: coming back onto a different one is the detour's answer, and adopted. */ - let steppedAside = $state<{ dataset: string | undefined } | undefined>(undefined) - - /** Hands the screen to the dataset drawer, to be given back when it closes. */ - function stepAside(go: () => void) { - steppedAside = { dataset: defaultDataset } - open = false - go() - } /** The agent's versions, for pinning one. Loaded when the dialog opens rather than held: a * version list goes stale the moment the agent is saved again. */ @@ -76,21 +66,26 @@ } } + /** Whether the dialog was already open on the previous run of the effect below, which reads it + * to tell an open from the pane's dataset moving underneath. */ + let wasOpen = false + $effect(() => { - if (!open) return + const isOpen = open + const pane = defaultDataset untrack(() => { - loadVersions() - const aside = steppedAside - steppedAside = undefined - if (aside) { - // Back from the drawer: a dataset created or edited there moves the pane onto it, and - // anything else leaves the field as it was left. - if (defaultDataset !== aside.dataset) dataset = defaultDataset + if (!isOpen) { + wasOpen = false return } - // Seeded on every open: the dataset last worked in, and the state of the agent there is - // most reason to measure. - dataset = defaultDataset + // Followed while the dialog stands, not only read at open: the dataset drawer opens over + // this dialog rather than in place of it, so creating, renaming or deleting a dataset + // there moves the pane's selection with the dialog still up. Nothing else moves it then. + dataset = pane + if (wasOpen) return + wasOpen = true + loadVersions() + // The state of the agent there is most reason to measure. choice = hasDraft ? 'draft' : 'deployed' }) }) @@ -184,7 +179,7 @@ unifiedSize="sm" variant="default" startIcon={{ icon: Plus }} - onclick={() => stepAside(onNewDataset)} + onclick={onNewDataset} > New dataset @@ -215,7 +210,7 @@ title="Edit this dataset" on:click={() => { close() - stepAside(() => onEditDataset(item.value ?? '')) + onEditDataset(item.value ?? '') }} /> {/snippet} @@ -228,7 +223,7 @@ btnClasses="w-full !h-auto !justify-start !rounded-none flex items-center gap-2 px-3 py-2 text-xs !font-normal text-secondary hover:bg-surface-hover" onClick={() => { close() - stepAside(onNewDataset) + onNewDataset() }} > @@ -247,7 +242,7 @@ startIcon={{ icon: Pencil }} iconOnly title="Edit this dataset" - on:click={() => stepAside(() => onEditDataset(dataset ?? ''))} + on:click={() => onEditDataset(dataset ?? '')} />
    {/if} diff --git a/frontend/src/lib/components/aiEvals/EvalRunsList.svelte b/frontend/src/lib/components/aiEvals/EvalRunsList.svelte index 5d84ef88ce..b544781902 100644 --- a/frontend/src/lib/components/aiEvals/EvalRunsList.svelte +++ b/frontend/src/lib/components/aiEvals/EvalRunsList.svelte @@ -7,17 +7,21 @@ import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte' import TimeAgo from '$lib/components/TimeAgo.svelte' import { Button } from '$lib/components/common' - import { Bot, Code2, Loader2, Plus } from 'lucide-svelte' + import { Bot, ChevronRight, Code2, Loader2, Plus } from 'lucide-svelte' + import { overlayHostActive, topmostSurface } from '$lib/components/common/overlayHost.svelte' import type { EvalDataset, EvalExperiment, ExperimentScore } from '$lib/gen' import { datasetSummary, experimentName, formatScore, subjectLabel } from './evalUtils' let { experiments, datasets, + caseProgress, loaded, + active = false, deployedHash = undefined, currentVersion = undefined, onOpen, + onHighlight, onEditDataset, onNew }: { @@ -25,17 +29,87 @@ experiments: EvalExperiment[] /** Whether the list has been read: an empty table is a statement about the agent. */ loaded: boolean + /** Whether this list is the page on screen. The keyboard is only answered while it is: the + * run page keeps its own rows, and both would otherwise move on one press. */ + active?: boolean /** The workspace's datasets, for naming the one a run is of by what it is for. */ datasets: EvalDataset[] + /** How many cases each still-running run has finished, keyed by run id. A run the flow has + * not been read for yet is at none of them rather than absent: the count is on the row from + * the moment it appears, so it never arrives late and shifts the column. */ + caseProgress: Record /** What the agent hashes to as deployed, and the version it is on: they resolve a run of * edits that were later saved, so a run is labelled here as the run picker labels it. */ deployedHash?: string currentVersion?: number onOpen: (experiment: EvalExperiment) => void + /** The highlighted run, reported up so the surface can act on it — arrowing into the run + * page opens the run under the highlight rather than whichever was opened last. */ + onHighlight?: (id: string | undefined) => void onEditDataset: (dataset: string) => void onNew: () => void } = $props() + /** The highlighted run, by id. One state for both the pointer and the keyboard, as a melt menu + * does it: hovering a row moves the highlight to it, so the arrows carry on from wherever the + * pointer left off instead of running a second, invisible cursor of their own. It says where + * the highlight is, not what is chosen — a run is not opened until Enter. + * + * By id and not by index: the list is newest-first and the poll prepends to it, so an index + * would quietly come to mean a different run and Enter would open the wrong one. */ + let cursorId = $state(undefined) + let cursor = $derived( + cursorId === undefined ? -1 : experiments.findIndex((e) => e.id === cursorId) + ) + let body: HTMLTableSectionElement | undefined = $state() + + // A window listener answers keys aimed anywhere, so it has to ask two questions the DOM cannot: + // is my host the visible one — session preview tabs stay mounted when hidden — and is my surface + // still the one on top, rather than under a drawer or a dialog opened since. + const hostActive = overlayHostActive() + const onTop = topmostSurface() + const listening = () => hostActive() && onTop() + + $effect(() => { + onHighlight?.(cursorId) + }) + + // A highlight on a run that has since gone, and the highlight itself when the list is not the + // page on screen. + $effect(() => { + if (!active || (cursorId !== undefined && cursor < 0)) cursorId = undefined + }) + + function move(by: number) { + if (experiments.length === 0) return + const from = cursor < 0 ? (by > 0 ? -1 : experiments.length) : cursor + const at = Math.max(0, Math.min(experiments.length - 1, from + by)) + cursorId = experiments[at]?.id + // `nearest`, so arrowing through a long list scrolls by a row rather than jumping the table. + requestAnimationFrame(() => + body?.querySelectorAll('tr')[at]?.scrollIntoView({ block: 'nearest' }) + ) + } + + function onKeydown(event: KeyboardEvent) { + if (!active || !listening() || event.metaKey || event.ctrlKey || event.altKey) return + const el = event.target as HTMLElement | null + if (el?.closest?.('input, textarea, select, [contenteditable="true"], [role="listbox"]')) return + if (event.key === 'ArrowDown') { + event.preventDefault() + move(1) + } else if (event.key === 'ArrowUp') { + event.preventDefault() + move(-1) + } else if (event.key === 'Enter' && experiments[cursor]) { + // Enter belongs to whatever is focused if that thing does something with it. A highlighted + // row is not a reason to swallow the press on `New evaluation` or a row's dataset button. + if (el?.closest?.('button, a[href], [role="button"], summary')) return + event.preventDefault() + onOpen(experiments[cursor]) + } + } + /** The one number a column reports: a pass rate where it has a line to pass, the mean where it * does not. */ function headline(score: ExperimentScore): string | undefined { @@ -44,13 +118,16 @@ } + + - + + @@ -58,12 +135,20 @@ Dataset Cases Scores - When + When + - - {#each experiments as experiment (experiment.id)} - onOpen(experiment)}> + + {#each experiments as experiment, i (experiment.id)} + + onOpen(experiment)} + on:hover={(e) => e.detail && (cursorId = experiment.id)} + >
    @@ -99,7 +184,18 @@ - {experiment.case_count} + {#if experiment.running} + + + + {caseProgress[experiment.id] ?? 0}/{experiment.case_count} + + {:else} + {experiment.case_count} + {/if}
    @@ -117,8 +213,6 @@ {value} {:else if score.failed > 0} failed - {:else if experiment.running} - {:else} — {/if} @@ -126,33 +220,34 @@ {/each} {#if (experiment.scores ?? []).length === 0} - {#if experiment.running} - - - scoring - - {:else} - not scored - {/if} + + not scored {/if}
    - + + + + {/each} {#if experiments.length === 0 && !loaded} - + {:else if experiments.length === 0} - +
    No runs yet diff --git a/frontend/src/lib/components/aiEvals/EvalScorers.svelte b/frontend/src/lib/components/aiEvals/EvalScorers.svelte index 68287b0fbc..d00e52d77e 100644 --- a/frontend/src/lib/components/aiEvals/EvalScorers.svelte +++ b/frontend/src/lib/components/aiEvals/EvalScorers.svelte @@ -196,26 +196,31 @@
    -
    +
    {#if scorers.length === 0} -
    +
    A scorer reads one run and returns a number. Every run of this dataset is measured by all of them, which is what makes two runs comparable.
    {:else} -
    + +
    {#each scorers as scorer (scorer.id)} -
    +
    {#if scorer.kind === 'agent'} - + {:else} - + {/if}
    - + {scorerLabel(scorer)} - {scorer.path} + {scorer.path}
    {#if scorer.pass_if != undefined} diff --git a/frontend/src/lib/components/aiEvals/EvalsPane.svelte b/frontend/src/lib/components/aiEvals/EvalsPane.svelte index d9e5b16fe4..55792ba8e2 100644 --- a/frontend/src/lib/components/aiEvals/EvalsPane.svelte +++ b/frontend/src/lib/components/aiEvals/EvalsPane.svelte @@ -9,9 +9,11 @@ import Label from '$lib/components/Label.svelte' import Popover from '$lib/components/Popover.svelte' import { Splitpanes, Pane } from 'svelte-splitpanes' + import AnimatedPane from '$lib/components/splitPanes/AnimatedPane.svelte' import { type AgentDraft, AiEvalsService, + JobService, type EvalCase, type EvalDataset, type EvalExperiment, @@ -35,9 +37,11 @@ Code2, ExternalLink } from 'lucide-svelte' + import PagedContent from '$lib/components/common/modal/PagedContent.svelte' import EvalDatasetDrawer from './EvalDatasetDrawer.svelte' import EvalRunsList from './EvalRunsList.svelte' import EvalRunDialog from './EvalRunDialog.svelte' + import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte' import GfmMarkdown from '$lib/components/GfmMarkdown.svelte' import { caseLabel, @@ -52,6 +56,10 @@ /** A dataset is capped at this many cases, so one page holds the whole set. */ const CASE_PAGE_SIZE = 1000 + /** The id the run's flow gives the loop over its cases (`CASES_NODE_ID` in `ai_evals/run.rs`). + * Looked up by id rather than by position: the flow has a step after the loop too. */ + const CASES_MODULE_ID = 'cases' + let { agentPath, opWorkspace = undefined, @@ -91,12 +99,56 @@ /** What the agent hashes to as deployed: a run of edits carrying it ran what was then saved. */ let deployedHash = $state(undefined) let running = $state(false) - let scorers = $derived(dataset?.scorers ?? []) + /** The run on screen belongs to a dataset still being read. Until it arrives the rows and the + * scorer columns would both be built from the *previous* dataset, so both are held back. */ + let datasetLoading = $state(false) + let scorers = $derived(datasetLoading ? [] : (dataset?.scorers ?? [])) let selectedCaseId = $state(undefined) let datasetDrawer: EvalDatasetDrawer | undefined = $state() let runDialogOpen = $state(false) - let resumeRunDialog = $state(false) + /** The run the list has highlighted, so arrowing into the run page opens that one. Without it + * the arrow could only fall back to whichever run was opened last, which on a dialog just + * opened is none at all. */ + let highlightedRunId = $state(undefined) + + /** How many cases each still-running run has finished, keyed by run id. Read from the flow + * executing the run: the list carries the case total, and counting the finished ones there + * would be a per-case query for every run listed. The flow already records it — one slot per + * case in `flow_jobs_success`, null until that case's iteration is over. */ + let caseProgress = $state>({}) + + async function readCaseProgress() { + const workspace = ws + const live = experiments.filter((e) => e.running) + if (!workspace || live.length === 0) { + if (Object.keys(caseProgress).length > 0) caseProgress = {} + return + } + const read = await Promise.all( + live.map(async (e) => { + try { + const update = await JobService.getJobUpdates({ + workspace, + id: e.run_job_id, + running: true, + noLogs: true + }) + const cases = update.flow_status?.modules?.find((m) => m.id === CASES_MODULE_ID) + if (!cases) return undefined + return [ + e.id, + (cases.flow_jobs_success ?? []).filter((s) => s != undefined).length + ] as const + } catch { + // Left out of the map, so the row reads `0/total` until a later poll answers. A + // flow that cannot be read is already the list's problem to report, not this one's. + return undefined + } + }) + ) + caseProgress = Object.fromEntries(read.filter((e) => e !== undefined)) + } let experiment = $derived(experiments.find((e) => e.id === experimentId)) @@ -120,6 +172,7 @@ runsLoadError = false try { experiments = await listSubjectExperiments() + await readCaseProgress() } catch (e) { runsLoadError = true sendUserToast(`Failed to load the runs: ${e}`, true) @@ -183,6 +236,8 @@ // Switching datasets leaves the previous request in flight; only the newest may write, or a // slow response for the dataset you just left replaces the one you are looking at. let loadGeneration = 0 + /** Which run the pane is opening; only the newest may clear `datasetLoading`. */ + let openGeneration = 0 async function loadDataset(path: string | undefined): Promise { const generation = ++loadGeneration @@ -345,6 +400,7 @@ await loadResults() } else { experiments = await listSubjectExperiments() + await readCaseProgress() } } finally { refreshing = false @@ -354,27 +410,46 @@ /** Opens a run, bringing its dataset with it and offering the run before it as the baseline. * Reading the cells is left to the effect on the selection, so every way in opens one alike. */ async function openRun(id: string) { - // Against the dataset that is loaded, not the one that is selected: skipping on the selection - // alone would leave a run open over a dataset whose cases and scorers were never read. const target = experiments.find((e) => e.id === id) - if (target && target.dataset !== dataset?.path) { - await useDataset(target.dataset) + // Only when the run itself changes: re-showing the one already open — arrowing back into it + // from the list — must keep whatever comparison the user picked. + if (id !== experimentId) { + const index = experiments.findIndex((e) => e.id === id) + // The run before it *of the same dataset*: the list spans datasets, and a run of another + // set of cases is not a baseline for this one. + baselineId = experiments.slice(index + 1).find((e) => e.dataset === target?.dataset)?.id } - const index = experiments.findIndex((e) => e.id === id) - // The run before it *of the same dataset*: the list spans datasets, and a run of another set - // of cases is not a baseline for this one. - baselineId = experiments.slice(index + 1).find((e) => e.dataset === target?.dataset)?.id experimentId = id - viewingRun = true selectedCaseId = undefined + // Opened first, read second: the dataset is a request, and waiting on it here is a click + // that does nothing at all until the network answers. The page carries the wait instead. + // + // Against what is *selected* as well as what is loaded. `selectedDataset` moves the moment a + // read starts, so a load still in flight for another dataset shows up here: without that + // test, opening a run of the dataset already committed would skip `useDataset` entirely and + // leave the in-flight one free to commit its cases under this run. + const needsDataset = + !!target && (target.dataset !== dataset?.path || target.dataset !== selectedDataset) + datasetLoading = needsDataset + viewingRun = true + if (needsDataset) { + // Numbered like `loadDataset`'s own read, and for the same reason: opening a second run + // while the first is still loading leaves two `finally`s racing, and the loser clearing + // the flag would uncover the table with neither dataset in hand. + const generation = ++openGeneration + try { + await useDataset(target!.dataset) + } finally { + if (generation === openGeneration) datasetLoading = false + } + } } async function runAll(runSubject: EvalSubject, path: string): Promise { if (!ws || !path) return false running = true - let id: string try { - id = await AiEvalsService.runExperiment({ + await AiEvalsService.runExperiment({ workspace: ws, requestBody: { dataset: path, subject: runSubject } }) @@ -386,9 +461,10 @@ // From here the run exists and is billing: what can still fail is reading it back, and // saying "failed to run" to that invites a second, duplicate run. try { + // Onto the list rather than into the run: a run that has just started has no answers and + // no scores, and the list already fills its row in as they land. Reading it is a click. if (path !== dataset?.path) await useDataset(path) await loadRuns() - await openRun(id) } catch (e) { sendUserToast( `The run started but could not be read back: ${e}. Reload the runs list to see it.`, @@ -407,8 +483,6 @@ /** The dataset is gone and every run of it with it: back to the list, on no dataset. */ async function datasetDeleted(path: string) { - // A run dialog waiting behind the drawer has nothing to come back to. - resumeRunDialog = false if (selectedDataset === path) { viewingRun = false selectedCaseId = undefined @@ -442,6 +516,13 @@ } let selectedRow = $derived(displayRows.find((row) => row.case_id === selectedCaseId)) + /** The case the side panel is showing. Held rather than read straight off the selection: the + * pane animates shut over a few hundred milliseconds, and the selection is gone on the first + * of them, which would empty the panel before it had finished closing. */ + let openRow = $state(undefined) + $effect(() => { + if (selectedRow) openRow = selectedRow + }) $effect(() => { if (!ws) return @@ -570,34 +651,52 @@
    + {#if loaded && loadError} +
    + Could not load evals + + The datasets or runs could not be read. Check your access to this agent and reload. + +
    + {:else} + + { + // Right opens the run under the highlight, falling back to whichever was open before; + // left is the way back, the same as the breadcrumb. + if (key === 'run') { + // Both branches go through `openRun`: it is what brings the run's own dataset back, + // and the fallback run may be of a dataset the list has since moved off. + const id = highlightedRunId ?? experimentId + if (id) openRun(id) + } else if (key === 'list') { + viewingRun = false + selectedCaseId = undefined + } + }} + pages={[ + { key: 'list', content: listPage }, + { key: 'run', content: runPage } + ]} + /> + {/if} +
    + +{#snippet listPage()} + +

    + Each run answers a dataset of cases with this agent and scores the answers, so runs can be + compared. +

    - {#if viewingRun} - - {/if}
    - {#if viewingRun && experiment?.run_job_id} - - Open the job - - - {/if} - {#if !viewingRun && loaded && datasets.length > 0} + {#if loaded && datasets.length > 0} -
    - {:else if !viewingRun || !loaded} - openRun(e.id)} - onEditDataset={async (path) => { - if (await useDataset(path)) datasetDrawer?.openDrawer('edit') - }} - onNew={() => (runDialogOpen = true)} - /> - {:else} - - - - - {#each scorers as scorer (scorer.id)} - - {/each} - - - - Case - Answer - {#each scorers as scorer, index (scorer.id)} - {@const mean = means.find((m) => m.scorer_id === scorer.id)} - {@const headline = columnHeadline(scorer, mean)} - - -
    - - {#if scorer.kind === 'agent'} - - {:else} - - {/if} - {scorerLabel(scorer)} - - - {#if headline} - - {headline.value} + + + + + {#each scorers as scorer (scorer.id)} + + {/each} + + + + Case + Answer + {#each scorers as scorer, index (scorer.id)} + {@const mean = means.find((m) => m.scorer_id === scorer.id)} + {@const headline = columnHeadline(scorer, mean)} + + +
    + + {#if scorer.kind === 'agent'} + + {:else} + + {/if} + {scorerLabel(scorer)} + + + {#if headline} + + {headline.value} + + {#if headline.delta && headline.direction !== 0} + 0 ? 'text-green-500' : headline.direction < 0 ? 'text-red-500' : 'text-tertiary'}`} + > + {headline.delta} - {#if headline.delta && headline.direction !== 0} - 0 ? 'text-green-500' : headline.direction < 0 ? 'text-red-500' : 'text-tertiary'}`} - > - {headline.delta} - - {/if} {/if} - -
    -     - {/each} + {/if} +     +
    +     + {/each} + + + + {#if datasetLoading} + + + + + - - + {:else} {#each displayRows as row (row.case_id)} {@const status = statusOf(row.status)} {/each} - -     - {/if} + {/if} + +
    - {#if selectedRow} - {@const openRow = selectedRow} - + + + {#if openRow}
    -
    - + +
    + {openRow.input?.user_message ?? caseLabel(openRow)} -
    - {#if openRow.job_id} -     - Open the case job - - - {/if} -
    -
    + +
    {#if openRow.expected != undefined && openRow.expected !== ''} - +
    +
    + Expected +
    +
    + + {typeof openRow.expected === 'string' + ? openRow.expected + : JSON.stringify(openRow.expected, null, 2)} + +
    +
    {/if} {#if scorers.length > 0 && openRow.scores.length > 0} -
    -
    +{/snippet} { - if (await useDataset(path)) { - resumeRunDialog = true - datasetDrawer?.openDrawer('edit') - } - }} - onNewDataset={() => { - resumeRunDialog = true - datasetDrawer?.openDrawer('new') + if (await useDataset(path)) datasetDrawer?.openDrawer('edit') }} + onNewDataset={() => datasetDrawer?.openDrawer('new')} /> { - if (!resumeRunDialog) return - resumeRunDialog = false - // On the dataset the drawer was just in: the dialog opens on the pane's own, which - // creating or editing one has already moved to it. - runDialogOpen = true - }} /> + + diff --git a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.css b/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.css deleted file mode 100644 index 1ac9f8b31c..0000000000 --- a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.css +++ /dev/null @@ -1,26 +0,0 @@ -/* MultilineCellEditor: a popup positioned over the cell, so it has to paint the cell's own frame - rather than inherit it. */ -.ag-theme-alpine .wm-multiline-cell-editor, -.ag-theme-alpine-dark .wm-multiline-cell-editor { - background-color: var(--ag-background-color); -} -.ag-theme-alpine .wm-multiline-cell-editor textarea, -.ag-theme-alpine-dark .wm-multiline-cell-editor textarea { - display: block; - box-sizing: border-box; - /* Horizontal only: the vertical padding is set by the editor, which knows the height of the row - it is replacing. `line-height` here is what it computes against. */ - padding: 0 calc(var(--ag-cell-horizontal-padding) - 1px); - border: 1px solid var(--ag-input-focus-border-color); - border-radius: 3px; - outline: none; - resize: none; - /* Past this it scrolls rather than growing. */ - max-height: 40vh; - overflow-y: auto; - background-color: var(--ag-background-color); - color: var(--ag-foreground-color); - font: inherit; - line-height: 20px; - white-space: pre-wrap; -} diff --git a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.ts b/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.ts deleted file mode 100644 index 00975955c9..0000000000 --- a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.ts +++ /dev/null @@ -1,108 +0,0 @@ -import type { ColDef, ICellEditorComp, ICellEditorParams } from 'ag-grid-community' -// Beside the editor rather than in the AgGrid theme: that file is the vendored theme, and a rule -// added to it is one the next copy of it drops. -import './multilineCellEditor.css' - -/** Kept in step with the `line-height` the stylesheet gives the textarea. */ -const LINE_HEIGHT = 20 - -/** - * A text cell editor that starts the height of the cell and grows as lines are added, for columns - * holding prose rather than a value. Enter commits, Shift+Enter adds a line, Escape cancels. - * - * Rendered as a popup positioned over the cell: an in-cell editor is clipped to the row height, so - * growing is only visible if the editor is allowed to paint outside it. - */ -export class MultilineCellEditor implements ICellEditorComp { - private eGui!: HTMLDivElement - private textarea!: HTMLTextAreaElement - private params!: ICellEditorParams - private wasEmpty = false - - init(params: ICellEditorParams) { - this.params = params - this.eGui = document.createElement('div') - this.eGui.className = 'wm-multiline-cell-editor' - - this.wasEmpty = params.value == undefined - - this.textarea = document.createElement('textarea') - this.textarea.rows = 1 - // A keystroke that opened the edit replaces the value, as it does in every other cell; F2 - // and double-click keep it to be edited. - this.textarea.value = params.eventKey?.length === 1 ? params.eventKey : (params.value ?? '') - this.textarea.style.width = `${params.column.getActualWidth() - 2}px` - // Padded so one line fills the cell it replaces and a second costs a line rather than a row. - // From the row rather than from `--ag-row-height`, which is the theme's figure and not - // necessarily this grid's. - const rowHeight = params.node.rowHeight ?? 28 - const padding = Math.max(0, (rowHeight - LINE_HEIGHT - 2) / 2) - this.textarea.style.paddingTop = `${padding}px` - this.textarea.style.paddingBottom = `${padding}px` - - this.textarea.addEventListener('input', () => this.resize()) - this.textarea.addEventListener('keydown', (e) => { - if (e.key === 'Escape') { - // Kept from whatever is around the grid: a grid in a drawer or a dialog is under a - // surface that closes on Escape, and leaving an edit is not asking to leave that. - e.preventDefault() - e.stopPropagation() - this.params.api.stopEditing(true) - return - } - if (e.key !== 'Enter' || e.isComposing) return - // Both branches keep the key from the grid, which ends the edit on Enter whether or not - // Shift is held: Shift+Enter falls through to the textarea's own newline, and plain Enter - // ends the edit here instead. - e.stopPropagation() - if (!e.shiftKey) { - e.preventDefault() - this.params.stopEditing() - } - }) - this.eGui.appendChild(this.textarea) - } - - private resize() { - this.textarea.style.height = 'auto' - this.textarea.style.height = `${this.textarea.scrollHeight}px` - } - - getGui() { - return this.eGui - } - - afterGuiAttached() { - this.resize() - this.textarea.focus() - // At the end rather than selected: a selection is a keystroke away from erasing the cell. - const end = this.textarea.value.length - this.textarea.setSelectionRange(end, end) - } - - getValue() { - // Nothing typed into a cell that held nothing is not an edit: returning '' here would write - // an empty string over a null, which the grid would see as a change and commit. - if (this.wasEmpty && this.textarea.value === '') return this.params.value - return this.textarea.value - } - - isPopup() { - return true - } - - getPopupPosition(): 'over' | 'under' { - return 'over' - } -} - -/** - * What a column of prose needs, ready to spread into a colDef. `suppressKeyboardEvent` as well as - * the editor: the grid ends an edit on Enter from a handler a popup editor's DOM does not sit - * under, so the editor cannot keep Shift+Enter for itself on its own. - */ -export const multilineCellColDef: Pick = { - cellEditor: MultilineCellEditor, - suppressKeyboardEvent: (p) => - p.editing && (p.event as KeyboardEvent).key === 'Enter' && (p.event as KeyboardEvent).shiftKey -} diff --git a/frontend/src/lib/components/apps/components/helpers/InputValue.svelte b/frontend/src/lib/components/apps/components/helpers/InputValue.svelte index aba0569228..7ec2336d3b 100644 --- a/frontend/src/lib/components/apps/components/helpers/InputValue.svelte +++ b/frontend/src/lib/components/apps/components/helpers/InputValue.svelte @@ -19,6 +19,7 @@ import { computeGlobalContext, eval_like } from './eval' import { deepEqual } from 'fast-equals' import { deepMergeWithPriority, isCodeInjection, readFieldsRecursively } from '$lib/utils' + import { escapeTemplateBackticks } from '$lib/utils/templateLiteral' import sum from 'hash-sum' import { createDispatcherIfMounted } from '$lib/createDispatcherIfMounted' @@ -271,7 +272,7 @@ if ((input.type === 'template' || input.type == 'templatev2') && isCodeInjection(input.eval)) { try { const r = await eval_like( - '`' + input.eval.replaceAll('`', '\\`') + '`', + '`' + escapeTemplateBackticks(input.eval) + '`', computeGlobalContext($worldStore, id, fullContext), $stateStore, $mode == 'dnd', diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelineInsertMenu.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelineInsertMenu.svelte index e3c6c04378..e341707250 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PipelineInsertMenu.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PipelineInsertMenu.svelte @@ -28,6 +28,7 @@ @@ -22,6 +24,7 @@ on:click={() => (dispatch('close'), onClick?.())} on:pointerdown={(e) => e.stopPropagation()} {id} + {title} startIcon={{ icon: Icon ?? X }} iconOnly unifiedSize="sm" diff --git a/frontend/src/lib/components/common/EditableTextarea.svelte b/frontend/src/lib/components/common/EditableTextarea.svelte new file mode 100644 index 0000000000..26a6574027 --- /dev/null +++ b/frontend/src/lib/components/common/EditableTextarea.svelte @@ -0,0 +1,173 @@ + + + +{#if editing} + + +{:else} + + +{/if} diff --git a/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte b/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte index 56244632ef..c858f9f08f 100644 --- a/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte +++ b/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte @@ -23,6 +23,12 @@ /** Tailwind z-index class for the modal root. Override to stack this modal * above another modal that's already open (both default to `z-[9999]`). */ zIndexClass?: string + /** Render into `body` instead of where this component sits. Needed when an ancestor + * creates a stacking context the dialog has to escape — a drawer paints over the page + * whatever the dialog's z-index, and a `transform`, `filter` or `overflow` on the way + * up confines it. Off by default: it moves the dialog out of its DOM position, so opt + * in per call site rather than assuming every caller wants it. */ + alwaysPortal?: boolean children?: Snippet onConfirmed?: () => void | Promise onCanceled?: () => void @@ -40,6 +46,7 @@ id, trashbin = false, zIndexClass = 'z-[9999]', + alwaysPortal = false, children, onConfirmed, onCanceled @@ -141,7 +148,11 @@ - + {#if open}
    disposable?.isTopmost() ?? true) + let reducedMotion = useReducedMotion() let duration = $derived(reducedMotion.val ? 0 : _duration) let durationMs = $derived(duration * 1000) diff --git a/frontend/src/lib/components/common/emptyState/EmptyState.svelte b/frontend/src/lib/components/common/emptyState/EmptyState.svelte index 8715065ad0..49c3cd2b77 100644 --- a/frontend/src/lib/components/common/emptyState/EmptyState.svelte +++ b/frontend/src/lib/components/common/emptyState/EmptyState.svelte @@ -10,6 +10,16 @@ label: string icon?: any onClick: () => void + /** + * `default` unless the surface has no other live call to action. Accent is for the case + * where this button is the only thing to press — a form whose submit is disabled until + * this is done, say — so it is not competing with one. + */ + variant?: 'default' | 'accent' + /** Same write lock the surface's other controls take. An empty state is still a live + * control: without this it stays clickable while a request that has already read the + * empty list is in flight, and whatever it adds is discarded when that request lands. */ + disabled?: boolean aiId?: string aiDescription?: string } @@ -32,11 +42,13 @@ {/if}
    {#if action} - +
    diff --git a/frontend/src/lib/components/common/modal/PagedContent.svelte b/frontend/src/lib/components/common/modal/PagedContent.svelte new file mode 100644 index 0000000000..430a53e66f --- /dev/null +++ b/frontend/src/lib/components/common/modal/PagedContent.svelte @@ -0,0 +1,202 @@ + + + + + + + +
    + {#each pages as page, i (page.key)} + {#if visited.includes(page.key) || warmed} + + +
    + {@render page.content()} +
    + {/if} + {/each} +
    + + diff --git a/frontend/src/lib/components/common/overlayHost.svelte.ts b/frontend/src/lib/components/common/overlayHost.svelte.ts index 0e76709cbf..7c4374115b 100644 --- a/frontend/src/lib/components/common/overlayHost.svelte.ts +++ b/frontend/src/lib/components/common/overlayHost.svelte.ts @@ -72,3 +72,23 @@ export function overlayHostActive(): () => boolean { const host = getOverlayHost() return () => host?.active() ?? true } + +const TOPMOST_SURFACE_KEY = 'topmostSurface' + +/** + * Declare whether the surface enclosing this subtree is the one on top. Set by whatever owns the + * stacking — a dialog, a drawer — so content inside it can tell a key meant for itself from one + * meant for something opened over it. + */ +export function setTopmostSurface(isTopmost: () => boolean) { + setContext(TOPMOST_SURFACE_KEY, isTopmost) +} + +/** + * Whether the enclosing surface is on top. True when nothing declared otherwise, so content that + * is not inside such a surface is not silently made deaf. + */ +export function topmostSurface(): () => boolean { + const isTopmost = getContext<(() => boolean) | undefined>(TOPMOST_SURFACE_KEY) + return () => isTopmost?.() ?? true +} diff --git a/frontend/src/lib/components/common/radioCard/RadioCard.svelte b/frontend/src/lib/components/common/radioCard/RadioCard.svelte index 6cbb9c78a0..f9b7f675a4 100644 --- a/frontend/src/lib/components/common/radioCard/RadioCard.svelte +++ b/frontend/src/lib/components/common/radioCard/RadioCard.svelte @@ -10,12 +10,14 @@ onSelect, disabled = false, icon = undefined, + showRadio = true, class: className = '' }: { /** Title shown in bold at the top of the card */ label: string - /** Optional supporting line under the label */ - description?: string + /** Optional supporting line under the label. A snippet when it needs markup + * of its own — an emphasised name, a count — rather than plain text. */ + description?: string | Snippet /** Whether this card is the selected option */ selected?: boolean /** Called when the card is clicked */ @@ -23,35 +25,55 @@ disabled?: boolean /** Optional leading icon, rendered after the radio */ icon?: Snippet + /** Draw the radio glyph. Turn it off where the card itself is the only + * control and the border and tint already say which one is picked — the dot + * is then a second, redundant answer to the same question. The group still + * reads as radios to a screen reader, which is what `role` carries. */ + showRadio?: boolean class?: string } = $props() + + // A snippet is a function; a description string is not. Checked rather than + // requiring callers to pick between two props. + const describedBySnippet = $derived(typeof description === 'function') - {/snippet} - +

    AI can help with these inputs

    + + {#snippet fallback()} + + {/snippet} + +
    +
    +

    + {instructions + ? 'Instructions: ' + instructions + : 'No AI instructions provided. Click edit to add guidance for AI form filling.'} +

    +
    -
    -

    - {instructions - ? 'Instructions: ' + instructions - : 'No AI instructions provided. Click edit to add guidance for AI form filling.'} -

    -
    -
    +{/if} diff --git a/frontend/src/lib/components/copilot/AIFormSettings.svelte b/frontend/src/lib/components/copilot/AIFormSettings.svelte index 44151a01b3..ef627ace4b 100644 --- a/frontend/src/lib/components/copilot/AIFormSettings.svelte +++ b/frontend/src/lib/components/copilot/AIFormSettings.svelte @@ -3,6 +3,7 @@ import Label from '../Label.svelte' import Toggle from '../Toggle.svelte' import Tooltip from '../Tooltip.svelte' + import { copilotInfo } from '$lib/aiStore' interface Props { prompt?: string | undefined @@ -12,35 +13,37 @@ let { prompt = $bindable(undefined), type = 'script' }: Props = $props() -
    - { - if (prompt !== undefined) { - prompt = undefined - } else { - prompt = '' - } - }} - options={{ right: `Enable filling ${type} inputs with AI` }} - /> - {#if prompt !== undefined} -
    - -
    - {/if} -
    +{#if !$copilotInfo.workspaceDisabled} +
    + { + if (prompt !== undefined) { + prompt = undefined + } else { + prompt = '' + } + }} + options={{ right: `Enable filling ${type} inputs with AI` }} + /> + {#if prompt !== undefined} +
    + +
    + {/if} +
    +{/if} diff --git a/frontend/src/lib/components/copilot/CronGen.svelte b/frontend/src/lib/components/copilot/CronGen.svelte index 6a8dc6892e..94715a0f18 100644 --- a/frontend/src/lib/components/copilot/CronGen.svelte +++ b/frontend/src/lib/components/copilot/CronGen.svelte @@ -79,66 +79,68 @@ }) - - {#snippet trigger()} -
    - {:else} -
    -

    Enable Windmill AI in the workspace settings

    -
    - {/if} -
    - {/snippet} - + }} + disabled={instructions.length == 0} + startIcon={{ icon: Wand2 }} + /> +
    + {:else} +
    +

    Enable Windmill AI in the workspace settings

    +
    + {/if} +
    + {/snippet} + +{/if} diff --git a/frontend/src/lib/components/copilot/RegexGen.svelte b/frontend/src/lib/components/copilot/RegexGen.svelte index e61ab5cbf2..f399726988 100644 --- a/frontend/src/lib/components/copilot/RegexGen.svelte +++ b/frontend/src/lib/components/copilot/RegexGen.svelte @@ -1,5 +1,5 @@ - - {#snippet trigger()} - +{#if !$copilotInfo.workspaceDisabled} + + {#snippet trigger()}
    - - {/snippet} - + {/snippet} + +{/if} diff --git a/frontend/src/lib/components/copilot/ResourceGen.svelte b/frontend/src/lib/components/copilot/ResourceGen.svelte index 34534285b4..2fc3cfa646 100644 --- a/frontend/src/lib/components/copilot/ResourceGen.svelte +++ b/frontend/src/lib/components/copilot/ResourceGen.svelte @@ -119,69 +119,71 @@ }) - - {#snippet trigger()} - -
    - {:else} -
    -

    Enable Windmill AI in the workspace settings

    -
    - {/if} -
    - {/snippet} - + }} + disabled={instructions.length == 0} + startIcon={{ icon: Wand2 }} + > + Generate + +
    + {:else} +
    +

    Enable Windmill AI in the workspace settings

    +
    + {/if} +
    + {/snippet} + +{/if} diff --git a/frontend/src/lib/components/copilot/ScriptFix.svelte b/frontend/src/lib/components/copilot/ScriptFix.svelte index bbd9341700..f586ad1aaf 100644 --- a/frontend/src/lib/components/copilot/ScriptFix.svelte +++ b/frontend/src/lib/components/copilot/ScriptFix.svelte @@ -13,6 +13,7 @@ import { getOpenInSessionHandoff } from '$lib/components/sessions/openInSessionContext' import { AIBtnClasses } from './chat/AIButtonStyle' import { getContext } from 'svelte' + import { logFeatureUsage } from '$lib/utils/featureUsage' let { lang, @@ -45,9 +46,28 @@ ? `Fix this error in ${what}:\n\n\`\`\`\n${error}\n\`\`\`` : `Fix the error from the last run of ${what}.` }) + // Anonymous counter for a failing run being handed to AI, keyed by where the run was. All + // three branches below report the same action: which one is on screen follows the session + // gate and whether a chat is already beside this panel, not a choice made here. + function logAiFix() { + logFeatureUsage('ai_fix', 'requested', { key: moduleId ? 'flow_step' : 'script' }) + } + const sessionSource = $derived.by(() => { const source = handoff?.source({ moduleId }) - return source ? { ...source, seedPrompt, autoSend: true } : undefined + if (!source) return undefined + // The counter wraps the editor's own hook rather than replacing it: that hook persists + // the draft the session opens on, so dropping it would fix an older copy of the code. + const editorBeforeOpen = source.beforeOpen + return { + ...source, + seedPrompt, + autoSend: true, + beforeOpen: async () => { + logAiFix() + await editorBeforeOpen?.() + } + } }) // Inside a session pane the chat is already beside this panel, so there is @@ -57,7 +77,7 @@ const sessionScopedManager = getContext('aiChatManager') -{#if SUPPORTED_LANGUAGES.has(lang)} +{#if SUPPORTED_LANGUAGES.has(lang) && !$copilotInfo.workspaceDisabled} {#if sessionScopedManager}
    • {/if} {/if} -{#if ($generatedCode.length === 0 || genLoading) && SUPPORTED_LANGUAGES.has(lang ?? '')} +{#if ($generatedCode.length === 0 || genLoading) && SUPPORTED_LANGUAGES.has(lang ?? '') && !$copilotInfo.workspaceDisabled} import { workspaceStore } from '$lib/stores' + import { copilotInfo } from '$lib/aiStore' import { ScriptService, type Script } from '$lib/gen' import { Wand2, Loader2 } from 'lucide-svelte' @@ -27,6 +28,7 @@ filteredItems = $bindable([]) }: Props = $props() let prefilteredItems = $derived(scripts ?? []) + let aiHidden = $derived(disableAi || $copilotInfo.workspaceDisabled) const dispatch = createEventDispatcher() @@ -46,6 +48,10 @@ let input: TextInput | undefined = $state() + export function focus() { + input?.focus() + } + $effect(() => { preFilter && setTimeout(() => { @@ -76,7 +82,7 @@ onkeydown: (e) => { if (e.key === 'Escape') dispatch('escape') }, - placeholder: `Search ${trigger ? 'triggers' : 'scripts'} ${disableAi ? '' : 'or AI gen'}` + placeholder: `Search ${trigger ? 'triggers' : 'scripts'} ${aiHidden ? '' : 'or AI gen'}` }} size="sm" /> @@ -85,7 +91,7 @@ {#if loading} {/if} - {#if funcDesc?.length === 0 && !loading && !disableAi} + {#if funcDesc?.length === 0 && !loading && !aiHidden} {/if} diff --git a/frontend/src/lib/components/copilot/StepInputGen.svelte b/frontend/src/lib/components/copilot/StepInputGen.svelte index 839792c5f0..5b912d4390 100644 --- a/frontend/src/lib/components/copilot/StepInputGen.svelte +++ b/frontend/src/lib/components/copilot/StepInputGen.svelte @@ -10,6 +10,7 @@ import { dfs } from '../flows/dfs' import { yamlStringifyExceptKeys } from './utils' import type { FlowCopilotContext } from './flow' + import { logStepInputFill } from './stepInputFillTelemetry' import { stepInputCompletionEnabled } from '$lib/stores' import type { SchemaProperty } from '$lib/common' import FlowCopilotInputsModal from './FlowCopilotInputsModal.svelte' @@ -66,6 +67,7 @@ if (generatedContent.length > 0 || loading) { return } + logStepInputFill('single') abortController = new AbortController() loading = true const flow: Flow = JSON.parse(JSON.stringify(flowStore.val)) diff --git a/frontend/src/lib/components/copilot/StepInputsGen.svelte b/frontend/src/lib/components/copilot/StepInputsGen.svelte index 99199812b5..c9ef616aa5 100644 --- a/frontend/src/lib/components/copilot/StepInputsGen.svelte +++ b/frontend/src/lib/components/copilot/StepInputsGen.svelte @@ -11,6 +11,7 @@ import { sendUserToast } from '$lib/toast' import Button from '../common/button/Button.svelte' import type { FlowCopilotContext } from './flow' + import { logStepInputFill } from './stepInputFillTelemetry' import { Check, ExternalLink, Loader2, Wand2 } from 'lucide-svelte' import { stepInputCompletionEnabled } from '$lib/stores' import { copilotInfo } from '$lib/aiStore' @@ -44,6 +45,7 @@ if (Object.keys($generatedExprs || {}).length > 0 || loading) { return } + logStepInputFill('all') abortController = new AbortController() loading = true stepInputsLoading?.set(true) @@ -224,7 +226,7 @@ input_name2: expression2 Fill inputs {/if} - {:else} + {:else if !$copilotInfo.workspaceDisabled} togglePanel() })} -{:else} +{:else if !$copilotInfo.workspaceDisabled} {#snippet trigger()} {@render button({ onPress: () => togglePanel() })} diff --git a/frontend/src/lib/components/copilot/chat/AIChat.svelte b/frontend/src/lib/components/copilot/chat/AIChat.svelte index 2a51b1b2e5..fd95613f2e 100644 --- a/frontend/src/lib/components/copilot/chat/AIChat.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChat.svelte @@ -44,27 +44,43 @@ const isAdmin = $derived($userStore?.is_admin || $userStore?.is_super_admin) const hasCopilot = $derived($copilotInfo.enabled) + // Another tab is running a turn on this session: transcript stays readable, + // composer locks, and the chat re-reads the shared record when the turn ends. + const runHeldElsewhere = $derived(aiChatManager.runHeldElsewhere) const disabled = $derived( forceDisabled || + runHeldElsewhere || !hasCopilot || (aiChatManager.mode === AIMode.SCRIPT && aiChatManager.scriptEditorOptions?.lang && !SUPPORTED_CHAT_SCRIPT_LANGUAGES.includes(aiChatManager.scriptEditorOptions.lang)) ) + // A spent free grant is not an unconfigured workspace: AIChatDisplay already shows an + // in-thread banner naming the real cause and linking to the key settings, so the generic + // "enable Windmill AI" line would both duplicate it and misstate why the chat is off. + const freeTierExhausted = $derived($copilotInfo.freeTier?.exhausted === true) const disabledMessage = $derived( forceDisabled ? forceDisabledMessage - : !hasCopilot - ? $aiUserDisabled - ? 'Windmill AI is disabled in your account settings' - : isAdmin - ? `Enable Windmill AI in your [workspace settings](${base}/workspace_settings?tab=ai) to use this chat` - : 'Ask an admin to enable Windmill AI in this workspace to use this chat' - : aiChatManager.mode === AIMode.SCRIPT && - aiChatManager.scriptEditorOptions?.lang && - !SUPPORTED_CHAT_SCRIPT_LANGUAGES.includes(aiChatManager.scriptEditorOptions.lang) - ? `Windmill AI does not support the ${aiChatManager.scriptEditorOptions.lang} language yet.` - : '' + : runHeldElsewhere + ? // The typing indicator and the composer placeholder already carry + // this state; a footer note would say it a third time. + '' + : freeTierExhausted + ? '' + : !hasCopilot + ? $copilotInfo.workspaceDisabled + ? 'Windmill AI is hidden in this workspace' + : $aiUserDisabled + ? 'Windmill AI is disabled in your account settings' + : isAdmin + ? `Enable Windmill AI in your [workspace settings](${base}/workspace_settings?tab=ai) to use this chat` + : 'Ask an admin to enable Windmill AI in this workspace to use this chat' + : aiChatManager.mode === AIMode.SCRIPT && + aiChatManager.scriptEditorOptions?.lang && + !SUPPORTED_CHAT_SCRIPT_LANGUAGES.includes(aiChatManager.scriptEditorOptions.lang) + ? `Windmill AI does not support the ${aiChatManager.scriptEditorOptions.lang} language yet.` + : '' ) const suggestions = [ diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index c68945695b..f610dd5ff4 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -7,6 +7,7 @@ AlertTriangle, ArrowDown, AtSign, + BookOpen, ChevronDown, ChevronsRight, CheckIcon, @@ -15,6 +16,7 @@ Folder, Hand, HistoryIcon, + KeyRound, MousePointer2, Plug, Plus, @@ -34,6 +36,7 @@ import ContextUsageIndicator from './ContextUsageIndicator.svelte' import AIChatModelSettings from './AIChatModelSettings.svelte' import McpConnections from './McpConnections.svelte' + import SkillsPicker from './SkillsPicker.svelte' import ChatMode from './ChatMode.svelte' import DatatableCreationPolicy from './DatatableCreationPolicy.svelte' import Tooltip from '$lib/components/meltComponents/Tooltip.svelte' @@ -59,9 +62,22 @@ readDroppedEntries } from './files/fsAccess' import { sendUserToast } from '$lib/toast' + import Alert from '$lib/components/common/alert/Alert.svelte' + import { copilotInfo } from '$lib/aiStore' + import { base } from '$lib/base' const MAX_YOLO_TOOLTIP_TOOLS = 8 const aiChatManager = getAiChatManager() + + // The user spent their one-time free Windmill AI grant: there is no model left to send + // to, so say so in the thread itself rather than only failing on send. + let freeTierExhausted = $derived($copilotInfo.freeTier?.exhausted === true) + // Still on the free grant: keep how much is left in view right above the composer, so + // running out isn't a surprise. Once spent, the exhausted banner replaces it. + let freeTier = $derived($copilotInfo.freeTier) + let freeTierUsedPct = $derived(Math.min(100, Math.round((freeTier?.used_ratio ?? 0) * 100))) + let showFreeTierUsage = $derived(!!freeTier && !freeTier.exhausted) + // One row per autonomy posture, in picker order, so adding one touches only this // table. `isAvailable` hides the postures that would do nothing in the current AI // mode, which is why the picker can be shorter than this list. @@ -192,6 +208,7 @@ let aiChatInput: AIChatInput | undefined = $state() let mcpConnections: McpConnections | undefined = $state() + let skillsPicker: SkillsPicker | undefined = $state() let plusMenuOpen = $state(false) let editingMessageIndex = $state(null) @@ -298,7 +315,10 @@ } }) - const showTypingIndicator = $derived(aiChatManager.loading) + // Also shown for a run held by another tab, labeled with where it is: the + // dots say a turn is in flight even before the reader reaches the footer + // note. Remote runs pause nothing and offer no Stop — this tab can't cancel. + const showTypingIndicator = $derived(aiChatManager.loading || aiChatManager.runHeldElsewhere) // The manual `@` context-picker button. Shown in SCRIPT/FLOW (workspace items + // code blocks) and APP (datatables, frontend files). Hidden in GLOBAL — there @@ -554,14 +574,58 @@ (aiChatManager.flowAiChatHelpers?.hasPendingChanges() ?? false) && !aiChatManager.autoAcceptEditsActive ) + // A disabled state with no message (a remote hold, a spent free grant) keeps + // the footer toolbar in place — swapping it for an empty strip would make + // the model/mode row flash out and back on every remote turn. A state with + // a real message (archived, AI off) still shows it, hold or not, matching + // the precedence disabledMessage itself encodes. + const footerMessageShown = $derived(disabled && disabledMessage !== '') const showFooterLeftControls = $derived( - !disabled && + !footerMessageShown && (showContextPicker || showAutonomyModeSelector || (aiChatManager.mode === AIMode.SCRIPT && hasDiff)) ) +{#snippet freeTierExhaustedBanner()} +
    + +
    + + You have used all of your free Windmill AI tokens. Add your own API key to keep using AI. + + +
    +     +
    +{/snippet} + +{#snippet freeTierUsageBanner()} +
    + + {freeTierUsedPct}% of your free Windmill AI used + + +
    +{/snippet} +
    {#each pastChats as chat (chat.id)}
    {/if} - {#if disabled} + {#if footerMessageShown}
    @@ -1040,6 +1145,7 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> {#if aiChatManager.mode === AIMode.GLOBAL} + {/if} diff --git a/frontend/src/lib/components/copilot/chat/AIChatInput.svelte b/frontend/src/lib/components/copilot/chat/AIChatInput.svelte index 14182d555f..7f4514cd41 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatInput.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatInput.svelte @@ -129,6 +129,12 @@ // Generate mode-specific placeholder const modePlaceholder = $derived.by(() => { + // The composer unlocks by itself when the other tab's turn ends, so the + // placeholder names what it is waiting on (the typing indicator says + // where the run is). + if (aiChatManager.runHeldElsewhere) { + return 'Waiting for the turn in the other tab to finish' + } if (pendingQuestionToolCallId !== undefined) { return 'Answer the question above' } @@ -688,6 +694,15 @@ selectedContext = [...selectedContext, contextToAdd] } + /** Consume inside the submit gesture, never after it: the manager's preflight + * awaits attachment upkeep and a session fork that can take seconds, and + * consuming past them would hand this message a mention the user picked for + * the next one. */ + function consumeMentionsIfGlobal() { + if (aiChatManager.mode !== AIMode.GLOBAL) return + aiChatManager.contextManager?.consumeMentionContext() + } + function sendRequest() { // The send button is disabled while decoding, but Enter reaches here directly. // Sending now would drop the in-flight attachments onto the following message. @@ -705,6 +720,10 @@ ]) ) { draft.take() + // The answer carries only its choice strings, so a mention here rides + // nothing — but clearForSend below keeps the selection, which would hand + // it to the next turn. + consumeMentionsIfGlobal() contextTextareaComponent?.clearForSend() return } @@ -728,6 +747,8 @@ [...selectedContext], sent.files ) + // Consumed at enqueue, not at flush: the entry above pinned them. + consumeMentionsIfGlobal() contextTextareaComponent?.clearForSend() } return @@ -748,17 +769,24 @@ onEditEnd() } else { const sent = draft.take() + // Pin before consuming: the manager falls back to the live selection only + // when given no override, and the consume below empties it. + const carried = aiChatManager.mode === AIMode.GLOBAL ? [...selectedContext] : undefined + consumeMentionsIfGlobal() aiChatManager.sendRequest({ instructions: sent.text, pastes: sent.pastes, images: sent.images, - files: sent.files + files: sent.files, + contextOverride: carried, + contextOverrideOrigin: carried ? 'pinned' : undefined }) - // clearForSend() pre-zaps the textarea's mention-sync so the wipe - // doesn't drop `selectedContext` before `AIChatManager.beforeSend` - // snapshots it. Only mounted in SCRIPT/FLOW/GLOBAL — APP and the - // fallback textarea still rely on the draft reset alone (no - // `@`-mention state to coordinate). + // clearForSend() pre-zaps the textarea's mention-sync so the wipe doesn't + // drop `selectedContext` before the send has settled its context: the pin + // above in GLOBAL, the manager's own read of the live selection in + // SCRIPT/FLOW. Only mounted in SCRIPT/FLOW/GLOBAL — APP and the fallback + // textarea still rely on the draft reset alone (no `@`-mention state to + // coordinate). contextTextareaComponent?.clearForSend() } } diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 5e56500cff..fb54204fe9 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -85,6 +85,8 @@ import { untrack } from 'svelte' import { get } from 'svelte/store' import { BROWSER } from 'esm-env' import { workspaceStore, type DBSchemas } from '$lib/stores' +import { copilotInfo } from '$lib/aiStore' +import { copilotWorkspaceRequested, loadCopilot } from '$lib/components/copilot/loadCopilot' import { askTools, prepareAskSystemMessage, prepareAskUserMessage } from './ask/core' import { readDocsPageTool, searchDocsTool } from './docs/core' import { TypewriterReveal } from './typewriterReveal' @@ -93,6 +95,7 @@ import { createAppBackendRunnableContextElement, createAppFrontendFileContextElement, flattenDatatablesToAppContextElements, + isMentionContext, isSameContextElement, type ContextElement, type AppDatatableElement @@ -102,11 +105,7 @@ import type AIChatInput from './AIChatInput.svelte' import { prepareApiSystemMessage, prepareApiUserMessage } from './api/core' import { closeInterruptedToolBatch, runChatLoop, truncateToToolPairedPrefix } from './chatLoop' import { sanitizeToolCallArguments } from './toolCallArguments' -import { - billedTokens, - normalizeContextUsage, - type ChatTokenUsage -} from './tokenUsage' +import { billedTokens, normalizeContextUsage, type ChatTokenUsage } from './tokenUsage' import { logAiUsage } from '$lib/utils/aiUsageReporter' import type { ReviewChangesOpts } from './monaco-adapter' import { @@ -369,6 +368,25 @@ function getSendRequestErrorMessage(err: unknown, webSearchUnavailable: boolean) return appendWebSearchErrorHint(message, webSearchUnavailable) } +/** Re-fetch copilotInfo after a free-tier turn so the usage banner tracks spend live and the + * exhausting turn flips `freeTier.exhausted`; otherwise these update only on the next workspace + * load. Scoped to a live (non-exhausted) free tier so configured-key users pay no extra request. */ +async function refreshFreeTierUsage(workspace: string | undefined) { + if (!workspace) return + // copilotInfo is a singleton shared across sessions: a warm session finishing after a + // workspace switch must not loadCopilot for its now-background workspace. Gate on the + // most-recently-*requested* workspace (set synchronously) so a refresh can't win the + // monotonic token over a newer load still in flight. + if (get(copilotWorkspaceRequested) !== workspace) return + const info = get(copilotInfo) + if (!info.freeTier || info.freeTier.exhausted) return + try { + await loadCopilot(workspace) + } catch (err) { + console.error('Failed to refresh free-tier usage', err) + } +} + /** A message queued while a turn streams: the draft lanes and the pinned * context snapshot always move together so a flush can't drop one. */ type QueuedEntry = { @@ -483,7 +501,24 @@ export class AIChatManager { openRunInPreview?: (a: { jobId: string; workspace: string; label: string }) => void openArtifact?: (artifactId: string, name: string, version?: ArtifactVersionTarget) => void closeArtifact?: (artifactId: string) => void - loading = $state(false) + #loading = $state(false) + get loading(): boolean { + return this.#loading + } + // An accessor so every run bracket — the send turn, manual compaction, a + // rollback — reports its transitions through one place, synchronously: the + // rising edge posts the cross-tab "running here" signal the moment the + // bracket opens (after the send's preflight awaits; the post-preflight + // guard covers that gap), and `loading` falls only after the turn's last + // saveChat, making the falling edge the "safe to re-read the record" signal. + set loading(v: boolean) { + if (v === this.#loading) return + this.#loading = v + this.onRunningChanged?.(v) + } + /** Sessions wiring (see sessionRuntime); undefined for the global + * side-panel chat, whose transcript no other tab renders. */ + onRunningChanged: ((running: boolean) => void) | undefined = undefined currentReply = $state('') currentReasoning = $state('') currentReasoningActive = $state(false) @@ -659,6 +694,14 @@ export class AIChatManager { // sessions modules — and re-read on every system-message rebuild; the send // path rebuilds after beforeSend, so a fork committed there is picked up. sessionContextResolver: (() => SessionPromptContext | undefined) | undefined = undefined + // Whether another tab is running a turn on this session right now (sessions + // wiring, same seam as above). The composer locks on it, and sendRequest + // refuses on it — the refusal covers the send already in flight when the + // other tab's run signal arrives, which no disabled input can stop. + runHeldElsewhereResolver: (() => boolean) | undefined = undefined + get runHeldElsewhere(): boolean { + return this.runHeldElsewhereResolver?.() ?? false + } // The page the side panel shows, stamped on each user message. Same seam as above: // a page tab is an iframe in its own realm, so the tab model is the only place the // chat can learn it. Undefined for a live editor — ACTIVE EDITOR covers those. @@ -670,8 +713,8 @@ export class AIChatManager { workspaceResolver: (() => string | undefined) | undefined = undefined // The workspace every workspace-scoped chat action targets — skills, tool - // loop, logging, user-message context, and commit. Session-resolved when a - // resolver is set, else the globally-active workspace. + // loop, logging, user-message context, message rendering, and commit. + // Session-resolved when a resolver is set, else the globally-active workspace. get operatingWorkspace(): string | undefined { return this.workspaceResolver?.() ?? get(workspaceStore) } @@ -1038,6 +1081,16 @@ export class AIChatManager { * doesn't spawn a new job leaves nothing to re-trigger on. A turn that DOES * spawn another job resumes again when that one finishes, which is the point. */ + #autoResumeRetry: ReturnType | undefined + + #scheduleAutoResumeRetry() { + clearTimeout(this.#autoResumeRetry) + this.#autoResumeRetry = setTimeout(() => { + this.#autoResumeRetry = undefined + void this.#maybeAutoResumeFromJobs() + }, 5_000) + } + async #maybeAutoResumeFromJobs() { if (this.#autoResuming) return // Global/sessions chat only (the only mode with a jobs tray + preamble). @@ -1048,6 +1101,17 @@ export class AIChatManager { // Nothing to continue (empty chat), or the user is mid-compose — don't // clobber their draft or auto-send it. Their eventual send carries the notes. if (this.messages.length === 0 || this.instructions.trim()) return + // Another tab is driving: the synthetic send would only be refused, and + // the instructions staged below would then block every later auto-resume + // in this tab. The notes stay pending; re-checked shortly, because the + // hold can clear silently (staleness after a driver crash) with nothing + // else to fire this. When the driver instead ends its turn normally, its + // own resume carries the notes and this tab's catch-up clears the local + // copy — the re-check then finds nothing and stands down. + if (this.runHeldElsewhere) { + this.#scheduleAutoResumeRetry() + return + } this.#autoResuming = true try { const count = this.pendingJobNotes.length @@ -1086,6 +1150,8 @@ export class AIChatManager { // Invalidate any in-flight poll so its post-await continuation can't write // into the conversation we're switching to. this.#jobPollGeneration++ + clearTimeout(this.#autoResumeRetry) + this.#autoResumeRetry = undefined this.backgroundJobs = [] this.pendingJobNotes = [] } @@ -1117,9 +1183,10 @@ export class AIChatManager { } } - // Workspace AI skills (name + description) advertised in the GLOBAL system - // prompt and surfaced as slash commands in session chat. Loaded - // asynchronously when entering GLOBAL mode; the system message is rebuilt + // The `ai_skill` resources this user turned on for the operating workspace, + // advertised in the GLOBAL system prompt and surfaced as slash commands in + // session chat. Loaded asynchronously when entering GLOBAL mode and again + // whenever the picker changes the selection; the system message is rebuilt // once they resolve. globalSkills = $state([]) private globalSkillsRefreshId = 0 @@ -1155,9 +1222,10 @@ export class AIChatManager { ] // Built-ins followed by workspace skills, with any skill whose name collides - // with a built-in dropped: the picker keys leaves by name, so a duplicate - // would break its keyed list and ambiguous-resolve nav. Built-ins win — they - // already shadow same-named skills at execution (the submit interception). + // with a built-in dropped. Built-ins win — they already shadow same-named + // skills at execution (the submit interception), so listing both would offer + // a row that cannot run. Two skills may still share a name; the picker keys + // those by path and the submit path declines to guess between them. sessionCommands: ChatCommandItem[] = $derived([ ...this.sessionBuiltinCommands, ...this.globalSkills @@ -1836,6 +1904,25 @@ export class AIChatManager { } } + /** Give back the mentions a send carried when its text returns to the composer, + * so its `@` tokens still have entries to bind to. Additive, unlike the DOM + * restore above: dropping the entries this send did not carry would strand the + * tokens naming them in a draft whose text now shares the same composer. + * + * `originMode` is the mode the send was submitted in, and is required: the + * composer only consumes in GLOBAL, so reading the mode at restore time would + * strand a send whose mode changed mid-turn and resurrect chips for one that + * never consumed. Every caller states which mode it means. */ + #restoreMentionContext(context: ContextElement[] | undefined, originMode: AIMode) { + if (originMode !== AIMode.GLOBAL) return + const mentions = (context ?? []).filter(isMentionContext) + if (mentions.length === 0) return + const selection = this.contextManager?.getSelectedContext() ?? [] + const missing = mentions.filter((m) => !selection.some((s) => isSameContextElement(s, m))) + if (missing.length === 0) return + this.contextManager?.setSelectedContext([...selection, ...missing]) + } + /** Send `text` as a turn, or queue it when one is already streaming. Callers * that send programmatically (an editor button, an arriving hand-off) must go * through this rather than `sendRequest`: a second concurrent loop shares this @@ -1868,6 +1955,10 @@ export class AIChatManager { // is selected now. If its text was prepended onto an existing draft, that // draft's chips are kept too — both instructions now share one composer. this.#restoreDomContext(queued.context, mergedIntoDraft) + // The queue aggregates several enqueues into one entry and records no + // originating mode, so the mode now is the closest signal available. A + // recall after a mid-turn mode switch can therefore miss a restore. + this.#restoreMentionContext(queued.context, this.mode) } /** Put what the user typed back where they can see it: into the input @@ -2095,7 +2186,11 @@ export class AIChatManager { if (refreshId !== this.globalSkillsRefreshId) { return } - this.globalSkills = skills + // Newest-wins is not enough: a refresh for the workspace just left can still + // hold the newest id, and installing it would advertise that workspace's + // skills to a chat now acting elsewhere. Same check the identity and MCP + // refreshes make. + this.globalSkills = workspace === (this.operatingWorkspace ?? '') ? skills : [] if (this.mode === AIMode.GLOBAL) { this.configureGlobalMode() } @@ -2183,16 +2278,25 @@ export class AIChatManager { if (!this.isSessionChat || this.mode !== AIMode.GLOBAL || !instructions.startsWith('/')) { return instructions } - const match = /^\/([a-z0-9-]+)(?:\s+([\s\S]*))?$/.exec(instructions) + // Accepts a bare name or a whole resource path: names are what people type, + // but the picker inserts the path when two folders answer to the same name. + // Unicode-aware rather than `\w`, which is ASCII-only — a resource path may + // hold any word character, and the picker can insert one the user must then + // be able to send (`f/équipe/deploy`). + const match = /^\/([\p{L}\p{N}_\-/]+)(?:\s+([\s\S]*))?$/u.exec(instructions) if (!match) { return instructions } - const skill = this.globalSkills.find((s) => s.name === match[1]) - if (!skill) { + // A path identifies one skill; a name shared by two would otherwise silently + // apply instructions the user did not choose, so it is left unexpanded. + const byPath = this.globalSkills.find((s) => s.path === match[1]) + const matches = byPath ? [byPath] : this.globalSkills.filter((s) => s.name === match[1]) + if (matches.length !== 1) { return instructions } const rest = match[2]?.trim() - return rest ? `Use the "${skill.name}" skill. ${rest}` : `Use the "${skill.name}" skill.` + const use = `Use the skill at "${matches[0].path}".` + return rest ? `${use} ${rest}` : use } canApplyCode = $derived(this.allowedModes.script && this.mode === AIMode.SCRIPT) @@ -2246,6 +2350,10 @@ export class AIChatManager { } openChat = () => { + // Nothing may open the docked pane in a workspace that hid the assistant. + if (get(copilotInfo).workspaceDisabled) { + return + } chatState.size = this.savedSize > 0 ? this.savedSize : DEFAULT_SIZE localStorage.setItem('ai-chat-open', 'true') } @@ -2257,6 +2365,9 @@ export class AIChatManager { } toggleOpen = () => { + if (chatState.size === 0 && get(copilotInfo).workspaceDisabled) { + return + } if (chatState.size > 0) { this.savedSize = chatState.size } @@ -2776,6 +2887,41 @@ export class AIChatManager { sendUserToast('This action needs the AI chat. Start an AI session to continue.', true) return } + // The workspace hid the assistant: every entry point is gone from the UI, so a turn + // reaching here comes from a path that missed the gate and would stream unseen. + if (!this.isSessionChat && get(copilotInfo).workspaceDisabled) { + sendUserToast('Windmill AI is hidden in this workspace.', true) + return + } + // Refused before anything mutates, so there is nothing to unwind: the + // draft (already taken by the composer) goes back where the user can see + // it, and the turn never starts. Only the message's own send restores it + // — a refused queued flush is re-queued by its caller (`accepted === + // false`), and a copy here would double it. Paste tokens are expanded + // into the text, as the queue does, because the restore lanes carry no + // pastes. + if (this.runHeldElsewhere) { + if (options.synthetic) { + // Client-authored prompt (a job auto-resume), not user input: nothing + // to hand back and no toast. Releasing the staged text un-blocks the + // next auto-resume attempt, scheduled for when the hold clears. + this.instructions = '' + this.#scheduleAutoResumeRetry() + } else { + if (!options.queued) { + // Programmatic prompts (askAi, fix) stage their text in + // `this.instructions` and pass no option — fall back to it so + // they are handed back too. + this.restoreToInput( + expanded(chatDraft(options.instructions ?? this.instructions, options.pastes ?? [])), + options.images, + options.files + ) + } + sendUserToast('This session is running in another tab. Your message was kept.', true) + } + return false + } this.#sendsInFlight++ try { return await this.sendRequestImpl(options) @@ -2796,8 +2942,8 @@ export class AIChatManager { lang?: ScriptLang | 'bunnative' isPreprocessor?: boolean // Use this selected-context snapshot for the turn instead of the live - // contextManager. Set when flushing a queued message that captured its - // context at submit time; the live selection is left untouched. + // contextManager. Set whenever a send settles its context ahead of the + // turn: a composer submit at the click, a queued message at enqueue. contextOverride?: ContextElement[] /** Where `contextOverride` came from. 'pinned' (default): the chips were * selected for THIS message, so they are consumed from the live selection @@ -2947,12 +3093,18 @@ export class AIChatManager { // re-reserves them, so release this send's outgoing-files reservation. this.#releaseOutgoingReservation(reservationKey) if (!options.queued) { - this.aiChatInput?.restoreInstructions( - this.instructions, - pastes, - options.images ?? [], - options.files ?? [] - ) + // Reached only once the mode has already moved off GLOBAL, so the + // restore is keyed to requestedMode: a GLOBAL submit whose mode + // flipped during the upkeep awaits above still gets its mentions + // back, while a send that started outside GLOBAL consumed none. + const taken = + this.aiChatInput?.restoreInstructions( + this.instructions, + pastes, + options.images ?? [], + options.files ?? [] + ) === true + if (taken) this.#restoreMentionContext(options.contextOverride, requestedMode) } return false } @@ -2973,6 +3125,11 @@ export class AIChatManager { // put them back in the composer instead of silently discarding them // (the input already cleared itself optimistically on send). Queued // drafts are the caller's to restore (it re-queues on false). + // + // No mention restore: an entry exists only while its `@` token is in the + // text (the picker adds both, the textarea's sync drops the entry when + // the token goes), and this branch requires empty text. A mention source + // that does not write a token would break that and need one here. if (!this.instructions.trim() && files.length === 0) { sendUserToast(`${sendModel.model} can't read images. Switch to a vision model first.`, true) if (!options.queued) this.restoreToInput('', requestedImages) @@ -2984,6 +3141,28 @@ export class AIChatManager { ) } const images = modelIsBlind ? [] : requestedImages + // Re-checks the wrapper's remote-run guard: a run announced by another tab + // during the upkeep awaits above would otherwise interleave two turns into + // one chat id. Resends are exempt — restartGeneration already truncated + // the transcript, so they run as the documented advisory race instead. + if (this.runHeldElsewhere && !options.resendReservationKey) { + this.#releaseOutgoingReservation(reservationKey) + if (options.synthetic) { + // Same as the wrapper guard: an internal prompt is released, not + // restored as a draft the user never wrote. + this.instructions = '' + this.#scheduleAutoResumeRetry() + } else { + // restoreToInput, not restoreInstructions: a draft typed during the + // awaits above occupies the composer, and this restore must merge + // into it (or queue), never be refused by it. + if (!options.queued) { + this.restoreToInput(expanded(chatDraft(this.instructions, pastes)), images, files) + } + sendUserToast('This session is running in another tab. Your message was kept.', true) + } + return false + } const optimisticIndex = this.displayMessages.length this.loading = true // Create the abort controller before the (possibly slow) beforeSend pre-flight, @@ -3029,7 +3208,13 @@ export class AIChatManager { console.error('AIChatManager beforeSend hook failed', e) rollbackOptimisticSend() if (!options.queued) { - this.aiChatInput?.restoreInstructions(this.instructions, pastes, images, files) + // Mentions were consumed at submit, so they come back with the text or + // not at all. Only when the composer took it: it declines when the + // user has started a new draft, and restoring then would put these + // mentions on that draft and every turn after it. + const taken = + this.aiChatInput?.restoreInstructions(this.instructions, pastes, images, files) === true + if (taken) this.#restoreMentionContext(options.contextOverride, requestedMode) } sendUserToast( `Could not prepare the session before sending: ${ @@ -3069,7 +3254,11 @@ export class AIChatManager { }) if (accepted === false) this.#restoreQueue(next) } else { - this.aiChatInput?.restoreInstructions(this.instructions, pastes, images, files) + // Same pairing as the beforeSend catch above: mentions ride back with the + // text, only if the composer took it. + const taken = + this.aiChatInput?.restoreInstructions(this.instructions, pastes, images, files) === true + if (taken) this.#restoreMentionContext(options.contextOverride, requestedMode) } return true } @@ -3164,9 +3353,10 @@ export class AIChatManager { hideTarget?.removeEventListener('visibilitychange', checkpointOnHide) } try { - // A queued message carries its own context snapshot (contextOverride); use - // it verbatim and leave the live selection alone (it belongs to whatever the - // user has selected since). Otherwise read the current selection. + // A pinned snapshot (a queued message, or a composer submit settling its + // context at the click) is used verbatim, leaving the live selection alone — + // it belongs to whatever the user has selected since. Otherwise read the + // current selection. const oldSelectedContext = options.contextOverride ?? this.contextManager?.getSelectedContext() ?? [] // DOM selector chips are one-shot: they ride with this message (captured in @@ -3177,9 +3367,9 @@ export class AIChatManager { // context, consumed on its original send. The live selection belongs to // the composer's own draft — touching it here would strip it. } else if (options.contextOverride) { - // Queued message: only the chips it carried are consumed. Drop just - // those from the live selection (still there if the user didn't - // re-select); a newer selection made since is left intact. + // A pinned submit consumes only the chips it carried. Drop just those + // from the live selection (still there if the user didn't re-select); a + // newer selection made since is left intact. for (const c of options.contextOverride) { if (c.type === 'app_dom_selector') { // Match appPath too: another app's live chip can share this @@ -3598,6 +3788,7 @@ export class AIChatManager { // retarget whatever draft is sitting there. if (textRestored) { this.#restoreDomContext(oldSelectedContext) + this.#restoreMentionContext(oldSelectedContext, requestedMode) } if (this.displayMessages.length === 0) { // saveChat no-ops on an empty transcript; the chat persisted earlier @@ -3713,6 +3904,9 @@ export class AIChatManager { // releases the loop; it never discards uncommitted text. this.replyReveal.reset() this.reasoningReveal.reset() + // Refresh the free-tier usage meter after every turn (success or error), and + // let the turn that exhausts the grant flip to the exhausted state live. + void refreshFreeTierUsage(this.operatingWorkspace) } // Flush the queued message. Send it after a cleanly committed turn OR a // deliberate user cancel (Esc / Stop) — in both cases the user is ready @@ -3810,6 +4004,29 @@ export class AIChatManager { throw new Error('No user message found at the specified index') } + // Refused before anything mutates: past this point the transcript is + // sliced and resend bytes are reserved, and the sendRequest guard could + // only refuse AFTER that damage — restoring nothing, since this path + // carries its text in `this.instructions`, not the options. The retry and + // edit controls check only local `loading`, so a remote run reaches here. + // An edit (newContent defined, even '': attachment-only edits exist) is + // restored with its pastes expanded into the text; a bare retry mutates + // nothing yet, so there is nothing to restore. Un-submitted context-chip + // edits are the one loss — the chips re-seed from the untouched message + // on the next edit. + if (this.runHeldElsewhere) { + if (newContent !== undefined) { + this.restoreToInput( + expanded(chatDraft(newContent, pastes ?? [])), + images ?? [], + files ?? [] + ) + } + // "Text", not "message": chip edits are the part that does not survive. + sendUserToast('This session is running in another tab. Your text was kept.', true) + return + } + // Resolve the API restart point BEFORE reserving bytes or truncating: a // stale index must fail while nothing has been mutated, or the transcript // would be left truncated with the reservation leaked. A negative index @@ -3933,7 +4150,7 @@ export class AIChatManager { this.onChatRotated?.(this.historyManager.getCurrentChatId()) } - loadPastChat = async (id: string) => { + loadPastChat = async (id: string, { preserveQueue = false } = {}) => { // A turn commits into whatever transcript it finds when it ends, so swapping // one in underneath it misfiles the turn — or duplicates it, when the loaded // chat already carries the turn's own checkpoint. Gated on `sendInFlight` @@ -3943,7 +4160,10 @@ export class AIChatManager { if (chat) { // Drop any message queued in the current conversation so it doesn't // auto-send into the loaded one or linger as a card across the switch. - this.#clearQueue() + // `preserveQueue` is for reloads that are NOT a switch — a cross-tab + // catch-up re-reading the conversation on screen — where the queued + // draft is unsent user input the reload must not destroy. + if (!preserveQueue) this.#clearQueue() // Stop the poller for the conversation being left before swapping in the // loaded chat's jobs below. this.clearBackgroundJobs() diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts index f788844135..084dc2c040 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts @@ -7,6 +7,7 @@ import type { ChatCompletionMessageParam } from 'openai/resources/chat/completio import type { DisplayMessage } from './shared' import type { AttachedImage } from './imageUtils' import { AIChatManager, AIMode, AIAutonomyMode } from './AIChatManager.svelte' +import { makePasteToken } from './pasteTokens' import { chatState } from './sharedChatState.svelte' import { PLAN_MODE_MESSAGES } from './planModeMessages' import { runChatLoop } from './chatLoop' @@ -33,7 +34,7 @@ const mocks = vi.hoisted(() => ({ getAnthropicClient: vi.fn(), getNonStreamingCompletion: vi.fn(), runChatLoop: vi.fn(), - listAiSkills: vi.fn(), + listResource: vi.fn(), getJob: vi.fn(), whoami: vi.fn(), workspace: 'test_workspace' as string | undefined, @@ -49,8 +50,9 @@ vi.mock('monaco-editor', () => ({ vi.mock('$lib/utils/featureUsage', () => ({ logFeatureUsage: vi.fn() })) vi.mock('$lib/gen', () => ({ - WorkspaceService: { - listAiSkills: mocks.listAiSkills + WorkspaceService: {}, + ResourceService: { + listResource: mocks.listResource }, ScriptService: {}, FlowService: {}, @@ -169,7 +171,7 @@ beforeEach(() => { mocks.isWebSearchEnabledForProvider.mockReturnValue(true) mocks.getOpenaiClient.mockReturnValue({}) mocks.getAnthropicClient.mockReturnValue({}) - mocks.listAiSkills.mockResolvedValue([]) + mocks.listResource.mockResolvedValue([]) mocks.workspace = 'test_workspace' mocks.runChatLoop.mockResolvedValue({ addedMessages: [], @@ -330,17 +332,29 @@ describe('AIChatManager global skills', () => { mocks.tryGetCurrentModel.mockReturnValue(model) }) + // Only selected skills reach the prompt, and the selection is keyed by + // workspace and account (see skills/enabledSkills.ts). + function selectSkills(workspace: string, ...paths: string[]) { + const stored = JSON.parse(localStorage.getItem('wm_skills_enabled') ?? '{}') + stored[`${workspace}:${TEST_EMAIL}`] = paths + localStorage.setItem('wm_skills_enabled', JSON.stringify(stored)) + } + it('loads skills after beforeSend commits the session workspace', async () => { - let resolveParentSkills: ((skills: { name: string; description: string }[]) => void) | undefined - const parentSkills = new Promise<{ name: string; description: string }[]>((resolve) => { + let resolveParentSkills: ((skills: unknown[]) => void) | undefined + const parentSkills = new Promise((resolve) => { resolveParentSkills = resolve }) mocks.workspace = 'parent' - mocks.listAiSkills.mockImplementation(({ workspace }: { workspace: string }) => { + selectSkills('parent', 'f/skills/parent-skill') + selectSkills('child', 'f/skills/child-skill') + mocks.listResource.mockImplementation(({ workspace }: { workspace: string }) => { if (workspace === 'parent') { return parentSkills } - return Promise.resolve([{ name: 'child-skill', description: 'child workspace skill' }]) + return Promise.resolve([ + { path: 'f/skills/child-skill', description: 'child workspace skill' } + ]) }) mocks.runChatLoop.mockImplementation(async (config: any) => { expect(config.workspace).toBe('child') @@ -362,22 +376,45 @@ describe('AIChatManager global skills', () => { } await manager.sendRequest({ instructions: 'first', mode: AIMode.GLOBAL }) - resolveParentSkills?.([{ name: 'parent-skill', description: 'parent workspace skill' }]) + resolveParentSkills?.([ + { path: 'f/skills/parent-skill', description: 'parent workspace skill' } + ]) await Promise.resolve() - expect(mocks.listAiSkills).toHaveBeenCalledWith({ workspace: 'parent' }) - expect(mocks.listAiSkills).toHaveBeenCalledWith({ workspace: 'child' }) + expect(mocks.listResource).toHaveBeenCalledWith( + expect.objectContaining({ workspace: 'parent', resourceType: 'ai_skill' }) + ) + expect(mocks.listResource).toHaveBeenCalledWith( + expect.objectContaining({ workspace: 'child', resourceType: 'ai_skill' }) + ) expect(manager.systemMessage.content).toContain('child-skill') expect(manager.systemMessage.content).not.toContain('parent-skill') }) - it('expands a leading slash skill command for the model while preserving the displayed text', async () => { - mocks.listAiSkills.mockResolvedValue([ - { name: 'review-code', description: 'review code for bugs' } + it('leaves a readable but unselected skill out of the prompt', async () => { + mocks.listResource.mockResolvedValue([ + { path: 'f/skills/selected', description: 'the one turned on' }, + { path: 'f/skills/unselected', description: 'readable but never turned on' } ]) + selectSkills('test_workspace', 'f/skills/selected') + + const manager = new AIChatManager() + manager.isSessionChat = true + await manager.refreshGlobalSkills('test_workspace') + await manager.changeMode(AIMode.GLOBAL) + + expect(manager.systemMessage.content).toContain('f/skills/selected') + expect(manager.systemMessage.content).not.toContain('f/skills/unselected') + }) + + it('expands a leading slash skill command for the model while preserving the displayed text', async () => { + mocks.listResource.mockResolvedValue([ + { path: 'u/admin/review-code', description: 'review code for bugs' } + ]) + selectSkills('test_workspace', 'u/admin/review-code') mocks.runChatLoop.mockImplementation(async (config: any) => { const userMessage = config.messages[config.messages.length - 1] - expect(userMessage.content).toContain('Use the "review-code" skill. find bugs') + expect(userMessage.content).toContain('Use the skill at "u/admin/review-code". find bugs') expect(userMessage.content).not.toContain('/review-code find bugs') const message = { role: 'assistant' as const, content: 'done' } config.addedMessages?.push(message) @@ -395,6 +432,33 @@ describe('AIChatManager global skills', () => { expect(manager.displayMessages[0]?.content).toBe('/review-code find bugs') }) + + it('does not expand a slash command two folders both answer to', async () => { + mocks.listResource.mockResolvedValue([ + { path: 'u/admin/deploy', description: 'personal deploy steps' }, + { path: 'f/team/deploy', description: 'the team deploy steps' } + ]) + selectSkills('test_workspace', 'u/admin/deploy', 'f/team/deploy') + mocks.runChatLoop.mockImplementation(async (config: any) => { + // Picking either one would silently apply instructions the user did not + // choose, so the text is left alone for the model to ask about. + const userMessage = config.messages[config.messages.length - 1] + expect(userMessage.content).toContain('/deploy ship it') + expect(userMessage.content).not.toContain('Use the skill at') + const message = { role: 'assistant' as const, content: 'done' } + config.addedMessages?.push(message) + return { + addedMessages: [message], + tokenUsage: { prompt: 0, completion: 0, total: 0 }, + hitMaxIterations: false + } + }) + + const manager = new AIChatManager() + manager.isSessionChat = true + + await manager.sendRequest({ instructions: '/deploy ship it', mode: AIMode.GLOBAL }) + }) }) describe('AIChatManager global prompt identity', () => { @@ -404,7 +468,7 @@ describe('AIChatManager global prompt identity', () => { localStorage.clear() mocks.getCurrentModel.mockReturnValue(model) mocks.tryGetCurrentModel.mockReturnValue(model) - mocks.listAiSkills.mockResolvedValue([]) + mocks.listResource.mockResolvedValue([]) }) afterEach(() => { @@ -2164,6 +2228,98 @@ describe('AIChatManager queued messages', () => { expect(chips.map((c) => c.selector)).toEqual(['div.card']) }) + // The composer consumes an `@` mention on send, so a send that never became a + // turn has to give it back — otherwise the restored text keeps its `@` token + // with nothing behind it. + it('restores a cancelled GLOBAL send’s @ mentions', async () => { + const manager = createManager(createInputMock()) + manager.mode = AIMode.GLOBAL + const cm = manager.contextManager + const mention = { + type: 'workspace_script' as const, + path: 'f/etl/sync', + title: 'f/etl/sync' + } + // What the composer does at submit: pin what it carries, then consume. + const carried = [mention] + cm.setSelectedContext([]) + mocks.runChatLoop.mockImplementationOnce(async ({ abortController }: any) => { + abortController.abort('user_cancelled') + throw new Error('aborted') + }) + + await manager.sendRequest({ + instructions: 'why does this retry', + contextOverride: carried, + contextOverrideOrigin: 'pinned' + }) + + expect(cm.getSelectedContext()).toEqual([mention]) + }) + + // The mode switcher stays live while a turn streams, so the restore is keyed + // to the mode the send was submitted in. Reading the mode at rollback time + // would strand the mention behind a `@` token that resolves to nothing. + it('restores a GLOBAL send’s @ mentions after a mid-turn switch to SCRIPT', async () => { + const manager = createManager(createInputMock()) + manager.mode = AIMode.GLOBAL + const cm = manager.contextManager + const mention = { type: 'workspace_script' as const, path: 'f/etl/sync', title: 'f/etl/sync' } + cm.setSelectedContext([]) + mocks.runChatLoop.mockImplementationOnce(async ({ abortController }: any) => { + // The user navigates to a script editor while the turn is streaming. + manager.mode = AIMode.SCRIPT + abortController.abort('user_cancelled') + throw new Error('aborted') + }) + + await manager.sendRequest({ + instructions: 'why does this retry', + contextOverride: [mention], + contextOverrideOrigin: 'pinned' + }) + + expect(cm.getSelectedContext()).toEqual([mention]) + }) + + // Attachments are refused outside GLOBAL, and the refusal sits past the + // preflight awaits — so it is reached exactly when a GLOBAL submit's mode + // changed underneath it, and owes that send its mentions back. + it('restores mentions when a GLOBAL send is refused for switching modes with attachments', async () => { + const manager = createManager(createInputMock()) + manager.mode = AIMode.GLOBAL + const cm = manager.contextManager + const mention = { type: 'workspace_script' as const, path: 'f/etl/sync', title: 'f/etl/sync' } + cm.setSelectedContext([]) + + const pending = manager.sendRequest({ + instructions: 'describe this image', + images: [{ id: 'img-1', dataUrl: 'data:image/png;base64,AAAA' } as any], + contextOverride: [mention], + contextOverrideOrigin: 'pinned' + }) + manager.mode = AIMode.SCRIPT + await pending + + expect(cm.getSelectedContext()).toEqual([mention]) + }) + + // The editor modes show mentions as chips the user deletes by hand, so the + // restore must never re-add one they removed. + it('leaves an editor mode’s context alone when a queued draft comes back', () => { + const manager = createManager(createInputMock()) + manager.mode = AIMode.SCRIPT + const cm = manager.contextManager + const mention = { type: 'workspace_script' as const, path: 'f/etl/sync', title: 'f/etl/sync' } + manager.queueMessage('fix this', [], [mention]) + // The user deletes the chip while the turn streams. + cm.setSelectedContext([]) + + manager.dequeueMessage() + + expect(cm.getSelectedContext()).toEqual([]) + }) + it('restores a dequeued inline prompt’s pinned DOM context, replacing the live selection', () => { const manager = createManager(createInputMock()) const cm = manager.contextManager @@ -2281,6 +2437,90 @@ describe('AIChatManager queued messages', () => { expect(manager.displayMessages).toHaveLength(0) }) + // The composer consumes mentions before calling in, so a send that never left + // the preflight has to give them back with the text. GLOBAL renders no chip for + // them, so a mention lost here is invisible: the restored `@` token silently + // resolves to nothing, and a mention-only draft comes back empty. + it('restores consumed mentions when beforeSend rejects a GLOBAL send', async () => { + const input = createInputMock() + const manager = createManager(input) + manager.mode = AIMode.GLOBAL + const mention = { type: 'workspace_script' as const, path: 'f/etl/sync', title: 'f/etl/sync' } + manager.contextManager.setSelectedContext([]) + manager.beforeSend = vi.fn().mockRejectedValue(new Error('workspace fork failed')) + + const accepted = await manager.sendRequest({ + instructions: '@f/etl/sync fix this', + contextOverride: [mention], + contextOverrideOrigin: 'pinned' + }) + + expect(accepted).toBe(false) + expect(manager.contextManager.getSelectedContext()).toEqual([mention]) + }) + + it('restores consumed mentions when a GLOBAL send is cancelled during the preflight', async () => { + const input = createInputMock() + const manager = createManager(input) + manager.mode = AIMode.GLOBAL + const mention = { type: 'workspace_script' as const, path: 'f/etl/sync', title: 'f/etl/sync' } + manager.contextManager.setSelectedContext([]) + // Stop/Escape while "Creating workspace fork..." is showing. + manager.beforeSend = vi.fn().mockImplementation(async () => { + manager.cancel('user_cancelled') + }) + + await manager.sendRequest({ + instructions: '@f/etl/sync fix this', + contextOverride: [mention], + contextOverrideOrigin: 'pinned' + }) + + expect(mocks.runChatLoop).not.toHaveBeenCalled() + expect(manager.contextManager.getSelectedContext()).toEqual([mention]) + }) + + // The gate, and the reason the restore is not unconditional: an occupied + // composer declines the dead draft's text, and its mentions would then ride the + // draft the user is writing now and every turn after it. + it('does not restore mentions into a draft the composer kept', async () => { + const input = createInputMock() + input.restoreInstructions.mockReturnValue(false) + const manager = createManager(input) + manager.mode = AIMode.GLOBAL + const mention = { type: 'workspace_script' as const, path: 'f/etl/sync', title: 'f/etl/sync' } + manager.contextManager.setSelectedContext([]) + manager.beforeSend = vi.fn().mockRejectedValue(new Error('workspace fork failed')) + + await manager.sendRequest({ + instructions: '@f/etl/sync fix this', + contextOverride: [mention], + contextOverrideOrigin: 'pinned' + }) + + expect(manager.contextManager.getSelectedContext()).toEqual([]) + }) + + // The bit-identity tripwire: an editor mode reaches these same bailouts with a + // contextOverride (a replay, a queued flush) and must come out of them with the + // selection main would have left. + it('leaves an editor mode’s selection untouched through the same bailout', async () => { + const input = createInputMock() + const manager = createManager(input) + manager.mode = AIMode.SCRIPT + const mention = { type: 'workspace_script' as const, path: 'f/etl/sync', title: 'f/etl/sync' } + manager.contextManager.setSelectedContext([]) + manager.beforeSend = vi.fn().mockRejectedValue(new Error('workspace fork failed')) + + await manager.sendRequest({ + instructions: 'fix this', + contextOverride: [mention], + contextOverrideOrigin: 'pinned' + }) + + expect(manager.contextManager.getSelectedContext()).toEqual([]) + }) + it('drops the queued message when switching conversations (no cross-chat leak)', async () => { const manager = createManager(createInputMock()) @@ -2923,8 +3163,8 @@ describe('AIChatManager manual compaction', () => { vi.clearAllMocks() mocks.getCurrentModel.mockReturnValue(model) mocks.tryGetCurrentModel.mockReturnValue(model) - // changeMode(GLOBAL) refreshes workspace skills; keep it a no-op here. - mocks.listAiSkills.mockResolvedValue([]) + // changeMode(GLOBAL) refreshes the selected skills; keep it a no-op here. + mocks.listResource.mockResolvedValue([]) }) function seedExchange(manager: AIChatManager) { @@ -3139,15 +3379,19 @@ describe('AIChatManager manual compaction', () => { expect(mocks.getNonStreamingCompletion).not.toHaveBeenCalled() }) - it('shadows a workspace skill that collides with a built-in command', () => { + it('shadows a selected skill that collides with a built-in command', () => { const manager = new AIChatManager() manager.globalSkills = [ - { name: 'compact', description: 'a workspace skill that happens to be named compact' }, - { name: 'review-code', description: 'review code for bugs' } + { + path: 'u/admin/compact', + name: 'compact', + description: 'a skill that happens to be named compact' + }, + { path: 'u/admin/review-code', name: 'review-code', description: 'review code for bugs' } ] - // Built-ins come first and the colliding skill is dropped, so the picker - // never renders two leaves with the same `skill:compact` key. + // Built-ins come first and the colliding skill is dropped: the built-in + // wins at execution too, so listing both would offer a row that cannot run. const names = manager.sessionCommands.map((c) => c.name) expect(names).toEqual(['compact', 'clear', 'review-code']) expect(manager.sessionCommands[0].description).toBe( @@ -3754,3 +3998,135 @@ describe('AIChatManager reasoning duration', () => { expect(assistantDurations(manager)).toEqual([3_000, 7_000]) }) }) + +describe('AIChatManager cross-tab run seams', () => { + // The whole cross-tab feature hangs off these two seams: `loading`'s edges + // are the "running here" / "safe to re-read" signals, and the resolver is + // the advisory lock. Reverting `loading` to a plain $state field would + // silently disconnect every tab. + it('reports loading transitions, and only transitions, through onRunningChanged', () => { + const manager = new AIChatManager() + const seen: boolean[] = [] + manager.onRunningChanged = (running) => seen.push(running) + manager.loading = true + manager.loading = true + manager.loading = false + manager.loading = false + expect(seen).toEqual([true, false]) + }) + + it('refuses a send while another tab holds the run, keeping the draft', async () => { + const manager = new AIChatManager() + manager.isSessionChat = true + manager.runHeldElsewhereResolver = () => true + + const accepted = await manager.sendRequest({ instructions: 'race loser' }) + + expect(accepted).toBe(false) + expect(mocks.runChatLoop).not.toHaveBeenCalled() + expect(manager.loading).toBe(false) + // restoreToInput falls back to the queued draft when no composer is + // mounted, so the refused text must surface there rather than vanish. + expect(manager.queuedMessage).toBe('race loser') + }) + + // A synthetic (auto-resume) prompt is client-authored: a refusal must + // release it rather than hand it back as a draft the user never wrote — + // staged instructions would otherwise block every later auto-resume. + it('releases a refused synthetic send instead of restoring it as a draft', async () => { + const manager = new AIChatManager() + manager.isSessionChat = true + manager.runHeldElsewhereResolver = () => true + manager.instructions = 'A background job just finished.' + + const accepted = await manager.sendRequest({ synthetic: true }) + + expect(accepted).toBe(false) + expect(manager.instructions).toBe('') + expect(manager.queuedMessage).toBe('') + }) + + // The restore lanes carry no pastes, so a refusal must expand the tokens + // into the text — dangling markers with the content gone otherwise. + it('expands paste tokens into the text a refusal hands back', async () => { + const manager = new AIChatManager() + manager.isSessionChat = true + manager.runHeldElsewhereResolver = () => true + const paste = { id: 1, lines: 1, content: 'the pasted block' } + + await manager.sendRequest({ + instructions: `see ${makePasteToken(paste)}`, + pastes: [paste] + }) + + expect(manager.queuedMessage).toBe('see the pasted block') + }) + + // The wrapper's check runs before the attachment upkeep awaits; a run + // announced by another tab during that upkeep must still be refused before + // the turn takes visible effect. + it('refuses a run announced by another tab during the preflight awaits', async () => { + const manager = new AIChatManager() + manager.isSessionChat = true + let held = false + manager.runHeldElsewhereResolver = () => held + let releaseUpkeep: (() => void) | undefined + vi.spyOn(manager.attachedFiles, 'refreshFolders').mockImplementation( + () => new Promise((resolve) => (releaseUpkeep = resolve)) + ) + + const sending = manager.sendRequest({ instructions: 'racing turn' }) + await vi.waitFor(() => expect(manager.sendInFlight).toBe(true)) + held = true + releaseUpkeep?.() + + expect(await sending).toBe(false) + expect(mocks.runChatLoop).not.toHaveBeenCalled() + expect(manager.loading).toBe(false) + }) + + it('refuses a retry/edit while another tab holds the run, before mutating the transcript', async () => { + const manager = new AIChatManager() + manager.isSessionChat = true + manager.displayMessages = [ + { role: 'user', content: 'original prompt', index: 0 }, + { role: 'assistant', content: 'original reply' } + ] as DisplayMessage[] + manager.messages = [ + { role: 'user', content: 'original prompt' } + ] as ChatCompletionMessageParam[] + manager.runHeldElsewhereResolver = () => true + + await manager.restartGeneration(0, 'edited prompt') + + expect(mocks.runChatLoop).not.toHaveBeenCalled() + expect(manager.displayMessages).toHaveLength(2) + expect(manager.messages).toHaveLength(1) + // The edited text survives the refusal via restoreToInput's queued-draft + // fallback. + expect(manager.queuedMessage).toBe('edited prompt') + }) + + // A cross-tab catch-up re-reads the conversation on screen; the queued + // draft is unsent user input (possibly the refusal's kept message) that + // this non-switch reload must not destroy — while a real conversation + // switch still drops it. + it('keeps the queued draft when a catch-up reload preserves it', async () => { + const manager = new AIChatManager() + manager.isSessionChat = true + vi.spyOn(manager.historyManager, 'loadPastChat').mockResolvedValue({ + id: 'c1', + actualMessages: [], + displayMessages: [], + title: '', + lastModified: 1 + } as never) + + manager.queueMessage('kept across catch-up') + await manager.loadPastChat('c1', { preserveQueue: true }) + expect(manager.queuedMessage).toBe('kept across catch-up') + + await manager.loadPastChat('c1') + expect(manager.queuedMessage).toBe('') + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/AIChatMessage.svelte b/frontend/src/lib/components/copilot/chat/AIChatMessage.svelte index 61d9a47966..12553be961 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatMessage.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatMessage.svelte @@ -13,9 +13,21 @@ import { messageDraft, segments } from './chatDraft' import { lineCountLabel } from './pasteTokens' import ExpandableImage from '$lib/components/common/image/ExpandableImage.svelte' + import { workspaceStore } from '$lib/stores' const aiChatManager = getAiChatManager() + // Paths in a message name items the chat's tools reach, so they resolve against the + // operating workspace, never `workspaceStore`: a fork session leaves the store on the + // navigated workspace, where a fork-only item resolves to nothing and the rest resolve + // to a different copy. + const messageWorkspace = $derived.by(() => { + // Registers the dependency that `operatingWorkspace`'s own untracked + // `get(workspaceStore)` cannot. + void $workspaceStore + return aiChatManager.operatingWorkspace + }) + // Per-message expand/collapse state for paste chips shown in the bubble. let expandedPastes = $state>(new Set()) @@ -119,7 +131,7 @@ {:else}
    {#if message.role === 'assistant'} -
    +
    {:else if message.role === 'tool'}
    = 80) + let capability = $derived( getReasoningCapability(providerModel.provider as AIProvider, providerModel.model) ) @@ -312,6 +319,13 @@ {#if effortLabel} · {effortLabel} {/if} + {#if freeTier && !freeTier.exhausted} + Free + {/if}
    diff --git a/frontend/src/lib/components/copilot/chat/AiChatLayout.svelte b/frontend/src/lib/components/copilot/chat/AiChatLayout.svelte index 1a9df4816c..4f3ef368ad 100644 --- a/frontend/src/lib/components/copilot/chat/AiChatLayout.svelte +++ b/frontend/src/lib/components/copilot/chat/AiChatLayout.svelte @@ -7,6 +7,7 @@ import { userStore, workspaceStore } from '$lib/stores' import { chatState } from './sharedChatState.svelte' import { loadCopilot } from '$lib/components/copilot/loadCopilot' + import { copilotInfo } from '$lib/aiStore' import { aiChatManager } from './AIChatManager.svelte' import { onDestroy } from 'svelte' import Button from '$lib/components/common/button/Button.svelte' @@ -66,6 +67,14 @@ } }) + // The pane restores its last open state from localStorage before the config can say + // the workspace hid the assistant; close it as soon as that is known. + $effect(() => { + if ($copilotInfo.workspaceDisabled && chatState.size > 0) { + aiChatManager.closeChat() + } + }) + const historyManager = aiChatManager.historyManager historyManager.init() diff --git a/frontend/src/lib/components/copilot/chat/AssistantMessage.svelte b/frontend/src/lib/components/copilot/chat/AssistantMessage.svelte index 787d9ef4fc..f06941c2a4 100644 --- a/frontend/src/lib/components/copilot/chat/AssistantMessage.svelte +++ b/frontend/src/lib/components/copilot/chat/AssistantMessage.svelte @@ -6,7 +6,6 @@ import { thinkingPreferences } from './thinkingPreferences.svelte' import CodeDisplay from './script/CodeDisplay.svelte' import LinkRenderer from './LinkRenderer.svelte' - import { workspaceStore } from '$lib/stores' import { extractCandidatePaths, remarkWindmillPaths, @@ -16,9 +15,12 @@ interface Props { message: DisplayMessage + // Workspace the message's paths are resolved against: the one the chat + // operates on, which is not always the one being navigated. + workspace: string | undefined } - let { message }: Props = $props() + let { message, workspace }: Props = $props() const reasoning = $derived( message.role === 'assistant' ? message.reasoning?.trim() || undefined : undefined @@ -69,12 +71,11 @@ // Only populate the registry for messages that contain path-shaped tokens. The // registry still dedups concurrent calls across messages and workspaces. $effect(() => { - const ws = $workspaceStore - if (ws && candidatePaths.length > 0) workspaceItemRegistry.ensureLoaded(ws) + if (workspace && candidatePaths.length > 0) workspaceItemRegistry.ensureLoaded(workspace) }) const plugins = $derived.by(() => { - const ws = $workspaceStore ?? '' + const ws = workspace ?? '' if (!ws || candidatePaths.length === 0) { return [gfmPlugin(), rendererPlugin] } diff --git a/frontend/src/lib/components/copilot/chat/ChatCommandPicker.svelte b/frontend/src/lib/components/copilot/chat/ChatCommandPicker.svelte index 995a84708c..5c507deb33 100644 --- a/frontend/src/lib/components/copilot/chat/ChatCommandPicker.svelte +++ b/frontend/src/lib/components/copilot/chat/ChatCommandPicker.svelte @@ -2,6 +2,7 @@ import DrillPicker from '$lib/components/DrillPicker.svelte' import type { DrillLeaf, DrillNode } from '$lib/components/drillPicker' import type { ChatCommandItem } from './global/core' + import { ambiguousSkillNames } from './skills/skillResources' interface Props { skills: ChatCommandItem[] @@ -24,15 +25,22 @@ skill: 'Skills' } - // No `secondary`: rows show just the command; the full description lives in - // the hover tooltip (rowTooltip below). It stays in `searchableText` so - // filtering by description keeps working. + // Two folders can each hold a skill of the same name, and `/name` cannot then + // say which one is meant. Those rows show their path so the two are at least + // distinguishable; unambiguous rows stay bare, with the description in the + // hover tooltip (rowTooltip below) and in `searchableText` so filtering by it + // keeps working. + const ambiguous = $derived(ambiguousSkillNames(skills.filter((s) => s.path !== undefined))) + const tree = $derived[]>( skills.map((skill) => ({ type: 'leaf' as const, - key: `skill:${skill.name}`, + // Keyed by path where there is one: names are not unique across folders, + // and a duplicate key breaks the keyed list and its ambiguous-resolve nav. + key: `skill:${skill.path ?? skill.name}`, label: `/${skill.name}`, - searchableText: `${skill.name} ${skill.description}`, + secondary: ambiguous.has(skill.name) ? skill.path : undefined, + searchableText: `${skill.name} ${skill.path ?? ''} ${skill.description}`, section: skill.kind ? SECTION_LABELS[skill.kind] : undefined, data: skill })) diff --git a/frontend/src/lib/components/copilot/chat/ContextManager.svelte.ts b/frontend/src/lib/components/copilot/chat/ContextManager.svelte.ts index af3b402f9c..9191de5408 100644 --- a/frontend/src/lib/components/copilot/chat/ContextManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/ContextManager.svelte.ts @@ -2,7 +2,12 @@ import { ResourceService, type Flow, type ListResourceResponse, type ScriptLang import { scriptLangToEditorLang } from '$lib/scripts' import { SQLSchemaLanguages, type DBSchemas } from '$lib/stores' import { diffLines } from 'diff' -import { createAppDomSelectorElement, type ContextElement, type FlowModuleElement } from './context' +import { + createAppDomSelectorElement, + isMentionContext, + type ContextElement, + type FlowModuleElement +} from './context' import type { FlowModule } from '$lib/gen' import type { DisplayMessage } from './shared' @@ -330,6 +335,13 @@ export default class ContextManager { this.selectedContext = newSelectedContext } + /** Callers must gate on mode: only a GLOBAL send owns its mentions. Outside + * GLOBAL these are chips the user removes by hand, and dropping them here + * would delete a selection they still expect to see. */ + consumeMentionContext() { + this.selectedContext = this.selectedContext.filter((c) => !isMentionContext(c)) + } + getAvailableContext() { return this.availableContext } diff --git a/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte b/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte index 2502cd075b..8ae3b51b56 100644 --- a/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte +++ b/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte @@ -575,7 +575,10 @@ function getCommandFilter(text: string): string | undefined { if (aiChatManager.mode !== AIMode.GLOBAL || !aiChatManager.isSessionChat) return undefined - const match = /^\/([a-z0-9-]*)$/.exec(text) + // Same character set the submit path expands, so a name the picker can insert + // does not close the picker as soon as it is typed. Paths reach here too, via + // the row inserted for an ambiguous name. + const match = /^\/([\p{L}\p{N}_\-/]*)$/u.exec(text) return match?.[1] } @@ -640,8 +643,12 @@ } } - function handleCommandSelection(skill: { name: string }) { - value = `/${skill.name} ` + function handleCommandSelection(skill: { name: string; path?: string }) { + // The picker lists a row per skill, so two folders holding the same name are + // two distinct rows — but `/name` could not say which one was clicked, and + // submission refuses to guess. Those insert the path the row stands for. + const ambiguous = commandSkills.filter((c) => c.name === skill.name).length > 1 + value = `/${ambiguous && skill.path ? skill.path : skill.name} ` showCommandTooltip = false setTimeout(() => textarea?.focus(), 0) } @@ -736,9 +743,9 @@ textarea?.focus() } - // Wipe after dispatching a send: pre-zero `prevMentionedTitles` so the - // effect above sees no diff when `value` clears, leaving `selectedContext` - // untouched until `AIChatManager.beforeSend` snapshots it. A manual + // Wipe after dispatching a send: pre-zero `prevMentionedTitles` so the effect + // above sees no diff when `value` clears, leaving `selectedContext` for the + // send that is already carrying it to settle (see the caller). A manual // textarea clear by the user keeps the old behaviour (badges drop). export function clearForSend() { prevMentionedTitles = new Set() @@ -763,8 +770,17 @@ +
    @@ -818,6 +834,7 @@ // @tailwindcss/forms border, focus ring, and background so only the // wrapper reads as the field. '!border-transparent !bg-transparent !shadow-none focus:!border-transparent focus:!ring-0', + 'disabled:cursor-not-allowed disabled:placeholder:text-disabled', CHAT_INPUT_PADDING, className )} diff --git a/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte b/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte index ffa06ca5df..96b452120b 100644 --- a/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte +++ b/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte @@ -3,7 +3,7 @@ import { getKnownModelContextWindow, getModelContextWindow } from '../modelConfig' import { getAiChatManager } from './aiChatManagerContext' import { AIMode } from './AIChatManager.svelte' - import Tooltip from '$lib/components/meltComponents/Tooltip.svelte' + import UsageMeter from './UsageMeter.svelte' import { formatTokenCount } from './tokenUsage' const aiChatManager = getAiChatManager() @@ -49,25 +49,11 @@ {#if visible} - - -
    -
    -
    -
    -
    - {#snippet text()} + + + {#snippet tooltip()}

    Context usage

    @@ -85,5 +71,5 @@ {/if}

    {/snippet} -     + {/if} diff --git a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts index 24d7284b95..592b6217c8 100644 --- a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts @@ -599,6 +599,35 @@ export default class HistoryManager { }).catch((err) => console.error('Could not delete chat', err)) } + /** Re-read one chat from the store into the in-memory mirror, for a record + * another tab wrote after this manager last read it. `init()` is the wrong + * tool: it re-reads the user's entire history to pick up a single chat. + * + * 'missing' is a fact about the conversation (the store holds nothing under + * this id); 'unavailable' is a fact about this browser. Callers act on the + * first and must not act on the second — treating a closed database as an + * empty chat would throw away a transcript that is merely unreadable. */ + async reloadChat(id: string): Promise<'loaded' | 'missing' | 'unavailable'> { + const db = await this.dbh.whenReady() + if (!db) return 'unavailable' + try { + const chat = await db.get('chats', id) + if (!chat) { + // Drop the mirror too. `loadPastChat` reads from it and never from the + // store, so a copy left behind here is a deleted chat that comes back + // on the next rotation onto this id. + const { [id]: _gone, ...rest } = this.savedChats + this.savedChats = rest + return 'missing' + } + this.savedChats = { ...this.savedChats, [id]: chat } + return 'loaded' + } catch (err) { + console.error('Could not reload chat', err) + return 'unavailable' + } + } + async loadPastChat(id: string) { const chat = this.savedChats[id] if (!chat) return diff --git a/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts b/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts index ffcc065157..cdbf6f2020 100644 --- a/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts @@ -757,3 +757,62 @@ describe('HistoryManager modified-items mask persistence', () => { expect(hm.getModifiedItems(id)).toBeUndefined() }) }) + +describe('HistoryManager.reloadChat', () => { + it('picks up another tab’s write, and tells an empty chat from an unreadable store', async () => { + const hm = new HistoryManager() + await hm.init() + const chatId = hm.getCurrentChatId() + await hm.saveChat( + [{ role: 'user', content: 'before the other tab ran' }] as DisplayMessage[], + [] as ChatCompletionMessageParam[] + ) + + // The other tab's turn, written straight to the store this one shares. + const db = await openDB('copilot-chat-history::admin@test') + const row = (await db.get('chats' as never, chatId)) as any + row.displayMessages = [{ role: 'user', content: 'written by the driving tab' }] + await db.put('chats' as never, row) + db.close() + + expect(await hm.reloadChat(chatId)).toBe('loaded') + const chat = await hm.loadPastChat(chatId) + expect((chat?.displayMessages[0] as any).content).toBe('written by the driving tab') + + // A chat the store does not hold — distinct from 'unavailable' below: + // 'missing' evicts the in-memory mirror, so conflating the two would let + // a store that merely failed to open erase transcripts this tab holds. + expect(await hm.reloadChat('no-such-chat')).toBe('missing') + }) + + it('evicts the mirrored copy of a chat the driver deleted', async () => { + const hm = new HistoryManager() + await hm.init() + const chatId = hm.getCurrentChatId() + await hm.saveChat( + [{ role: 'user', content: 'deleted by the driving tab' }] as DisplayMessage[], + [] as ChatCompletionMessageParam[] + ) + + const db = await openDB('copilot-chat-history::admin@test') + await db.delete('chats' as never, chatId) + db.close() + + expect(await hm.reloadChat(chatId)).toBe('missing') + // loadPastChat serves the mirror, so a copy left behind would resurrect the + // deleted transcript the next time this id came round again. + expect(await hm.loadPastChat(chatId)).toBeUndefined() + }) + + it('reports a store it cannot open as unavailable, never as missing', async () => { + ;(globalThis as any).indexedDB = { + open: () => { + throw new Error('blocked') + } + } + const hm = new HistoryManager() + await hm.init() + + expect(await hm.reloadChat(hm.getCurrentChatId())).toBe('unavailable') + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte b/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte index 5d79251fd2..1e885fa304 100644 --- a/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte +++ b/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte @@ -3,7 +3,10 @@ import { ExternalLink, PanelRight } from 'lucide-svelte' import { Button } from '$lib/components/common' import RowIcon from '$lib/components/common/table/RowIcon.svelte' - import { runToolDisplayAction } from './createdResourceActions.svelte' + import { + hasToolDisplayActionHandler, + runToolDisplayAction + } from './createdResourceActions.svelte' import { workspaceItemAction, type WindmillItemKind, @@ -16,6 +19,7 @@ 'data-wm-kind'?: WindmillItemKind 'data-wm-path'?: string 'data-wm-target-kind'?: WorkspaceItemTargetKind + 'data-wm-raw-app'?: string title?: string } let { @@ -24,10 +28,25 @@ 'data-wm-kind': wmKind, 'data-wm-path': wmPath, 'data-wm-target-kind': wmTargetKind, + 'data-wm-raw-app': wmRawApp, title }: Props = $props() - const drawerAction = $derived(workspaceItemAction(wmKind, wmPath, wmTargetKind)) + // The drawers ride with the docked chat, so a surface can render this pill with nothing + // able to open one. + const available = $derived.by(() => { + const action = workspaceItemAction(wmKind, wmPath, wmTargetKind, wmRawApp === 'true') + return action && hasToolDisplayActionHandler(action.type) ? action : undefined + }) + // Only the preview panel takes the plain click. A drawer keeps its own button beside an + // outbound link: the docked chat mounts drawer handlers on nearly every page, so claiming + // that click would redirect these pills far outside the sessions page. + const previewAction = $derived(available?.type === 'open_item_preview' ? available : undefined) + const drawerAction = $derived(available?.type === 'open_created_resource' ? available : undefined) + + const hint = $derived( + previewAction ? `Open ${wmPath} in the preview panel` : `Open ${wmPath} in a new tab` + ) async function openDrawer(event?: Event) { event?.preventDefault() @@ -36,6 +55,14 @@ await runToolDisplayAction(drawerAction) } } + + async function onclick(event: MouseEvent) { + // Modifier clicks are the only remaining route to the tab once the plain click is + // spoken for, so leave them to the browser. + if (!previewAction || event.metaKey || event.ctrlKey || event.shiftKey || event.altKey) return + event.preventDefault() + await runToolDisplayAction(previewAction) + } {#if href} @@ -43,20 +70,29 @@ - - + + + + + + + {#if previewAction} + + {:else} + + {/if} + {@render children?.()} - - - {#if drawerAction} + {/snippet} + + {#if listNotice} + + {listNotice} + + {/if} + + {#if forkPending} + + Skills are read-only until the first message creates this session's fork. Editing or + selecting one now would apply to the parent workspace and stop applying once the fork is + created. + + {/if} + + + + Drop a folder of SKILL.md files to import, or click to choose + one + + + + {#if loading} +
    Loading skills…
    + {:else if loadError} +
    + Failed to load skills: {loadError} +
    + {:else if skills.length === 0} +
    + No skills in this workspace yet. Paste a SKILL.md or import a folder of them. +
    + {:else} +
    + {#each skills as skill (skill.path)} +
    + +
    +
    + {ambiguous.has(skill.name) ? skill.path : skill.name} +
    + {#if skill.description} +
    {skill.description}
    + {/if} +
    + await toggle(skill.path, e.detail)} + /> + openSkill(skill, skill.canWrite ? 'edit' : 'view') + }, + { + displayName: 'Delete', + icon: Trash2, + type: 'delete', + disabled: !skill.canWrite || forkPending, + action: () => (toDelete = skill) + } + ]} + /> +
    + {/each} +
    + {/if} + + { + const skill = toDelete + toDelete = undefined + if (skill) await remove(skill) + }} + onCanceled={() => (toDelete = undefined)} + > + + This deletes the resource at {toDelete?.path}, so + everyone who selected it loses the skill. + + + + { + const toImport = [ + ...pendingNew.map((skill) => ({ skill, overwrite: false })), + ...pendingConflicts + .filter((s) => overwriteChoices[s.name]) + .map((skill) => ({ skill, overwrite: true })) + ] + const skipped = pendingSkipped + pendingImport = undefined + pendingSkipped = [] + overwriteChoices = {} + if (toImport.length) await importSkills(toImport, skipped) + else sendUserToast('No skills imported.') + }} + onCanceled={() => { + pendingImport = undefined + pendingSkipped = [] + overwriteChoices = {} + }} + > +
    + + Skills are added under {defaultOwner()}. Move one to a + shared folder from the resources page to share it. + + {#if pendingNew.length} +
    + Add {pendingNew.length} new skill(s): + {pendingNew.map((s) => s.name).join(', ')} +
    + {/if} + {#if pendingConflicts.length} +
    + + {pendingConflicts.length} skill(s) already exist — choose which to overwrite: + +
    + {#each pendingConflicts as conflict (conflict.name)} +
    + {conflict.name} + +
    + {/each} +
    +
    + {/if} + {#if pendingSkipped.length} + {pendingSkipped.length} file(s) will be skipped. + {/if} +
    +     + + + + + {#snippet headerRight()} + {#if editing} + + {#snippet children({ item })} + + + {/snippet} + + {/if} + {/snippet} +
    + {#if detailMode === 'view'} + {#if parsed.description} +

    {parsed.description}

    + {/if} +
    + +
    + {:else} + + +
    + +
    +
    + {contentError ?? ''} +
    + + +
    +
    + {/if} +
    +     diff --git a/frontend/src/lib/components/copilot/chat/ToolMessageActions.svelte b/frontend/src/lib/components/copilot/chat/ToolMessageActions.svelte index 02612492a5..db2a0e22c4 100644 --- a/frontend/src/lib/components/copilot/chat/ToolMessageActions.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolMessageActions.svelte @@ -28,7 +28,10 @@ import MqttIcon from '$lib/components/icons/MqttIcon.svelte' import AmqpIcon from '$lib/components/icons/AmqpIcon.svelte' import NatsIcon from '$lib/components/icons/NatsIcon.svelte' - import { runToolDisplayAction } from './createdResourceActions.svelte' + import { + hasToolDisplayActionHandler, + runToolDisplayAction + } from './createdResourceActions.svelte' import type { CreatedResourceTriggerKind, ToolDisplayAction } from './shared' interface Props { @@ -122,17 +125,21 @@
    {card.title}
    {card.subtitle}
    - + + {#if hasToolDisplayActionHandler(action.type)} + + {/if} {/each} diff --git a/frontend/src/lib/components/copilot/chat/UsageMeter.svelte b/frontend/src/lib/components/copilot/chat/UsageMeter.svelte new file mode 100644 index 0000000000..26e4758953 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/UsageMeter.svelte @@ -0,0 +1,39 @@ + + + +
    +
    +
    +
    +
    + {#snippet text()} + {@render tooltip()} + {/snippet} +     diff --git a/frontend/src/lib/components/copilot/chat/artifacts/artifactsState.svelte.ts b/frontend/src/lib/components/copilot/chat/artifacts/artifactsState.svelte.ts index b8e543a720..2455948e22 100644 --- a/frontend/src/lib/components/copilot/chat/artifacts/artifactsState.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/artifacts/artifactsState.svelte.ts @@ -90,6 +90,15 @@ export class SessionArtifactsStore { await this.#load() } + /** Re-read the loaded session's artifacts from the store, for records another + * tab wrote after this one loaded. Forces the read setSession skips: that + * skip protects local edits whose best-effort persist failed, while a tab + * catching up on another tab's finished turn wants the store's truth. */ + async resyncFromStore(): Promise { + if (this.#sessionId === undefined) return + await this.#load() + } + async #load(): Promise { const token = ++this.#seq const id = this.#sessionId diff --git a/frontend/src/lib/components/copilot/chat/context.ts b/frontend/src/lib/components/copilot/chat/context.ts index bf915515db..f57862c179 100644 --- a/frontend/src/lib/components/copilot/chat/context.ts +++ b/frontend/src/lib/components/copilot/chat/context.ts @@ -331,6 +331,13 @@ export function contextElementKey(c: ContextElement): string { return c.type === 'app_dom_selector' ? `dom:${c.appPath}:${c.selector}` : `${c.type}:${c.title}` } +/** An `@`-mentioned workspace item, attached to one message the way a DOM pick + * is. Membership only: this does not gate on mode, so every caller must. Outside + * GLOBAL these stay selected as chips the user removes by hand. */ +export function isMentionContext(c: ContextElement): boolean { + return c.type === 'workspace_script' || c.type === 'workspace_flow' || c.type === 'workspace_app' +} + export function isSameContextElement(a: ContextElement, b: ContextElement): boolean { if (a.type !== b.type) return false if (a.type === 'app_dom_selector' && b.type === 'app_dom_selector') { diff --git a/frontend/src/lib/components/copilot/chat/createdResourceActions.svelte.ts b/frontend/src/lib/components/copilot/chat/createdResourceActions.svelte.ts index 725385d3ee..bd0f3b3d0e 100644 --- a/frontend/src/lib/components/copilot/chat/createdResourceActions.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/createdResourceActions.svelte.ts @@ -25,6 +25,15 @@ export function registerToolDisplayActionHandler( } } +/** + * Reactive: reads the `$state` registry, so a component re-renders when a page mounts or + * unmounts its handler. Offering an action without checking this yields an affordance whose + * only outcome is the unavailable-action toast. + */ +export function hasToolDisplayActionHandler(type: ToolDisplayAction['type']): boolean { + return toolDisplayActionHandlers[type] !== undefined +} + export async function runToolDisplayAction(action: ToolDisplayAction): Promise { const handler = toolDisplayActionHandlers[action.type] if (!handler) { diff --git a/frontend/src/lib/components/copilot/chat/enabledPathsPreference.ts b/frontend/src/lib/components/copilot/chat/enabledPathsPreference.ts new file mode 100644 index 0000000000..5290528dba --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/enabledPathsPreference.ts @@ -0,0 +1,74 @@ +import { get } from 'svelte/store' +import { userStore } from '$lib/stores' + +/** + * A set of workspace-object paths the chat may act through, remembered per + * workspace and per account. + * + * Being able to read a resource is not the same as wanting the chat to use it: a + * resource in a shared folder is readable by a whole team, and each enabled entry + * costs something on every turn — an MCP server puts its tool descriptions in the + * model's context and reaches an external system, a skill puts its description + * there. So an entry is off until it is turned on. + * + * Stored per browser, like the chat's other per-user preferences, but keyed by + * email as well as workspace: browser storage outlives a logout, and inheriting + * the previous account's selection would hand the next person capabilities they + * never turned on. Workspace ids cannot contain `:`, so the composite key is + * unambiguous. + */ +export type EnabledPathsPreference = { + enabledPaths: (workspace: string) => string[] + isEnabled: (workspace: string, path: string) => boolean + /** Returns false when there is no account to record the preference against, so + * a caller that just created the object can say it did not stay on. */ + setEnabled: (workspace: string, path: string, enabled: boolean) => boolean +} + +export function createEnabledPathsPreference(storageKey: string): EnabledPathsPreference { + function scope(workspace: string): string | undefined { + const email = get(userStore)?.email + return email ? `${workspace}:${email}` : undefined + } + + function read(): Record { + if (typeof localStorage === 'undefined') return {} + try { + return JSON.parse(localStorage.getItem(storageKey) ?? '{}') + } catch { + return {} + } + } + + function write(all: Record) { + try { + localStorage.setItem(storageKey, JSON.stringify(all)) + } catch (e) { + console.error(`Failed to persist ${storageKey}`, e) + } + } + + function enabledPaths(workspace: string): string[] { + const key = scope(workspace) + return key ? (read()[key] ?? []) : [] + } + + return { + enabledPaths, + isEnabled: (workspace, path) => enabledPaths(workspace).includes(path), + setEnabled: (workspace, path, enabled) => { + const key = scope(workspace) + if (!key) return false + const all = read() + const current = new Set(all[key] ?? []) + if (enabled) { + current.add(path) + } else { + current.delete(path) + } + all[key] = [...current] + write(all) + return true + } + } +} diff --git a/frontend/src/lib/components/copilot/chat/global/core.test.ts b/frontend/src/lib/components/copilot/chat/global/core.test.ts index 90301f5493..19050bb7da 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.test.ts @@ -210,7 +210,8 @@ vi.mock('$lib/gen', async () => { }), createResource: vi.fn(async () => 'created'), updateResource: vi.fn(async () => 'updated'), - deleteResource: vi.fn(async () => 'deleted') + deleteResource: vi.fn(async () => 'deleted'), + getResourceValue: vi.fn(async () => ({ content: 'skill body' })) }), VariableService: wrapService(actual.VariableService, { existsVariable: vi.fn(async () => false), @@ -5435,6 +5436,18 @@ describe('session-only preview tools gating', () => { }) }) +describe('read_skill', () => { + it('refuses a path the user has not selected, without reading it', async () => { + localStorage.clear() + userStore.set({ username: 'bob', email: 'bob@windmill.dev', workspace_id: WORKSPACE } as any) + + const res = await callGlobalTool('read_skill', { path: 'u/someone/private-notes' }) + + expect(res).toContain('not one of the skills selected') + expect(vi.mocked(ResourceService.getResourceValue)).not.toHaveBeenCalled() + }) +}) + describe('update_user_instructions', () => { function makeHelpers(initial = '') { let value = initial diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index 52704e7cb8..e919769e7a 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -17,8 +17,7 @@ import { ScriptService, SqsTriggerService, VariableService, - WebsocketTriggerService, - WorkspaceService + WebsocketTriggerService } from '$lib/gen' import { createTwoFilesPatch } from 'diff' import type { ArtifactVersionTarget } from '$lib/components/sessions/previewRouter' @@ -83,6 +82,16 @@ import { } from '../flow/inlineScriptsUtils' import { searchNpmPackagesTool } from '../script/core' import type { McpServer } from './mcpTools' +import { logFeatureUsage } from '$lib/utils/featureUsage' +import { enabledSkillPaths } from '../skills/enabledSkills' +import { + listSkillResources, + readSkillBody, + skillNameFromPath, + truncateChars, + truncateForPrompt +} from '../skills/skillResources' +import { MAX_SKILL_DESCRIPTION_LENGTH, MAX_SKILL_INSTRUCTIONS_LENGTH } from '../skills/skillMd' import { getDatatableSdkReference, getFlowPrompt, @@ -636,7 +645,7 @@ const writeVariableSchema = variableRequestSchema.extend({ .string() .optional() .describe( - 'The value of the variable. Omit it to leave the value alone — required only when creating a new variable, or when changing a secret variable into a non-secret one. Never invent or guess the value of an existing variable: you cannot read it, and a "$var:..." reference is NOT a valid value (that syntax only references a variable from inside a resource). Omitting it keeps whatever the draft already holds, so a value you set earlier in this conversation stays set; discard_local_draft abandons it.' + 'The value of the variable. Omit it to leave the value alone — required only when creating a new variable, or when changing a secret variable into a non-secret one. Never invent or guess the value of an existing variable: you cannot read it, and a "$var:..." reference is NOT a valid value (a variable cannot reference itself). Omitting it keeps whatever the draft already holds, so a value you set earlier in this conversation stays set; discard_local_draft abandons it.' ), is_secret: z .boolean() @@ -845,7 +854,9 @@ const testRunArgsSchema = z .record(z.string(), z.any()) .nullable() .optional() - .describe('Arguments to pass to the runnable. Omit or pass null when no arguments are needed.') + .describe( + 'Arguments to pass to the runnable. Omit or pass null when no arguments are needed. An argument typed as a resource (format "resource-" in the input schema) takes the bare string "$res:" as its whole value — never an object wrapper like {"$res": ""}, and never a plain path, both of which reach the runnable unresolved. Same for a variable, with "$var:". The prefixed string can also sit in a nested field, e.g. {"gh_auth": {"token": "$var:g/all/gh_token"}}.' + ) const backgroundArgSchema = z .boolean() @@ -1362,9 +1373,9 @@ Data Tables: ? ` Skills: -- Skills are reusable instruction sets curated for this workspace, each covering a specific kind of task. The available skills are listed below by name and description. -- When a user's request matches a skill's description, call read_skill with its exact name to load the full instructions BEFORE acting, then follow them. -${skills.map((s) => `- ${s.name}: ${s.description}`).join('\n')}` +- Skills are reusable instruction sets the user selected for this chat, each covering a specific kind of task. The available skills are listed below by resource path and description. +- When a user's request matches a skill's description, call read_skill with its exact path to load the full instructions BEFORE acting, then follow them. +${skills.map((s) => `- ${s.path}: ${s.description}`).join('\n')}` : '' }${ mcpServers.length > 0 @@ -2205,7 +2216,7 @@ function getResourceInstructions(): string { - Reading a variable returns \`{ type: 'variable', path, summary?, isSecret, isDraft }\` — never its value, secret or not. \`isSecret\` tells you whether the value is encrypted. - \`write_variable\` takes \`{ path, value?, is_secret?, description?, account?, is_oauth?, expires_at?, labels? }\`. Creating a variable needs \`value\` and \`is_secret\`; editing one needs only the fields you are changing. Omitting \`value\` keeps the stored value, which is the only way to edit a secret variable — you cannot read its value, so passing any \`value\` you did not get from the user destroys it. - For secret fields in a resource value, do NOT inline the raw secret. Create a Variable first with \`is_secret: true\`, then in the resource value reference it as \`"$var:path/to/variable"\`. -- Reference formats inside resource values: \`$var:g/all/name\` (global), \`$var:u/user/name\` (user), \`$var:f/folder/name\` (folder). Reference another resource with \`$res:path/to/resource\`. These are references FROM a resource value; never store a \`$var:\` string as a variable's own value. +- Reference formats inside resource values: \`$var:g/all/name\` (global), \`$var:u/user/name\` (user), \`$var:f/folder/name\` (folder). Reference another resource with \`$res:path/to/resource\`. The same strings are also how a resource or variable is passed as a run argument (see the run-argument rule in the resource reference below); what they are never valid as is a variable's own value. - When deploying drafts that depend on each other (e.g., a resource and the variables it references), deploy the variables first. - Use \`search_resource_types\` to discover valid \`resource_type\` names and their JSON Schemas. Match the resource value to that schema. - For OAuth resources, the \`is_oauth: true\` flag is managed by Windmill's OAuth flow; global mode generally creates manual resources, not OAuth ones. @@ -2253,7 +2264,9 @@ function getInstructions( } } -export type AiSkillListItem = { name: string; description: string } +/** A skill the user turned on, as the prompt and the `/` picker see it. `path` + * is the `ai_skill` resource and the model-facing id; `name` is its basename. */ +export type AiSkillListItem = { path: string; name: string; description: string } /** Live session facts appended to the GLOBAL system prompt for session chats. * Provided by the session runtime as a resolver (copilot must not import the @@ -2316,15 +2329,43 @@ export function getSessionContextPromptSection(ctx: SessionPromptContext): strin return lines.join('\n') } -/** `/` picker entry: a workspace skill or a built-in session action. The kind - * drives the picker's category grouping; entries without one are ungrouped. */ -export type ChatCommandItem = AiSkillListItem & { kind?: 'action' | 'skill' } +/** `/` picker entry: a selected skill or a built-in session action. The kind + * drives the picker's category grouping; entries without one are ungrouped. + * Only skills carry a `path` — built-in actions run locally and have no resource. */ +export type ChatCommandItem = { + name: string + description: string + path?: string + kind?: 'action' | 'skill' +} -/** Fetch the workspace's AI skills (name + description) for the global system prompt. */ +/** + * The skills this user turned on in this workspace, for the global system prompt. + * A readable `ai_skill` resource is only a candidate — enabling one is a personal + * choice, since each enabled skill spends context on every turn. + */ export async function loadWorkspaceSkills(workspace: string): Promise { if (!workspace) return [] try { - return await WorkspaceService.listAiSkills({ workspace }) + const enabled = new Set(enabledSkillPaths(workspace)) + if (enabled.size === 0) return [] + // Filtered against what is actually readable now, so a skill that was + // deleted or whose folder access was revoked drops out instead of being + // advertised to the model as something read_skill can load. + // A truncated listing still carries most of the workspace, and the drawer is + // where that is surfaced; dropping everything here would silently empty the + // Skills section instead. + return (await listSkillResources(workspace)).skills + .filter((s) => enabled.has(s.path)) + .map(({ path, name, description }) => ({ + path, + name, + // Every description goes into the system prompt on every turn, and any + // resource of this type can be selected — including ones written through + // git sync or the resource editor, which never saw the authoring form's + // bounds. One unbounded description would crowd out the conversation. + description: truncateChars(description, MAX_SKILL_DESCRIPTION_LENGTH) + })) } catch (e) { console.error('Failed to load AI skills', e) return [] @@ -2332,32 +2373,52 @@ export async function loadWorkspaceSkills(workspace: string): Promise = { def: createToolDef( readSkillSchema, 'read_skill', - 'Load the full instructions for a workspace AI skill by name. Skills are listed in the system prompt under "Skills"; call this before acting on a task a skill covers, then follow its instructions.' + 'Load the full instructions for a selected AI skill by resource path. Skills are listed in the system prompt under "Skills"; call this before acting on a task a skill covers, then follow its instructions.' ), planModeSafe: true, fn: async ({ args, workspace, toolId, toolCallbacks }) => { const parsed = readSkillSchema.parse(args) - toolCallbacks.setToolStatus(toolId, { content: `Reading skill "${parsed.name}"...` }) + const name = skillNameFromPath(parsed.path) + // The prompt lists only selected skills, but the tool takes a path the model + // composed, so the selection is enforced here too rather than assumed. Without + // it the tool reads any resource holding a string `content` — the user's own + // access, but not what "load a selected skill" says it does. + if (!enabledSkillPaths(workspace).includes(parsed.path)) { + toolCallbacks.setToolStatus(toolId, { content: `Skill "${name}" is not selected` }) + return `"${parsed.path}" is not one of the skills selected for this chat. Only the paths listed under "Skills" in the system prompt can be read.` + } + toolCallbacks.setToolStatus(toolId, { content: `Reading skill "${name}"...` }) try { - const skill = await WorkspaceService.getAiSkill({ workspace, name: parsed.name }) - toolCallbacks.setToolStatus(toolId, { content: `Read skill "${parsed.name}"` }) - return `Skill: ${skill.name}\nDescription: ${skill.description}\n\nInstructions:\n${skill.instructions}` + // Bounded here rather than in the reader: any `ai_skill` resource can be + // selected, including ones written through git sync or the resource editor + // that never passed the authoring form's limits, and an unbounded body + // would exhaust the context on one tool call. The editor reads the same + // resource untruncated, so opening a long skill cannot rewrite it short. + const instructions = truncateForPrompt( + await readSkillBody(workspace, parsed.path), + MAX_SKILL_INSTRUCTIONS_LENGTH + ) + toolCallbacks.setToolStatus(toolId, { content: `Read skill "${name}"` }) + // Whether a selected skill is actually reached for. No key: the path is + // workspace-authored text. + logFeatureUsage('ai_session', 'skill_read', { workspace }) + return `Skill: ${parsed.path}\n\nInstructions:\n${instructions}` } catch (e) { const msg = e instanceof Error ? e.message : String(e) toolCallbacks.setToolStatus(toolId, { - content: `Error reading skill "${parsed.name}"`, + content: `Error reading skill "${name}"`, error: msg }) - return `Failed to read skill "${parsed.name}": ${msg}. Check the name against the Skills list in the system prompt.` + return `Failed to read skill "${parsed.path}": ${msg}. Check the path against the Skills list in the system prompt.` } } } @@ -5046,7 +5107,7 @@ function writeVariableDraft(args: WriteVariableArgs, ctx: WriteDraftCtx): Promis // is always the model echoing the reference syntax back instead of a real value. if (args.value === `$var:${args.path}`) { throw new Error( - `"${args.value}" is not a valid value for variable "${args.path}" — it is a self-reference. The "$var:" syntax only references a variable from inside a resource value. Omit value to keep the current one.` + `"${args.value}" is not a valid value for variable "${args.path}" — it is a self-reference. Omit value to keep the current one.` ) } return writeDraft(VARIABLE_SPEC, 'variable', args.path, args, ctx, { override: args.override }) diff --git a/frontend/src/lib/components/copilot/chat/global/gate.ts b/frontend/src/lib/components/copilot/chat/global/gate.ts index 3321d5a0cd..8fb650aa36 100644 --- a/frontend/src/lib/components/copilot/chat/global/gate.ts +++ b/frontend/src/lib/components/copilot/chat/global/gate.ts @@ -9,8 +9,8 @@ * * When the beta ends, replace every call to `isGlobalAiEnabled()` with `true` * and delete this file. The references are intentionally narrow (chat mode - * visibility, custom prompt settings, the `change_mode` tool enum, and the - * AI skills workspace settings tab) so the rip-out is a small grep. + * visibility, custom prompt settings, and the `change_mode` tool enum) so the + * rip-out is a small grep. */ import { logFeatureUsage } from '$lib/utils/featureUsage' diff --git a/frontend/src/lib/components/copilot/chat/itemPreview.ts b/frontend/src/lib/components/copilot/chat/itemPreview.ts new file mode 100644 index 0000000000..ef9f14ddc9 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/itemPreview.ts @@ -0,0 +1,30 @@ +// The session preview panel's action, kept out of `shared.ts` so the chat message render +// path can import it at runtime without pulling in that module's graph and risking the +// chunk cycles docs/frontend-import-cycles.md exists to prevent. Keep this file import-free. + +/** Item kinds a session preview can host: the three live editors, which are also the + * subset a write tool can land. */ +export type PreviewCardKind = 'script' | 'flow' | 'raw_app' + +// Dispatched by a preview card on a tool call that created or updated a workspace item, +// and by a path link in a chat message. Opens the item's live editor in the session side +// panel — or focuses the tab if it is already open. The handler is registered by the +// sessions page (the only surface with a preview panel). +export type OpenItemPreviewAction = { + id: string + type: 'open_item_preview' + label: string + previewKind: PreviewCardKind + path: string +} + +/** Build the action a preview card or path link dispatches from its (kind, path). */ +export function openItemPreviewAction(kind: PreviewCardKind, path: string): OpenItemPreviewAction { + return { + id: `open-item-preview:${kind}:${path}`, + type: 'open_item_preview', + label: `Open ${kind === 'raw_app' ? 'app' : kind} preview`, + previewKind: kind, + path + } +} diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index b485be51b3..6d6794b56c 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -7,6 +7,12 @@ import type { UserDraftItemKind } from '$lib/gen' // The gate's two refusals, from a module that holds prose and one size limit: under the // shallow-import rule below, the rest of plan mode is not reachable from here. import { PLAN_MODE_MESSAGES } from './planModeMessages' +// Import-free leaf, so it satisfies the shallow-import rule below. +import { + openItemPreviewAction, + type OpenItemPreviewAction, + type PreviewCardKind +} from './itemPreview' // The tool modules that import this one (workspaceTools, flow/core, global/core, ...) // call createToolDef and read SPECIAL_MODULE_IDS at *module scope*, so if a chunk cycle @@ -526,35 +532,11 @@ export type NavigateAction = { page: string } -/** Kinds of previewable item a write tool can land — the subset of draft item - * kinds a session preview can host. */ -export type PreviewCardKind = 'script' | 'flow' | 'raw_app' - -// A discrete card shown on a tool call that created or updated a workspace item. -// Clicking it opens the item's live preview in the session side panel — or focuses -// the tab if it is already open. The handler is registered by the sessions page -// (the only surface with a preview panel). -export type OpenItemPreviewAction = { - id: string - type: 'open_item_preview' - label: string - previewKind: PreviewCardKind - path: string -} +// Re-exported: most consumers reach these through this module. +export { openItemPreviewAction, type PreviewCardKind, type OpenItemPreviewAction } export type ToolDisplayAction = CreatedResourceAction | NavigateAction | OpenItemPreviewAction -/** Build the action a preview card dispatches from its (kind, path). */ -export function openItemPreviewAction(kind: PreviewCardKind, path: string): OpenItemPreviewAction { - return { - id: `open-item-preview:${kind}:${path}`, - type: 'open_item_preview', - label: `Open ${kind === 'raw_app' ? 'app' : kind} preview`, - previewKind: kind, - path - } -} - export type UserQuestionDisplay = { question: string choices: string[] diff --git a/frontend/src/lib/components/copilot/chat/skills/enabledSkills.ts b/frontend/src/lib/components/copilot/chat/skills/enabledSkills.ts new file mode 100644 index 0000000000..e91d40bff0 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/skills/enabledSkills.ts @@ -0,0 +1,10 @@ +import { createEnabledPathsPreference } from '../enabledPathsPreference' + +/** Which `ai_skill` resources the chat may follow, per workspace and per account. + * Every enabled skill spends context on every turn, so selecting one is a personal + * choice rather than a consequence of being able to read it. */ +const preference = createEnabledPathsPreference('wm_skills_enabled') + +export const enabledSkillPaths = preference.enabledPaths +export const isSkillEnabled = preference.isEnabled +export const setSkillEnabled = preference.setEnabled diff --git a/frontend/src/lib/components/workspaceSettings/aiSkills.test.ts b/frontend/src/lib/components/copilot/chat/skills/skillMd.test.ts similarity index 99% rename from frontend/src/lib/components/workspaceSettings/aiSkills.test.ts rename to frontend/src/lib/components/copilot/chat/skills/skillMd.test.ts index 87f84d865a..b498cc2d24 100644 --- a/frontend/src/lib/components/workspaceSettings/aiSkills.test.ts +++ b/frontend/src/lib/components/copilot/chat/skills/skillMd.test.ts @@ -8,7 +8,7 @@ import { parseAndValidateSkill, parseSkillMd, validateSkill -} from './aiSkills' +} from './skillMd' describe('parseSkillMd', () => { it('splits frontmatter name/description from the body', () => { diff --git a/frontend/src/lib/components/workspaceSettings/aiSkills.ts b/frontend/src/lib/components/copilot/chat/skills/skillMd.ts similarity index 90% rename from frontend/src/lib/components/workspaceSettings/aiSkills.ts rename to frontend/src/lib/components/copilot/chat/skills/skillMd.ts index aeb148f7c1..93916c023f 100644 --- a/frontend/src/lib/components/workspaceSettings/aiSkills.ts +++ b/frontend/src/lib/components/copilot/chat/skills/skillMd.ts @@ -1,10 +1,13 @@ import YAML from 'yaml' import { z } from 'zod' +/** A SKILL.md split into the three parts a `skills` resource stores: `name` + * becomes the resource path's basename, `description` its description column, + * `instructions` its file body. */ export type SkillUpload = { name: string; description: string; instructions: string } -// `name` + `description` mirror the Claude SKILL.md spec (counted in characters); -// the body is a byte-bounded payload. Keep these in sync with backend `validate_skill`. +// `name` + `description` mirror the Claude SKILL.md spec (counted in characters), +// so a skill stays portable with Claude Code; the body is a byte-bounded payload. export const MAX_SKILL_NAME_LENGTH = 64 export const MAX_SKILL_DESCRIPTION_LENGTH = 1_024 export const MAX_SKILL_INSTRUCTIONS_LENGTH = 64 * 1024 @@ -12,8 +15,8 @@ export const MAX_SKILL_INSTRUCTIONS_LENGTH = 64 * 1024 const textEncoder = new TextEncoder() // Single source of truth for skill field validation, shared by the paste/edit -// modal and the folder importer. Lengths are code-point / byte bounded to match -// the backend, so `.refine` (not `.max`, which counts UTF-16 units) is used. +// modal and the folder importer. Lengths are code-point / byte bounded, so +// `.refine` (not `.max`, which counts UTF-16 units) is used. export const skillSchema = z.object({ name: z .string() diff --git a/frontend/src/lib/components/copilot/chat/skills/skillResources.ts b/frontend/src/lib/components/copilot/chat/skills/skillResources.ts new file mode 100644 index 0000000000..20737caffa --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/skills/skillResources.ts @@ -0,0 +1,159 @@ +import { ResourceService } from '$lib/gen' +import { canWrite } from '$lib/utils' +import type { UserExt } from '$lib/stores' + +/** + * Skills are resources of this type: a file resource (`format_extension = 'md'`) + * whose `value.content` is the SKILL.md body, whose description column is what the + * assistant reads when deciding the skill applies, and whose path names it. + */ +export const SKILLS_RESOURCE_TYPE = 'ai_skill' + +/** A skill as the picker and the system prompt see it — never the body, which + * `read_skill` fetches only once the model commits to using the skill. */ +export type SkillResource = { + path: string + /** Path basename: what the `/` command and the picker row show. */ + name: string + description: string + editedAt?: string + canWrite: boolean +} + +/** The `/`-command and display name for a skill. Paths are `[ufg]/x/y…`, so the + * last segment is always present. */ +export function skillNameFromPath(path: string): string { + return path.split('/').pop() ?? path +} + +/** Basenames carried by more than one of these skills. Two folders can each hold + * a `deploy`, and then the name alone no longer says which one — the picker shows + * the path for these, and the `/` command refuses to guess. */ +export function ambiguousSkillNames(skills: readonly { name: string }[]): Set { + const seen = new Map() + for (const s of skills) seen.set(s.name, (seen.get(s.name) ?? 0) + 1) + return new Set([...seen].filter(([, n]) => n > 1).map(([name]) => name)) +} + +const SKILLS_PAGE_SIZE = 100 +/** Pages to walk before giving up. Ordinary resources and repeated imports can + * make any number of skills, and a single page would drop the rest — including a + * selected one, which would then vanish from the prompt with nothing to explain + * it. The bound is a guard against a paging bug looping forever, not a product + * cap, so reaching it is reported rather than passed off as the whole set. */ +const MAX_SKILLS_PAGES = 100 + +/** The rows read, and whether the walk stopped at the bound rather than the end. + * Reported rather than thrown: a truncated read is still most of the skills, and + * dropping them all would take every selected skill out of the prompt at once. */ +export type SkillListing = { skills: SkillResource[]; truncated: boolean } + +/** Every skill resource readable in the workspace. + * + * `user` decides which rows the drawer offers to edit rather than only view; pass + * the account the workspace is being browsed as. Ownership is mostly implicit in + * the path (`u//…`, a folder the user owns), which is why this goes through + * the shared `canWrite` rather than reading `extra_perms` alone. */ +export async function listSkillResources( + workspace: string, + user?: UserExt +): Promise { + if (!workspace) return { skills: [], truncated: false } + const rows: SkillResource[] = [] + for (let page = 1; page <= MAX_SKILLS_PAGES; page++) { + const resources = await ResourceService.listResource({ + workspace, + resourceType: SKILLS_RESOURCE_TYPE, + page, + perPage: SKILLS_PAGE_SIZE + }) + rows.push( + ...resources.map((r) => ({ + path: r.path, + name: skillNameFromPath(r.path), + description: r.description ?? '', + editedAt: r.edited_at, + canWrite: canWrite(r.path, r.extra_perms ?? {}, user) + })) + ) + if (resources.length < SKILLS_PAGE_SIZE) return { skills: rows, truncated: false } + } + return { skills: rows, truncated: true } +} + +/** Cut `text` to `maxChars` code points. For the description, whose cap is stated + * in characters — cutting that one by bytes would reduce a legal 1,024-character + * CJK description to about a third of itself. */ +export function truncateChars(text: string, maxChars: number): string { + const points = [...text] + return points.length <= maxChars ? text : `${points.slice(0, maxChars).join('')}… [truncated]` +} + +/** Cut `text` to `maxBytes` of UTF-8, marking the cut so a reader (the model + * included) can tell truncation from a body that simply ends there. + * + * For the body, whose cap is a byte budget: 64k CJK characters are ~192 KiB, so a + * code-unit cut would let three times the intended payload through. */ +export function truncateForPrompt(text: string, maxBytes: number): string { + const encoded = new TextEncoder().encode(text) + if (encoded.byteLength <= maxBytes) return text + // `fatal: false` replaces the partial code point a byte-aligned cut can leave + // with U+FFFD; dropping it keeps the tail clean. + const cut = new TextDecoder('utf-8').decode(encoded.slice(0, maxBytes)).replace(/�$/, '') + return `${cut}… [truncated]` +} + +/** The SKILL.md body of one skill. Throws rather than returning `''` when the + * resource holds no readable body: an empty string reaches the model as a + * successful read of a skill with no instructions, which it would then act on. + * + * Deliberately unbounded — the editor loads through here and saves what it loaded, + * so truncating would rewrite an over-long skill the first time someone opened it. + * Bounding belongs at the prompt boundary, where the cost actually is. */ +export async function readSkillBody(workspace: string, path: string): Promise { + const value = (await ResourceService.getResourceValue({ workspace, path })) as + | { content?: unknown } + | undefined + if (typeof value?.content !== 'string') { + throw new Error(`resource ${path} has no string "content" — is it an ${SKILLS_RESOURCE_TYPE}?`) + } + return value.content +} + +export async function saveSkillResource( + workspace: string, + path: string, + description: string, + instructions: string, + { overwrite = false }: { overwrite?: boolean } = {} +): Promise { + await ResourceService.createResource({ + workspace, + updateIfExists: overwrite, + requestBody: { + path, + description, + value: { content: instructions }, + resource_type: SKILLS_RESOURCE_TYPE + } + }) +} + +/** Save an edit to an existing skill, moving it when the path changed. */ +export async function updateSkillResource( + workspace: string, + currentPath: string, + path: string, + description: string, + instructions: string +): Promise { + await ResourceService.updateResource({ + workspace, + path: currentPath, + requestBody: { path, description, value: { content: instructions } } + }) +} + +export async function deleteSkillResource(workspace: string, path: string): Promise { + await ResourceService.deleteResource({ workspace, path }) +} diff --git a/frontend/src/lib/components/copilot/chat/skills/skills.test.ts b/frontend/src/lib/components/copilot/chat/skills/skills.test.ts new file mode 100644 index 0000000000..5b0d2196ca --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/skills/skills.test.ts @@ -0,0 +1,69 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { session } = vi.hoisted(() => ({ + session: { email: 'first@windmill.dev' } as { email?: string } +})) + +vi.mock('$lib/stores', () => ({ + // Read at call time, so a test can switch accounts the way a logout does. + userStore: { subscribe: (run: (v: unknown) => void) => (run({ ...session }), () => {}) } +})) + +import { enabledSkillPaths, isSkillEnabled, setSkillEnabled } from './enabledSkills' +import { ambiguousSkillNames, truncateChars, truncateForPrompt } from './skillResources' + +describe('enabledSkills', () => { + beforeEach(() => { + localStorage.clear() + session.email = 'first@windmill.dev' + }) + + it('keeps the selection separate per workspace', () => { + setSkillEnabled('ws_a', 'u/me/deploy', true) + expect(isSkillEnabled('ws_a', 'u/me/deploy')).toBe(true) + expect(isSkillEnabled('ws_b', 'u/me/deploy')).toBe(false) + }) + + it('does not hand the next account the previous one’s selection', () => { + setSkillEnabled('ws_a', 'u/me/deploy', true) + session.email = 'second@windmill.dev' + expect(enabledSkillPaths('ws_a')).toEqual([]) + }) + + it('reports failure when there is no account to record the choice against', () => { + session.email = undefined + expect(setSkillEnabled('ws_a', 'u/me/deploy', true)).toBe(false) + expect(enabledSkillPaths('ws_a')).toEqual([]) + }) +}) + +describe('skill names', () => { + it('flags a basename two folders both use, so /name is not resolved by chance', () => { + const ambiguous = ambiguousSkillNames([ + { name: 'deploy' }, + { name: 'deploy' }, + { name: 'release' } + ]) + expect([...ambiguous]).toEqual(['deploy']) + }) +}) + +describe('prompt truncation', () => { + // The two caps are stated in different units, and using one truncator for both + // either lets three times the payload through or cuts a legal value to a third. + it('bounds a skill body by utf-8 bytes, not code units', () => { + const body = '漢'.repeat(100) // 300 bytes + expect(truncateForPrompt(body, 3000)).toBe(body) + const cut = truncateForPrompt(body, 30) + expect(new TextEncoder().encode(cut.replace('… [truncated]', '')).byteLength).toBeLessThanOrEqual(30) + expect(cut).toContain('[truncated]') + // A byte-aligned cut must not leave a broken code point behind. + expect(cut).not.toContain('\ufffd') + }) + + it('bounds a description by code points, so a CJK one is not cut to a third', () => { + const description = '漢'.repeat(100) + expect(truncateChars(description, 100)).toBe(description) + expect([...truncateChars(description, 10)].slice(0, 10).join('')).toBe('漢'.repeat(10)) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/workspaceItems.svelte.ts b/frontend/src/lib/components/copilot/chat/workspaceItems.svelte.ts index b1a6f695d3..1e02573a32 100644 --- a/frontend/src/lib/components/copilot/chat/workspaceItems.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/workspaceItems.svelte.ts @@ -22,6 +22,8 @@ import { findAndReplace } from 'mdast-util-find-and-replace' import { visit } from 'unist-util-visit' import type { Root, InlineCode, Link } from 'mdast' import type { CreatedResourceAction, ToolDisplayAction } from './shared' +// Leaf module, deliberately not './shared' — see the header of itemPreview.ts. +import { openItemPreviewAction } from './itemPreview' export type WindmillItemKind = | 'script' @@ -46,6 +48,9 @@ export interface WorkspaceItemEntry { kind: WindmillItemKind path: string targetKind?: WorkspaceItemTargetKind + /** Apps only. Both kinds share the `app` kind and the same `/apps/get/` route, + * so this flag is all that tells them apart downstream. */ + rawApp?: boolean } export type WorkspaceItemTargetKind = 'script' | 'flow' @@ -96,6 +101,7 @@ export function itemHref(entry: WorkspaceItemEntry, workspace?: string): string type WorkspaceItemListResult = Array<{ path: string is_flow?: boolean | null + raw_app?: boolean | null }> const workspaceItemLoaders: Array<{ @@ -165,7 +171,8 @@ class WorkspaceItemRegistry { kind, path: it.path, targetKind: - typeof it.is_flow === 'boolean' ? (it.is_flow ? 'flow' : 'script') : undefined + typeof it.is_flow === 'boolean' ? (it.is_flow ? 'flow' : 'script') : undefined, + rawApp: kind === 'app' ? it.raw_app === true : undefined }) } } @@ -213,13 +220,30 @@ export function extractCandidatePaths(text: string | undefined | null): string[] return [...seen] } +/** + * The in-app action a resolved path link runs instead of opening a new tab, or undefined + * when the kind has none (its link then stays outbound). Returning an action does not mean + * it can run on this surface — only the sessions page hosts a preview panel, so callers + * gate on `hasToolDisplayActionHandler(action.type)`. + */ export function workspaceItemAction( kind: WindmillItemKind | undefined, path: string | undefined, - targetKind?: WorkspaceItemTargetKind + targetKind?: WorkspaceItemTargetKind, + rawApp?: boolean ): ToolDisplayAction | undefined { if (!kind || !path) return undefined + if (kind === 'script' || kind === 'flow') { + return openItemPreviewAction(kind, path) + } + + // Raw apps only. A legacy drag-and-drop app has no editor the panel can host, and + // legacy items are not extended onto new surfaces — its link stays outbound. + if (kind === 'app') { + return rawApp ? openItemPreviewAction('raw_app', path) : undefined + } + const base = { id: `open_workspace_item:${kind}:${path}`, type: 'open_created_resource' as const, @@ -253,6 +277,9 @@ function buildPathLinkNode( if (entry.targetKind) { hProperties['data-wm-target-kind'] = entry.targetKind } + if (entry.rawApp) { + hProperties['data-wm-raw-app'] = 'true' + } return { type: 'link', diff --git a/frontend/src/lib/components/copilot/chat/workspaceItems.test.ts b/frontend/src/lib/components/copilot/chat/workspaceItems.test.ts index 9bb477fa83..70810dad9a 100644 --- a/frontend/src/lib/components/copilot/chat/workspaceItems.test.ts +++ b/frontend/src/lib/components/copilot/chat/workspaceItems.test.ts @@ -153,10 +153,27 @@ describe('workspaceItemAction', () => { }) }) - it('skips non-drawerable items and trigger items without target kind', () => { - expect(workspaceItemAction('script', 'f/a/b')).toBeUndefined() - expect(workspaceItemAction('flow', 'f/a/b')).toBeUndefined() + it('creates preview actions for scripts, flows and raw apps', () => { + expect(workspaceItemAction('script', 'f/a/b')).toMatchObject({ + type: 'open_item_preview', + previewKind: 'script', + path: 'f/a/b' + }) + expect(workspaceItemAction('flow', 'f/a/b')).toMatchObject({ + type: 'open_item_preview', + previewKind: 'flow' + }) + expect(workspaceItemAction('app', 'f/a/b', undefined, true)).toMatchObject({ + previewKind: 'raw_app' + }) + }) + + it('leaves legacy apps as plain links', () => { + expect(workspaceItemAction('app', 'f/a/b', undefined, false)).toBeUndefined() expect(workspaceItemAction('app', 'f/a/b')).toBeUndefined() + }) + + it('skips trigger items without target kind', () => { expect(workspaceItemAction('schedule', 'f/a/b')).toBeUndefined() expect(workspaceItemAction('http_trigger', 'f/a/b')).toBeUndefined() }) @@ -172,6 +189,7 @@ const SAMPLE_ENTRIES: Record = { path: 'u/admin/cleanup_old_jobs' }, 'f/ops/dashboard': { kind: 'app', path: 'f/ops/dashboard' }, + 'f/ops/live_board': { kind: 'app', path: 'f/ops/live_board', rawApp: true }, 'f/etl/daily': { kind: 'schedule', path: 'f/etl/daily', @@ -249,6 +267,23 @@ describe('remarkWindmillPaths (mdast)', () => { expect(props['data-wm-target-kind']).toBe('flow') }) + // Both app kinds reach the renderer as the same `app` kind and route, so this flag is + // the only thing that keeps a legacy app off the preview panel. + it('marks raw apps so the renderer can tell them from legacy apps', () => { + const processor = buildProcessor('admins') + const tree = processor.runSync( + processor.parse('Open f/ops/live_board and f/ops/dashboard.') + ) as MdastRoot + const byPath = Object.fromEntries( + findLinks(tree).map((l) => [ + (l.data?.hProperties as Record)['data-wm-path'], + l.data?.hProperties as Record + ]) + ) + expect(byPath['f/ops/live_board']['data-wm-raw-app']).toBe('true') + expect(byPath['f/ops/dashboard']['data-wm-raw-app']).toBeUndefined() + }) + it('leaves unknown paths as plain text', () => { const processor = buildProcessor() const tree = processor.runSync( diff --git a/frontend/src/lib/components/copilot/chat/workspaceToolsZod.gen.ts b/frontend/src/lib/components/copilot/chat/workspaceToolsZod.gen.ts index 47237a0ae1..42773873c2 100644 --- a/frontend/src/lib/components/copilot/chat/workspaceToolsZod.gen.ts +++ b/frontend/src/lib/components/copilot/chat/workspaceToolsZod.gen.ts @@ -69,7 +69,7 @@ export const httpTriggerRequestSchema = z.object({ "wrap_body": z.boolean().describe("If true, wraps the request body in a 'body' parameter").optional(), "mode": z.enum(["enabled", "disabled", "suspended"]).describe("job trigger mode").optional(), "raw_string": z.boolean().describe("If true, passes the request body as a raw string instead of parsing as JSON").optional(), - "error_handler_path": z.string().describe("Path to a script or flow to run when the triggered job fails").optional(), + "error_handler_path": z.string().describe("Path to a script to run when the triggered job fails. A bare path, without the script/ or flow/ prefix a schedule error handler takes; it cannot be a flow.").optional(), "error_handler_args": z.record(z.string(), z.any()).describe("Arguments to pass to the error handler").optional(), "retry": z.object({ "constant": z.object({ @@ -128,7 +128,7 @@ export const websocketTriggerRequestSchema = z.object({ "message": z.string().describe("Message to send as heartbeat. Use {{state}} as a placeholder for a value extracted from incoming messages (see state_field)."), "state_field": z.string().describe("Optional. Top-level JSON field to extract from incoming messages. The extracted value replaces {{state}} in the heartbeat message.").optional() }).describe("Optional periodic heartbeat message configuration").nullable().optional(), - "error_handler_path": z.string().describe("Path to a script or flow to run when the triggered job fails").optional(), + "error_handler_path": z.string().describe("Path to a script to run when the triggered job fails. A bare path, without the script/ or flow/ prefix a schedule error handler takes; it cannot be a flow.").optional(), "error_handler_args": z.record(z.string(), z.any()).describe("Arguments to pass to the error handler").optional(), "retry": z.object({ "constant": z.object({ @@ -174,7 +174,7 @@ export const kafkaTriggerRequestSchema = z.object({ "auto_offset_reset": z.enum(["latest", "earliest"]).describe("Initial offset behavior when consumer group has no committed offset.").default("latest").optional(), "auto_commit": z.boolean().describe("When true (default), offsets are committed automatically after receiving each message. When false, you must manually commit offsets using the commit_offsets endpoint.").default(true).optional(), "mode": z.enum(["enabled", "disabled", "suspended"]).describe("job trigger mode").optional(), - "error_handler_path": z.string().describe("Path to a script or flow to run when the triggered job fails").optional(), + "error_handler_path": z.string().describe("Path to a script to run when the triggered job fails. A bare path, without the script/ or flow/ prefix a schedule error handler takes; it cannot be a flow.").optional(), "error_handler_args": z.record(z.string(), z.any()).describe("Arguments to pass to the error handler").optional(), "retry": z.object({ "constant": z.object({ @@ -206,7 +206,7 @@ export const natsTriggerRequestSchema = z.object({ "consumer_name": z.string().describe("JetStream consumer name (required when use_jetstream is true)").nullable().optional(), "subjects": z.array(z.string()).describe("Array of NATS subjects to subscribe to"), "mode": z.enum(["enabled", "disabled", "suspended"]).describe("job trigger mode").optional(), - "error_handler_path": z.string().describe("Path to a script or flow to run when the triggered job fails").optional(), + "error_handler_path": z.string().describe("Path to a script to run when the triggered job fails. A bare path, without the script/ or flow/ prefix a schedule error handler takes; it cannot be a flow.").optional(), "error_handler_args": z.record(z.string(), z.any()).describe("Arguments to pass to the error handler").optional(), "retry": z.object({ "constant": z.object({ @@ -247,7 +247,7 @@ export const postgresTriggerRequestSchema = z.object({ })).optional(), "transaction_to_track": z.array(z.string()) }).describe("Configuration for creating/managing the publication (tables, operations)").optional(), - "error_handler_path": z.string().describe("Path to a script or flow to run when the triggered job fails").optional(), + "error_handler_path": z.string().describe("Path to a script to run when the triggered job fails. A bare path, without the script/ or flow/ prefix a schedule error handler takes; it cannot be a flow.").optional(), "error_handler_args": z.record(z.string(), z.any()).describe("Arguments to pass to the error handler").optional(), "retry": z.object({ "constant": z.object({ @@ -289,7 +289,7 @@ export const mqttTriggerRequestSchema = z.object({ "script_path": z.string().describe("Path to the script or flow to execute when a message is received"), "is_flow": z.boolean().describe("True if script_path points to a flow, false if it points to a script"), "mode": z.enum(["enabled", "disabled", "suspended"]).describe("job trigger mode").optional(), - "error_handler_path": z.string().describe("Path to a script or flow to run when the triggered job fails").optional(), + "error_handler_path": z.string().describe("Path to a script to run when the triggered job fails. A bare path, without the script/ or flow/ prefix a schedule error handler takes; it cannot be a flow.").optional(), "error_handler_args": z.record(z.string(), z.any()).describe("Arguments to pass to the error handler").optional(), "retry": z.object({ "constant": z.object({ @@ -326,7 +326,7 @@ export const amqpTriggerRequestSchema = z.object({ "script_path": z.string().describe("Path to the script or flow to execute when a message is received"), "is_flow": z.boolean().describe("True if script_path points to a flow, false if it points to a script"), "mode": z.enum(["enabled", "disabled", "suspended"]).describe("job trigger mode").optional(), - "error_handler_path": z.string().describe("Path to a script or flow to run when the triggered job fails").optional(), + "error_handler_path": z.string().describe("Path to a script to run when the triggered job fails. A bare path, without the script/ or flow/ prefix a schedule error handler takes; it cannot be a flow.").optional(), "error_handler_args": z.record(z.string(), z.any()).describe("Arguments to pass to the error handler").optional(), "retry": z.object({ "constant": z.object({ @@ -357,7 +357,7 @@ export const sqsTriggerRequestSchema = z.object({ "script_path": z.string().describe("Path to the script or flow to execute when a message is received"), "is_flow": z.boolean().describe("True if script_path points to a flow, false if it points to a script"), "mode": z.enum(["enabled", "disabled", "suspended"]).describe("job trigger mode").optional(), - "error_handler_path": z.string().describe("Path to a script or flow to run when the triggered job fails").optional(), + "error_handler_path": z.string().describe("Path to a script to run when the triggered job fails. A bare path, without the script/ or flow/ prefix a schedule error handler takes; it cannot be a flow.").optional(), "error_handler_args": z.record(z.string(), z.any()).describe("Arguments to pass to the error handler").optional(), "retry": z.object({ "constant": z.object({ diff --git a/frontend/src/lib/components/copilot/loadCopilot.ts b/frontend/src/lib/components/copilot/loadCopilot.ts index 4634833b0b..496fb89fdc 100644 --- a/frontend/src/lib/components/copilot/loadCopilot.ts +++ b/frontend/src/lib/components/copilot/loadCopilot.ts @@ -1,6 +1,14 @@ import { WorkspaceService } from '$lib/gen' import { copilotWorkspace, setCopilotInfo } from '$lib/aiStore' import { workspaceAIClients } from './lib' +import { writable } from 'svelte/store' + +// The workspace of the most recent loadCopilot *request*, set synchronously before the +// await — as opposed to `copilotWorkspace`, which only updates once a load resolves. A +// background refresh (e.g. free-tier usage) compares against this so it can't supersede an +// in-flight load for a newer workspace (which would otherwise win the token and restore +// stale state). +export const copilotWorkspaceRequested = writable(undefined) // Lives here, not in $lib/aiStore, purely so that module needs no import of the AI // client — it is the one thing that wanted both. Moving it back recreates the @@ -20,6 +28,7 @@ let loadCopilotToken = 0 let inFlight: { workspace: string; promise: Promise } | undefined export function loadCopilot(workspace: string): Promise { + copilotWorkspaceRequested.set(workspace) if (inFlight?.workspace === workspace) { return inFlight.promise } diff --git a/frontend/src/lib/components/copilot/stepInputFillTelemetry.ts b/frontend/src/lib/components/copilot/stepInputFillTelemetry.ts new file mode 100644 index 0000000000..19efc2b932 --- /dev/null +++ b/frontend/src/lib/components/copilot/stepInputFillTelemetry.ts @@ -0,0 +1,13 @@ +import { logFeatureUsage } from '$lib/utils/featureUsage' + +// Anonymous counters for the AI filling of a step's inputs. Same rules as every other +// `logFeatureUsage` caller: aggregated counts only, and the two keys below are the whole +// vocabulary — no argument name, expression or step id ever reaches here. + +/** Which filler the user reached for: the per-field one, or the one above the whole form. */ +export type StepInputFillScope = 'single' | 'all' + +/** Counted where the user asks for a suggestion, not where one is accepted. */ +export function logStepInputFill(scope: StepInputFillScope): void { + logFeatureUsage('flow_step', 'ai_fill', { key: scope }) +} diff --git a/frontend/src/lib/components/flows/agentTelemetry.ts b/frontend/src/lib/components/flows/agentTelemetry.ts new file mode 100644 index 0000000000..fcfad440ff --- /dev/null +++ b/frontend/src/lib/components/flows/agentTelemetry.ts @@ -0,0 +1,19 @@ +import { logFeatureUsage } from '$lib/utils/featureUsage' + +// Anonymous counters for the reusable-agent lifecycle (`docs/reusable-ai-agents.md`). Same rules +// as every other `logFeatureUsage` caller: aggregated counts only, and the four keys below are +// the whole vocabulary — no agent path, prompt, model or tool ever reaches here. + +export type ReusableAgentEvent = + /** A step was saved as a new reusable agent. */ + | 'saved' + /** Edits to a linked agent were written back, propagating to every flow using it. */ + | 'updated' + /** A saved agent was picked into a new step. */ + | 'linked' + /** A linked step was forked back into a standalone agent. */ + | 'unlinked' + +export function logReusableAgentUsage(event: ReusableAgentEvent): void { + logFeatureUsage('ai_agent', 'reusable', { key: event }) +} diff --git a/frontend/src/lib/components/flows/content/AgentResourceBar.svelte b/frontend/src/lib/components/flows/content/AgentResourceBar.svelte index 687dc9d98c..f5d5ec19a1 100644 --- a/frontend/src/lib/components/flows/content/AgentResourceBar.svelte +++ b/frontend/src/lib/components/flows/content/AgentResourceBar.svelte @@ -31,6 +31,7 @@ linkedToolsScope } from '../linkedAgentToolsStore.svelte' import { getAgentEdit, getAgentEditingPath, setAgentEditingPath } from '../agentEditStore.svelte' + import { logReusableAgentUsage } from '../agentTelemetry' import { claimLinkedToolsFetch } from '../flowState' import type { AgentTool as AgentToolStrict } from '../agentToolUtils' import { resource } from 'runed' @@ -338,6 +339,7 @@ const linked = await persist(newPath, description) saveDrawer?.closeDrawer() if (linked) { + logReusableAgentUsage(updating ? 'updated' : 'saved') sendUserToast(updating ? `Updated agent ${newPath}` : `Saved reusable agent ${newPath}`) } } catch (e) { @@ -356,6 +358,7 @@ const path = editingPath try { if (await persist(path)) { + logReusableAgentUsage('updated') sendUserToast(`Updated agent ${path}`) } } catch (e) { @@ -419,6 +422,7 @@ const fork = await forkFromResource(true) if (fork) { setAgentEditingPath(tools, undefined) + logReusableAgentUsage('unlinked') sendUserToast('Forked agent. Its configuration was copied into this step') } else { sendUserToast('The step changed while loading the agent, so nothing was unlinked', true) diff --git a/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte index cf9d008a27..eb875973dc 100644 --- a/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte +++ b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte @@ -15,12 +15,11 @@ workspaceStore } from '$lib/stores' import type { SupportedLanguage } from '$lib/common' - import { createEventDispatcher, getContext, onDestroy, onMount, untrack } from 'svelte' + import { createEventDispatcher, getContext, untrack } from 'svelte' import type { FlowBuilderWhitelabelCustomUi } from '$lib/components/custom_ui' import { type Script, type ScriptLang, type HubScriptKind } from '$lib/gen' import ListFiltersQuick from '$lib/components/home/ListFiltersQuick.svelte' import { ExternalLink, Folder, User, X } from 'lucide-svelte' - import type { FlowEditorContext } from '../../flows/types' import { fade } from 'svelte/transition' import { flip } from 'svelte/animate' import { Button } from '$lib/components/common' @@ -87,8 +86,6 @@ let hubCompletions: HubCompletion[] = $state([]) - const { insertButtonOpen } = getContext('FlowEditorContext') - let selected: { kind: 'owner' | 'integrations'; name: string | undefined } | undefined = $state(undefined) @@ -221,13 +218,6 @@ selectedByKeyboard = index } - onMount(() => { - $insertButtonOpen = true - }) - - onDestroy(() => { - $insertButtonOpen = false - }) let langs = $derived( processInlineLangs(undefined, $defaultScripts?.order ?? Object.keys(defaultScriptLanguages)) .map((l) => [defaultScriptLanguages[l], l]) @@ -255,6 +245,7 @@ // on indices that render nothing. let showAiRows = $derived( !disableAi && + !$copilotInfo.workspaceDisabled && funcDesc?.length > 0 && kind != 'failure' && kind != 'preprocessor' && diff --git a/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte b/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte index 6aeaa14f35..e6d7223fcb 100644 --- a/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte +++ b/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte @@ -464,7 +464,7 @@ btnClasses="px-1 py-1.5 bg-surface" on:click={() => { outputPicker?.toggleOpen(true) - moduleTest?.loadArgsAndRunTest() + moduleTest?.runTestWithStepArgs() }} dropdownItems={[ { diff --git a/frontend/src/lib/components/flows/map/InsertModuleInner.svelte b/frontend/src/lib/components/flows/map/InsertModuleInner.svelte index 3316463bbb..7c75d97fab 100644 --- a/frontend/src/lib/components/flows/map/InsertModuleInner.svelte +++ b/frontend/src/lib/components/flows/map/InsertModuleInner.svelte @@ -3,7 +3,7 @@ +
    dispatch('close')} {disableAi} on:insert @@ -229,7 +307,10 @@ selected={selectedKind === 'aiagent'} onSelect={() => { selectedKind = 'aiagent' + selectedByKeyboard = 0 loadSavedAgents() + // Clicking leaves focus on this button, where Enter would only re-select it. + stepGen?.focus() }} /> {/if} @@ -239,6 +320,8 @@ selected={selectedKind === 'aisandbox'} onSelect={() => { selectedKind = 'aisandbox' + selectedByKeyboard = 0 + stepGen?.focus() }} /> {/if} @@ -247,19 +330,25 @@ {/if} {#if selectedKind === 'aiagent'} -
    +
    {#if savedAgentsLoading}
    @@ -267,20 +356,23 @@
    {:else if filteredAgents.length > 0}
    Saved agents
    - {#each filteredAgents as agent (agent.path)} + {#each filteredAgents as agent, i (agent.path)} {/each} {:else} @@ -295,17 +387,11 @@
    { - dispatch('close') - dispatch('new', { - kind: 'script', - inlineScript: { - language: 'bun', - kind: 'script', - subkind: 'claudesandbox' - } - }) - }} + neutral + returnIcon + selected={aiSelected === 0} + onSelect={newClaudeSandbox} + onHover={() => (selectedByKeyboard = 0)} />
    {:else} diff --git a/frontend/src/lib/components/flows/stepsInputArgs.svelte.ts b/frontend/src/lib/components/flows/stepsInputArgs.svelte.ts index 8aa7a1f69f..6db6a4eb23 100644 --- a/frontend/src/lib/components/flows/stepsInputArgs.svelte.ts +++ b/frontend/src/lib/components/flows/stepsInputArgs.svelte.ts @@ -125,12 +125,15 @@ export class StepsInputArgs { initializeFromSchema( mod: FlowModule, schema: { properties?: Record }, - pickableProperties: PickableProperties | undefined + pickableProperties: PickableProperties | undefined, + // Off for the reactive re-evaluations that follow every flow edit; on for an explicit + // run, where a failing expression would otherwise become `undefined` unseen. + showError: boolean = false ) { const args = Object.fromEntries( Object.keys(schema.properties ?? {}).map((k) => [ k, - evalValue(k, mod, pickableProperties, false) + evalValue(k, mod, pickableProperties, showError) ]) ) @@ -158,14 +161,16 @@ export class StepsInputArgs { id: string, flowState: FlowState | undefined, flow: OpenFlow | undefined, - previewArgs: Record | undefined + previewArgs: Record | undefined, + showError: boolean = false ) { if (id === 'failure' && flow && flow.value.failure_module && flowState) { const picker = getFailureStepPropPicker(flowState, flow, previewArgs) this.initializeFromSchema( flow.value.failure_module, flowState['failure']?.schema ?? {}, - picker.pickableProperties + picker.pickableProperties, + showError ) return } @@ -193,7 +198,12 @@ export class StepsInputArgs { false ) const pickableProperties = stepPropPicker.pickableProperties - this.initializeFromSchema(modules[0], flowState[id]?.schema ?? {}, pickableProperties) + this.initializeFromSchema( + modules[0], + flowState[id]?.schema ?? {}, + pickableProperties, + showError + ) } removeExtraKey(moduleId: string, keys: string[]) { diff --git a/frontend/src/lib/components/flows/utils.svelte.ts b/frontend/src/lib/components/flows/utils.svelte.ts index d62138868f..8d8e4366e5 100644 --- a/frontend/src/lib/components/flows/utils.svelte.ts +++ b/frontend/src/lib/components/flows/utils.svelte.ts @@ -12,6 +12,7 @@ import { } from '$lib/gen' import { workspaceStore } from '$lib/stores' import { cleanExpr, emptySchema } from '$lib/utils' +import { unescapeTemplateBackticks } from '$lib/utils/templateLiteral' import { get } from 'svelte/store' import type { FlowModuleState } from './flowState' import { type PickableProperties, dfs } from './previousResults' @@ -219,7 +220,7 @@ export function codeToStaticTemplate(code?: string): string | undefined { if (lines.length == 1) { const line = lines[0].trim() if (line[0] == '`' && line.charAt(line.length - 1) == '`') { - return line.slice(1, line.length - 1).replaceAll('\\`', '`') + return unescapeTemplateBackticks(line.slice(1, line.length - 1)) } else { return `\$\{${line}\}` } diff --git a/frontend/src/lib/components/home/HomeAIChat.svelte b/frontend/src/lib/components/home/HomeAIChat.svelte new file mode 100644 index 0000000000..441489db65 --- /dev/null +++ b/frontend/src/lib/components/home/HomeAIChat.svelte @@ -0,0 +1,332 @@ + + + + +
    +
    + {#if showComposer && !collapsed} + {#if !disabled} + +
    + setCollapsed(true)} /> +
    + {/if} +
    +

    Build with AI

    + Beta +
    + +
    +
    + + {#if !value} + + + {/if} + +
    + +
    +
    + {#if disabled} + +
    +

    + {#if $aiUserDisabled} + Windmill AI is disabled in your account settings + {:else if freeTierExhausted} + You have used all of your free Windmill AI tokens + {:else} + No AI provider is configured + {/if} +

    +
    + {#if $aiUserDisabled} + + + {:else} + + {/if} + +
    +
    + {/if} +
    + {/if} + +
    + {#if showComposer && !collapsed} +
    + {#each homeAIExamples as example (example.label)} + + {/each} +
    + {:else if showComposer} + + + {:else} +
    + {/if} + + +
    + + {#if !$userStore?.operator && HOME_SHOW_HUB} + + {/if} +
    +
    +
    +
    + + diff --git a/frontend/src/lib/components/home/ItemsList.svelte b/frontend/src/lib/components/home/ItemsList.svelte index d1f3b9b761..172482f060 100644 --- a/frontend/src/lib/components/home/ItemsList.svelte +++ b/frontend/src/lib/components/home/ItemsList.svelte @@ -1,7 +1,7 @@ {#if Array.isArray(filtersAndSelected) && filtersAndSelected.length > 0} -
    +
    {#each displayedFilters as filter (filter)}
    - {#if resourceType} + {#if icon} + {@const Icon = icon} + + {:else if resourceType} {@const SvelteComponent = appIconComponent(filter)} {:else if filter.startsWith('u/')} @@ -123,7 +140,7 @@
    (expanded = !expanded)} diff --git a/frontend/src/lib/components/home/NoItemFound.svelte b/frontend/src/lib/components/home/NoItemFound.svelte index dd88ea9d4f..94d6dfa1b8 100644 --- a/frontend/src/lib/components/home/NoItemFound.svelte +++ b/frontend/src/lib/components/home/NoItemFound.svelte @@ -30,8 +30,7 @@ {:else}
    -
    Welcome to Windmill
    -
    +
    Get started by creating your first script, flow, or app
    diff --git a/frontend/src/lib/components/home/TreeView.svelte b/frontend/src/lib/components/home/TreeView.svelte index 4af936b987..8e872eebcd 100644 --- a/frontend/src/lib/components/home/TreeView.svelte +++ b/frontend/src/lib/components/home/TreeView.svelte @@ -1,6 +1,7 @@ {#if !isDismissed} -
    -
    - -
    -
    - {#if hasCompletedAny} - New tutorial available! - {:else} - Learn with interactive tutorials - {/if} -
    -
    - {#if hasCompletedAny} - Continue your learning journey and master new Windmill skills. - {:else} - Get started quickly with step-by-step guides on building flows, scripts, and more. - {/if} -
    -
    -
    -
    - - -
    + +
    + + {#if hasCompletedAny} + New tutorial available! + {:else} + First time? + {/if} + + +
    {/if} diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 6c66af7d84..eb8c4c86c4 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -30,6 +30,10 @@ export interface Setting { placeholder?: string cloudonly?: boolean ee_only?: string + /** Ceiling a `seconds` field enforces on a build without a license, when CE genuinely caps + * the value. Not implied by `ee_only`: a setting can be EE-badged because the feature it + * configures is EE while the value itself has the same range on either edition. */ + ceMaxSeconds?: number tooltip?: string key: string // If value is not specified for first element, it will automatcally use undefined @@ -283,6 +287,8 @@ export const settings: Record = { placeholder: '30', storage: 'setting', ee_only: 'You can only adjust this setting to above 30 days in the EE version', + // Mirrors CE_MAX_RETENTION_PERIOD_SECS, which the backend clamps to on write. + ceMaxSeconds: 60 * 60 * 24 * 30, cloudonly: false }, { @@ -971,6 +977,24 @@ export const settings: Record = { triggersRestart: true, defaultValue: () => ({ enabled: false, enabled_languages: [...OTEL_TRACING_PROXY_LANGUAGES] }) }, + { + label: 'HTTP Request Tracing retention in secs', + key: 'otel_traces_retention_secs', + description: + 'How long a captured HTTP request span is kept in the database, and therefore how far back the job details view can show a job its requests. Independent of the job retention period, so a span may outlive its job or be swept while the job remains. Defaults to 7 days. Leave it empty for the default.', + fieldType: 'seconds', + storage: 'setting', + cloudonly: false, + // Badged EE because only the EE proxy captures spans, but deliberately no + // `ceMaxSeconds`: a CE build still sweeps rows an EE-era instance left behind, and + // the backend accepts the same range on either edition. + ee_only: 'HTTP Request Tracing is an EE feature', + error: + 'HTTP Request Tracing retention must be between 1 second and 100 years, leave it empty for the default', + isValid: (value: any) => + value == undefined || + (typeof value === 'number' && value > 0 && value <= 60 * 60 * 24 * 365 * 100) + }, { label: 'Prometheus', description: @@ -982,6 +1006,23 @@ export const settings: Record = { triggersRestart: true } ], + 'Service logs': [ + { + label: 'Retention in secs', + key: 'service_log_retention_secs', + description: + 'How long a service log is kept, across every copy of it: the entry in the database, the file on the disk of the process that wrote it, and — once instance object storage is configured and the indexer has ingested it — its line in the columnar store that search and the log viewer read. Search reaches back at most this far, and less when the indexer time window under Indexer is shorter. Defaults to 14 days. There is no keep-forever setting here — leave it empty for the default.', + fieldType: 'seconds', + storage: 'setting', + cloudonly: false, + error: + 'Service log retention must be between 1 second and 100 years — leave it empty for the default', + isValid: (value: any) => + value == undefined || + (typeof value === 'number' && value > 0 && value <= 60 * 60 * 24 * 365 * 100) + } + ], + Indexer: [ { label: '', @@ -1173,6 +1214,12 @@ export const instanceSettingsNavigationGroups = [ aiDescription: 'Instance OTEL/Prometheus settings', isEE: true }, + { + id: 'service_logs', + label: 'Service logs', + aiId: 'instance-settings-service-logs', + aiDescription: 'Service log retention settings' + }, { id: 'indexer', label: 'Indexer', @@ -1256,6 +1303,7 @@ export const tabToCategoryMap: Record = { webhooks: 'Webhooks', otel_prom: 'OTEL/Prom', indexer: 'Indexer', + service_logs: 'Service logs', telemetry: 'Telemetry', secret_storage: 'Secret Storage', object_storage: 'Object Storage', @@ -1291,6 +1339,7 @@ export const categoryToTabMap: Record = { Webhooks: 'webhooks', 'OTEL/Prom': 'otel_prom', Indexer: 'indexer', + 'Service logs': 'service_logs', Telemetry: 'telemetry', 'Secret Storage': 'secret_storage', 'Object Storage': 'object_storage', diff --git a/frontend/src/lib/components/instanceSettings/GhesAppSettings.svelte b/frontend/src/lib/components/instanceSettings/GhesAppSettings.svelte index da63e8d64a..acd18130da 100644 --- a/frontend/src/lib/components/instanceSettings/GhesAppSettings.svelte +++ b/frontend/src/lib/components/instanceSettings/GhesAppSettings.svelte @@ -283,7 +283,17 @@ placeholder: '12345', disabled: fieldsDisabled }} - bind:value={$values['github_enterprise_app'].app_id} + bind:value={ + () => $values['github_enterprise_app'].app_id, + (v) => { + // The backend expects app_id as a positive integer (i64). Reject + // fractional/out-of-range values instead of truncating them, and store + // undefined (never a string or 0) so the config omits the key when unset. + const n = typeof v === 'string' ? Number(v.trim() || NaN) : (v ?? NaN) + $values['github_enterprise_app'].app_id = + Number.isSafeInteger(n) && n > 0 ? n : undefined + } + } />
    diff --git a/frontend/src/lib/components/mcp/enabledServers.ts b/frontend/src/lib/components/mcp/enabledServers.ts index 4a7bb7bae2..b49b788c67 100644 --- a/frontend/src/lib/components/mcp/enabledServers.ts +++ b/frontend/src/lib/components/mcp/enabledServers.ts @@ -1,66 +1,11 @@ -import { get } from 'svelte/store' -import { userStore } from '$lib/stores' +import { createEnabledPathsPreference } from '$lib/components/copilot/chat/enabledPathsPreference' -/** - * Which MCP servers the chat may use, per workspace and per account. - * - * Being able to read an `mcp` resource is not the same as wanting the chat to - * act through it: a resource in a shared folder is readable by a whole team, and - * each server's tools both reach an external system and put their descriptions - * in the model's context. So a server is off until it is turned on here, and - * connecting one through the chat turns it on for the person who connected it. - * - * Stored per browser, like the chat's other per-user preferences, but keyed by - * email as well as workspace: browser storage outlives a logout, and inheriting - * the previous account's enabled servers would hand the next person tools they - * never turned on. - */ -const KEY = 'wm_mcp_enabled' +/** Which MCP servers the chat may act through, per workspace and per account. A + * server's tools both reach an external system and put their descriptions in the + * model's context, so one is off until it is turned on; connecting one through the + * chat turns it on for the person who connected it. */ +const preference = createEnabledPathsPreference('wm_mcp_enabled') -function scope(workspace: string): string | undefined { - const email = get(userStore)?.email - return email ? `${workspace}:${email}` : undefined -} - -function read(): Record { - if (typeof localStorage === 'undefined') return {} - try { - return JSON.parse(localStorage.getItem(KEY) ?? '{}') - } catch { - return {} - } -} - -function write(all: Record) { - try { - localStorage.setItem(KEY, JSON.stringify(all)) - } catch (e) { - console.error('Failed to persist enabled MCP servers', e) - } -} - -export function enabledMcpPaths(workspace: string): string[] { - const key = scope(workspace) - return key ? (read()[key] ?? []) : [] -} - -export function isMcpEnabled(workspace: string, path: string): boolean { - return enabledMcpPaths(workspace).includes(path) -} - -/** Returns false when there is no account to record the preference against, so a - * caller that just connected a server can say it did not stay on. */ -export function setMcpEnabled(workspace: string, path: string, enabled: boolean): boolean { - const key = scope(workspace) - if (!key) return false - const all = read() - const current = new Set(all[key] ?? []) - if (enabled) { - current.add(path) - } else { - current.delete(path) - } - all[key] = [...current] - write(all) - return true -} +export const enabledMcpPaths = preference.enabledPaths +export const isMcpEnabled = preference.isEnabled +export const setMcpEnabled = preference.setEnabled diff --git a/frontend/src/lib/components/oauthRegistry.ts b/frontend/src/lib/components/oauthRegistry.ts new file mode 100644 index 0000000000..2a18b2661c --- /dev/null +++ b/frontend/src/lib/components/oauthRegistry.ts @@ -0,0 +1,41 @@ +import oauthConnectRegistry from '$oauth_connect_registry' + +/** + * Reads of the static OAuth connect registry (`backend/oauth_connect.json`), shared by the + * connect dialog and by anything deciding whether to offer connecting at all. + * + * Kept here rather than inside `AppConnectInner` because two places have to agree on the + * answer: the dialog decides whether a type gets the OAuth flow, and a caller deciding + * whether to open the dialog has to reach the same verdict — or it offers Connect where the + * dialog would fall back to a manual form, or hides it where the dialog would have worked. + */ + +const SANDBOX_SUFFIX = '_sandbox' + +export function stripSandboxSuffix(name: string): string { + return name.endsWith(SANDBOX_SUFFIX) ? name.slice(0, -SANDBOX_SUFFIX.length) : name +} + +/** + * The registry entry for the first of `names` that has one. Callers pass the client name and + * the resource type, which differ for sandbox clients (`salesforce_sandbox` vs `salesforce`); + * both are resolved to the parent entry so a sandbox connection sees the same metadata. + */ +export function registryEntryFor(...names: (string | undefined)[]): any { + const reg = oauthConnectRegistry as Record + for (const n of names) { + if (!n) continue + const entry = reg[stripSandboxSuffix(n)] + if (entry) return entry + } + return undefined +} + +/** + * The registry declares this provider supports client credentials — which is what makes it + * connectable with no OAuth client configured on the instance, since the credentials are + * entered per resource rather than held by a superadmin. + */ +export function registryCcCapableFor(...names: (string | undefined)[]): boolean { + return registryEntryFor(...names)?.grant_types?.includes('client_credentials') ?? false +} diff --git a/frontend/src/lib/components/raw_apps/FileTreeNode.svelte b/frontend/src/lib/components/raw_apps/FileTreeNode.svelte index 3d428b9aaf..0e02701347 100644 --- a/frontend/src/lib/components/raw_apps/FileTreeNode.svelte +++ b/frontend/src/lib/components/raw_apps/FileTreeNode.svelte @@ -1,5 +1,15 @@ @@ -125,16 +127,8 @@ {runnable} isSelected={selectedRunnable === id} isEditing={editingId === id} - onSelect={() => { - selectedRunnable = id - onSelect?.(id) - }} - onDelete={() => { - delete runnables[id] - if (selectedRunnable === id) { - selectedRunnable = undefined - } - }} + onSelect={() => onSelect?.(id)} + onDelete={() => onDelete(id)} onRename={(newId) => renameRunnable(id, newId)} onRequestEdit={() => (editingId = id)} onCancelEdit={() => (editingId = undefined)} diff --git a/frontend/src/lib/components/raw_apps/RawAppSidebar.svelte b/frontend/src/lib/components/raw_apps/RawAppSidebar.svelte index b8d6e33c87..fbe02e1dc9 100644 --- a/frontend/src/lib/components/raw_apps/RawAppSidebar.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppSidebar.svelte @@ -3,6 +3,7 @@ SUBTLE_PANEL_TITLE } from '../apps/editor/settingsPanel/common/PanelSection.svelte' import type { Runnable } from '../apps/inputType' + import { WMILL_TS_PATH } from './utils' import RawAppInlineScriptPanelList from './RawAppInlineScriptPanelList.svelte' import FileExplorer from '../FileExplorer.svelte' import { Plus, File, Folder, Camera } from 'lucide-svelte' @@ -18,10 +19,14 @@ interface Props { runnables: Record + /** Read-only; the editor switches selection through `onSelectRunnable`. */ selectedRunnable: string | undefined files: Record modules?: Modules - onSelectFile?: (path: string) => void + onSelectRunnable?: (key: string) => void + onDeleteRunnable: (key: string) => void + onSelectPath?: (path: string) => void + /** Read-only; the editor switches selection through `onSelectPath`. */ selectedDocument: string | undefined historyManager?: RawAppHistoryManager historySelectedId?: number | undefined @@ -39,11 +44,13 @@ let { runnables, - selectedRunnable = $bindable(), + selectedRunnable, files = $bindable({}), modules, - onSelectFile, - selectedDocument = $bindable(), + onSelectRunnable, + onDeleteRunnable, + onSelectPath, + selectedDocument, historyManager, historySelectedId, onHistorySelect, @@ -79,13 +86,6 @@ } let fileExplorer: FileExplorer | undefined = $state() - - function handleSelectPath(path: string) { - selectedDocument = path - if (!path.endsWith('/')) { - onSelectFile?.(path) - } - }
    -
    -

    - - Start with AI - (optional) -

    + {#if !$copilotInfo.workspaceDisabled} +
    +

    + + Start with AI + (optional) +

    - {#if !aiConfigLoaded} -
    - - Loading AI settings... -
    - {:else if !isAiEnabled} - - You can still create an app manually but using AI is highly recommended. -
    - {#if $userStore?.is_admin} - Configure AI in - workspace settings - - {#if $superadmin} - or + {#if !aiConfigLoaded} +
    + + Loading AI settings... +
    + {:else if !isAiEnabled} + + You can still create an app manually but using AI is highly recommended. +
    + {#if $userStore?.is_admin} + Configure AI in + workspace settings + + {#if $superadmin} + or + instance settings + + {/if} to enable this feature. + {:else if $superadmin} + Configure AI in instance settings - - {/if} to enable this feature. - {:else if $superadmin} - Configure AI in - instance settings - to enable this feature. - {:else} - Ask your workspace admin to configure AI in workspace settings to enable this feature. - {/if} -     - {:else} -
    - -

    - {handsOffToSession - ? 'Leave empty to start with a blank template, or describe your app to open an AI session that builds it.' - : 'Leave empty to start with a blank template, or describe your app to get AI assistance right away.'} -

    -
    - {/if} -
    + to enable this feature. + {:else} + Ask your workspace admin to configure AI in workspace settings to enable this + feature. + {/if} + + {:else} +
    + +

    + {handsOffToSession + ? 'Leave empty to start with a blank template, or describe your app to open an AI session that builds it.' + : 'Leave empty to start with a blank template, or describe your app to get AI assistance right away.'} +

    +
    + {/if} +
    + {/if}
    {#if isAiEnabled}
    - {#if (itemMap[tab] ?? []).length === 0 && searchTerm.length > 0} + {#if (itemMap[tab] ?? []).length === 0 && searchTerm.length > 0 && !$copilotInfo.workspaceDisabled} - { - askAiButton?.onClick() - }} - id={'ai:no-results-ask-ai'} - hovered={true} - label={`Try asking \`${searchTerm}\` to AI`} - icon={WandSparkles} - bind:mouseMoved - /> + {#if !$copilotInfo.workspaceDisabled} + { + askAiButton?.onClick() + }} + id={'ai:no-results-ask-ai'} + hovered={true} + label={`Try asking \`${searchTerm}\` to AI`} + icon={WandSparkles} + bind:mouseMoved + /> + {/if}
    Tip: press `esc` to quickly clear the search bar
    diff --git a/frontend/src/lib/components/select/GenericDropdown.svelte b/frontend/src/lib/components/select/GenericDropdown.svelte index fc738d5f3f..78a88a6457 100644 --- a/frontend/src/lib/components/select/GenericDropdown.svelte +++ b/frontend/src/lib/components/select/GenericDropdown.svelte @@ -15,6 +15,7 @@ class: className = '', innerClass = '', maxHeight = 256, + instantClose = false, getInputRect, children }: { @@ -26,6 +27,9 @@ class?: string innerClass?: string maxHeight?: number + // Close with no height-collapse animation (still animates open). Used where the + // dropdown is toggled off as the user types, so the panel vanishes at once. + instantClose?: boolean getInputRect?: () => DOMRect children?: Snippet } = $props() @@ -71,7 +75,11 @@ // We do not use Svelte transitions because they can not animate in the opposite direction // when the dropdown is opens above the input // Also CSS transitions are smoother because they do not rely on JS / animation frames - let uiState = $state({ domExists: untrack(() => open), visible: untrack(() => open), timeout: null as number | null }) + let uiState = $state({ + domExists: untrack(() => open), + visible: untrack(() => open), + timeout: null as number | null + }) let initial = true watch( () => open && !disabled, @@ -81,7 +89,10 @@ initial = false return } - if (reducedMotion.val) { + // Reduced motion skips all animation; instantClose skips only the closing + // one (open still animates) so the panel disappears the instant it's toggled off. + if (reducedMotion.val || (instantClose && !isOpen)) { + if (uiState.timeout) clearTimeout(uiState.timeout) uiState = { domExists: open && !disabled, visible: open && !disabled, diff --git a/frontend/src/lib/components/sessions/OpenInSessionButton.svelte b/frontend/src/lib/components/sessions/OpenInSessionButton.svelte index 00bf1c829e..03ce21589e 100644 --- a/frontend/src/lib/components/sessions/OpenInSessionButton.svelte +++ b/frontend/src/lib/components/sessions/OpenInSessionButton.svelte @@ -40,6 +40,7 @@ import AIButton from '$lib/components/copilot/chat/AIButton.svelte' import { AIBtnClasses } from '$lib/components/copilot/chat/AIButtonStyle' import { prefersSessionHandoff } from '$lib/components/copilot/chat/global/gate' + import { copilotInfo } from '$lib/aiStore' import { userStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' import { openSourceInSession } from './sessionSwitch.svelte' @@ -80,7 +81,9 @@ // them, so an entry point on a page they can reach (Runs, the trigger lists) // would only route them into that refusal. const show = $derived( - !inSessionPanel && !!(source?.target || source?.page) && prefersSessionHandoff($userStore?.operator) + !inSessionPanel && + !!(source?.target || source?.page) && + prefersSessionHandoff($userStore?.operator) ) // Not $state: only read inside open() as a re-entrancy latch, never rendered. @@ -102,7 +105,10 @@ } -{#if show} +{#if $copilotInfo.workspaceDisabled} + +{:else if show} void } = $props() + // The group's highlighted side. Melt moves it on click, before the navigation + // that would change `mode`, so it is derived from the route (which wins once + // a switch navigates) and pushed back when one does not, or the rail would + // read "AI Sessions" on an editor page with the clicked side inert until + // "Workspace" was pressed first. + let selected: string | string[] | null | undefined = $derived(mode) + function onSelected(next: 'nav' | 'session') { if (next === mode) return onToggle?.() - if (next === 'session') void enterSessionMode() - else void exitSessionMode() + if (next === 'session') { + // An editor whose draft could not be persisted keeps the user on the + // page, as its own "Open in AI session" button does, rather than open a + // session on an older draft than the one on screen. + void enterSessionModeFromNav().catch((e) => { + selected = mode + sendUserToast(e instanceof Error ? e.message : String(e), true) + }) + } else void exitSessionMode() } // Pressing the already-active "Workspace" side goes home, so the toggle doubles @@ -41,7 +56,7 @@ child of the group's track — so the buttons fill the rail width only if those wrappers grow. `[&>*]:flex-1` makes every direct child split the track evenly. --> *]:w-full' : 'w-full [&>*]:flex-1'} > diff --git a/frontend/src/lib/components/sessions/SessionPicker.svelte b/frontend/src/lib/components/sessions/SessionPicker.svelte index 67b53aeff4..e5b62da0eb 100644 --- a/frontend/src/lib/components/sessions/SessionPicker.svelte +++ b/frontend/src/lib/components/sessions/SessionPicker.svelte @@ -1,5 +1,6 @@ {#snippet externalLinkHint()} @@ -315,6 +342,32 @@
    Session not found
    {:else} {#snippet inputPreface()} + {#if showRecoveryNotice} + + + +
    +
    + + + We couldn't find that session +
    +
    + {/if} {#if !hasFirstUserMessage} {/if} @@ -329,9 +382,7 @@ and reconcile would re-archive a workspace-archived one anyway. When the workspace is unavailable the SessionChangesBar below shows the move/discard banner instead (its actions are the real recovery path). --> -
    +
    This session is archived diff --git a/frontend/src/lib/components/sessions/openInSessionContext.ts b/frontend/src/lib/components/sessions/openInSessionContext.ts index 27284802cc..ddecb01b4f 100644 --- a/frontend/src/lib/components/sessions/openInSessionContext.ts +++ b/frontend/src/lib/components/sessions/openInSessionContext.ts @@ -1,5 +1,6 @@ -import { getContext, setContext } from 'svelte' +import { getContext, onDestroy, setContext } from 'svelte' import type { OpenInSessionSource } from './OpenInSessionButton.svelte' +import type { PreviewItemRoute } from './previewPaths' // The "Open in AI session" hand-off, published by the component that owns the // item being edited (FlowBuilder, RawAppEditor) for AI entry points too deep in @@ -14,10 +15,43 @@ export type OpenInSessionHandoff = { source: (opts?: { moduleId?: string }) => OpenInSessionSource | undefined } +// The hand-offs of every editor currently mounted. The navigation rail's "AI +// Sessions" switch sits above every page, out of reach of the context, and +// looks the editor of the item it is leaving up here instead. +const mounted = new Set() + export function setOpenInSessionHandoff(handoff: OpenInSessionHandoff): void { setContext(KEY, handoff) + onDestroy(registerMountedOpenInSessionHandoff(handoff)) +} + +/** Count `handoff` as mounted until the returned unregister runs. Split from + * setOpenInSessionHandoff so the registry can be driven outside a component. */ +export function registerMountedOpenInSessionHandoff(handoff: OpenInSessionHandoff): () => void { + mounted.add(handoff) + return () => { + mounted.delete(handoff) + } } export function getOpenInSessionHandoff(): OpenInSessionHandoff | undefined { return getContext(KEY) } + +/** The mounted editor's hand-off for the item `route` names, or undefined when + * no editor on screen publishes one for it (a legacy app, a detail page). + * Matched on the item rather than taken as "the latest registered": a script + * editor mounted in a flow's drawer registers too, and the rail wants the + * page's own item. */ +export function findMountedOpenInSessionSource( + route: PreviewItemRoute +): OpenInSessionSource | undefined { + const kind = route.kind === 'app' ? (route.raw_app ? 'raw_app' : undefined) : route.kind + if (!kind) return undefined + for (const handoff of mounted) { + const source = handoff.source() + const target = source?.target + if (target && target.kind === kind && target.path === route.itemPath) return source + } + return undefined +} diff --git a/frontend/src/lib/components/sessions/sessionRecoveryNotice.svelte.ts b/frontend/src/lib/components/sessions/sessionRecoveryNotice.svelte.ts new file mode 100644 index 0000000000..635bd68601 --- /dev/null +++ b/frontend/src/lib/components/sessions/sessionRecoveryNotice.svelte.ts @@ -0,0 +1,18 @@ +import { SvelteSet } from 'svelte/reactivity' + +// Session ids opened as a stand-in for a `session_name` this browser doesn't +// hold. Kept in memory rather than on the Session record: persisted, the notice +// would replay on every reload of a session the user has since made their own. +const recovered = new SvelteSet() + +export function markSessionRecovered(id: string): void { + recovered.add(id) +} + +export function isSessionRecovered(id: string): boolean { + return recovered.has(id) +} + +export function clearSessionRecovered(id: string): void { + recovered.delete(id) +} diff --git a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts index fc0d291190..9bae07288d 100644 --- a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts @@ -29,6 +29,12 @@ import { userWorkspaces, workspaceStore } from '$lib/stores' import { copilotWorkspace } from '$lib/aiStore' import { loadCopilot } from '$lib/components/copilot/loadCopilot' import { emptySchema, type StateStore } from '$lib/utils' +import { + localRunEnded, + localRunStarted, + onRemoteTurnEnd, + runHeldElsewhere +} from './sessionSync.svelte' import { commitSessionWorkspace, deleteSession as deleteSessionState, @@ -338,6 +344,15 @@ function createRuntime(session: Session): SessionRuntime { // Carried into the tool helpers so this session's preview/deploy tool calls // dispatch to THIS session even when another session is the UI-active one. manager.sessionId = session.id + // Cross-tab awareness: heartbeat while this tab runs a turn, composer lock + // (and send refusal) while another tab does. The chat id is read at turn + // end, not captured at start — the turn may have rotated it, and the other + // tabs re-read whichever record it ended on. + manager.runHeldElsewhereResolver = () => runHeldElsewhere(session.id) + manager.onRunningChanged = (running) => { + if (running) localRunStarted(session.id, manager.historyManager.getCurrentChatId()) + else localRunEnded(session.id, manager.historyManager.getCurrentChatId()) + } // The chat targets the session's OWN (possibly forked) workspace without // switching the global workspaceStore. Resolved live from the session record // so it tracks the pending → committed (and staged-fork) transitions. @@ -958,6 +973,65 @@ export function getRuntime(sessionId: string): SessionRuntime | undefined { return runtimes.get(sessionId) } +// --------------------------------------------------------------------------- +// Cross-tab catch-up +// --------------------------------------------------------------------------- + +// Chained per session so two turn-ends close together (a turn plus its queued +// follow-up) re-read sequentially: the later read starts after the earlier +// one's loadPastChat, so the newest record is what ends up on screen. +const catchUps = new Map>() + +onRemoteTurnEnd((sessionId, chatId) => { + const next = (catchUps.get(sessionId) ?? Promise.resolve()) + .then(() => applyRemoteTurnEnd(sessionId, chatId)) + .catch((e) => console.error('Failed to catch up on a turn from another tab', e)) + catchUps.set(sessionId, next) + void next.finally(() => { + if (catchUps.get(sessionId) === next) catchUps.delete(sessionId) + }) + // Awaited by the caller: the composer unlock rides on this settling. + return next +}) + +async function applyRemoteTurnEnd(sessionId: string, chatId: string): Promise { + const runtime = runtimes.get(sessionId) + if (!runtime) return + const m = runtime.manager + // Two transient states get a short retry rather than a skip, because the + // composer unlocks when this promise settles and a skip would unlock it on + // stale history: a send of this tab's own still in preflight (it may yet be + // refused, leaving no turn to converge on), and a store that failed to + // open. A turn actually running here owns the transcript instead — its own + // end converges — and the pruner caps the whole hold at STALE_MS anyway. + for (let attempt = 0; ; attempt++) { + if (m.loading) return + if (!m.sendInFlight) { + const res = await m.historyManager.reloadChat(chatId) + if (res === 'missing') return + if (res === 'loaded') break + } + if (attempt >= 7) return + await new Promise((r) => setTimeout(r, 500)) + if (runtimes.get(sessionId) !== runtime) return + } + // Disposed (session deleted, teardown) while the read was in flight. + if (runtimes.get(sessionId) !== runtime) return + // Adopts the driver's chat unconditionally, current view included: watching + // a session means following where its activity is, and it is also how tabs + // converge after an unsynced /clear rotation. A watcher browsing an older + // conversation is pulled along — deliberate, and the price of not syncing + // rotation as its own message. + // + // preserveQueue: this reload is a catch-up, not a conversation switch — a + // draft queued here (a refused send's kept message, a failed turn's card) + // is unsent user input the re-read must not destroy. + await m.loadPastChat(chatId, { preserveQueue: true }) + // loadPastChat's own artifact sync no-ops for an unchanged session id, so + // artifacts the driver wrote during the turn need this forced re-read. + await m.artifacts.resyncFromStore() +} + // Point a session's preview at a single seed tab. For re-pointing an existing // draft session at a new destination ("Open in AI session" / new-session-from- // page on a reused transient): its previous tabs — persisted with the draft diff --git a/frontend/src/lib/components/sessions/sessionState.svelte.ts b/frontend/src/lib/components/sessions/sessionState.svelte.ts index d8ef56b273..0f22a23974 100644 --- a/frontend/src/lib/components/sessions/sessionState.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionState.svelte.ts @@ -21,6 +21,7 @@ import { import { getLocalSetting, storeLocalSetting } from '$lib/utils' import { logFeatureUsage } from '$lib/utils/featureUsage' import { workspaceRootId } from './sessionScope.svelte' +import { clearSessionRecovered } from './sessionRecoveryNotice.svelte' import { type DBSchema, type IDBPDatabase } from 'idb' import { userScopedDb } from '$lib/userScopedDb' import { deleteItemsForSession } from '../copilot/chat/files/attachedFilesDB' @@ -343,8 +344,8 @@ export function setSessionDraftPrompt(sessionId: string, text: string): void { if ((s.draftPrompt ?? '') === text) return // Keep `transient` (means "in-memory only") set until the flush persists the // draft, so hydrateSessions preserves it across a reconcile inside this window; - // isReusableBlank, not `transient`, is what stops createSession reusing a typed - // draft. Only the IndexedDB write is debounced. + // isDiscardableDraft, not `transient`, is what stops createSession reusing a + // typed draft. Only the IndexedDB write is debounced. s.draftPrompt = text clearTimeout(draftPromptFlushHandles.get(sessionId)) draftPromptFlushHandles.set( @@ -404,8 +405,9 @@ export function takeSessionAutoSend(sessionId: string): boolean { // Persist a session on a genuine user edit, promoting an in-memory-only // (transient) pending session to a durable IndexedDB record on first touch. -// Non-touch writers (runtime chatId seeding, unread watermark) call putSession -// directly, so an untouched draft stays in memory and vanishes on reload. +// Non-touch writers (runtime chatId seeding via patchStoredSessionChatId, the +// unread watermark via putSession) persist directly, so an untouched draft +// stays in memory and vanishes on reload. function persistTouched(s: Session): void { if (s.transient) delete s.transient s.lastActivityAt = Date.now() @@ -436,10 +438,10 @@ async function deleteSessionRow(db: IDBPDatabase, id: string): Pr await db.delete('sessions', id) } -// The one way to write a session's record, and the other half of the invariant above: -// every caller reaches its write across an await — putSession on the DB handle, the -// reconcile and hydrate passes on a getAll() snapshot that an interleaved delete -// invalidates — so the tombstone has to be consulted here, not only at the entry points. +// The way a session record is written (patchStoredSessionChatId is the one +// exception: it re-checks the tombstone inline to stay inside its own +// transaction). Every caller reaches its write across an await, so the +// tombstone has to be consulted here, not only at the entry points. async function putSessionRow(db: IDBPDatabase, s: Session): Promise { if (deletedSessionIds.has(s.id)) return await db.put('sessions', s) @@ -748,30 +750,52 @@ export function requestComposerFocus(): void { composerFocusRequest.nonce++ } -// An untouched in-memory blank that `+` may reuse/discard. `draftPrompt === -// undefined` (never edited), not falsiness: a draft typed then erased to '' still -// has a pending flush and is a real session, so it must survive both. Every other +// An untouched in-memory blank that `+` may reuse and createSession may silently +// drop. `draftPrompt === undefined` (never edited), not falsiness: a draft typed +// then erased to '' still has a pending flush and is a real session. Every other // touch clears `transient` synchronously, so only the draft prompt needs checking. -function isReusableBlank(s: Session): boolean { +function isDiscardableDraft(s: Session): boolean { return !!s.transient && s.draftPrompt === undefined } +// Somewhere empty to put the user, for a URL naming a session this browser +// doesn't hold. Only `transient` makes "empty" trustworthy: chat seeding and +// attached-file persistence both key off `!transient` and leave every field +// below untouched, so a persisted session can hold a conversation regardless. +export function findEmptyLandingSession(): Session | undefined { + return sessionState.sessions.find( + (s) => + !!s.transient && + !s.archived && + !s.workspace_id && + // Falsiness, not `=== undefined`: we only navigate into the session, so a + // draft erased back to '' is still an empty composer to land on. + !s.draftPrompt?.trim() && + !s.pending_fork && + sessionInCurrentFamily(s) + ) +} + export function createSession(): Session { // Reuse an existing untouched draft from the active family rather than pile a // blank entry on every `+`, so several pending sessions can still be built up // in parallel, one touch at a time. A cross-family leftover blank is dropped // instead of reused (reusing it would act on that family). const reusable = sessionState.sessions.find( - (s) => isReusableBlank(s) && sessionInCurrentFamily(s) + (s) => isDiscardableDraft(s) && sessionInCurrentFamily(s) ) if (reusable) { sessionState.currentSessionId = reusable.id + // The blank recovery just landed on is exactly what this reuses, so `+` + // would otherwise hand back a session still carrying the recovery notice: + // asking for a new session must not be answered with "we couldn't find it". + clearSessionRecovered(reusable.id) // Reusing an already-active draft doesn't change currentSessionId, so ask // the composer to focus explicitly — the caller still navigates/redirects. requestComposerFocus() return reusable } - sessionState.sessions = sessionState.sessions.filter((s) => !isReusableBlank(s)) + sessionState.sessions = sessionState.sessions.filter((s) => !isDiscardableDraft(s)) const existingNumbers = sessionState.sessions .map((s) => /^session-(\d+)$/.exec(s.name)?.[1]) .map((n) => (n ? parseInt(n, 10) : 0)) @@ -1087,6 +1111,24 @@ export function setSessionArchived(id: string, archived: boolean) { persistTouched(s) } +// A counter rather than a flag: an inner teardown finishing must not reopen the +// gate while an outer one is still running. Released in a finally, so a delete +// that throws can't wedge it shut. +let openSessionTeardowns = $state(0) + +export function isTearingDownOpenSession(): boolean { + return openSessionTeardowns > 0 +} + +export async function withOpenSessionTeardown(run: () => Promise): Promise { + openSessionTeardowns++ + try { + return await run() + } finally { + openSessionTeardowns-- + } +} + export function deleteSession(id: string) { const s = sessionState.sessions.find((x) => x.id === id) if (!s) return @@ -1110,7 +1152,36 @@ export function setSessionChatId(sessionId: string, chatId: string) { const s = sessionState.sessions.find((x) => x.id === sessionId) if (s && s.chatId !== chatId) { s.chatId = chatId - void putSession(s) + void patchStoredSessionChatId(s, chatId) + } +} + +// Persists the pointer through the STORED row, not this tab's copy: another +// tab may have written newer fields (summary, tabs, archive state) since this +// tab last read the record, and a whole-object put would roll them back — a +// watcher adopting the driver's rotation reaches here with exactly that copy. +async function patchStoredSessionChatId(s: Session, chatId: string): Promise { + if (!BROWSER || s.transient || deletedSessionIds.has(s.id)) return + const db = await sessionsDb.whenReady() + if (!db) return + try { + const tx = db.transaction('sessions', 'readwrite') + const stored = await tx.store.get(s.id) + // Inline tombstone re-check in place of putSessionRow's: routing through + // it would put outside this transaction and lose the read's atomicity. + if (stored && !deletedSessionIds.has(s.id)) { + stored.chatId = chatId + await tx.store.put(stored) + await tx.done + return + } + await tx.done + // No stored row: either the record is not yet persisted — its own + // materialization writes it later with the chatId already set in memory — + // or another tab deleted it, and an upsert here would resurrect it. No + // write either way. + } catch (e) { + console.error('Failed to persist session chat id', e) } } diff --git a/frontend/src/lib/components/sessions/sessionState.test.ts b/frontend/src/lib/components/sessions/sessionState.test.ts index 7732c44746..1a54c86e78 100644 --- a/frontend/src/lib/components/sessions/sessionState.test.ts +++ b/frontend/src/lib/components/sessions/sessionState.test.ts @@ -4,14 +4,22 @@ import { commitSessionWorkspace, createSession, decideSessionLifecycle, + findEmptyLandingSession, isForkSession, + isTearingDownOpenSession, renameSession, sessionInCurrentFamily, setGeneratedSessionSummary, setSessionDraftPrompt, sessionState, + withOpenSessionTeardown, type Session } from './sessionState.svelte' +import { + clearSessionRecovered, + isSessionRecovered, + markSessionRecovered +} from './sessionRecoveryNotice.svelte' import { enterpriseLicense, usersWorkspaceStore, @@ -362,6 +370,28 @@ describe('createSession — reuses an untouched draft, family-scoped', () => { } }) + it('clears the recovery notice off the draft it reuses, so `+` is not answered with "not found"', () => { + const restore = withTwoFamilies('rootA') + const prevCurrent = sessionState.currentSessionId + const landed = session({ + id: 'recovered-blank', + name: 'session-903', + pending_workspace_id: 'forkA', + transient: true + }) + sessionState.sessions.push(landed) + markSessionRecovered(landed.id) + try { + expect(createSession().id).toBe('recovered-blank') + expect(isSessionRecovered('recovered-blank')).toBe(false) + } finally { + clearSessionRecovered('recovered-blank') + sessionState.sessions = sessionState.sessions.filter((s) => s.id !== 'recovered-blank') + sessionState.currentSessionId = prevCurrent + restore() + } + }) + it('drops an untouched draft left over from another family and starts in the active workspace', () => { const restore = withTwoFamilies('rootB') const prevCurrent = sessionState.currentSessionId @@ -488,3 +518,78 @@ describe('createSession — reuses an untouched draft, family-scoped', () => { } }) }) + +describe('findEmptyLandingSession — where an unresolvable session link lands', () => { + it('takes an untouched draft', () => { + const restore = withTwoFamilies('forkA') + const blank = session({ + id: 'landing-blank', + name: 'session-910', + pending_workspace_id: 'forkA', + transient: true + }) + sessionState.sessions.push(blank) + try { + expect(findEmptyLandingSession()?.id).toBe('landing-blank') + } finally { + sessionState.sessions = sessionState.sessions.filter((s) => s.id !== 'landing-blank') + restore() + } + }) + + it('passes over a persisted session, onto which chat seeding can graft a conversation', () => { + const restore = withTwoFamilies('forkA') + // ensureChatIdsSeeded assigns untagged legacy chats to `!transient` sessions + // and initRuntime loads them, without touching a field checked here. + const abandoned = session({ + id: 'landing-abandoned', + name: 'session-910', + pending_workspace_id: 'forkA' + }) + const others = sessionState.sessions + sessionState.sessions = [abandoned] + try { + expect(findEmptyLandingSession()).toBeUndefined() + } finally { + sessionState.sessions = others + restore() + } + }) + + it('passes over a session that has been sent, so recovery never reopens a conversation', () => { + const restore = withTwoFamilies('forkA') + const sent = session({ id: 'landing-sent', name: 'session-911', workspace_id: 'forkA' }) + // Sole candidate, so `undefined` pins the exclusion: `not.toBe` would also + // pass on any unrelated session the shared module state happens to hold. + const others = sessionState.sessions + sessionState.sessions = [sent] + try { + expect(findEmptyLandingSession()).toBeUndefined() + } finally { + sessionState.sessions = others + restore() + } + }) +}) + +describe('withOpenSessionTeardown — the gate that holds recovery off during a delete', () => { + it('stays shut until the outermost teardown finishes', async () => { + let innerDone = false + await withOpenSessionTeardown(async () => { + await withOpenSessionTeardown(async () => {}) + innerDone = true + expect(isTearingDownOpenSession()).toBe(true) + }) + expect(innerDone).toBe(true) + expect(isTearingDownOpenSession()).toBe(false) + }) + + it('reopens when the teardown throws, so a failed delete cannot wedge recovery shut', async () => { + await expect( + withOpenSessionTeardown(async () => { + throw new Error('fork deletion failed') + }) + ).rejects.toThrow('fork deletion failed') + expect(isTearingDownOpenSession()).toBe(false) + }) +}) diff --git a/frontend/src/lib/components/sessions/sessionStateIndexedDb.test.ts b/frontend/src/lib/components/sessions/sessionStateIndexedDb.test.ts index 6a88fd53fe..d0063f0a97 100644 --- a/frontend/src/lib/components/sessions/sessionStateIndexedDb.test.ts +++ b/frontend/src/lib/components/sessions/sessionStateIndexedDb.test.ts @@ -50,6 +50,7 @@ import { getSessionDraftPrompt, setSessionDraftPrompt, setSessionTabs, + setSessionChatId, reconcileSessionsLifecycle, __resetDeletedSessionIdsForTesting, setSessionArchived, @@ -117,6 +118,30 @@ describe('sessionState IndexedDB persistence', () => { await vi.waitFor(() => expect(sessionState.sessions.map((s) => s.id)).toEqual(['s2', 's1'])) }) + // A watcher adopting the driver's chat rotation holds a stale in-memory + // record; persisting the pointer must not roll back fields another tab + // wrote to the store since. + it('setSessionChatId patches the stored row instead of writing back a stale copy', async () => { + const user = freshUser() + await login(user) + + const stale = session({ id: 's1', createdAt: 100, summary: 'old summary' }) + await putSession(stale) + // A newer write from another tab, landing directly in the store. + await putSession(session({ id: 's1', createdAt: 100, summary: 'newer summary' })) + + sessionState.sessions = [stale] + setSessionChatId('s1', 'chat-2') + await flush() + + await rehydrate(user) + await vi.waitFor(() => { + const s = sessionState.sessions.find((x) => x.id === 's1') + expect(s?.chatId).toBe('chat-2') + expect(s?.summary).toBe('newer summary') + }) + }) + it('does not persist a transient (untouched) session — it is in-memory only', async () => { const user = freshUser() await login(user) diff --git a/frontend/src/lib/components/sessions/sessionSwitch.svelte.ts b/frontend/src/lib/components/sessions/sessionSwitch.svelte.ts index 0d30088767..a7a492b874 100644 --- a/frontend/src/lib/components/sessions/sessionSwitch.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionSwitch.svelte.ts @@ -5,13 +5,17 @@ import { createSession, selectSession, sessionInCurrentFamily, + sessionLastActivityAt, sessionState, setSessionAutoSend, setSessionDraftPrompt, setSessionPendingWorkspace, + type Session, type SessionTarget } from './sessionState.svelte' import { sessionTargetHref, withPreviewParams } from './sessionMode.svelte' +import { parsePreviewItemRoute, type PreviewItemRoute } from './previewPaths' +import { findMountedOpenInSessionSource } from './openInSessionContext' // Type-only: erased at compile time, so the component graph stays out of this // navigation seam (see the dynamic import in openEditorInSession). import type { OpenInSessionSource } from './OpenInSessionButton.svelte' @@ -27,33 +31,100 @@ import type { OpenInSessionSource } from './OpenInSessionButton.svelte' // is safe. Plain module state: it is read only inside click handlers, never // rendered, so it needs no reactivity. let lastNavRoute = '/' +// Whether `lastNavRoute` has been offered as a new session's starting point since +// it was remembered (see takeNewSessionSeed). +let navRouteOffered = false export function rememberNavRoute(pathnameWithSearch: string): void { lastNavRoute = pathnameWithSearch + navRouteOffered = false } -// Enter session mode: open the active session if one is selected, else the most -// recent non-archived session, else spin up a fresh one — then route to it. -// Restore candidates are scoped to the active workspace family: reviving a -// session from another family would pull that family's scope (sidebar list, -// "Acting on" workspace) into the one the user is actually in. +/** An item editor the user reached session mode from, as the in-app href a new + * session's preview can open, plus the item it edits (for naming it). */ +export type NewSessionSeed = { url: string; route: PreviewItemRoute } + +// The item (flow, script, app) the user came to session mode from, or undefined +// when they came from anywhere else. A "New session" asked for after arriving +// from an item is usually a session about that item, but the arrival route +// (`enterSessionMode`) resumes whatever session was open, so the picker offers +// the item back. Offered once per remembered route: the first "New session" of +// a visit gets the question, however long the visit has run and whatever was +// done in between; later ones start empty without asking, and a dismissed offer +// is not repeated until the user leaves for another page and comes back. +export function takeNewSessionSeed(): NewSessionSeed | undefined { + if (navRouteOffered) return undefined + navRouteOffered = true + const route = parsePreviewItemRoute(lastNavRoute) + return route ? { url: lastNavRoute, route } : undefined +} + +// The session entering session mode resumes: the active one if selected, else +// the most recent non-archived one. Scoped to the active workspace family: +// reviving a session from another family would pull that family's scope +// (sidebar list, "Acting on" workspace) into the one the user is actually in. +function resumableSession(): Session | undefined { + const current = sessionState.currentSessionId + ? sessionState.sessions.find((s) => s.id === sessionState.currentSessionId) + : undefined + return ( + (current && sessionInCurrentFamily(current) ? current : undefined) ?? + sessionState.sessions.find((s) => !s.archived && sessionInCurrentFamily(s)) + ) +} + +// Enter session mode: resume the session `resumableSession` picks, else spin up +// a fresh one — then route to it. // `replace` swaps the current history entry instead of pushing — for the // sessions page's family reconcile, where Back must not return to the // redirected-away URL just to bounce here again. export async function enterSessionMode(opts?: { replace?: boolean }): Promise { - const current = sessionState.currentSessionId - ? sessionState.sessions.find((s) => s.id === sessionState.currentSessionId) - : undefined - const target = - (current && sessionInCurrentFamily(current) ? current : undefined) ?? - sessionState.sessions.find((s) => !s.archived && sessionInCurrentFamily(s)) ?? - createSession() + const target = resumableSession() ?? createSession() selectSession(target.id) await goto(`/sessions?session_name=${encodeURIComponent(target.name)}`, { replaceState: opts?.replace ?? false }) } +// How long the resumable session may have sat idle before entering from an item +// editor starts a session on that item instead. Long enough that stepping out to +// the editor in the middle of a conversation comes back to the same chat; short +// enough that a session left since the previous day is not taken for the task +// the user is now on. +const RESUME_IDLE_LIMIT_MS = 60 * 60 * 1000 + +// Enter session mode from the navigation rail. Coming from an item editor with +// no session to resume, or one idle past RESUME_IDLE_LIMIT_MS, opens a fresh +// session on that item straight away: a session that old is rarely what a visit +// from a flow or app is about, and landing in it would only lead to "New +// session" and the offer takeNewSessionSeed makes. Anything else resumes as +// enterSessionMode does. Rejects when the editor could not persist its draft, +// so the caller can stay on the page and say so. +export async function enterSessionModeFromNav(): Promise { + const route = parsePreviewItemRoute(lastNavRoute) + if (route) { + const resumable = resumableSession() + if (!resumable || Date.now() - sessionLastActivityAt(resumable) > RESUME_IDLE_LIMIT_MS) { + // The editor's own hand-off is what its "Open in AI session" button uses: + // it persists the draft the preview loads (an edit still inside the + // autosave debounce, the row of a never-saved new item) and names the + // workspace the item lives in. Only an item with no such editor on + // screen (a legacy app, a detail page) is opened by route, as last + // persisted, in the workspace the route was scoped to. + const source = findMountedOpenInSessionSource(route) + if (source) await openSourceInSession(source) + else await openPageInSession(lastNavRoute, workspaceParamOf(lastNavRoute)) + return + } + } + await enterSessionMode() +} + +function workspaceParamOf(pathnameWithSearch: string): string | undefined { + const query = pathnameWithSearch.split('?')[1] + return query ? (new URLSearchParams(query).get('workspace') ?? undefined) : undefined +} + // Exit session mode: back to the last navigation route (home as a fallback). export async function exitSessionMode(): Promise { let target = lastNavRoute || '/' @@ -83,12 +154,16 @@ export async function openEditorInSession( previewParams?: Record, opts?: { seedPrompt?: string; autoSend?: boolean } ): Promise { - await openInSession(withPreviewParams(sessionTargetHref(target), previewParams), workspaceId, opts) + await openInSession( + withPreviewParams(sessionTargetHref(target), previewParams), + workspaceId, + opts + ) } -// Open a fresh AI session showing a workspace page (Runs, a trigger list) in its -// preview. A page is not an editable item, so callers hand over the in-app href -// they want the tab to load rather than a SessionTarget. +// Open a fresh AI session showing an in-app href in its preview: a workspace +// page (Runs, a trigger list), which is not an editable item and so has no +// SessionTarget, or a location captured as the user left it. export async function openPageInSession( href: string, workspaceId?: string, @@ -117,6 +192,9 @@ async function openInSession( // node-run unit tests. const { resetSessionPreviewTabs } = await import('./sessionRuntime.svelte') resetSessionPreviewTabs(session.id, url) + // Hand-offs seed the page they leave, so the arrival has opened the item + // itself and "New session" need not offer it again. + navRouteOffered = true } selectSession(session.id) await goto(`/sessions?session_name=${encodeURIComponent(session.name)}`) diff --git a/frontend/src/lib/components/sessions/sessionSwitch.test.ts b/frontend/src/lib/components/sessions/sessionSwitch.test.ts index 616f5fe751..23ca52a693 100644 --- a/frontend/src/lib/components/sessions/sessionSwitch.test.ts +++ b/frontend/src/lib/components/sessions/sessionSwitch.test.ts @@ -2,8 +2,11 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { get } from 'svelte/store' import { enterSessionMode, + enterSessionModeFromNav, openSourceInSession, - startSessionWithPrompt + rememberNavRoute, + startSessionWithPrompt, + takeNewSessionSeed } from './sessionSwitch.svelte' import { peekSessionAutoSend, @@ -20,6 +23,8 @@ import { goto } from '$lib/navigation' // monaco (hence that import being dynamic in the first place) and cannot load // under node. vi.mock('./sessionRuntime.svelte', () => ({ resetSessionPreviewTabs: vi.fn() })) +import { resetSessionPreviewTabs } from './sessionRuntime.svelte' +import { registerMountedOpenInSessionHandoff } from './openInSessionContext' function session(over: Partial = {}): Session { return { id: 's1', name: 'sess', createdAt: 0, ...over } @@ -248,3 +253,159 @@ describe('startSessionWithPrompt', () => { } }) }) + +describe('takeNewSessionSeed', () => { + it('offers the item editor the user came from, once per arrival', () => { + rememberNavRoute('/flows/edit/u/me/my_flow?workspace=ws') + expect(takeNewSessionSeed()).toEqual({ + url: '/flows/edit/u/me/my_flow?workspace=ws', + route: { kind: 'flow', raw_app: false, itemPath: 'u/me/my_flow' } + }) + // A dismissed or taken offer is not repeated until the user leaves again. + expect(takeNewSessionSeed()).toBeUndefined() + rememberNavRoute('/apps_raw/edit/f/team/dashboard?workspace=ws') + expect(takeNewSessionSeed()?.route).toEqual({ + kind: 'app', + raw_app: true, + itemPath: 'f/team/dashboard' + }) + }) + + it('offers nothing for a non-item page', () => { + rememberNavRoute('/runs?workspace=ws') + expect(takeNewSessionSeed()).toBeUndefined() + }) +}) + +describe('enterSessionModeFromNav', () => { + const HOUR = 60 * 60 * 1000 + beforeEach(() => { + vi.mocked(goto).mockClear() + vi.mocked(resetSessionPreviewTabs).mockClear() + }) + + // One session in the active family (rootA), as the one the rail would resume. + // Restores everything the test touched, the module-level nav route included, + // and drops whatever session the call under test created. + function withResumable(over: Partial): { restore: () => void } { + const restoreFamilies = withTwoFamilies('rootA') + const prevCurrent = sessionState.currentSessionId + const before = new Set(sessionState.sessions.map((s) => s.id)) + const s = session({ workspace_id: 'rootA', ...over }) + sessionState.sessions.push(s) + sessionState.currentSessionId = s.id + return { + restore: () => { + sessionState.sessions = sessionState.sessions.filter((x) => before.has(x.id)) + sessionState.currentSessionId = prevCurrent + rememberNavRoute('/') + restoreFamilies() + } + } + } + + it('resumes a recently active session even when coming from an item', async () => { + const { restore } = withResumable({ + id: 'nav-recent', + name: 'session-921', + lastActivityAt: Date.now() - HOUR / 2 + }) + try { + rememberNavRoute('/flows/edit/u/me/f?workspace=rootA') + await enterSessionModeFromNav() + expect(sessionState.currentSessionId).toBe('nav-recent') + expect(resetSessionPreviewTabs).not.toHaveBeenCalled() + } finally { + restore() + } + }) + + it('starts a session on the item, by route, instead of resuming one idle for hours', async () => { + const { restore } = withResumable({ + id: 'nav-stale', + name: 'session-922', + lastActivityAt: Date.now() - 3 * HOUR + }) + try { + rememberNavRoute('/flows/edit/u/me/f?workspace=forkA') + await enterSessionModeFromNav() + const createdId = sessionState.currentSessionId + expect(createdId).not.toBe('nav-stale') + expect(resetSessionPreviewTabs).toHaveBeenCalledWith( + createdId, + '/flows/edit/u/me/f?workspace=forkA' + ) + // The route's workspace, not createSession's pick for the active one. + const created = sessionState.sessions.find((s) => s.id === createdId) + expect(created?.pending_workspace_id).toBe('forkA') + // The arrival opened the item itself, so "New session" does not offer it. + expect(takeNewSessionSeed()).toBeUndefined() + } finally { + restore() + } + }) + + // The editor's hand-off is what its own "Open in AI session" button uses: its + // beforeOpen persists the draft the preview loads, and it names the item's + // workspace. The rail must take it over opening the route as last persisted. + it('hands off through the mounted editor of the item, running its beforeOpen first', async () => { + const { restore } = withResumable({ + id: 'nav-stale-editor', + name: 'session-924', + lastActivityAt: Date.now() - 3 * HOUR + }) + const order: string[] = [] + vi.mocked(goto).mockImplementation((async () => { + order.push('goto') + }) as never) + // A script editor mounted alongside (a flow's drawer) must not be taken + // for the page's own item. + const unregisterScript = registerMountedOpenInSessionHandoff({ + source: () => ({ target: { kind: 'script', path: 'u/me/s' }, workspaceId: 'rootB' }) + }) + const unregisterFlow = registerMountedOpenInSessionHandoff({ + source: () => ({ + target: { kind: 'flow', path: 'u/me/f' }, + workspaceId: 'forkA', + beforeOpen: () => { + order.push('beforeOpen') + } + }) + }) + try { + rememberNavRoute('/flows/edit/u/me/f?workspace=rootA') + await enterSessionModeFromNav() + const createdId = sessionState.currentSessionId + expect(createdId).not.toBe('nav-stale-editor') + expect(order).toEqual(['beforeOpen', 'goto']) + expect(resetSessionPreviewTabs).toHaveBeenCalledWith( + createdId, + expect.stringMatching(/\/flows\/edit\/u\/me\/f$/) + ) + const created = sessionState.sessions.find((s) => s.id === createdId) + expect(created?.pending_workspace_id).toBe('forkA') + } finally { + unregisterFlow() + unregisterScript() + vi.mocked(goto).mockReset() + vi.mocked(goto).mockResolvedValue(undefined as never) + restore() + } + }) + + it('resumes a stale session when coming from a non-item page', async () => { + const { restore } = withResumable({ + id: 'nav-stale-runs', + name: 'session-923', + lastActivityAt: Date.now() - 3 * HOUR + }) + try { + rememberNavRoute('/runs?workspace=rootA') + await enterSessionModeFromNav() + expect(sessionState.currentSessionId).toBe('nav-stale-runs') + expect(resetSessionPreviewTabs).not.toHaveBeenCalled() + } finally { + restore() + } + }) +}) diff --git a/frontend/src/lib/components/sessions/sessionSync.svelte.ts b/frontend/src/lib/components/sessions/sessionSync.svelte.ts new file mode 100644 index 0000000000..842bb8961c --- /dev/null +++ b/frontend/src/lib/components/sessions/sessionSync.svelte.ts @@ -0,0 +1,206 @@ +import { BROWSER } from 'esm-env' +import { SvelteMap } from 'svelte/reactivity' +import { onUserChange, scopedKey } from '$lib/userScopedStorage' +import { randomUUID } from '$lib/utils/uuid' + +// Cross-tab awareness for AI sessions. Invariant: no message carries state — +// a heartbeat is presence, turn-end triggers an idempotent re-read of the +// shared IndexedDB record — so tabs converge on the store, never on delivery +// order. The lock is advisory (a broadcast-latency race stays last-writer- +// wins, as with no channel), and the channel is per-user like the stores. + +const CHANNEL_BASE = 'windmill-sessions-sync' + +/** Silence past STALE_MS unlocks watchers a dead driver would strand. The + * window sits above the 1/min floor browsers throttle a hidden tab's timers + * to — and a hidden driver is the normal case here. Only an uncleanly killed + * tab waits it out; a closed one says goodbye via the pagehide farewell. */ +const HEARTBEAT_MS = 3_000 +const STALE_MS = 90_000 +const PRUNE_MS = 2_000 + +// `from` identifies the driving tab: two drivers racing on one session (the +// documented advisory race) hold separate slots, so one's turn-end can never +// unlock a watcher the other still holds. +export type SyncMsg = + | { kind: 'run-heartbeat'; sessionId: string; from: string } + | { kind: 'turn-end'; sessionId: string; chatId: string; from: string } + +/** This tab's identity on the channel (a tab never receives its own posts). */ +const TAB_ID = randomUUID() + +// One slot per (session, driving tab), keyed with a separator no UUID contains. +// The value is a fresh object per message: turn-end's deferred cleanup asks +// "is this slot still mine?" by identity — a timestamp can't, since a same- +// millisecond follow-up heartbeat would compare equal and be deleted. +const remoteRuns = new SvelteMap() + +function runKey(sessionId: string, from: string): string { + return sessionId + ':' + from +} + +export function runHeldElsewhere(sessionId: string): boolean { + const prefix = sessionId + ':' + for (const key of remoteRuns.keys()) { + if (key.startsWith(prefix)) return true + } + return false +} + +let remoteTurnEnd: ((sessionId: string, chatId: string) => void | Promise) | undefined + +/** Registered by sessionRuntime, which already imports this module — a + * callback rather than an import keeps that edge one-way. The returned + * promise is when the catch-up has been applied; the composer stays locked + * until it settles. */ +export function onRemoteTurnEnd( + fn: (sessionId: string, chatId: string) => void | Promise +): void { + remoteTurnEnd = fn +} + +let channel: BroadcastChannel | undefined +let channelName: string | undefined + +function openChannel(): void { + const name = scopedKey(CHANNEL_BASE) + if (name === channelName) return + channel?.close() + channel = undefined + channelName = name + if (!name) return + try { + const ch = new BroadcastChannel(name) + ch.onmessage = (ev: MessageEvent) => receive(ev.data) + channel = ch + } catch (e) { + // No BroadcastChannel (or blocked): every tab simply stays independent, + // which is the pre-sync behavior rather than a broken one. + console.error('sessionSync: could not open channel', e) + } +} + +if (BROWSER) { + // A user switch rescopes the channel name, so the previous identity's + // channel is closed before the next one opens. + onUserChange(() => openChannel()) +} + +function receive(msg: SyncMsg): void { + switch (msg.kind) { + case 'run-heartbeat': + remoteRuns.set(runKey(msg.sessionId, msg.from), { at: Date.now() }) + ensurePruner() + break + case 'turn-end': { + // Unlocking on receipt would let a send here start from history missing + // the turn that just ended, so the slot holds until the catch-up + // settles — unless the driver's next turn replaced it meanwhile (object + // identity, see remoteRuns). The pruner caps a wedged reload at STALE_MS. + const key = runKey(msg.sessionId, msg.from) + const hold = { at: Date.now() } + remoteRuns.set(key, hold) + ensurePruner() + Promise.resolve() + .then(() => remoteTurnEnd?.(msg.sessionId, msg.chatId)) + .catch((e) => console.error('sessionSync: turn-end handler failed', e)) + .finally(() => { + if (remoteRuns.get(key) === hold) remoteRuns.delete(key) + }) + break + } + } +} + +function post(msg: SyncMsg): void { + if (!channel) return + try { + channel.postMessage(msg) + } catch (e) { + // A failed post must never take the turn down with it. + console.error('sessionSync: could not post message', e) + } +} + +let pruneTimer: ReturnType | undefined + +function ensurePruner(): void { + if (pruneTimer) return + pruneTimer = setInterval(() => { + const cutoff = Date.now() - STALE_MS + for (const [id, entry] of remoteRuns) { + if (entry.at < cutoff) remoteRuns.delete(id) + } + if (remoteRuns.size === 0) { + clearInterval(pruneTimer) + pruneTimer = undefined + } + }, PRUNE_MS) +} + +// --------------------------------------------------------------------------- +// Driving side +// --------------------------------------------------------------------------- + +// The chat id rides along for the pagehide farewell below, which cannot ask +// the manager for it. Taken at run start; only a mid-turn rotation could make +// it stale, and a farewell pointing at the pre-rotation record still converges +// (the re-read is idempotent and the next turn-end names the right one). +const heartbeats = new Map; chatId: string }>() + +/** Posted when the run's loading bracket opens — after the send's attachment + * upkeep awaits, so a competing send can start during them; the sender's own + * post-preflight re-check is what refuses one that did. */ +export function localRunStarted(sessionId: string, chatId: string): void { + if (heartbeats.has(sessionId)) return + post({ kind: 'run-heartbeat', sessionId, from: TAB_ID }) + heartbeats.set(sessionId, { + timer: setInterval( + () => post({ kind: 'run-heartbeat', sessionId, from: TAB_ID }), + HEARTBEAT_MS + ), + chatId + }) +} + +/** `chatId` is read at turn end, not reused from the start: a rotation + * mid-turn means the transcript now lives under a different record, and the + * watchers' re-read must follow it there. */ +export function localRunEnded(sessionId: string, chatId: string): void { + const entry = heartbeats.get(sessionId) + if (entry !== undefined) { + clearInterval(entry.timer) + heartbeats.delete(sessionId) + } + post({ kind: 'turn-end', sessionId, chatId, from: TAB_ID }) +} + +if (BROWSER) { + // The run dies with the page: a turn-end farewell (which also has watchers + // re-read the last checkpoint) beats making them wait out STALE_MS. Not on + // a bfcache freeze (persisted) — that turn resumes with the page, and + // nothing would re-arm a farewelled heartbeat. + window.addEventListener('pagehide', (ev) => { + if (ev.persisted) return + for (const [sessionId, entry] of [...heartbeats]) { + localRunEnded(sessionId, entry.chatId) + } + }) +} + +/** Test seam: deliver a message as if it arrived on the channel. */ +export function __receiveForTest(msg: SyncMsg): void { + receive(msg) +} + +/** Test seam: clear the module's state between tests. */ +export function __resetForTest(): void { + remoteRuns.clear() + if (pruneTimer) { + clearInterval(pruneTimer) + pruneTimer = undefined + } + for (const entry of heartbeats.values()) clearInterval(entry.timer) + heartbeats.clear() + remoteTurnEnd = undefined +} diff --git a/frontend/src/lib/components/sessions/sessionSync.test.ts b/frontend/src/lib/components/sessions/sessionSync.test.ts new file mode 100644 index 0000000000..e00eab8307 --- /dev/null +++ b/frontend/src/lib/components/sessions/sessionSync.test.ts @@ -0,0 +1,90 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + __receiveForTest, + __resetForTest, + onRemoteTurnEnd, + runHeldElsewhere +} from './sessionSync.svelte' + +// Exercises the receive-side state machine directly (the module opens no +// BroadcastChannel outside the browser). The channel itself is glue that only +// a real browser can prove; what these pin are the invariants a refactor could +// silently break: the identity-token cleanup, the staleness prune, and the +// turn-end hold. + +beforeEach(() => { + __resetForTest() + vi.useFakeTimers() +}) + +afterEach(() => { + __resetForTest() + vi.useRealTimers() +}) + +describe('sessionSync receive-side state', () => { + it('locks on a heartbeat and unlocks by staleness when the driver dies silently', async () => { + __receiveForTest({ kind: 'run-heartbeat', sessionId: 's1', from: 'driver-a' }) + expect(runHeldElsewhere('s1')).toBe(true) + + // Refreshed heartbeats keep the lock past the original entry's window. + await vi.advanceTimersByTimeAsync(60_000) + __receiveForTest({ kind: 'run-heartbeat', sessionId: 's1', from: 'driver-a' }) + await vi.advanceTimersByTimeAsync(60_000) + expect(runHeldElsewhere('s1')).toBe(true) + + // Silence past STALE_MS (90s) prunes the entry. + await vi.advanceTimersByTimeAsync(40_000) + expect(runHeldElsewhere('s1')).toBe(false) + }) + + it('holds the lock through the turn-end catch-up and releases when it settles', async () => { + let releaseCatchUp: (() => void) | undefined + onRemoteTurnEnd(() => new Promise((resolve) => (releaseCatchUp = resolve))) + + __receiveForTest({ kind: 'run-heartbeat', sessionId: 's1', from: 'driver-a' }) + __receiveForTest({ kind: 'turn-end', sessionId: 's1', chatId: 'c1', from: 'driver-a' }) + await vi.advanceTimersByTimeAsync(0) + // Unlocking on receipt would let a send here start from history missing + // the turn that just ended; the lock must outlive the re-read. + expect(runHeldElsewhere('s1')).toBe(true) + + releaseCatchUp?.() + await vi.advanceTimersByTimeAsync(0) + expect(runHeldElsewhere('s1')).toBe(false) + }) + + it("keeps the lock when the driver's next turn arrives during the catch-up", async () => { + let releaseCatchUp: (() => void) | undefined + onRemoteTurnEnd(() => new Promise((resolve) => (releaseCatchUp = resolve))) + + __receiveForTest({ kind: 'turn-end', sessionId: 's1', chatId: 'c1', from: 'driver-a' }) + // Flush so the catch-up handler has started (releaseCatchUp is assigned) + // before the follow-up arrives — otherwise the release below no-ops and + // the lock would survive for the wrong reason (a catch-up that never + // settled), passing even with the identity comparison broken. + await vi.advanceTimersByTimeAsync(0) + // The queued-follow-up sequence: the next turn's first heartbeat lands + // while this tab's catch-up is still reading — in the same millisecond, + // which is why the cleanup must compare identity, not timestamps. + __receiveForTest({ kind: 'run-heartbeat', sessionId: 's1', from: 'driver-a' }) + + releaseCatchUp?.() + await vi.advanceTimersByTimeAsync(0) + expect(runHeldElsewhere('s1')).toBe(true) + }) + + // Two drivers on one session is the documented advisory race; a watcher + // must not compound it by unlocking when only one of them finishes. + it('stays locked when one of two drivers ends its turn', async () => { + onRemoteTurnEnd(() => Promise.resolve()) + + __receiveForTest({ kind: 'run-heartbeat', sessionId: 's1', from: 'driver-b' }) + __receiveForTest({ kind: 'turn-end', sessionId: 's1', chatId: 'c1', from: 'driver-a' }) + await vi.advanceTimersByTimeAsync(0) + await vi.advanceTimersByTimeAsync(0) + + // Driver A's slot released with its catch-up; driver B still holds its own. + expect(runHeldElsewhere('s1')).toBe(true) + }) +}) diff --git a/frontend/src/lib/components/settings/CreateToken.svelte b/frontend/src/lib/components/settings/CreateToken.svelte index 444e245bee..20e4891189 100644 --- a/frontend/src/lib/components/settings/CreateToken.svelte +++ b/frontend/src/lib/components/settings/CreateToken.svelte @@ -287,7 +287,9 @@ { label: '1 day', value: 1 * 24 * 60 * 60 }, { label: '7 days', value: 7 * 24 * 60 * 60 }, { label: '30 days', value: 30 * 24 * 60 * 60 }, - { label: '90 days', value: 90 * 24 * 60 * 60 } + { label: '90 days', value: 90 * 24 * 60 * 60 }, + { label: '180 days', value: 180 * 24 * 60 * 60 }, + { label: '365 days', value: 365 * 24 * 60 * 60 } ]} />
    diff --git a/frontend/src/lib/components/settings/ForkMemberSettings.svelte b/frontend/src/lib/components/settings/ForkMemberSettings.svelte index 6b75018cae..45de0bf3f0 100644 --- a/frontend/src/lib/components/settings/ForkMemberSettings.svelte +++ b/frontend/src/lib/components/settings/ForkMemberSettings.svelte @@ -102,7 +102,7 @@ @@ -124,12 +124,12 @@ nonCaptureEvent={true} startIcon={{ icon: UserPlus }} > - Add collaborator + Add member {/snippet} {#snippet content()}
    - Add a collaborator + Add a member They join as a developer of this fork. Only members of {parentWorkspaceId} who are developers or admins there can be added. diff --git a/frontend/src/lib/components/settings/PremiumInfo.svelte b/frontend/src/lib/components/settings/PremiumInfo.svelte index a1036a0ff6..42bfef9d65 100644 --- a/frontend/src/lib/components/settings/PremiumInfo.svelte +++ b/frontend/src/lib/components/settings/PremiumInfo.svelte @@ -1,11 +1,11 @@ @@ -154,17 +145,15 @@
    {#snippet actions()} - - - + {/snippet} @@ -260,8 +249,8 @@
    Developers

    - Calculated on the MAXIMUM number of users in a given billing - period, see the Customer Portal for more info. + Calculated on the MAXIMUM number of users in a given billing period, see the + Customer Portal for more info.

    @@ -276,8 +265,8 @@
    Operators

    - Calculated on the MAXIMUM number of operators in a given - billing period, see the Customer Portal for more info. + Calculated on the MAXIMUM number of operators in a given billing period, see + the Customer Portal for more info.

    @@ -295,8 +284,9 @@ 1 developer = 1 seat, 2 operators = 1 seat.

    - u = ceil({formatNumber(premiumInfo.developerNb)} + {formatNumber(premiumInfo.operatorNb)}/2) - = {formatNumber(premiumInfo.seatsFromUsers)} + u = ceil({formatNumber(premiumInfo.developerNb)} + {formatNumber( + premiumInfo.operatorNb + )}/2) = {formatNumber(premiumInfo.seatsFromUsers)}

    @@ -311,9 +301,8 @@
    Executions this month

    - One execution equals one job - up to 1 second on a worker with 2GB of memory, with each additional - second counting as an extra execution. + One execution equals one job up to 1 second on a worker with 2GB of memory, + with each additional second counting as an extra execution.

    @@ -365,8 +354,8 @@ Used seats (billed)

    - Highest between seats from 'Developers + Operators' and 'Seats from executions'. - This is the number of seats used for billing this month. + Highest between seats from 'Developers + Operators' and 'Seats from + executions'. This is the number of seats used for billing this month.

    u + c = {formatNumber(premiumInfo.usedSeats)} @@ -398,8 +387,8 @@

    Estimate your monthly cost

    - This is a rough estimate based on your expected team size and workload. Actual billing is based - on the maximum number of users and executions in a given month. + This is a rough estimate based on your expected team size and workload. Actual billing is + based on the maximum number of users and executions in a given month.

    @@ -420,13 +409,13 @@
    Operators
    - {estimatedOps} operator{estimatedOps === 1 ? '' : 's'} + {estimatedOps} operator{estimatedOps === 1 + ? '' + : 's'}
    -

    - 2 operators = 1 seat -

    +

    2 operators = 1 seat

    @@ -434,8 +423,8 @@
    Monthly executions
    - One execution equals one job up to 1 second on a virtual CPU with 2 GB of memory, with - each additional second counting as an extra execution. + One execution equals one job up to 1 second on a virtual CPU with 2 GB of memory, + with each additional second counting as an extra execution.
    @@ -449,9 +438,7 @@ format={(v) => `${v * 10}k`} hideInput /> -

    - Each seat includes 10k executions per month. -

    +

    Each seat includes 10k executions per month.

    @@ -558,8 +545,8 @@
  • Every seat includes 10 000 executions - One execution equals one job up to 1 second on a virtual CPU with 2 GB of memory, with - each additional second counting as an extra execution. + One execution equals one job up to 1 second on a virtual CPU with 2 GB of memory, + with each additional second counting as an extra execution.
    • {:else} @@ -586,9 +573,7 @@
    {/if} {:else} -
    - Workspace is on the team plan -
    +
    Workspace is on the team plan
    {/if} {:else if planTitle == 'Enterprise'} {#if plan != 'enterprise'} @@ -601,9 +586,7 @@ See more {:else} -
    - Workspace is on enterprise plan -
    +
    Workspace is on enterprise plan
    {/if} {:else if planTitle === 'Free'} {#if plan} @@ -611,9 +594,7 @@ Cancel your plan in the Customer Portal to downgrade to the free plan
    {:else} -
    - Workspace is on the free plan -
    +
    Workspace is on the free plan
    {/if} {/if}
    diff --git a/frontend/src/lib/components/sidebar/SidebarUsage.svelte b/frontend/src/lib/components/sidebar/SidebarUsage.svelte index e62cd02565..1625ed83e2 100644 --- a/frontend/src/lib/components/sidebar/SidebarUsage.svelte +++ b/frontend/src/lib/components/sidebar/SidebarUsage.svelte @@ -2,21 +2,18 @@ import { resource } from 'runed' import { goto } from '$lib/navigation' import { isCloudHosted } from '$lib/cloud' - import { UserService } from '$lib/gen' + import { WorkspaceService } from '$lib/gen' import { isPremiumStore, usageStore, userStore, - userWorkspaces, workspaceMembershipVersion, workspaceStore, - workspaceUsageStore, - type UserWorkspace + workspaceUsageStore } from '$lib/stores' import { refreshExecutions } from '$lib/usage.svelte' import { logFeatureUsage } from '$lib/utils/featureUsage' import { scopedValue, tagged } from '$lib/utils/scopedValue' - import { findWorkspaceAncestors } from '$lib/utils/workspaceHierarchy' import { Button } from '$lib/components/common' import Modal from '$lib/components/common/modal/Modal.svelte' import { Tooltip } from '$lib/components/meltComponents' @@ -30,51 +27,30 @@ let open = $state(false) - // A fork's usage and tier resolve to its billing root while its member list is a - // subset of the root's, so seats must come from the root or the cap is fork-sized - // against root usage. `undefined` when the root isn't visible from here: the cap - // is then unknowable, and the caller hides the meter rather than guessing. - function billingRoot(workspace: string, all: UserWorkspace[]): string | undefined { - const self = all.find((w) => w.id === workspace) - if (!self) return undefined - if (!self.parent_workspace_id) return workspace - const top = findWorkspaceAncestors(workspace, all).at(-1) - return top && !top.parent_workspace_id ? top.id : undefined - } + // Seat count for a paid workspace, the basis of its included executions. The server + // resolves a fork to the workspace its plan is billed on and counts the seats there, + // because neither is answerable from here: a fork's member list is a subset of that + // root's, and a fork member need not be a member of the root at all. + const fetchSeats = tagged( + async (workspace: string) => (await WorkspaceService.getBillableSeats({ workspace })).seats + ) - // Seat count for a paid workspace, the basis of its included executions. Only the - // user list is needed: `premium_info` carries the same usage number as - // `workspaceUsageStore` but requires admin and only exists when Stripe is - // configured, so it would leave regular members with no block at all. - const fetchSeats = tagged(async (root: string) => { - // Throws for a fork member with no seat in the root, which is the same answer as - // an unresolvable root: leave the paid meter hidden. - const users = await UserService.listUsers({ workspace: root }) - // Same basis as the backend's `count_paid_seats`: disabled members and service - // accounts are not billed, so counting them inflates the cap and hides a real - // overage. 1 developer = 1 seat, 2 operators = 1 seat. - const billable = users.filter((u) => !u.disabled && !u.is_service_account) - const developers = billable.filter((u) => !u.operator).length - const operators = billable.length - developers - return Math.ceil(developers + operators / 2) - }) - - const billingRootId = $derived.by(() => { - const workspace = $workspaceStore - if (!isCloudHosted() || !$isPremiumStore || !workspace) return undefined - return billingRoot(workspace, $userWorkspaces ?? []) - }) + const meteredWorkspace = $derived( + isCloudHosted() && $isPremiumStore ? $workspaceStore : undefined + ) // The membership version is in the key so a change re-resolves the cap, but not in // the tag: tagging by it would blank the bar on every change. const seatsResource = resource( () => - billingRootId ? { root: billingRootId, version: $workspaceMembershipVersion } : undefined, - async (key) => (key ? await fetchSeats(key.root) : undefined) + meteredWorkspace + ? { workspace: meteredWorkspace, version: $workspaceMembershipVersion } + : undefined, + async (key) => (key ? await fetchSeats(key.workspace) : undefined) ) const scopedSeats = scopedValue() - const seats = $derived(scopedSeats(billingRootId, seatsResource.current)) + const seats = $derived(scopedSeats(meteredWorkspace, seatsResource.current)) type QuotaKey = 'user' | 'workspace' diff --git a/frontend/src/lib/components/table/Cell.svelte b/frontend/src/lib/components/table/Cell.svelte index 94aac6db7a..1fc8d1d8fd 100644 --- a/frontend/src/lib/components/table/Cell.svelte +++ b/frontend/src/lib/components/table/Cell.svelte @@ -11,7 +11,9 @@ shouldStopPropagation?: boolean selected?: boolean sticky?: boolean - stickyEnd?: boolean + /** The column holding a row's action buttons. It hugs its content at the table's + * right edge instead of absorbing the width the other columns leave over. */ + actions?: boolean wrap?: boolean children?: import('svelte').Snippet [key: string]: any @@ -25,7 +27,7 @@ shouldStopPropagation = false, selected = false, sticky = false, - stickyEnd = false, + actions = false, wrap = false, children, ...rest @@ -55,11 +57,11 @@ last && size === 'xs' ? 'sm:pr-3' : '', numeric ? 'text-right' : '', - // Pin an actions column to the right so it stays visible when a wide table - // scrolls horizontally. The background must be opaque so cells sliding under it - // are occluded — the row's hover tint is translucent and would bleed through. - stickyEnd ? 'sticky right-0 border-l' : '', - stickyEnd ? (head ? 'bg-surface-secondary' : 'bg-surface') : '', + // `w-0` shrinks the column to its buttons instead of taking the leftover width, and + // the pin keeps them reachable while a wide table scrolls. The background must stay + // opaque for the cells passing under it to be occluded — see `wm-cell-pinned` below. + actions ? 'w-0 text-right [&>*]:ml-auto sticky right-0 wm-cell-pinned' : '', + actions ? (head ? 'bg-surface-secondary' : 'bg-surface') : '', sticky ? `!p-0 sticky ${first ? 'left-0' : 'right-0'}` : 'px-2 py-2', size === 'sm' ? 'px-1.5 py-2.5' : '', size === 'lg' ? 'px-3 py-4' : '', @@ -77,3 +79,32 @@ {@render children?.()} {/if} + + diff --git a/frontend/src/lib/components/table/DataTable.svelte b/frontend/src/lib/components/table/DataTable.svelte index b6ab8d00e6..d86d165883 100644 --- a/frontend/src/lib/components/table/DataTable.svelte +++ b/frontend/src/lib/components/table/DataTable.svelte @@ -15,6 +15,24 @@ let tableHeight: number = $state(0) const dispatch = createEventDispatcher() let tableContainer: HTMLDivElement | undefined = $state() + let tableEl: HTMLTableElement | undefined = $state() + + // A pinned actions column only earns a seam once something can actually pass under it, + // so the overflow is measured rather than assumed: on a table that fits, the column + // should be indistinguishable from an ordinary one. + let xOverflowing = $state(false) + $effect(() => { + const container = tableContainer + const table = tableEl + if (!container || !table) return + // Sub-pixel widths make an exactly-fitting table read as 0.5px over. + const measure = () => (xOverflowing = container.scrollWidth > container.clientWidth + 1) + measure() + const observer = new ResizeObserver(measure) + observer.observe(container) + observer.observe(table) + return () => observer.disconnect() + }) interface Props { paginated?: boolean currentPage?: number @@ -130,11 +148,15 @@ >
    - +
    {@render children?.()}
    {@render emptyMessage?.()} diff --git a/frontend/src/lib/components/table/Row.svelte b/frontend/src/lib/components/table/Row.svelte index c3cbd68665..5854a88d3b 100644 --- a/frontend/src/lib/components/table/Row.svelte +++ b/frontend/src/lib/components/table/Row.svelte @@ -28,7 +28,9 @@ = $props() let underlyingInputEl = $derived(_underlyingInputEl ?? ('input' as const)) @@ -152,7 +158,7 @@ onpointerdown={(e) => e.stopImmediatePropagation()} bind:this={inputEl} bind:value - use:autosize + use:autosize={autosizeParams} > {:else if underlyingInputEl === 'input'} void ) => Promise>> create?: (workspace: string, requestBody: any) => Promise + /** + * Write back an existing trigger's config. Used to point an imported trigger at a + * resource the workspace already has; `schedule` has none because its own service + * takes a different body shape, handled by the caller. + */ + update?: (workspace: string, path: string, requestBody: any) => Promise } > = { http: { @@ -101,7 +107,9 @@ export const TRIGGER_KINDS: Record< resourceField: 'authentication_resource_path', list: (workspace) => HttpTriggerService.listHttpTriggers({ workspace }), create: (workspace, requestBody) => - HttpTriggerService.createHttpTrigger({ workspace, requestBody }) + HttpTriggerService.createHttpTrigger({ workspace, requestBody }), + update: (workspace, path, requestBody) => + HttpTriggerService.updateHttpTrigger({ workspace, path, requestBody }) }, websocket: { configFields: [ @@ -119,7 +127,9 @@ export const TRIGGER_KINDS: Record< note: 'Reconnect WebSocket auth after import if external service requires it.', list: (workspace) => WebsocketTriggerService.listWebsocketTriggers({ workspace }), create: (workspace, requestBody) => - WebsocketTriggerService.createWebsocketTrigger({ workspace, requestBody }) + WebsocketTriggerService.createWebsocketTrigger({ workspace, requestBody }), + update: (workspace, path, requestBody) => + WebsocketTriggerService.updateWebsocketTrigger({ workspace, path, requestBody }) }, schedule: { configFields: [ @@ -180,7 +190,9 @@ export const TRIGGER_KINDS: Record< eeOnly: true, list: (workspace) => KafkaTriggerService.listKafkaTriggers({ workspace }), create: (workspace, requestBody) => - KafkaTriggerService.createKafkaTrigger({ workspace, requestBody }) + KafkaTriggerService.createKafkaTrigger({ workspace, requestBody }), + update: (workspace, path, requestBody) => + KafkaTriggerService.updateKafkaTrigger({ workspace, path, requestBody }) }, nats: { configFields: [ @@ -197,7 +209,9 @@ export const TRIGGER_KINDS: Record< eeOnly: true, list: (workspace) => NatsTriggerService.listNatsTriggers({ workspace }), create: (workspace, requestBody) => - NatsTriggerService.createNatsTrigger({ workspace, requestBody }) + NatsTriggerService.createNatsTrigger({ workspace, requestBody }), + update: (workspace, path, requestBody) => + NatsTriggerService.updateNatsTrigger({ workspace, path, requestBody }) }, sqs: { configFields: [ @@ -212,7 +226,9 @@ export const TRIGGER_KINDS: Record< eeOnly: true, list: (workspace) => SqsTriggerService.listSqsTriggers({ workspace }), create: (workspace, requestBody) => - SqsTriggerService.createSqsTrigger({ workspace, requestBody }) + SqsTriggerService.createSqsTrigger({ workspace, requestBody }), + update: (workspace, path, requestBody) => + SqsTriggerService.updateSqsTrigger({ workspace, path, requestBody }) }, mqtt: { configFields: [ @@ -228,7 +244,9 @@ export const TRIGGER_KINDS: Record< resourceField: 'mqtt_resource_path', list: (workspace) => MqttTriggerService.listMqttTriggers({ workspace }), create: (workspace, requestBody) => - MqttTriggerService.createMqttTrigger({ workspace, requestBody }) + MqttTriggerService.createMqttTrigger({ workspace, requestBody }), + update: (workspace, path, requestBody) => + MqttTriggerService.updateMqttTrigger({ workspace, path, requestBody }) }, amqp: { configFields: ['amqp_resource_path', 'queue_name', 'exchange', 'options'], @@ -238,7 +256,9 @@ export const TRIGGER_KINDS: Record< resourceField: 'amqp_resource_path', list: (workspace) => AmqpTriggerService.listAmqpTriggers({ workspace }), create: (workspace, requestBody) => - AmqpTriggerService.createAmqpTrigger({ workspace, requestBody }) + AmqpTriggerService.createAmqpTrigger({ workspace, requestBody }), + update: (workspace, path, requestBody) => + AmqpTriggerService.updateAmqpTrigger({ workspace, path, requestBody }) }, gcp: { configFields: [ @@ -256,7 +276,9 @@ export const TRIGGER_KINDS: Record< eeOnly: true, list: (workspace) => GcpTriggerService.listGcpTriggers({ workspace }), create: (workspace, requestBody) => - GcpTriggerService.createGcpTrigger({ workspace, requestBody }) + GcpTriggerService.createGcpTrigger({ workspace, requestBody }), + update: (workspace, path, requestBody) => + GcpTriggerService.updateGcpTrigger({ workspace, path, requestBody }) }, azure: { configFields: [ @@ -274,7 +296,9 @@ export const TRIGGER_KINDS: Record< eeOnly: true, list: (workspace) => AzureTriggerService.listAzureTriggers({ workspace }), create: (workspace, requestBody) => - AzureTriggerService.createAzureTrigger({ workspace, requestBody }) + AzureTriggerService.createAzureTrigger({ workspace, requestBody }), + update: (workspace, path, requestBody) => + AzureTriggerService.updateAzureTrigger({ workspace, path, requestBody }) }, postgres: { configFields: [ @@ -288,7 +312,9 @@ export const TRIGGER_KINDS: Record< resourceField: 'postgres_resource_path', list: (workspace) => PostgresTriggerService.listPostgresTriggers({ workspace }), create: (workspace, requestBody) => - PostgresTriggerService.createPostgresTrigger({ workspace, requestBody }) + PostgresTriggerService.createPostgresTrigger({ workspace, requestBody }), + update: (workspace, path, requestBody) => + PostgresTriggerService.updatePostgresTrigger({ workspace, path, requestBody }) }, email: { configFields: ['local_part', 'workspaced_local_part'], @@ -297,7 +323,9 @@ export const TRIGGER_KINDS: Record< note: 'Email address regenerates on import.', list: (workspace) => EmailTriggerService.listEmailTriggers({ workspace }), create: (workspace, requestBody) => - EmailTriggerService.createEmailTrigger({ workspace, requestBody }) + EmailTriggerService.createEmailTrigger({ workspace, requestBody }), + update: (workspace, path, requestBody) => + EmailTriggerService.updateEmailTrigger({ workspace, path, requestBody }) } } diff --git a/frontend/src/lib/components/wizards/SetupChecklist.svelte b/frontend/src/lib/components/wizards/SetupChecklist.svelte index 9da6d3a5ad..95f278b0c1 100644 --- a/frontend/src/lib/components/wizards/SetupChecklist.svelte +++ b/frontend/src/lib/components/wizards/SetupChecklist.svelte @@ -31,9 +31,12 @@ type Props = { steps: SetupStep[] class?: string + /** Applied to each step's substep block, for a caller whose substeps are a long + * list rather than a handful of checks and need their own scroll. */ + substepsClass?: string } - let { steps, class: className = '' }: Props = $props() + let { steps, class: className = '', substepsClass = '' }: Props = $props() /** * Only the steps the user has actually toggled. A failed step opens itself, so recording @@ -108,8 +111,8 @@
    {#if step.substeps?.length} -
    - +
    +
    {/if}
    diff --git a/frontend/src/lib/components/workspaceSettings/AISettings.svelte b/frontend/src/lib/components/workspaceSettings/AISettings.svelte index 5979e8a535..65187d98c8 100644 --- a/frontend/src/lib/components/workspaceSettings/AISettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/AISettings.svelte @@ -15,8 +15,6 @@ import { supportsAutocomplete } from '../copilot/utils' import TestAiKey from '../copilot/TestAIKey.svelte' import Label from '../Label.svelte' - import AiSkillsSettings from './AiSkillsSettings.svelte' - import { isGlobalAiEnabled } from '../copilot/chat/global/gate' import SettingsPageHeader from '../settings/SettingsPageHeader.svelte' import ResourcePicker from '../ResourcePicker.svelte' import Toggle from '../Toggle.svelte' @@ -80,6 +78,7 @@ let modelPricing: Record = $state({}) let usingOpenaiClientCredentialsOauth = $state(false) let workspaceOverrideEditorOpened = $state(false) + let copilotDisabled = $state(false) // --- Initial state for dirty tracking --- let initialAiProviders: Exclude = $state({}) @@ -90,6 +89,7 @@ let initialMaxTokensPerModel: Record = $state({}) let initialModelPricing: Record = $state({}) let initialPrompts: Record = $state({}) + let initialCopilotDisabled = $state(false) let lastLoadedConfigKey = $state(undefined) function clone(v: T): T { @@ -117,6 +117,7 @@ customPrompts = clone(config?.custom_prompts ?? {}) maxTokensPerModel = clone(config?.max_tokens_per_model ?? {}) modelPricing = clone(config?.model_pricing ?? {}) + copilotDisabled = config?.copilot_disabled === true for (const mode of ['edit', 'fix', 'gen']) { if (!(mode in customPrompts)) { customPrompts[mode] = '' @@ -133,6 +134,7 @@ initialMaxTokensPerModel = clone(maxTokensPerModel) initialModelPricing = clone(modelPricing) initialPrompts = clone(customPrompts) + initialCopilotDisabled = copilotDisabled } export function loadFromConfig(config: AIConfig | undefined) { @@ -148,6 +150,7 @@ customPrompts = clone(initialCustomPrompts) maxTokensPerModel = clone(initialMaxTokensPerModel) modelPricing = clone(initialModelPricing) + copilotDisabled = initialCopilotDisabled } $effect(() => { @@ -182,7 +185,8 @@ codeCompletionModel !== initialCodeCompletionModel || JSON.stringify(customPrompts) !== JSON.stringify(initialCustomPrompts) || JSON.stringify(maxTokensPerModel) !== JSON.stringify(initialMaxTokensPerModel) || - JSON.stringify(modelPricing) !== JSON.stringify(initialModelPricing) + JSON.stringify(modelPricing) !== JSON.stringify(initialModelPricing) || + copilotDisabled !== initialCopilotDisabled ) $effect(() => { @@ -287,6 +291,8 @@ .filter(([_, prompt]) => prompt.trim().length > 0) .reduce((acc, [mode, prompt]) => ({ ...acc, [mode]: prompt }), {}) + // The flag is the one thing a workspace on instance defaults still stores of its own. + const copilot_disabled = copilotDisabled ? true : undefined return Object.keys(aiProviders ?? {}).length > 0 ? { providers: aiProviders, @@ -296,9 +302,10 @@ custom_prompts: Object.keys(custom_prompts).length > 0 ? custom_prompts : undefined, max_tokens_per_model: Object.keys(maxTokensPerModel).length > 0 ? maxTokensPerModel : undefined, - model_pricing: Object.keys(modelPricing).length > 0 ? modelPricing : undefined + model_pricing: Object.keys(modelPricing).length > 0 ? modelPricing : undefined, + copilot_disabled } - : {} + : { copilot_disabled } } function isSaveDisabled(): boolean { @@ -607,10 +614,6 @@
    {/if} - - {#if promptScope === 'workspace' && isGlobalAiEnabled()} - - {/if}
    {/if} -{#if showWorkspaceOverrideEditor} - +{#if promptScope === 'workspace'} + + { + copilotDisabled = e.detail + }} + options={{ right: 'Hide AI sessions in this workspace' }} + /> + {/if} + + + diff --git a/frontend/src/lib/components/workspaceSettings/AddDataTableWizard.svelte b/frontend/src/lib/components/workspaceSettings/AddDataTableWizard.svelte index 2e19cbea82..3495bdbee0 100644 --- a/frontend/src/lib/components/workspaceSettings/AddDataTableWizard.svelte +++ b/frontend/src/lib/components/workspaceSettings/AddDataTableWizard.svelte @@ -21,6 +21,8 @@ FolderService, OauthService, ResourceService, + UserService, + type User, VariableService, WorkspaceService } from '$lib/gen' @@ -85,6 +87,38 @@ customInstanceDbs: ResourceReturn confirmationModal: ConfirmationModalHandle defaultInstanceDbName: () => string + /** Name to open with, when the caller needs a table of a particular name rather + * than whatever the user picks — the import wizard configures the one a project's + * migrations target. + * + * Locked when `onFinishAlso` is also given, because that work targets this name and + * nothing carries an edit through to it: renaming `main` to `other` would create + * `other`, then run the migrations against `main`, fail, and leave a data table + * nobody asked for. Editable without one, where the name is only a name. */ + initialName?: string + /** Where the dialog portals to. `#content` is the app shell's scroll container, + * which only exists inside the `(logged)` layout; a page reparented out of it + * (the hub import wizard) has to say `body` or the portal finds nothing and the + * dialog never appears. */ + modalTarget?: string + /** What the caller does once the table exists, named on the final button so the + * user is told before pressing it — the import wizard runs the project's + * migrations, which is otherwise invisible until it has already happened. */ + finishAlso?: string + /** The work `finishAlso` names. Run as the last checklist step, so it reports + * where the rest of the run does instead of starting after the dialog closes. + * Throwing marks that step failed; the table itself is already made either way. */ + onFinishAlso?: () => Promise + /** The workspace everything here is created in and checked against. + * + * Defaults to `$workspaceStore`, which is right for the settings page — it is the + * workspace being looked at. The import wizard is the exception: its page is + * reparented out of `(logged)`, so nothing re-runs the layout's workspace + * persistence, and after a reload the store still names whatever workspace the + * user came from while the plan in the URL names the destination. Left ambient, + * this would create the data table in one workspace and run the project's + * migrations in the other. */ + workspace?: string } let { @@ -95,9 +129,62 @@ onDone, customInstanceDbs, confirmationModal, - defaultInstanceDbName + defaultInstanceDbName, + initialName, + modalTarget = '#content', + finishAlso, + onFinishAlso, + workspace: workspaceProp }: Props = $props() + /** + * The caller needs this exact table, and has follow-up work bound to its name. + * + * Captured when the dialog opens rather than read live: `initialName` is the caller's + * `wizardFor`, which it clears from `onDone` — and that fires after a *failed* run too, + * while the dialog stays up offering Back. Reading it live releases the lock exactly when + * the user is most likely to edit, which is the divergence the lock exists to stop. + */ + let nameLocked = $state(false) + + /** Every write and every check goes through this, never `$workspaceStore` directly. */ + const targetWorkspace = $derived(workspaceProp ?? $workspaceStore ?? '') + + /** + * Who the caller is *in the destination*, which is not who `$userStore` describes. + * + * `$userStore` is the membership of the workspace the app is in. Routing the API calls + * elsewhere without routing this leaves the username behind: after a reload on the import + * wizard's step 4 it names the workspace the user came from, and a resource path built + * from it lands on `u/` inside the destination — failing an ownership check, + * or for an admin, quietly putting database credentials in another member's namespace. + */ + let targetUser = $state(undefined) + const aimedElsewhere = $derived(!!workspaceProp && workspaceProp !== $workspaceStore) + const ambientUsername = $derived($userStore?.username ?? '') + const targetUsername = $derived(aimedElsewhere ? (targetUser?.username ?? '') : ambientUsername) + /** The destination's membership could not be read, so nothing here knows who the user is. */ + let membershipFailed = $state(false) + + async function loadTargetUser(): Promise { + const ws = workspaceProp + if (!ws || ws === $workspaceStore) { + targetUser = undefined + membershipFailed = false + return + } + try { + targetUser = await UserService.whoami({ workspace: ws }) + membershipFailed = false + } catch { + // Recorded rather than swallowed: an unknown username silently becomes `admin` in + // the default path, which is the wrong namespace to write credentials into. Setup + // is blocked instead. + targetUser = undefined + membershipFailed = true + } + } + const STEPS = ['Choose a database', 'Set it up', 'Review'] let wiz: WizardState = $state( @@ -146,7 +233,7 @@ } clearTimeout(variableCheck) variableCheck = setTimeout(async () => { - const taken = await VariableService.existsVariable({ workspace: $workspaceStore!, path }) + const taken = await VariableService.existsVariable({ workspace: targetWorkspace, path }) // Two checks can be in flight at once and resolve out of order. A `false` for a path // nobody is on any more would clear the error guarding the one about to be written; // a `true` would disable Finish over a path this run stopped caring about. @@ -163,7 +250,7 @@ * in flight when Finish is pressed. */ async function pathConflictMessage(path: string): Promise { - const workspace = $workspaceStore! + const workspace = targetWorkspace // Each namespace answers to its own claim. Holding the secret says nothing about who owns // the resource beside it, so one claim must not wave the other's check through. const [variable, resource] = await Promise.all([ @@ -178,11 +265,14 @@ let maxStep = $state(1) function defaultProjectName(): string { - return `windmill-${$workspaceStore ?? 'workspace'}` + return `windmill-${targetWorkspace || 'workspace'}` } function defaultTableName(): string { - return existingNames.includes('main') ? `${$workspaceStore ?? 'data'}_datatable` : 'main' + // A caller that needs a specific name wins over the usual "main, unless taken": + // the import wizard's migrations only apply to a table of the name they target. + if (initialName) return initialName + return existingNames.includes('main') ? `${targetWorkspace || 'data'}_datatable` : 'main' } // Takes the list rather than reading it, so the fetch that loads it can seed off its own @@ -190,7 +280,7 @@ function defaultFolder(list: string[] = folders): string { // The first folder this admin can write to, so the resource lands somewhere the team // can find and repair. A workspace with no folders falls back to the personal space. - return list.length ? `f/${list[0]}` : `u/${$userStore?.username ?? 'admin'}` + return list.length ? `f/${list[0]}` : `u/${targetUsername || 'admin'}` } // A row this run wrote and could not take back out is still its own: `removeRow` reports @@ -236,7 +326,7 @@ ) const pgResources = resource( - () => (opened && wiz.provider === 'resource' ? ($workspaceStore ?? '') : ''), + () => (opened && wiz.provider === 'resource' ? targetWorkspace : ''), async (workspace) => { if (!workspace) return undefined const list = await ResourceService.listResource({ workspace, resourceType: 'postgresql' }) @@ -330,7 +420,7 @@ ) const folderNames = resource( - () => (opened ? ($workspaceStore ?? '') : ''), + () => (opened ? targetWorkspace : ''), async (workspace) => { if (!workspace) return [] const all = await FolderService.listFolderNames({ workspace }) @@ -374,7 +464,11 @@ let resumedPath = $state(undefined) function reset(from: WizardResume | undefined) { + nameLocked = !!initialName && !!onFinishAlso resumedPath = from?.resourcePath + // A pending confirmation that never settled leaves `dismissing` true, and `finally` + // cannot clear what never resolves — so a fresh open always starts dismissable. + dismissing = false wiz = newWizardState({ name: from?.name || defaultTableName(), projectName: from?.projectName || defaultProjectName(), @@ -391,6 +485,7 @@ createdProjects = [] nameConflictFor = undefined lastFailure = '' + finishAlsoFailed = false pathTakenError = '' poolerUnavailable = undefined if (from) { @@ -424,7 +519,11 @@ * what keeps the restore independent of when the prop it was assigned to reaches this * component. */ - export function open(parked?: WizardResume) { + export async function open(parked?: WizardResume) { + // Awaited before `reset`, which seeds the resource path from `defaultFolder()` and so + // needs the destination's username. Seeding first and correcting later loses whenever + // the folder list resolves first, and never corrects at all if `whoami` fails. + await loadTargetUser() reset(parked ?? resume) opened = true } @@ -472,12 +571,14 @@ // Also retires any check still in flight, so its answer cannot land on the edited value. probeToken++ clearProbe(wiz) - // Read off one attempt against one project; the review step would otherwise warn about - // a limitation that no longer applies while claiming session pooling right above it. + // Read off one attempt against one project, so it does not survive a change of inputs: + // the review step would otherwise warn about a limitation that does not apply to what + // it is describing, while claiming session pooling right above it. poolerUnavailable = undefined // Same for the failure carried back to the review step: it names inputs that have since // been edited, so it would describe a run nobody can still act on. lastFailure = '' + finishAlsoFailed = false if (maxStep > wiz.step) maxStep = wiz.step } @@ -527,7 +628,7 @@ settle({ checking: false, report: undefined, error: undefined }) return } - const report = await probeDatatableConnection($workspaceStore!, database) + const report = await probeDatatableConnection(targetWorkspace, database) settle({ checking: false, report, error: undefined }) } catch (err: any) { settle({ @@ -579,6 +680,13 @@ ) /** Why the last run failed, kept on the review step after the checklist is dropped. */ let lastFailure = $state('') + /** + * The appended `onFinishAlso` step failed while `runSetup` itself succeeded. Tracked apart + * from `run.result`, which stays the setup's own verdict: the data table really was + * created, so a retry must re-run only this last step. Re-running the setup would ask for + * the table name it has just taken, and be refused as a duplicate. + */ + let finishAlsoFailed = $state(false) /** * A refused pre-flight means nothing ran, so the checklist from a previous attempt has to @@ -631,7 +739,7 @@ const name = wiz.review.name.trim() try { if (claimedName !== name) { - const settings = await WorkspaceService.getSettings({ workspace: $workspaceStore! }) + const settings = await WorkspaceService.getSettings({ workspace: targetWorkspace }) if (settings.datatable?.datatables?.[name]) { nameConflictFor = { name, @@ -666,6 +774,7 @@ } run = { steps: planSteps(wiz), running: true } lastFailure = '' + finishAlsoFailed = false // The database is registered by the call whatever it answers, so asking for one is // already leaving something behind. if (wiz.provider === 'instance' && wiz.instance.mode === 'create') { @@ -675,7 +784,7 @@ let result: RunResult | undefined = undefined try { result = await runSetup(wiz, { - workspace: $workspaceStore!, + workspace: targetWorkspace, supabaseToken: supaOauth.token, onInstanceDbsChanged: async () => { await customInstanceDbs.refetch() @@ -684,9 +793,27 @@ onPoolerUnavailable: (reason) => (poolerUnavailable = reason), createdProjects, claims, - username: $userStore?.username ?? '' + username: targetUsername }) } finally { + // The caller's own finishing work, appended to the same checklist. It only runs + // on a clean setup: there is no table for it to act on otherwise. + if (result?.ok && onFinishAlso && finishAlso) { + const title = finishAlso.charAt(0).toUpperCase() + finishAlso.slice(1) + run.steps = [...run.steps, { title, status: 'running' }] + try { + await onFinishAlso() + run.steps = run.steps.map((s, i) => + i === run.steps.length - 1 ? { ...s, status: 'done' as const } : s + ) + } catch (err: any) { + const description = err?.body ?? err?.message ?? String(err) + run.steps = run.steps.map((s, i) => + i === run.steps.length - 1 ? { ...s, status: 'failed' as const, description } : s + ) + finishAlsoFailed = true + } + } // `runSetup` catches per step, but anything escaping it would otherwise leave the // button spinning with a page reload the only way out. // Kept, not replaced: what an earlier attempt wrote is still out there, so a later @@ -713,7 +840,10 @@ /** * Whether closing would throw away work. A failed run counts: its inputs are still editable * and it may have left something behind. A run in flight cannot be closed at all, and one - * that succeeded has nothing left to lose. + * that made its data table has nothing left to lose — including when `onFinishAlso` failed + * afterwards, because the table is real and working and the caller owns what is left. The + * import step, the only caller that passes one, shows that failure on its own row with a + * way to run it again and will not let Finish through while it stands. */ function hasUnfinishedIntent(): boolean { return wiz.provider !== undefined && !run.running && !run.result?.ok @@ -730,19 +860,52 @@ return } dismissing = true - const confirmed = await confirmationModal.ask({ - title: 'Leave without adding a data table?', - // A run that failed and was sent back to be edited leaves whatever it got through - // behind it, so promising otherwise would be a lie exactly when it matters most. - children: leftBehind - ? 'The setup that already ran left what it created behind, and what you have filled in here will be lost.' - : 'Nothing has been created yet, and what you have filled in here will be lost.', - confirmationText: 'Discard' - }) - dismissing = false - // Re-read rather than trust the entry check: a run can start while the dialog is up, and - // answering Discard would otherwise tear the modal down in the middle of it. - if (confirmed && !preventClose) close() + // `finally`, because the flag is what blocks a second attempt: an `ask` that throws + // would otherwise leave the dialog permanently undismissable — the backdrop, Escape + // and the close button all return early here, so the only way out would be a reload. + try { + const confirmed = await confirmationModal.ask({ + title: 'Leave without adding a data table?', + // A run that failed and was sent back to be edited leaves whatever it got through + // behind it, so promising otherwise would be a lie exactly when it matters most. + children: leftBehind + ? 'The setup that already ran left what it created behind, and what you have filled in here will be lost.' + : 'Nothing has been created yet, and what you have filled in here will be lost.', + confirmationText: 'Discard' + }) + // Re-read rather than trust the entry check: a run can start while the dialog is up, and + // answering Discard would otherwise tear the modal down in the middle of it. + if (confirmed && !preventClose) close() + } finally { + dismissing = false + } + } + + /** Re-runs only the appended step, which is the only thing that failed. */ + async function retryFinishAlso() { + if (!onFinishAlso || !finishAlso) return + const title = finishAlso.charAt(0).toUpperCase() + finishAlso.slice(1) + finishAlsoFailed = false + run = { + ...run, + running: true, + steps: [...run.steps.slice(0, -1), { title, status: 'running' as const }] + } + try { + await onFinishAlso() + run.steps = run.steps.map((s, i) => + i === run.steps.length - 1 ? { ...s, status: 'done' as const } : s + ) + } catch (err: any) { + const description = err?.body ?? err?.message ?? String(err) + run.steps = run.steps.map((s, i) => + i === run.steps.length - 1 ? { ...s, status: 'failed' as const, description } : s + ) + finishAlsoFailed = true + } finally { + run = { ...run, running: false } + onDone() + } } function close() { @@ -752,10 +915,18 @@ // The single primary action. Its label says what it is about to do, and doing it is what // moves the wizard on. let primary = $derived.by(() => { + // Ahead of everything: without the destination's membership the resource path would be + // guessed, and a guess here writes database credentials into somebody else's namespace. + if (aimedElsewhere && membershipFailed) + return { label: 'Cannot read your access to this workspace', disabled: true } if (submitting && !run.running) return { label: 'Setting things up', disabled: true, busy: true } if (run.steps.length) { if (run.running) return { label: 'Setting things up', disabled: true, busy: true } + // Before the `ok` check: the setup succeeded and the step after it did not, so + // "Done" would be offered over a failed row. + if (finishAlsoFailed) + return { label: 'Try again', disabled: false, act: retryFinishAlso } if (run.result?.ok) return { label: 'Done', disabled: false, act: close } // A run that died because the Supabase token expired would retry into the same 401 // forever; authorizing again is the only thing that can move it on. @@ -809,11 +980,12 @@ act: enterReview } } + const created = + wiz.provider === 'supabase' && wiz.supabase.mode === 'create' + ? 'Create project and data table' + : 'Create data table' return { - label: - wiz.provider === 'supabase' && wiz.supabase.mode === 'create' - ? 'Create project and data table' - : 'Create data table', + label: finishAlso ? `${created} and ${finishAlso}` : created, disabled: // Guards the way back as well as the way forward: the stepper can return to step 2, // and not every control there invalidates the review it just made stale. @@ -842,7 +1014,7 @@ else opened = v } } - target="#content" + target={modalTarget} formStyling title="Add a data table" contentClasses="flex flex-col" @@ -1034,7 +1206,7 @@ {#if wiz.instance.mode === 'existing'} {@const shared = ( customInstanceDbs.current?.[wiz.instance.dbName ?? '']?.used_by_workspaces ?? [] - ).filter((w) => w !== $workspaceStore)} + ).filter((w) => w !== targetWorkspace)} {#if shared.length} @@ -1047,7 +1219,7 @@
    {#each instanceDbs as { name, db } (name)} {@const selected = wiz.instance.dbName === name} - {@const others = (db.used_by_workspaces ?? []).filter((w) => w !== $workspaceStore)} + {@const others = (db.used_by_workspaces ?? []).filter((w) => w !== targetWorkspace)} -
    - {/if} -{/snippet} - - - {#snippet headerAction()} -
    - {#if manageMode} - - - {:else} - {#if skills.length > 1} - - {/if} - - {/if} -
    - {#if $addMenuOpen} -
    - - -
    - {/if} - {/snippet} - -
    - {#if skills.length === 0} -
    - No custom skills yet -
    - {:else} -
    - {#if manageMode} -
    - 0 && !allSelected} - onChange={toggleSelectAll} - /> - - {selectedCount ? `${selectedCount} selected` : 'Select all'} - -
    - {/if} - {#each skills as skill (skill.name)} -
    - {#if manageMode} - - {:else} -
    -
    {skill.name}
    -
    {skill.description}
    - -
    - openSkill(skill.name, 'edit') - }, - { - displayName: 'Delete', - icon: Trash2, - type: 'delete', - action: () => (toDelete = skill.name) - } - ]} - /> - {/if} -
    - {/each} -
    - {/if} -
    -     - - - - - - {#snippet headerRight()} - {#if editingOriginalName} - - {#snippet children({ item })} - - - {/snippet} - - {/if} - {/snippet} -
    - {#if detailMode === 'view'} -
    - {#if viewParsed.description} -

    {viewParsed.description}

    - {/if} -
    - -
    -
    - {:else} - {@render pasteZone()} - {/if} -
    -     - - { - const toImport = [...pendingNew, ...pendingConflicts.filter((s) => overwriteChoices[s.name])] - const skipped = pendingSkipped - pendingImport = undefined - pendingSkipped = [] - overwriteChoices = {} - if (toImport.length) await uploadSkills(toImport, skipped) - else sendUserToast('No skills imported.') - }} - onCanceled={() => { - pendingImport = undefined - pendingSkipped = [] - overwriteChoices = {} - }} -> -
    - {#if pendingNew.length} -
    - Add {pendingNew.length} new skill(s): - {pendingNew.map((s) => s.name).join(', ')} -
    - {/if} - {#if pendingConflicts.length} -
    - - {pendingConflicts.length} skill(s) already exist — choose which to overwrite: - -
    - {#each pendingConflicts as conflict (conflict.name)} -
    - {conflict.name} - -
    - {/each} -
    -
    - {/if} - {#if pendingSkipped.length} - {pendingSkipped.length} file(s) will be skipped. - {/if} -
    -     - - { - const name = toDelete - toDelete = undefined - if (name) await deleteSkill(name) - }} - onCanceled={() => (toDelete = undefined)} -> - - Delete the skill {toDelete}? The AI chat will no longer be able to use it. - - - - { - confirmBatchDelete = false - await deleteSelected() - }} - onCanceled={() => (confirmBatchDelete = false)} -> - - Delete {selectedCount} selected skill(s)? The AI chat will no longer be able to use them. - - diff --git a/frontend/src/lib/components/workspaceSettings/CreateWorkspaceInner.svelte b/frontend/src/lib/components/workspaceSettings/CreateWorkspaceInner.svelte index 7e02a1b7ae..22a2fc889b 100644 --- a/frontend/src/lib/components/workspaceSettings/CreateWorkspaceInner.svelte +++ b/frontend/src/lib/components/workspaceSettings/CreateWorkspaceInner.svelte @@ -4,7 +4,6 @@ import { JobService, ResourceService, - SettingService, UserService, VariableService, WorkspaceService, @@ -47,7 +46,12 @@ import { onMount } from 'svelte' import { sendUserToast } from '$lib/toast' import TestAIKey from '$lib/components/copilot/TestAIKey.svelte' - import { switchWorkspace } from '$lib/storeUtils' + import { + enterNewWorkspace, + loadUsernamePolicy, + refreshWorkspaceList + } from '$lib/workspaceCreation' + import { validateWorkspaceId } from '$lib/utils/workspaceId' import { deleteSessionsForWorkspace } from '$lib/components/sessions/sessionState.svelte' import { isCloudHosted } from '$lib/cloud' import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte' @@ -270,13 +274,10 @@ errorId = forkIdTaken ? `A workspace with id '${effectiveId}' already exists. It may be an archived fork: archiving keeps the id reserved.` : 'ID already exists' - } else if (id != '' && !/^\w+(-\w+)*$/.test(id)) { - errorId = 'ID can only contain letters, numbers and dashes and must not finish by a dash' - } else if (effectiveId.length > 50) { - // `wm-fork-` prefix included: matches the backend's 50-char (git-branch / DB) limit. - errorId = `ID '${effectiveId}' is too long (${effectiveId.length} chars). Maximum is 50.` } else { - errorId = '' + // `effectiveId` carries the `wm-fork-` prefix into the length check, since + // that is what the backend stores. + errorId = (id != '' && validateWorkspaceId(id, effectiveId)) || '' } checking = false } @@ -482,8 +483,7 @@ : `Successfully forked workspace ${baseWorkspaceId} as: wm-fork-${id}` ) - usersWorkspaceStore.set(await WorkspaceService.listUserWorkspaces()) - switchWorkspace(prefixed_id) + await enterNewWorkspace(prefixed_id) onFinish?.() } @@ -552,15 +552,11 @@ } : {} }) - - usersWorkspaceStore.set(await WorkspaceService.listUserWorkspaces()) - switchWorkspace(id) } sendUserToast(`Created workspace id: ${id}`) - usersWorkspaceStore.set(await WorkspaceService.listUserWorkspaces()) - switchWorkspace(id) + await enterNewWorkspace(id) onFinish?.() } @@ -575,7 +571,7 @@ async function loadWorkspaces() { if (!$usersWorkspaceStore) { try { - usersWorkspaceStore.set(await WorkspaceService.listUserWorkspaces()) + await refreshWorkspaceList() } catch {} } if (!$usersWorkspaceStore) { @@ -587,20 +583,10 @@ let automateUsernameCreation = $state(true) async function getAutomateUsernameCreationSetting() { - automateUsernameCreation = - ((await SettingService.getGlobal({ key: 'automate_username_creation' })) as any) ?? true - - if (!automateUsernameCreation) { - UserService.globalWhoami().then((x) => { - let uname = '' - if (x.name) { - uname = x.name.split(' ')[0] - } else { - uname = x.email.split('@')[0] - } - uname = uname.replace(/\./gi, '') - username = uname.toLowerCase() - }) + const policy = await loadUsernamePolicy() + automateUsernameCreation = policy.automate + if (policy.suggested) { + username = policy.suggested } } getAutomateUsernameCreationSetting() diff --git a/frontend/src/lib/components/workspaceSettings/DataTableMigrationsButton.svelte b/frontend/src/lib/components/workspaceSettings/DataTableMigrationsButton.svelte index eac0153993..fc9bddcf4e 100644 --- a/frontend/src/lib/components/workspaceSettings/DataTableMigrationsButton.svelte +++ b/frontend/src/lib/components/workspaceSettings/DataTableMigrationsButton.svelte @@ -52,6 +52,8 @@ let loadError = $state(undefined) let loading = $state(false) let busy = $state(false) + // pg_dump can take a while, so the snapshot gets its own flag to spin its button. + let generatingInitial = $state(false) // Only workspace admins and super admins can opt a data table in or out. const canManage = $derived(!!$userStore?.is_admin || !!$superadmin) @@ -280,6 +282,7 @@ }) if (!confirmed) return busy = true + generatingInitial = true try { await WorkspaceService.generateInitialDatatableMigration({ workspace, @@ -291,6 +294,7 @@ sendUserToast(`Failed to generate initial migration: ${e?.body ?? e?.message ?? e}`, true) } finally { busy = false + generatingInitial = false } } @@ -493,12 +497,13 @@
    {#if migrations.length === 0}
    - No migrations yet + {generatingInitial ? 'Snapshotting current schema…' : 'No migrations yet'} {:else if s.phase === 'draft'} + {/if} + {/if} + {#if s.liveOnHub && s.phase === 'draft'} + {/if}
    @@ -270,8 +319,10 @@ {#if s.phase === 'predeploy'}
    - Bundling creates a draft project on the Hub from the selected scripts, flows and - apps of {s.selectedFolder}/. + Bundling creates {s.liveOnHub + ? 'an update to your Hub project' + : 'a draft project on the Hub'} from the selected scripts, flows and apps of + {s.selectedFolder}/. {s.selectedItems.length} of {s.filteredWorkspaceItems.length} items selected.
    @@ -351,6 +402,31 @@ {/if}
    {/if} + {#if s.liveOnHub && s.phase !== 'live' && s.phase !== 'under_review'} + + Visitors keep seeing the published version, with its stars, forks and comments, + until this update is approved. Approving replaces it in place; discarding leaves it + exactly as it is. + + {/if} + {#if s.pipelineReplayMayBeStale && s.phase === 'draft'} + + This update carries the cascade recorded for the version that is live, and at least + one item has changed since. Record it again below, or visitors will replay the old + run as though it were this version. + + {/if} + {#if s.rejectionReason && s.phase === 'draft'} + + {s.rejectionReason} + + {/if} {#if s.phase === 'draft'}
    @@ -477,18 +553,21 @@
    {/if} {#if s.phase === 'under_review'} -
    - -
    - Locked while under review - - The Windmill team is reviewing this submission. Editing, recording, and sharing - actions are disabled. Estimated turnaround: 1-2 business days. - -
    -
    + The Windmill team is reviewing your project. Submission is locked until they answer + — no new version can be sent to the Hub, and no recording added to this one. + Estimated turnaround: 1-2 business days{#if s.hubSupportsUpdates}; cancel the + submission to get back to it sooner{/if}.{#if s.liveOnHub} + Visitors keep seeing the published version meanwhile, with its stars, forks and + comments; approving replaces it in place.{/if} Your folder itself is untouched — keep + editing your scripts and flows as usual. + {/if} {#if s.phase === 'draft'} {@const recordedCount = s.recordableItems.filter((i) => i.rec === 'recorded').length} @@ -668,7 +747,11 @@ Waiting for the Windmill team to review the submission. {:else} - Iterate further by starting a new draft. + + {s.liveOnHub + ? 'Publish an update to change it — this stays live until the update is approved.' + : 'Iterate further by starting a new draft.'} + {/if}
    {/snippet} diff --git a/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte b/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte index 1e6c990819..a2ccf0b958 100644 --- a/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte @@ -44,6 +44,14 @@ onDiscard?: () => void } = $props() + const creatableStorageTypes = [ + { value: 's3', label: 'S3' }, + { value: 'azure_blob', label: 'Azure Blob' }, + { value: 's3_aws_oidc', label: 'AWS OIDC' }, + { value: 'azure_workload_identity', label: 'Azure Workload Identity' }, + { value: 'gcloud_storage', label: 'Google Cloud Storage' } + ] + let advancedPermissionModalState: | { open: false } | { open: true; storage: S3ResourceSettingsItem } = $state({ open: false }) @@ -294,31 +302,38 @@
    - {#if tableRow[1].resourceType === 'filesystem'} - +
    + - {/if} + {#if tableRow[1].resourceType === 'filesystem'} + + Filesystem storage points the workspace at a directory on the server's own + disk. Only development builds of Windmill accept it — switch this storage to + S3, Azure Blob or Google Cloud Storage to configure it here. + + {/if} +
    {#if tableRow[1].resourceType === 'filesystem'} diff --git a/frontend/src/lib/components/workspaceSettings/deployToHubSession.svelte.ts b/frontend/src/lib/components/workspaceSettings/deployToHubSession.svelte.ts index 5cc7f0d1d7..c421553a7c 100644 --- a/frontend/src/lib/components/workspaceSettings/deployToHubSession.svelte.ts +++ b/frontend/src/lib/components/workspaceSettings/deployToHubSession.svelte.ts @@ -198,8 +198,31 @@ export class DeployToHubSession { // Whether the Hub currently has a custom logo for this project (from // rehydration) — drives the "Remove current logo" affordance. hubHasRemoteLogo = $state(false) + // A pipeline recording is attached on the Hub. An update inherits the published + // one, which is only a demo of the new version if nothing it runs changed. + hubHasPipelineRecording = $state(false) + // The Hub's own verdict: this update runs different content from the published + // version. False when there is no update in flight. + hubItemsChanged = $state(false) + // The attached pipeline recording is the published version's, copied when this + // update started, rather than one recorded for it. Authoritative across reloads, + // unlike `pipelineRecorded`, which only remembers this session. + hubPipelineRecordingInherited = $state(false) effectiveSlug = $state('') hubItemIds = $state>({}) + // Set once the project is published: everything the wizard shows from here on + // describes an update to it, and the published version keeps serving until that + // update is approved. `phase` is the update's own status, not the project's. + liveOnHub = $state(false) + /** This Hub knows about pending updates — it answers rehydration with a `live` + * key. An older one takes a project offline to republish and has neither the + * withdraw nor the discard endpoint, so the actions built on them stay hidden. */ + hubSupportsUpdates = $state(false) + // A reviewer's verdict on the current draft, shown so the publisher knows what + // to fix before resubmitting. + rejectionReason = $state(undefined) + discardingUpdate = $state(false) + withdrawing = $state(false) // Best-effort data table migrations for the bundle, editable in the drawer and // pushed on deploy. Regenerated when the bundle drawer opens. @@ -228,6 +251,10 @@ export class DeployToHubSession { submitting = $state(false) syncing = $state(false) + // Set from the Hub's answer to the draft request: this push went into an update + // rather than over the published project. + #publishedAsUpdate = false + // Intra-session tokens: latest call wins among competing calls on this session. #triggerLoadTok = 0 #recordRunTok = 0 @@ -316,6 +343,18 @@ export class DeployToHubSession { ) pipelineScriptPathSet = $derived(new Set(this.pipelineScriptPaths)) isPipelineProject = $derived(this.pipelineScriptPaths.length > 0) + /** The pipeline replay this update carries came from the published version, and + * something it runs has changed since — so it is a recording of another version. + * `hubItemsChanged` is the Hub comparing content, not a guess from which items + * carry recordings: an item nobody ever recorded has not changed. */ + pipelineReplayMayBeStale = $derived( + this.liveOnHub && + this.isPipelineProject && + this.hubHasPipelineRecording && + this.hubPipelineRecordingInherited && + this.hubItemsChanged && + !this.pipelineRecorded + ) hubSlug = $derived(this.effectiveSlug || sanitizeSlug(this.hubName)) relevantTriggers = $derived.by(() => { @@ -573,6 +612,17 @@ export class DeployToHubSession { this.hubSummary = p.summary ?? '' this.hubReadme = p.readme ?? '' this.hubHasRemoteLogo = p.has_logo === true + this.hubHasPipelineRecording = p.has_pipeline_recording === true + this.hubItemsChanged = p.items_changed === true + this.hubPipelineRecordingInherited = p.pipeline_recording_inherited === true + this.rejectionReason = p.rejection_reason ?? undefined + // `live` is a key this Hub always sends — null unless an update is in + // flight, in which case the fields above describe that update and the + // project itself is still published. Its absence means a Hub old enough to + // still take a project offline while it re-publishes, so the wizard must + // not promise otherwise. + this.hubSupportsUpdates = 'live' in p + this.liveOnHub = this.hubSupportsUpdates && (p.live?.approved === true || p.status === 'live') this.phase = p.status === 'live' ? 'live' : p.status === 'under_review' ? 'under_review' : 'draft' const ids: Record = {} @@ -898,6 +948,9 @@ export class DeployToHubSession { try { const parsed = JSON.parse(text) if (typeof parsed?.slug === 'string') returnedSlug = parsed.slug + // The Hub decides this: publishing over an approved project goes into a + // pending update instead, and the project keeps serving meanwhile. + this.#publishedAsUpdate = parsed?.pending_revision === true } catch {} if (!returnedSlug) { sendUserToast(`Hub did not return a slug. Aborting publish to avoid path drift.`, true) @@ -1252,8 +1305,13 @@ export class DeployToHubSession { // UI stuck in `predeploy`; rehydrate then upgrades to authoritative state. this.draftItems = itemsSnapshot.map((i) => ({ ...i, rec: 'none' })) this.phase = 'draft' + const asUpdate = this.#publishedAsUpdate await this.rehydrateFromHub() - sendUserToast(`Draft created on the Hub. Add recordings before submitting for review.`) + sendUserToast( + asUpdate + ? `Update ready on the Hub. Your published project stays live until it is approved.` + : `Draft created on the Hub. Add recordings before submitting for review.` + ) } finally { this.deploying = false } @@ -1313,12 +1371,82 @@ export class DeployToHubSession { } } + /** Go back to picking items, to publish again. Local only — nothing reaches the + * Hub until the bundle is confirmed, and where the Hub supports updates the + * published version keeps serving even then. */ startNewDraft = () => { this.draftItems = [] this.recordings = {} + this.rejectionReason = undefined + // All of it belongs to the update just finished, not the one starting. The + // captured cascade especially: left in place, the next update could save a + // replay of the version it replaces. Bumping the token first abandons a run + // still in flight, which would otherwise write its result back over this. + this.#pipelineRunTok++ + this.pipelineRecorded = false + this.pipelineRecordingResult = undefined + this.pipelineRunState = 'idle' + this.pipelineRunError = undefined this.phase = 'predeploy' } + /** Take the submission back out of review. Everything pushed for it is kept, so + * it can be fixed and submitted again. */ + cancelSubmission = async () => { + if (this.withdrawing) return + const slug = this.effectiveSlug + if (!slug) return + this.withdrawing = true + try { + const res = await fetch( + `/api/w/${this.workspace}/hub/projects/${encodeURIComponent(slug)}/withdraw${this.#folderQs()}`, + { method: 'POST', credentials: 'include' } + ) + if (!res.ok) { + sendUserToast(`Could not cancel the submission: ${await res.text()}`, true) + return + } + if (this.#disposed) return + this.phase = 'draft' + await this.rehydrateFromHub() + sendUserToast(`Submission cancelled. Everything you pushed is still here.`) + } catch (e: any) { + sendUserToast(`Could not cancel the submission: ${e?.message ?? e}`, true) + } finally { + this.withdrawing = false + } + } + + /** Throw away an update in progress and go back to what is published. */ + discardUpdate = async () => { + if (this.discardingUpdate) return + const slug = this.effectiveSlug + if (!slug) return + this.discardingUpdate = true + try { + const res = await fetch( + `/api/w/${this.workspace}/hub/projects/${encodeURIComponent(slug)}/discard_update${this.#folderQs()}`, + { method: 'POST', credentials: 'include' } + ) + if (!res.ok) { + sendUserToast(`Could not discard the update: ${await res.text()}`, true) + return + } + if (this.#disposed) return + this.draftItems = [] + this.recordings = {} + this.deploymentStatus = {} + this.rejectionReason = undefined + this.phase = 'live' + await this.rehydrateFromHub() + sendUserToast(`Update discarded. The published project is unchanged.`) + } catch (e: any) { + sendUserToast(`Could not discard the update: ${e?.message ?? e}`, true) + } finally { + this.discardingUpdate = false + } + } + /** Reset record-drawer state and load the target's schema. */ async openRecord(it: DeployItem) { const tok = ++this.#recordRunTok diff --git a/frontend/src/lib/components/workspaceSettings/projectBundle.test.ts b/frontend/src/lib/components/workspaceSettings/projectBundle.test.ts index 669c0a43de..1833a33b33 100644 --- a/frontend/src/lib/components/workspaceSettings/projectBundle.test.ts +++ b/frontend/src/lib/components/workspaceSettings/projectBundle.test.ts @@ -16,6 +16,8 @@ import { collectExportVarPaths, extractTriggerConfigResourceRefs, extractVarRefsFromValue, + projectReferencesResource, + textHoldsBarePath, type ProjectExport, type FetchedItem, type ItemRef @@ -867,3 +869,46 @@ describe('flow_env and preprocessor_module', () => { expect(out.flow_env.PLAIN).toBe('not-a-ref') }) }) + +describe('projectReferencesResource', () => { + /** + * A project declares one resource per `resource-` input schema as well as one per + * `$res:` reference, so an app pinning `f/proj/google_calendar` for a script whose schema + * says `resource-gcal` ships both it and an unreferenced `f/proj/gcal`. Only the pinned + * one has to hold a credential for the project to work. + */ + const bundle = { + project: { slug: 'proj', name: 'Proj', summary: '', readme: null }, + scripts: [{ path: 'f/proj/send', content: 'const c = "$res:f/proj/google_calendar"' }], + flows: [], + apps: [], + resources: [ + { path: 'f/proj/gcal', resource_type: 'gcal' }, + { path: 'f/proj/google_calendar', resource_type: 'gcal' }, + { path: 'f/proj/db', resource_type: 'postgresql' } + ], + triggers: [{ path: 'f/proj/ingest', config: { postgres_resource_path: 'f/proj/db' } }] + } as unknown as ProjectExport + + it("sees a $res: token and a trigger's bare path, and not a stub nothing points at", () => { + expect(projectReferencesResource(bundle, 'f/proj/google_calendar')).toBe(true) + expect(projectReferencesResource(bundle, 'f/proj/db')).toBe(true) + // Declared only because a script's input schema names the type. + expect(projectReferencesResource(bundle, 'f/proj/gcal')).toBe(false) + }) +}) + +describe('textHoldsBarePath', () => { + // Gates three deletion decisions, and its two directions cost differently: a false yes + // keeps a stub nobody needed, a false no deletes one something still reads. + const P = 'f/proj/db' + it.each([ + ['a token only', `const c = "$res:${P}"`, false], + ['a path written in code', `await getResource("${P}")`, true], + ['both spellings', `"$res:${P}"; getResource("${P}")`, true], + ['a longer path that starts the same', `await getResource("${P}_prod")`, false], + ['the same path inside a longer token', `"$res:${P}_prod"`, false] + ])('%s', (_label, text, expected) => { + expect(textHoldsBarePath(text, P)).toBe(expected) + }) +}) diff --git a/frontend/src/lib/components/workspaceSettings/projectBundle.ts b/frontend/src/lib/components/workspaceSettings/projectBundle.ts index 1fd62498d5..8adedeb559 100644 --- a/frontend/src/lib/components/workspaceSettings/projectBundle.ts +++ b/frontend/src/lib/components/workspaceSettings/projectBundle.ts @@ -238,6 +238,89 @@ export function extractTriggerConfigResourceRefs(config: any): string[] { return extractScriptRefs(JSON.stringify(config ?? {})).map((r) => r.path) } +/** + * Whether a value reaches a resource, in either spelling a rewrite has to handle: a + * `$res:`/`res://` token embedded in content, or the bare path standing alone as a string + * the way trigger configs hold it (`kafka_resource_path: "f/slug/db"`). + * + * Matching the parsed structure rather than its serialization is what keeps + * `f/slug/db` out of `$res:f/slug/db_prod`. + */ +export function referencesResourcePath(value: unknown, path: string): boolean { + const walk = (v: any): boolean => { + if (typeof v === 'string') { + if (v === path) return true + RES_TOKEN_RE.lastIndex = 0 + let m: RegExpExecArray | null + while ((m = RES_TOKEN_RE.exec(v)) !== null) if (m[1] === path) return true + return false + } + if (Array.isArray(v)) return v.some(walk) + if (v && typeof v === 'object') return Object.values(v).some(walk) + return false + } + return walk(value) +} + +/** + * Whether a `$res:`/`res://` token for this path survives anywhere in the value — the one + * spelling the rewriters relocate, and so the only one whose survival means a rewrite did + * not take. A bare path is deliberately not matched: nothing here moves one, so its presence + * says nothing about whether the rewrite worked. + */ +export function holdsResourceToken(value: unknown, path: string): boolean { + const walk = (v: any): boolean => { + if (typeof v === 'string') { + RES_TOKEN_RE.lastIndex = 0 + let m: RegExpExecArray | null + while ((m = RES_TOKEN_RE.exec(v)) !== null) if (m[1] === path) return true + return false + } + if (Array.isArray(v)) return v.some(walk) + if (v && typeof v === 'object') return Object.values(v).some(walk) + return false + } + return walk(value) +} + +/** + * Whether the text names the resource somewhere no rewriter reaches — a path written on its + * own rather than inside a `$res:` token, the way `getResource("f/proj/db")` does. The + * tokens are stripped first so the ones a rewrite would move do not count, and the match is + * bounded so `f/proj/db` is not found inside `f/proj/db_prod`. + */ +export function textHoldsBarePath(text: string, path: string): boolean { + const withoutTokens = text.replace(RES_TOKEN_RE, '') + const boundary = /[\w\-./]/ + for (let i = withoutTokens.indexOf(path); i !== -1; i = withoutTokens.indexOf(path, i + 1)) { + const before = withoutTokens[i - 1] ?? '' + const after = withoutTokens[i + path.length] ?? '' + if (!boundary.test(before) && !boundary.test(after)) return true + } + return false +} + +/** + * Whether anything the project ships points at one of its own resources. + * + * A project declares two kinds of resource. One is referenced — an app pins `$res:` for a + * script argument, a trigger names it — and the project does not work until it holds a + * credential. The other is minted from a `resource-` input schema: it names a type + * a script accepts, nothing points at it, and a standalone run picks a resource in the + * argument picker instead. Only the first kind is worth asking anyone to fill in. + * + * The `resources` list is excluded from the walk because a stub's own declaration carries + * its path, which would make every stub look referenced. + */ +export function projectReferencesResource(bundle: ProjectExport, path: string): boolean { + const { resources: _resources, ...rest } = bundle as any + if (referencesResourcePath(rest, path)) return true + // A script that reads the resource by name rather than through a `$res:` token still needs + // it filled in. Asked about is the safe side of this answer: the cost of a wrong yes is a + // row nobody had to act on, and of a wrong no a credential nobody was told to set up. + return textHoldsBarePath(JSON.stringify(rest ?? {}), path) +} + /** * Trigger configs reference resources as plain path strings (e.g. * `kafka_resource_path: "f/slug/db"`), not `$res:` tokens, so token rewriting diff --git a/frontend/src/lib/components/workspaceSettings/projectInstall.ts b/frontend/src/lib/components/workspaceSettings/projectInstall.ts index abc48a3ad6..6fd4bdd728 100644 --- a/frontend/src/lib/components/workspaceSettings/projectInstall.ts +++ b/frontend/src/lib/components/workspaceSettings/projectInstall.ts @@ -44,6 +44,11 @@ export interface InstallResult { path: string ok: boolean error?: string + /** + * Already in the destination, so nothing was written. Not a failure and not an import — + * reporting it as either would be a lie, and the difference is what a retry is for. + */ + skipped?: boolean } // Guarding an item's own path is not enough: the `$res:`/script/flow refs baked @@ -201,7 +206,11 @@ async function importApp(workspace: string, a: ExportItem): Promise { // Apply one migration to the target data table. If the data table opted into // migrations, record it (datatable_migrations + _wm_migrations, run only this // version); otherwise run the SQL once as a preview job (unrecorded). -async function applyOneMigration( +// +// Exported because the import can leave migrations unapplied: a data table the +// project needs may not be configured in the destination yet, and the wizard's setup +// step runs them once it is. +export async function applyOneMigration( workspace: string, projectSlug: string, m: ProjectMigration @@ -254,15 +263,73 @@ async function applyOneMigration( * reviewed) migrations. Each item's outcome is reported through `onResult`; * failures never abort the remaining items. */ +/** + * The kinds an import writes that carry a path and can therefore already be there. + * + * Triggers carry their own kind too (`trigger:schedule`, `trigger:http`, … — the values of + * `WorkspaceTriggerKind`): each trigger kind is a separate table keyed on + * `(path, workspace_id)`, so one workspace can hold a schedule and an HTTP trigger both + * called `f/cal/sync`. Flattening them to `trigger` would let whichever exists answer for + * the other. + */ +export type ImportedKind = + | 'script' + | 'flow' + | 'app' + | 'resource' + | `trigger:${string}` + +/** + * The key `alreadyPresent` is built and read with. Kind and path together, because the kinds + * share one path namespace and a bare path cannot say which of them is already there. + */ +export function presenceKey(kind: ImportedKind, path: string): string { + return `${kind}:${path}` +} + export async function installProject(args: { workspace: string exportData: ProjectExport folder: string migrations: ProjectMigration[] + /** Called once, before the reviewed migrations are applied, when there are any. Lets a + * caller show them as their own step rather than folding them into the item import. */ + onMigrationsStart?: () => void + /** + * Asked before each write. Returning true stops the run where it is — the writes already + * made stay, the rest never start. Nothing here can cancel a request already in flight, + * so this is the granularity available without threading an `AbortSignal` through every + * service call: the import wizard uses it when the user confirms leaving mid-run. + */ + stopped?: () => boolean + /** + * What is already in the destination, as `presenceKey` keys — so a retry writes only what + * is missing instead of replaying the bundle into a wall of "already exists". Built from + * retargeted paths, because that is what these items will actually be called. + * + * Keyed by kind and not by path alone: the five kinds share one `f//` namespace, so + * a trigger and a script may legitimately both be called `f/cal/sync`. A flat path set + * would let either one mask the other and silently skip an item that was never imported. + * + * Never a way to *replace* anything: an item that is there is left exactly as it is, + * which is the same promise `updateIfExists: false` makes for a resource whose value + * someone has since filled in. + */ + alreadyPresent?: Set hasEeLicense: boolean onResult: (r: InstallResult) => void }): Promise { - const { workspace, exportData, folder, migrations, hasEeLicense, onResult } = args + const { + workspace, + exportData, + folder, + migrations, + hasEeLicense, + onResult, + onMigrationsStart, + stopped, + alreadyPresent + } = args const record = (path: string, p: Promise): Promise => p.then( @@ -270,6 +337,20 @@ export async function installProject(args: { (e: any) => onResult({ path, ok: false, error: errorMessage(e) }) ) + /** Every write goes through here, so one check covers items, variables and migrations. */ + const halted = () => stopped?.() === true + + /** + * True when the destination already has this path, so the write is not attempted. Reported + * rather than dropped: the checklist has to account for every item the project ships, and + * "already there" is a different thing from "imported". + */ + const present = (kind: ImportedKind, path: string): boolean => { + if (!alreadyPresent?.has(presenceKey(kind, path))) return false + onResult({ path, ok: true, skipped: true }) + return true + } + try { await FolderService.createFolder({ workspace, requestBody: { name: folder } }) } catch {} @@ -309,6 +390,8 @@ export async function installProject(args: { } for (const s of proj.scripts) { + if (halted()) return + if (present('script', s.path)) continue // `$var:` is resolved in job args (flow inputs, schedule args, trigger config), // not in script source, so there is no variable arg to contain here. await checkedItem(s.path, extractScriptRefs(s.content ?? ''), undefined, () => @@ -316,19 +399,26 @@ export async function installProject(args: { ) } for (const f of proj.flows) { + if (halted()) return + if (present('flow', f.path)) continue await checkedItem(f.path, extractFlowRefs(f.value), f.value, () => importFlow(workspace, f)) } for (const r of proj.resources) { + if (halted()) return + if (present('resource', r.path)) continue await checked(r.path, () => importResourceStub(workspace, r)) } // Placeholders for the project's internal `$var:`/`$jsonvar:` refs (retargeted // into this folder). External refs are rejected per-item, so only stub in-folder // ones; guard again in case an out-of-folder ref slipped through retargeting. for (const p of collectExportVarPaths(proj)) { + if (halted()) return if (!p.startsWith(prefix)) continue await record(`variable: ${p}`, importVariablePlaceholder(workspace, p)) } for (const a of proj.apps) { + if (halted()) return + if (present('app', a.path)) continue const isRaw = a.app_type === 'raw' const refs = isRaw ? extractRawAppRefs(a.value?.raw ?? '') : extractAppRefs(a.value) // Raw apps hold their runnables in the `value.raw` JSON string; parse it so the @@ -368,6 +458,8 @@ export async function installProject(args: { return varContainmentViolation(cfg, folder) } for (const t of proj.triggers) { + if (halted()) return + if (present(`trigger:${t.kind}`, String(t.path))) continue const violation = guard(t.path, t.runnable_path) ?? triggerConfigViolation(t) await record( String(t.path), @@ -389,7 +481,10 @@ export async function installProject(args: { } // Apply the reviewed data table migrations after items exist. + if (halted()) return + if (migrations.length) onMigrationsStart?.() for (const m of migrations) { + if (halted()) return await record( `data table: ${m.datatable_name}`, applyOneMigration(workspace, exportData.project.slug, m) diff --git a/frontend/src/lib/components/workspaceSettings/projectInstallPresence.test.ts b/frontend/src/lib/components/workspaceSettings/projectInstallPresence.test.ts new file mode 100644 index 0000000000..186d581d58 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectInstallPresence.test.ts @@ -0,0 +1,106 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +/** + * A retry must not re-write what the destination already has. The interesting kind is the + * trigger: it is the one item the import creates through an API that rejects an existing + * path outright, so replaying it turns "already there" into a reported failure. + */ + +const created = vi.hoisted(() => ({ triggers: [] as string[] })) + +vi.mock('$lib/gen', () => { + const nothing = vi.fn(async () => []) + return { + AppService: { createApp: vi.fn(), listApps: nothing }, + FlowService: { createFlow: vi.fn(), listFlows: nothing }, + FolderService: { createFolder: vi.fn() }, + ResourceService: { createResource: vi.fn(), listResource: nothing }, + ScriptService: { createScript: vi.fn(), listScripts: nothing }, + VariableService: { createVariable: vi.fn(), listVariable: nothing }, + WorkspaceService: { listDataTables: vi.fn(async () => []) } + } +}) + +vi.mock('../triggers/workspaceTriggersList', () => ({ + TRIGGER_KINDS: { + schedule: { badge: 'Schedule', resourceField: undefined }, + http: { badge: 'HTTP', resourceField: undefined } + }, + createWorkspaceTriggerDisabled: vi.fn(async (_ws: string, t: { path: string; kind: string }) => { + created.triggers.push(`${t.kind}:${t.path}`) + }), + triggerHandlerRefs: () => [] +})) + +import { installProject, presenceKey } from './projectInstall' + +const exportData = { + project: { slug: 'calendly', name: 'Calendly', summary: '', readme: null }, + scripts: [], + flows: [], + apps: [], + resources: [], + variables: [], + triggers: [ + { + kind: 'schedule', + path: 'f/calendly/nightly', + runnable_path: 'f/calendly/sync', + runnable_kind: 'script', + summary: null, + config: {} + } + ], + migrations: [] +} as any + +async function run(alreadyPresent?: Set) { + const results: any[] = [] + await installProject({ + workspace: 'w', + exportData, + folder: 'calendly', + migrations: [], + hasEeLicense: true, + alreadyPresent, + onResult: (r) => results.push(r) + }) + return results +} + +describe('installProject presence', () => { + beforeEach(() => { + created.triggers = [] + }) + + it('creates a trigger the destination does not have', async () => { + const results = await run(new Set()) + expect(created.triggers).toEqual(['schedule:f/calendly/nightly']) + expect(results).toContainEqual({ path: 'f/calendly/nightly', ok: true }) + }) + + // Without the skip the retry calls the create API again, which rejects the existing + // path, and the row reads as a failure for something that is already there. + it('skips a trigger that is already there instead of re-creating it', async () => { + const results = await run(new Set([presenceKey('trigger:schedule', 'f/calendly/nightly')])) + expect(created.triggers).toEqual([]) + expect(results).toContainEqual({ path: 'f/calendly/nightly', ok: true, skipped: true }) + }) + + // Kinds share one `f//` namespace, so the key has to carry the kind: a script of + // the same name is not this trigger and must not stand in for it. + it('does not let another kind at the same path mask the trigger', async () => { + const results = await run(new Set([presenceKey('script', 'f/calendly/nightly')])) + expect(created.triggers).toEqual(['schedule:f/calendly/nightly']) + expect(results).toContainEqual({ path: 'f/calendly/nightly', ok: true }) + }) + + // Each trigger kind is its own table keyed on (path, workspace_id), so a workspace can + // hold a schedule and an HTTP trigger both called `f/calendly/nightly`. The one that + // exists must not answer for the one that does not. + it('does not let another trigger kind at the same path mask this one', async () => { + const results = await run(new Set([presenceKey('trigger:http', 'f/calendly/nightly')])) + expect(created.triggers).toEqual(['schedule:f/calendly/nightly']) + expect(results).toContainEqual({ path: 'f/calendly/nightly', ok: true }) + }) +}) diff --git a/frontend/src/lib/folderDraft.test.ts b/frontend/src/lib/folderDraft.test.ts new file mode 100644 index 0000000000..75b8fa6dd0 --- /dev/null +++ b/frontend/src/lib/folderDraft.test.ts @@ -0,0 +1,148 @@ +import { describe, it, expect } from 'vitest' +import { + folderPermissionDiff, + isFolderDraftDirty, + type FolderDraft, + type FolderMember, + type FolderRole +} from './folderDraft' + +function member(role: FolderRole): FolderMember { + return { owner_name: 'u/alice', role } +} + +function baseline(): FolderDraft { + return { + summary: 'Reporting jobs', + labels: ['prod'], + defaultPermissionedAs: [{ path_glob: '**', permissioned_as: 'u/admin' }], + perms: [ + { owner_name: 'u/admin', role: 'admin' }, + { owner_name: 'g/all', role: 'viewer' } + ] + } +} + +describe('folderPermissionDiff', () => { + // The whole transition matrix: which endpoint each role change maps to. `admin` lives in + // `owners` and the other two in `extra_perms`, so leaving admin is the one transition that + // cannot go through the ACL endpoint. + const transitions: Array<[from: FolderRole | 'absent', to: FolderRole, expected: unknown]> = [ + ['absent', 'viewer', { kind: 'setAcl', owner: 'u/alice', write: false }], + ['absent', 'writer', { kind: 'setAcl', owner: 'u/alice', write: true }], + ['absent', 'admin', { kind: 'grantAdmin', owner: 'u/alice' }], + ['viewer', 'writer', { kind: 'setAcl', owner: 'u/alice', write: true }], + ['viewer', 'admin', { kind: 'grantAdmin', owner: 'u/alice' }], + ['writer', 'viewer', { kind: 'setAcl', owner: 'u/alice', write: false }], + ['writer', 'admin', { kind: 'grantAdmin', owner: 'u/alice' }], + ['admin', 'viewer', { kind: 'demoteAdmin', owner: 'u/alice', write: false }], + ['admin', 'writer', { kind: 'demoteAdmin', owner: 'u/alice', write: true }] + ] + + it.each(transitions)('%s → %s', (from, to, expected) => { + const prev = from === 'absent' ? [] : [member(from)] + expect(folderPermissionDiff(prev, [member(to)])).toEqual([expected]) + }) + + it.each(['viewer', 'writer', 'admin'] as const)('%s → removed drops owner and acl', (role) => { + expect(folderPermissionDiff([member(role)], [])).toEqual([{ kind: 'remove', owner: 'u/alice' }]) + }) + + it.each(['viewer', 'writer', 'admin'] as const)('%s unchanged calls nothing', (role) => { + expect(folderPermissionDiff([member(role)], [member(role)])).toEqual([]) + }) + + // The caller is a folder admin only through `g/ops`, so that demotion is the one the write + // policy refuses. Sent first it takes the rest of the save down with it. + it('gives up the caller own admin last', () => { + const prev: FolderMember[] = [ + { owner_name: 'g/ops', role: 'admin' }, + { owner_name: 'u/bob', role: 'viewer' } + ] + const next: FolderMember[] = [ + { owner_name: 'g/ops', role: 'viewer' }, + { owner_name: 'u/bob', role: 'admin' } + ] + expect(folderPermissionDiff(prev, next, ['u/alice', 'g/ops'])).toEqual([ + { kind: 'grantAdmin', owner: 'u/bob' }, + { kind: 'demoteAdmin', owner: 'g/ops', write: false } + ]) + }) + + // `g/z` is a group the caller belongs to but holds no admin through, so removing it is an + // ordinary call — queued behind the refused one it would never run. + it('defers only the rows the caller is an admin through', () => { + const prev: FolderMember[] = [ + { owner_name: 'g/a', role: 'admin' }, + { owner_name: 'g/z', role: 'viewer' } + ] + expect(folderPermissionDiff(prev, [], ['u/alice', 'g/a', 'g/z'])).toEqual([ + { kind: 'remove', owner: 'g/z' }, + { kind: 'remove', owner: 'g/a' } + ]) + }) + + it('touches only the members that changed', () => { + const prev: FolderMember[] = [ + { owner_name: 'u/admin', role: 'admin' }, + { owner_name: 'g/all', role: 'viewer' }, + { owner_name: 'g/ops', role: 'writer' } + ] + const next: FolderMember[] = [ + { owner_name: 'u/admin', role: 'admin' }, + { owner_name: 'g/all', role: 'writer' } + ] + expect(folderPermissionDiff(prev, next)).toEqual([ + { kind: 'setAcl', owner: 'g/all', write: true }, + { kind: 'remove', owner: 'g/ops' } + ]) + }) +}) + +describe('isFolderDraftDirty', () => { + it('is clean against its own baseline', () => { + expect(isFolderDraftDirty(baseline(), baseline())).toBe(false) + }) + + it('is clean before anything has loaded', () => { + expect(isFolderDraftDirty(baseline(), undefined)).toBe(false) + }) + + // A reload rebuilds the members in the server's order, which is not the order they were + // added in. Order-sensitive, an applied change would keep Save lit with nothing to send. + it('ignores the order the members are held in', () => { + const reordered = baseline() + reordered.perms = [...reordered.perms].reverse() + expect(isFolderDraftDirty(reordered, baseline())).toBe(false) + }) + + // Enumerated from the value itself rather than a hand-written list: a field added to + // `FolderDraft` and to `baseline()` is covered here without anyone remembering to add a + // case. An edit this misses is one the drawer discards without asking. + it.each(Object.keys(baseline()) as Array)('notices a change to %s', (key) => { + const edited = baseline() + if (key === 'summary') edited.summary = 'Something else' + else if (key === 'labels') edited.labels = [...edited.labels, 'staging'] + else if (key === 'defaultPermissionedAs') edited.defaultPermissionedAs = [] + else if (key === 'perms') edited.perms[1].role = 'writer' + else throw new Error(`no edit defined for ${key} — add one so the field stays covered`) + + expect(isFolderDraftDirty(edited, baseline())).toBe(true) + }) + + it('notices a member added and a member removed', () => { + const added = baseline() + added.perms.push({ owner_name: 'g/ops', role: 'writer' }) + expect(isFolderDraftDirty(added, baseline())).toBe(true) + + const removed = baseline() + removed.perms.pop() + expect(isFolderDraftDirty(removed, baseline())).toBe(true) + }) + + it('is clean again once the baseline catches up', () => { + const saved = baseline() + saved.summary = 'Renamed' + expect(isFolderDraftDirty(saved, structuredClone(saved))).toBe(false) + }) +}) diff --git a/frontend/src/lib/folderDraft.ts b/frontend/src/lib/folderDraft.ts new file mode 100644 index 0000000000..ade1b43d71 --- /dev/null +++ b/frontend/src/lib/folderDraft.ts @@ -0,0 +1,96 @@ +import { deepEqual } from 'fast-equals' +import type { FolderDefaultPermissionedAs } from '$lib/gen' + +/** What a member may hold on a folder. `admin` is the `owners` array server-side; `writer` + * and `viewer` are the `true`/`false` entries of `extra_perms`. */ +export type FolderRole = 'viewer' | 'writer' | 'admin' + +export type FolderMember = { owner_name: string; role: FolderRole } + +/** Everything the folder editor can change, held as one value so the whole edit is one + * comparison against the loaded folder and one Save. */ +export type FolderDraft = { + summary: string + labels: string[] + defaultPermissionedAs: FolderDefaultPermissionedAs + perms: FolderMember[] +} + +/** Whether the draft still matches the folder it was loaded from. Every field of + * `FolderDraft` participates, so a field added to the type is covered by construction — + * which is what the discard guard depends on: an edit this misses is an edit the drawer + * throws away without asking. No baseline means nothing has loaded yet, so nothing to lose. */ +export function isFolderDraftDirty(draft: FolderDraft, baseline: FolderDraft | undefined): boolean { + return baseline != undefined && !deepEqual(sortedMembers(draft), sortedMembers(baseline)) +} + +/** Members are a set, but a reload rebuilds them in the server's `extra_perms` key order while + * the draft keeps the order they were added in. Compared as-is, a change that has already been + * applied still reads as dirty. Labels and rules keep their order, which is meaningful. */ +function sortedMembers(value: FolderDraft): FolderDraft { + return { + ...value, + perms: [...value.perms].sort((a, b) => a.owner_name.localeCompare(b.owner_name)) + } +} + +/** One backend call the folder's members need. Kept as data so the mapping from role + * transitions to endpoints can be read — and tested — without a server. */ +export type FolderPermissionCall = + /** `addowner`: appends to `owners` and sets `extra_perms[owner] = true`. */ + | { kind: 'grantAdmin'; owner: string } + /** `removeowner` with a write flag: takes the member out of `owners` and sets their + * level. The only way down from admin. */ + | { kind: 'demoteAdmin'; owner: string; write: boolean } + /** `acls/add`: sets `extra_perms[owner]`, for a member who is not an admin. */ + | { kind: 'setAcl'; owner: string; write: boolean } + /** Both removals. `removeowner` without a write only drops the member from `owners`, + * leaving their `extra_perms` entry — alone it demotes an admin rather than removing + * them, so the ACL delete is not optional. */ + | { kind: 'remove'; owner: string } + +/** The calls that turn `prev` into `next`. Members whose role is unchanged produce none. + * + * `callerOwners` is the caller's own `u/name` plus every group they belong to. Giving up the + * last of those that is in `owners` goes last: the write policy checks the row the update + * would produce, so that call is refused for anyone but a workspace admin, and sent early it + * takes the rest of the save with it. */ +export function folderPermissionDiff( + prev: FolderMember[], + next: FolderMember[], + callerOwners?: string[] +): FolderPermissionCall[] { + const previousRole = new Map(prev.map((p) => [p.owner_name, p.role])) + const calls: FolderPermissionCall[] = [] + + for (const member of next) { + const before = previousRole.get(member.owner_name) + if (before === member.role) continue + if (member.role === 'admin') { + calls.push({ kind: 'grantAdmin', owner: member.owner_name }) + } else if (before === 'admin') { + calls.push({ + kind: 'demoteAdmin', + owner: member.owner_name, + write: member.role === 'writer' + }) + } else { + calls.push({ kind: 'setAcl', owner: member.owner_name, write: member.role === 'writer' }) + } + } + + const kept = new Set(next.map((n) => n.owner_name)) + for (const member of prev) { + if (kept.has(member.owner_name)) continue + calls.push({ kind: 'remove', owner: member.owner_name }) + } + + // `previousRole === 'admin'` is what makes it a handle: `callerOwners` lists every group + // the caller belongs to, and one holding only a viewer or writer row is not in `owners`, + // so removing it is an ordinary call that should not queue behind the fatal one. + const revokesCaller = (call: FolderPermissionCall) => + (call.kind === 'demoteAdmin' || call.kind === 'remove') && + (callerOwners?.includes(call.owner) ?? false) && + previousRole.get(call.owner) === 'admin' + return [...calls.filter((c) => !revokesCaller(c)), ...calls.filter(revokesCaller)] +} diff --git a/frontend/src/lib/groupDraft.test.ts b/frontend/src/lib/groupDraft.test.ts new file mode 100644 index 0000000000..2b8ef8c180 --- /dev/null +++ b/frontend/src/lib/groupDraft.test.ts @@ -0,0 +1,112 @@ +import { describe, it, expect } from 'vitest' +import { groupMemberDiff, isGroupDraftDirty, type GroupDraft, type GroupRole } from './groupDraft' + +function baseline(): GroupDraft { + return { + summary: 'On-call engineers', + members: [ + { member_name: 'admin', role: 'admin' }, + { member_name: 'alice', role: 'member' } + ] + } +} + +describe('groupMemberDiff', () => { + // The whole transition matrix: which endpoints each role change maps to. A role is a + // membership row plus an ACL entry, so only the halves that actually change are sent — + // an extra call would log a permission-history row for something that did not move. + const transitions: Array< + [from: GroupRole | 'absent', to: GroupRole | 'absent', expected: unknown[]] + > = [ + ['absent', 'member', [{ kind: 'addUser', username: 'bob' }]], + ['absent', 'manager', [{ kind: 'setAcl', username: 'bob' }]], + [ + 'absent', + 'admin', + [ + { kind: 'addUser', username: 'bob' }, + { kind: 'setAcl', username: 'bob' } + ] + ], + ['member', 'admin', [{ kind: 'setAcl', username: 'bob' }]], + [ + 'member', + 'manager', + [ + { kind: 'removeUser', username: 'bob' }, + { kind: 'setAcl', username: 'bob' } + ] + ], + ['manager', 'admin', [{ kind: 'addUser', username: 'bob' }]], + [ + 'manager', + 'member', + [ + { kind: 'addUser', username: 'bob' }, + { kind: 'removeAcl', username: 'bob' } + ] + ], + ['admin', 'member', [{ kind: 'removeAcl', username: 'bob' }]], + ['admin', 'manager', [{ kind: 'removeUser', username: 'bob' }]], + ['member', 'absent', [{ kind: 'removeUser', username: 'bob' }]], + ['manager', 'absent', [{ kind: 'removeAcl', username: 'bob' }]], + [ + 'admin', + 'absent', + [ + { kind: 'removeUser', username: 'bob' }, + { kind: 'removeAcl', username: 'bob' } + ] + ] + ] + + for (const [from, to, expected] of transitions) { + it(`${from} to ${to}`, () => { + const prev = from === 'absent' ? [] : [{ member_name: 'bob', role: from }] + const next = to === 'absent' ? [] : [{ member_name: 'bob', role: to }] + expect(groupMemberDiff(prev, next)).toEqual(expected) + }) + } + + it('sends nothing for an unchanged member', () => { + expect(groupMemberDiff(baseline().members, baseline().members)).toEqual([]) + }) + + it('revokes the caller last so the rest of the save stays authorized', () => { + const prev = [{ member_name: 'admin', role: 'admin' as GroupRole }] + const next = [ + { member_name: 'admin', role: 'member' as GroupRole }, + { member_name: 'bob', role: 'admin' as GroupRole } + ] + expect(groupMemberDiff(prev, next, 'admin')).toEqual([ + { kind: 'addUser', username: 'bob' }, + { kind: 'setAcl', username: 'bob' }, + { kind: 'removeAcl', username: 'admin' } + ]) + }) +}) + +describe('isGroupDraftDirty', () => { + it('is clean against an equal baseline and dirty on any field', () => { + expect(isGroupDraftDirty(baseline(), baseline())).toBe(false) + expect(isGroupDraftDirty({ ...baseline(), summary: 'Other' }, baseline())).toBe(true) + expect( + isGroupDraftDirty( + { ...baseline(), members: [{ member_name: 'admin', role: 'member' }] }, + baseline() + ) + ).toBe(true) + }) + + it('is clean while nothing has loaded', () => { + expect(isGroupDraftDirty(baseline(), undefined)).toBe(false) + }) + + // A reload rebuilds the members in the server's order, which is not the order they were + // added in. Order-sensitive, an applied change would keep Save lit with nothing to send. + it('ignores the order the members are held in', () => { + const reordered = baseline() + reordered.members = [...reordered.members].reverse() + expect(isGroupDraftDirty(reordered, baseline())).toBe(false) + }) +}) diff --git a/frontend/src/lib/groupDraft.ts b/frontend/src/lib/groupDraft.ts new file mode 100644 index 0000000000..dfd1767c30 --- /dev/null +++ b/frontend/src/lib/groupDraft.ts @@ -0,0 +1,99 @@ +import { deepEqual } from 'fast-equals' + +/** What a member may hold on a group. `member` is the `usr_to_group` row server-side and + * `manager` is the `true` entry in `extra_perms`; `admin` is both at once. */ +export type GroupRole = 'member' | 'manager' | 'admin' + +export type GroupMember = { member_name: string; role: GroupRole } + +/** Everything the group editor can change, held as one value so the whole edit is one + * comparison against the loaded group and one Save. */ +export type GroupDraft = { + summary: string + members: GroupMember[] +} + +/** Whether the draft still matches the group it was loaded from. Every field of `GroupDraft` + * participates, so a field added to the type is covered by construction — which is what the + * discard guard depends on: an edit this misses is an edit the drawer throws away without + * asking. No baseline means nothing has loaded yet, so nothing to lose. */ +export function isGroupDraftDirty(draft: GroupDraft, baseline: GroupDraft | undefined): boolean { + return baseline != undefined && !deepEqual(sortedMembers(draft), sortedMembers(baseline)) +} + +/** Members are a set, but a reload rebuilds them in the server's order while the draft keeps + * the order they were added in. Compared as-is, a change that has already been applied still + * reads as dirty. */ +function sortedMembers(value: GroupDraft): GroupDraft { + return { + ...value, + members: [...value.members].sort((a, b) => a.member_name.localeCompare(b.member_name)) + } +} + +/** One backend call a group's members need. Kept as data so the mapping from role + * transitions to endpoints can be read — and tested — without a server. */ +export type GroupMemberCall = + /** `addUserToGroup` / `removeUserToGroup`: the `usr_to_group` row. */ + | { kind: 'addUser'; username: string } + | { kind: 'removeUser'; username: string } + /** `acls/add` / `acls/remove` on kind `group_`: the write entry that lets someone + * manage the group. */ + | { kind: 'setAcl'; username: string } + | { kind: 'removeAcl'; username: string } + +/** The two independent things a role is made of: belonging to the group, and holding the + * write entry that lets you manage it. Every role is one combination of the two, which is + * why a transition needs at most one call per flag. */ +function flagsOf(role: GroupRole | undefined): { belongs: boolean; manages: boolean } { + return { + belongs: role === 'member' || role === 'admin', + manages: role === 'manager' || role === 'admin' + } +} + +/** The calls that turn `prev` into `next`. Members whose role is unchanged produce none, and + * a member dropped from `next` is treated as holding neither flag — which is what removing + * one means. + * + * `require_is_owner` authorizes each of these against `extra_perms['u/']`, so the + * caller's own revocation goes last: in row order it lands first and the rest 403s. */ +export function groupMemberDiff( + prev: GroupMember[], + next: GroupMember[], + caller?: string +): GroupMemberCall[] { + const previousRole = new Map(prev.map((p) => [p.member_name, p.role])) + const calls: GroupMemberCall[] = [] + + const transition = ( + username: string, + before: GroupRole | undefined, + after: GroupRole | undefined + ) => { + const from = flagsOf(before) + const to = flagsOf(after) + if (to.belongs !== from.belongs) { + calls.push({ kind: to.belongs ? 'addUser' : 'removeUser', username }) + } + if (to.manages !== from.manages) { + calls.push({ kind: to.manages ? 'setAcl' : 'removeAcl', username }) + } + } + + for (const member of next) { + const before = previousRole.get(member.member_name) + if (before === member.role) continue + transition(member.member_name, before, member.role) + } + + const kept = new Set(next.map((n) => n.member_name)) + for (const member of prev) { + if (kept.has(member.member_name)) continue + transition(member.member_name, member.role, undefined) + } + + const revokesCaller = (call: GroupMemberCall) => + call.kind === 'removeAcl' && call.username === caller + return [...calls.filter((c) => !revokesCaller(c)), ...calls.filter(revokesCaller)] +} diff --git a/frontend/src/lib/hubPaths.json b/frontend/src/lib/hubPaths.json index 2862fc7768..ee60298db4 100644 --- a/frontend/src/lib/hubPaths.json +++ b/frontend/src/lib/hubPaths.json @@ -1,6 +1,6 @@ { "gitSyncTest": "hub/28184/git-repo-test-read-write-windmill", - "gitInitRepo": "hub/28910/git-sync-init-repository-windmill", + "gitInitRepo": "hub/28930/git-sync-init-repository-windmill", "slackErrorHandler": "hub/28794/workspace-or-schedule-error-handler-slack", "emailErrorHandler": "hub/19795/workspace-or-error-handler-email", "slackRecoveryHandler": "hub/28791/slack/schedule-recovery-handler-slack", diff --git a/frontend/src/lib/hubProject.ts b/frontend/src/lib/hubProject.ts new file mode 100644 index 0000000000..b58b338f7d --- /dev/null +++ b/frontend/src/lib/hubProject.ts @@ -0,0 +1,87 @@ +import type { Component } from 'svelte' +import { appIconComponent } from '$lib/components/icons' +import { SettingService } from '$lib/gen' +import { DEFAULT_HUB_BASE_URL } from '$lib/hub' +import type { ImportProjectSummary } from '$lib/components/ImportProjectCard.svelte' + +/** + * The browser-reachable hub. `hub_accessible_url` exists precisely for this: on a + * private instance `hub_base_url` may be an address only the server can resolve. + * The (logged) layout does the same lookup, but the import wizard renders outside + * it, so it has to ask for itself. + */ +export async function hubBrowserUrl(): Promise { + try { + const accessible = (await SettingService.getGlobal({ key: 'hub_accessible_url' })) as string + if (accessible) return accessible.replace(/\/+$/, '') + const base = (await SettingService.getGlobal({ key: 'hub_base_url' })) as string + if (base) return base.replace(/\/+$/, '') + } catch { + // Unset or unreadable — the public hub is the right default either way. + } + return DEFAULT_HUB_BASE_URL.replace(/\/+$/, '') +} + +/** Shape of `GET /projects/` — the hub's own summary endpoint. */ +interface HubProject { + slug: string + name: string + summary: string + author: string + apps: string[] + logoApp: string | null + hasLogo: boolean + counts: { scripts: number; flows: number; apps: number; resources: number; total: number } +} + +/** + * Fetches one project's presentation straight from the hub. Cross-origin and + * unauthenticated by design: this runs before the wizard has a workspace, so the + * workspace-scoped `/api/w//hub/...` proxy is not available yet. The endpoint + * is public and sends `Access-Control-Allow-Origin: *`. + */ +export async function fetchHubProject(slug: string): Promise { + const hub = await hubBrowserUrl() + const res = await fetch(`${hub}/projects/${encodeURIComponent(slug)}`, { + headers: { accept: 'application/json' } + }) + if (!res.ok) throw new Error(`hub returned ${res.status}`) + const p = (await res.json()) as HubProject + return { + slug: p.slug, + name: p.name, + summary: p.summary, + author: p.author, + apps: p.apps ?? [], + // A project with an uploaded logo shows that; otherwise the icon of the + // integration it is filed under, otherwise its first integration. + logoUrl: p.hasLogo ? `${hub}/projects/${encodeURIComponent(p.slug)}/logo` : undefined, + iconApps: [p.logoApp, ...(p.apps ?? [])].filter( + (a, i, all): a is string => !!a && all.indexOf(a) === i + ), + counts: { + apps: p.counts?.apps ?? 0, + flows: p.counts?.flows ?? 0, + scripts: p.counts?.scripts ?? 0, + resources: p.counts?.resources ?? 0 + } + } +} + +/** + * The icon for a hub integration slug, resolved from the icons Windmill already bundles. + * + * Not fetched from the hub: the hub renders these out of `@windmill-labs/components`, which + * is this frontend's own package, so asking it over HTTP is a round trip to get our own + * assets back — and it made the card depend on a cross-origin request that an `API_SECRET` + * hub refuses anyway. + * + * The alias exists because the two repos disagree on one slug: the hub files Postgres scripts + * under `postgres`, the icon set ships the mark as `postgresql`. The hub bridges it in + * `aliasApp`; this is the same bridge on the consuming side. + */ +const HUB_APP_ICON_ALIAS: Record = { postgres: 'postgresql' } + +export function hubAppIcon(app: string): Component | undefined { + return appIconComponent(HUB_APP_ICON_ALIAS[app] ?? app) +} diff --git a/frontend/src/lib/importWizard/abandon.test.ts b/frontend/src/lib/importWizard/abandon.test.ts new file mode 100644 index 0000000000..abf0ff61b9 --- /dev/null +++ b/frontend/src/lib/importWizard/abandon.test.ts @@ -0,0 +1,237 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +// Everything the executor reaches over the network, stubbed. The two behaviours under test +// are decisions it makes around those calls, not the calls themselves. +vi.mock('$lib/gen', () => ({ + WorkspaceService: { + createWorkspace: vi.fn(), + listDataTables: vi.fn(async () => []), + // No workspace of ours at that id: the existing-workspace plan these tests use never + // reaches the create, and an empty list is the honest answer for a fresh instance. + listWorkspaces: vi.fn(async () => []) + }, + UserService: { + whoami: vi.fn(async () => ({ username: 'u' })), + globalWhoami: vi.fn(async () => ({ email: 'u@example.com' })) + } +})) +vi.mock('$lib/storeUtils', () => ({ switchWorkspace: vi.fn() })) +// What the destination already holds. A test sets this to stand in for a workspace that has +// some of the bundle in it — a half-finished run, or an existing workspace. +const present = vi.hoisted(() => ({ + paths: new Set(), + /** Fires when the probe is entered, so a test can abandon while it is in flight. */ + onProbe: undefined as (() => void) | undefined +})) +vi.mock('./probe', async (orig) => ({ + ...(await orig()), + probeWorkspace: vi.fn(async () => ({ exists: false, ours: false })), + probeImportedPaths: vi.fn(async () => { + present.onProbe?.() + return present.paths + }) +})) +vi.mock('$lib/user', () => ({ getUserExt: vi.fn(async () => ({ username: 'u' })) })) +// Let a test abandon *during* a write loop, which is the only way it happens for real: +// `run()` clears the flag on entry so a retry can proceed. Two hooks, because the item and +// migration phases stop in different places and the second is what pins the migrate row. +const hooks = vi.hoisted(() => ({ + afterFirstItem: undefined as (() => void) | undefined, + afterMigrationsStart: undefined as (() => void) | undefined +})) + +vi.mock('$lib/components/workspaceSettings/projectInstall', () => ({ + installProject: vi.fn(async (args: any) => { + // Ordered as the real one is: every item loop, then `onMigrationsStart`, then the + // migrations — and `stopped` checked before each write, returning the same way it + // returns on success. + for (const path of ['a', 'b', 'c']) { + if (args.stopped?.() === true) return + // Keyed exactly as the real `installProject` keys it, so this stand-in cannot drift + // into testing a contract the production code does not have. + if (args.alreadyPresent?.has(`script:${path}`)) { + args.onResult({ path, ok: true, skipped: true }) + continue + } + args.onResult({ path, ok: true }) + hooks.afterFirstItem?.() + hooks.afterFirstItem = undefined + } + if (args.stopped?.() === true) return + if (args.migrations?.length) { + args.onMigrationsStart?.() + hooks.afterMigrationsStart?.() + hooks.afterMigrationsStart = undefined + for (const m of args.migrations) { + if (args.stopped?.() === true) return + args.onResult({ path: `data table: ${m.datatable_name}`, ok: true }) + } + } + }) +})) + +// The export the run fetches from the hub proxy. Two items so an abandoned run can stop +// partway through, which is the case under test. +const EXPORT = { + project: { slug: 'calendly', name: 'Calendly', summary: '', readme: null }, + scripts: [], + flows: [], + apps: [], + resources: [], + triggers: [], + migrations: [] +} +vi.stubGlobal( + 'fetch', + vi.fn(async () => ({ ok: true, status: 200, text: async () => JSON.stringify(EXPORT) })) +) + +import { ImportExecution } from './execution.svelte' + +const PLAN = { slug: 'calendly', destination: { kind: 'existing' as const, workspaceId: 'ws-a' } } +const deps = { reviewMigrations: async () => [], hasEeLicense: false } + +/** A project that ships one, so `#import` appends the `migrate` row at all. */ +const MIGRATION = { + datatable_name: 'main', + sql: 'CREATE TABLE IF NOT EXISTS "calendly"."config" (id int)', + sql_down: '', + enabled: true +} +const depsWithMigration = { reviewMigrations: async () => [MIGRATION], hasEeLicense: false } + +describe('planTag', () => { + // A run handed back after a remount is checked against the plan being rendered. Tagging + // it with that plan instead of its own would make the check pass by construction. + it('identifies the plan the run belongs to', () => { + const a = new ImportExecution(PLAN, deps) + const b = new ImportExecution( + { ...PLAN, destination: { kind: 'existing', workspaceId: 'ws-b' } }, + deps + ) + expect(a.planTag).not.toBe(b.planTag) + }) + + it('ignores the folder, which stays editable after the run is made', () => { + const a = new ImportExecution(PLAN, deps) + const tag = a.planTag + a.setFolder('somewhere-else') + expect(a.planTag).toBe(tag) + }) + + it('separates two projects going to the same workspace', () => { + const a = new ImportExecution(PLAN, deps) + const b = new ImportExecution({ ...PLAN, slug: 'bitly' }, deps) + expect(a.planTag).not.toBe(b.planTag) + }) +}) + +describe('abandoning mid-import', () => { + beforeEach(() => { + hooks.afterFirstItem = undefined + hooks.afterMigrationsStart = undefined + present.paths = new Set() + }) + + it('does not report done, so the resumed step offers Retry rather than Continue', async () => { + const run = new ImportExecution(PLAN, deps) + hooks.afterFirstItem = () => run.abandon() + await run.run() + // `installProject` returned early exactly as it does on success; calling that done + // would report a clean import over items that never started. + expect(run.done).toBe(false) + expect(run.error).toMatch(/stopped/i) + }) + + it('reports done when nothing abandoned it', async () => { + const run = new ImportExecution(PLAN, deps) + await run.run() + expect(run.done).toBe(true) + expect(run.itemResults.length).toBe(3) + }) + + it('stops the migrate row spinning when it is abandoned mid-migration', async () => { + const run = new ImportExecution(PLAN, depsWithMigration) + // After `onMigrationsStart`, which is where the row is actually set to running — + // the real one fires it at the head of the migration loop, past every item loop. + hooks.afterMigrationsStart = () => run.abandon() + await run.run() + const migrate = run.tasks.find((t) => t.key === 'migrate') + // Guards the test itself: without a migration in the export there is no row, and the + // assertion below would pass over a branch that never ran. + expect(migrate).toBeDefined() + // A row left on `running` reads as work still in progress on a run that has stopped. + expect(migrate?.status).not.toBe('running') + expect(run.done).toBe(false) + }) +}) + +/** + * A retry must write only what is missing. Resending the whole bundle turns every item that + * already landed into an "already exists" failure — a wall of red over work that succeeded. + * What is already there is skipped, and reported as skipped rather than as imported, because + * the checklist has to account for every item the project ships. + */ +describe('retrying over what is already there', () => { + beforeEach(() => { + present.paths = new Set() + }) + + it('writes nothing for a path the destination already holds', async () => { + present.paths = new Set(['script:a', 'script:b']) + const run = new ImportExecution(PLAN, deps) + await run.run() + const byPath = new Map(run.itemResults.map((r) => [r.path, r])) + expect(byPath.get('a')?.skipped).toBe(true) + expect(byPath.get('b')?.skipped).toBe(true) + expect(byPath.get('c')?.skipped).toBeUndefined() + }) + + it('still accounts for every item, so the checklist stays complete', async () => { + present.paths = new Set(['script:a', 'script:b']) + const run = new ImportExecution(PLAN, deps) + await run.run() + expect(run.itemResults.length).toBe(3) + expect(run.done).toBe(true) + }) + + it('says what it did rather than claiming to have imported all of it', async () => { + present.paths = new Set(['script:a', 'script:b']) + const run = new ImportExecution(PLAN, deps) + await run.run() + const importRow = run.tasks.find((t) => t.key === 'import') + expect(importRow?.detail).toMatch(/1 imported/) + expect(importRow?.detail).toMatch(/2 already there/) + }) + + it('imports everything when the destination is empty', async () => { + const run = new ImportExecution(PLAN, deps) + await run.run() + expect(run.itemResults.every((r) => !r.skipped)).toBe(true) + expect(run.tasks.find((t) => t.key === 'import')?.detail).toMatch(/3 imported/) + }) +}) + +describe('abandoning while the presence probe is in flight', () => { + beforeEach(() => { + present.paths = new Set() + present.onProbe = undefined + }) + + /** + * `import` goes `running` before the probe is asked, so returning straight out of an + * abandon here leaves a spinner on a run that has stopped — next to an enabled Retry and + * with no explanation of why it stopped. + */ + it('leaves no task running', async () => { + const run = new ImportExecution(PLAN, deps as any) + // Abandoned from inside the probe: `import` is already `running` by then, and the + // executor's next look at the flag is the early return under test. + present.onProbe = () => run.abandon() + await run.run() + expect(run.tasks.find((t) => t.status === 'running')).toBeUndefined() + expect(run.tasks.find((t) => t.key === 'import')?.status).toBe('failed') + expect(run.error).toBeTruthy() + expect(run.done).toBe(false) + }) +}) diff --git a/frontend/src/lib/importWizard/execution.svelte.ts b/frontend/src/lib/importWizard/execution.svelte.ts new file mode 100644 index 0000000000..6a2c39b0b6 --- /dev/null +++ b/frontend/src/lib/importWizard/execution.svelte.ts @@ -0,0 +1,520 @@ +import { UserService, WorkspaceService } from '$lib/gen' +import { switchWorkspace } from '$lib/storeUtils' +import { userStore, workspaceStore } from '$lib/stores' +import { getUserExt } from '$lib/user' +import { get } from 'svelte/store' +import { enterNewWorkspace, refreshWorkspaceList } from '$lib/workspaceCreation' +import { + installProject, + type InstallResult +} from '$lib/components/workspaceSettings/projectInstall' +import { + projectReferencesResource, + type ProjectExport, + type ProjectMigration +} from '$lib/components/workspaceSettings/projectBundle' +import { planWorkspaceId, type ImportPlan } from './plan' +import { probeImportedPaths, probeWorkspace } from './probe' + +/** + * The only thing in the wizard that changes anything. It takes a finished plan and + * runs it as an ordered, observable list of tasks, so the last step can show what + * is happening and exactly where it stopped. + * + * Everything the run needs from the outside — reviewing data table migrations, + * the EE licence — is injected, so the wizard's UI decisions stay in the wizard + * and this file stays testable without a browser. + */ + +/** + * Whether an import is mid-flight, readable by anything that can navigate away + * from it. The run outlives no component: leaving the last step unmounts the + * migration review the executor is awaiting, so a run in progress has to block + * the stepper and the browser rather than be silently detached. + */ +const runState = $state({ active: false }) +export function importIsRunning(): boolean { + return runState.active +} + +export type TaskStatus = 'pending' | 'running' | 'done' | 'failed' | 'skipped' + +export interface TaskView { + key: string + label: string + status: TaskStatus + detail?: string +} + +/** + * What a plan will do, as the same task list the run reports against. Exported so the + * last step can show it before the run starts: the checklist is what the step says it + * is going to do, and the run then fills in the same rows rather than replacing them. + * + * Derived from the plan alone — no network — so it is safe to call while rendering. + */ +export function plannedTasks(plan: ImportPlan): TaskView[] { + const d = plan.destination + const tasks: TaskView[] = [] + if (d?.kind === 'new') { + tasks.push({ key: 'create', label: `Create workspace ${d.id}`, status: 'pending' }) + } + tasks.push({ key: 'fetch', label: 'Fetch the project from the hub', status: 'pending' }) + // The destination rides on this row when nothing creates it, so the list still says + // where the items are going in the existing-workspace case. + tasks.push({ + key: 'import', + label: d?.kind === 'existing' ? `Import the items into ${d.workspaceId}` : 'Import the items', + status: 'pending' + }) + return tasks +} + +export interface ExecutionDeps { + /** + * Chooses which data table migrations to run. Returns the migrations to apply, + * or null to abort the whole import (the user backed out at the warning). + */ + reviewMigrations: ( + workspace: string, + migrations: ProjectMigration[] + ) => Promise + hasEeLicense: boolean +} + +export class ImportExecution { + #plan: ImportPlan + #deps: ExecutionDeps + + /** Carried between attempts so a retry does not redo finished work. */ + // The hub's summary endpoint counts scripts/flows/apps/resources; only the export + // carries triggers and data table migrations. Surfaced so the last step can say + // what it is about to create — the warning below it talks about triggers, and the + // page this replaced did show both. + #export = $state(undefined) + + /** + * Data tables the project's migrations target. The wizard compares these with the + * destination's configured tables to decide whether a setup step is needed — + * `installProject` skips a migration whose data table does not exist. + */ + get datatableNames(): string[] { + const e = this.#export + if (!e) return [] + return [ + ...new Set( + (e.migrations ?? []) + .filter((m) => m.enabled && (m.sql ?? '').trim() !== '') + .map((m) => m.datatable_name) + ) + ] + } + + /** + * How many of the project's resources the setup step will ask about. Every resource + * arrives as an empty stub — the hub never publishes resource values — but only the ones + * something in the project points at have to hold a credential for it to work, and those + * are the ones the step lists. Counting all of them here would offer a fourth step that + * then has nothing on it. + */ + get resourceCount(): number { + const e = this.#export + if (!e) return 0 + return (e.resources ?? []).filter((r) => projectReferencesResource(e, String(r.path))).length + } + + get extraCounts(): { triggers: number; migrations: number } | undefined { + const e = this.#export + if (!e) return undefined + return { + triggers: e.triggers?.length ?? 0, + migrations: (e.migrations ?? []).filter((m) => m.enabled && (m.sql ?? '').trim() !== '') + .length + } + } + // $state, not a plain field: the UI offers to delete the workspace this run + // created, and a plain field would never re-render that button. + #workspaceCreated = $state(false) + + tasks = $state([]) + results = $state([]) + running = $state(false) + /** Set when a run stopped early; cleared when a retry starts. */ + error = $state(undefined) + done = $state(false) + + // Where the app pointed before this run switched away from it, so undoing the run + // can put it back. Captured at construction rather than at switch time: by then + // `$workspaceStore` already holds the workspace being entered. + #priorWorkspace = get(workspaceStore) + + constructor(plan: ImportPlan, deps: ExecutionDeps) { + this.#plan = plan + this.#deps = deps + this.tasks = this.#initialTasks() + } + + /** + * Identifies the plan this run belongs to — destination and project, not the folder, + * which stays editable on the last step and is pushed onto the run instead. A caller + * handing a run back after a remount compares this against the plan it is rendering; + * computing the tag from *that* plan would make the check pass by construction. + */ + get planTag(): string { + return JSON.stringify(this.#plan.destination) + this.#plan.slug + } + + get workspaceId(): string | undefined { + return planWorkspaceId(this.#plan) + } + + /** + * True once this run created a workspace — the only case where deleting is ours to offer. + * + * Deliberately not satisfied by having *adopted* one. `workspace.owner` is enough to know + * a create can be skipped — the id is one this user made — but not enough to offer to + * delete it, because `owner` is an identity, not a run: a second import by the same person + * into the same id looks identical. Skipping a create wrongly is recoverable; deleting a + * workspace is not, so an adopted run finishes the import and leaves the undo to the run + * that did the creating. + */ + get createdWorkspace(): boolean { + return this.#workspaceCreated + } + + get failedCount(): number { + return this.results.filter((r) => !r.ok).length + } + + /** `installProject` reports migrations through the same channel as items, tagged by this + * prefix. Split so the import row counts what it imported and the migrate row counts + * what it migrated — one failure should not be attributed to both. */ + static readonly MIGRATION_PREFIX = 'data table: ' + get itemResults(): InstallResult[] { + return this.results.filter((r) => !r.path.startsWith(ImportExecution.MIGRATION_PREFIX)) + } + get migrationResults(): InstallResult[] { + return this.results.filter((r) => r.path.startsWith(ImportExecution.MIGRATION_PREFIX)) + } + + #initialTasks(): TaskView[] { + return plannedTasks(this.#plan) + } + + #set(key: string, status: TaskStatus, detail?: string) { + this.tasks = this.tasks.map((t) => (t.key === key ? { ...t, status, detail } : t)) + } + + /** + * Set when the user confirms leaving mid-run. Nothing here can abort a request already + * in flight — `installProject` takes no signal — so this stops the run at the next phase + * boundary instead, which is as far as "stops where it is" can honestly go. + * + * The workspace it created stays, and the run stays resumable: coming back to the link + * re-probes the instance, finds the workspace, and carries on rather than trying to create + * it a second time. + */ + #abandoned = false + + /** The user has left. Stop at the next phase boundary and leave the run resumable. */ + abandon() { + this.#abandoned = true + } + + /** + * Runs every task that has not already succeeded. Safe to call again after a failure: the + * destination is asked what it already holds, so a workspace that exists is entered rather + * than recreated and an item that landed is skipped rather than rewritten. What a second + * run costs is the reads, not the writes. + */ + async run(): Promise { + if (this.running) return + this.#abandoned = false + this.running = true + runState.active = true + this.error = undefined + try { + const workspace = await this.#ensureWorkspace() + if (!workspace || this.#abandoned) return + const exportData = await this.#ensureExport(workspace) + if (!exportData || this.#abandoned) return + await this.#import(workspace, exportData) + } finally { + this.running = false + runState.active = false + } + } + + /** + * The folder is the one part of the plan still editable on the last step, so a + * retry after changing it must import where the field now says — not where the + * first attempt was told to. + */ + setFolder(folder: string) { + this.#plan = { ...this.#plan, folder } + } + + async #ensureWorkspace(): Promise { + const d = this.#plan.destination + if (!d) { + this.error = 'No destination' + return undefined + } + if (d.kind === 'existing') { + if (!d.workspaceId) { + this.error = 'No destination workspace' + return undefined + } + switchWorkspace(d.workspaceId) + await this.#adoptUser(d.workspaceId) + return d.workspaceId + } + // Asked of the instance, not remembered. A retry after entering it failed must not + // run the create again — that would only report the id as taken by the workspace this + // run just made — and after a reload the field is false again while the workspace is + // still there. `ours` is what makes adopting it safe: an id that exists but belongs to + // someone else is not this run's work, and importing into it would be importing into + // a stranger's workspace. + const already = await probeWorkspace(d.id, await this.#email()) + if (!this.#workspaceCreated && !(already.exists && already.ours)) { + this.#set('create', 'running') + try { + await WorkspaceService.createWorkspace({ + requestBody: { id: d.id, name: d.name, username: d.username } + }) + } catch (e: any) { + const detail = e?.body?.toString?.() ?? String(e) + this.#set('create', 'failed', detail) + this.error = `Could not create the workspace: ${detail}` + return undefined + } + this.#workspaceCreated = true + } + try { + await enterNewWorkspace(d.id) + await this.#adoptUser(d.id) + } catch (e: any) { + const detail = e?.body?.toString?.() ?? String(e) + this.#set('create', 'failed', `created, but could not be entered: ${detail}`) + this.error = `Created ${d.id}, but could not enter it: ${detail}` + return undefined + } + this.#set('create', 'done') + return d.id + } + + async #ensureExport(workspace: string): Promise { + if (this.#export) { + this.#set('fetch', 'done') + return this.#export + } + this.#set('fetch', 'running') + try { + // Workspace-scoped on purpose: this is the same proxy the rest of the app + // uses, so a private hub reachable only from the server still works. + const res = await fetch( + `/api/w/${encodeURIComponent(workspace)}/hub/projects/${encodeURIComponent(this.#plan.slug)}/export`, + { credentials: 'include', headers: { accept: 'application/json' } } + ) + const text = await res.text() + if (!res.ok) throw new Error(`export ${res.status}: ${text}`) + this.#export = JSON.parse(text) as ProjectExport + this.#set('fetch', 'done', `${itemCount(this.#export)} items`) + return this.#export + } catch (e: any) { + const detail = e?.message ?? String(e) + this.#set('fetch', 'failed', detail) + this.error = `Could not read the project: ${detail}` + return undefined + } + } + + /** + * Leave the checklist saying what actually happened, from wherever the run stopped. + * + * Reached from every point after `import` goes `running`, so nothing is left spinning on a + * run that has ended. A partial import is failed rather than done: calling it done reports + * a clean import over items that never started, and the resumed step offers Continue where + * it should offer Retry. + */ + #settleAbandoned() { + const landed = this.itemResults.length + this.#set('import', 'failed', `stopped after ${landed} item${landed === 1 ? '' : 's'}`) + // The migrate row is appended once the review settles and set running by + // `onMigrationsStart`. Stopping before its loop leaves it spinning forever, which + // reads as work still in progress on a run that has stopped. + if (this.tasks.some((t) => t.key === 'migrate' && t.status === 'running')) { + this.#set('migrate', 'pending') + } + this.error = 'Import stopped. Retry to import what is left.' + } + + async #import(workspace: string, exportData: ProjectExport): Promise { + this.#set('import', 'running') + const folder = this.#plan.folder?.trim() || exportData.project.slug + + let migrations: ProjectMigration[] | null + try { + migrations = await this.#deps.reviewMigrations(workspace, exportData.migrations ?? []) + } catch (e: any) { + this.#set('import', 'failed', String(e)) + this.error = `Could not plan the data table migrations: ${e}` + return + } + if (migrations === null) { + // The user backed out at the missing-data-table warning. + this.#set('import', 'skipped', 'cancelled at the data table warning') + this.error = 'Import cancelled.' + return + } + + // Appended only once the review has settled: until then nothing knows whether any + // migration is runnable here, and a row that might not apply is worse than none. + if (migrations.length && !this.tasks.some((t) => t.key === 'migrate')) { + const n = migrations.length + this.tasks = [ + ...this.tasks, + { + key: 'migrate', + label: `Run ${n} data table migration${n === 1 ? '' : 's'}`, + status: 'pending' + } + ] + } + + this.results = [] + // Asked every run, not only on a retry: the destination may be a workspace that already + // holds some of these paths, and a run interrupted halfway is indistinguishable from + // one that never started. On a workspace this run just created the answer is empty and + // nothing is skipped. + const alreadyPresent = await probeImportedPaths(workspace, folder, { + triggers: exportData.triggers.length > 0, + hasEeLicense: this.#deps.hasEeLicense + }) + // Settled, not just returned: `import` has been `running` since before the probe, and + // leaving it there shows a spinner on a run that has stopped, next to a Retry button. + if (this.#abandoned) { + this.#settleAbandoned() + return + } + try { + await installProject({ + alreadyPresent, + workspace, + exportData, + folder, + migrations, + hasEeLicense: this.#deps.hasEeLicense, + onResult: (r) => (this.results = [...this.results, r]), + onMigrationsStart: () => this.#set('migrate', 'running'), + // Checked before every write, so leaving mid-run stops the remaining items + // rather than only the phases. What already landed stays and is listed. + stopped: () => this.#abandoned + }) + } catch (e: any) { + this.#set('import', 'failed', String(e)) + this.error = `The import stopped: ${e}` + return + } + + // `installProject` returns early when `stopped` goes true, and it returns the same way + // it does on success — so the tail has to ask why. An abandoned run has written only + // what it got through; calling that `done` reports a clean import over items that + // never started, and the resumed step would offer Continue instead of Retry. + if (this.#abandoned) { + this.#settleAbandoned() + return + } + + const items = this.itemResults + const failed = items.filter((r) => !r.ok).length + const skipped = items.filter((r) => r.skipped).length + // Three outcomes, so the row says which: written, left alone because it was already + // there, and failed. Rolling the second into the first would report an import that + // did not happen. + const wrote = items.length - failed - skipped + const parts: string[] = [] + if (wrote > 0 || (failed === 0 && skipped === 0)) parts.push(`${wrote} imported`) + if (skipped > 0) parts.push(`${skipped} already there`) + if (failed > 0) parts.push(`${failed} failed`) + this.#set('import', failed > 0 ? 'failed' : 'done', parts.join(', ')) + + const migrated = this.migrationResults + const badMigrations = migrated.filter((r) => !r.ok).length + if (migrated.length) { + const badly = migrated.filter((r) => !r.ok) + this.#set( + 'migrate', + badly.length ? 'failed' : 'done', + badly.length ? badly.map((r) => r.error).join('; ') : undefined + ) + } + // A partial import is finished, not broken: the items that landed are real, + // and the failures are listed. Only a hard stop leaves `done` false. + this.done = true + // Both kinds of failure, because `error` is what offers Retry. A migration that fails + // against an existing data table is as retryable as a failed item; leaving it out here + // would present the run as a clean finish with no way to run it again. + const problems: string[] = [] + if (failed > 0) problems.push(`${failed} item${failed === 1 ? '' : 's'} failed to import`) + if (badMigrations > 0) { + problems.push(`${badMigrations} data table migration${badMigrations === 1 ? '' : 's'} failed`) + } + if (problems.length) this.error = `${problems.join(', ')}.` + } + + /** + * Load the membership for the workspace this run just entered. + * + * The wizard's page is reparented out of `(logged)`, so it never gets that + * layout's `getUserExt` call and `$userStore` stays undefined. Anything deciding + * what the user may do then reads "no user" and refuses: `canWrite` returns false + * without one, which renders every field of the resource editor disabled — the + * setup step could show the credentials to fill and then not let anyone fill them. + */ + async #adoptUser(workspace: string): Promise { + try { + userStore.set(await getUserExt(workspace)) + } catch { + // Leave it unset; the step degrades to read-only rather than failing the run. + } + } + + /** + * Who we are, for the ownership check. `$userStore` is workspace-scoped and this page is + * reparented out of `(logged)`, so it is unset until a run adopts one — `globalWhoami` is + * the identity that exists before any workspace does. + */ + async #email(): Promise { + const known = get(userStore)?.email + if (known) return known + try { + return (await UserService.globalWhoami()).email + } catch { + return undefined + } + } + + /** Undoes the one thing this run created, when the user asks for it. */ + async deleteCreatedWorkspace(): Promise { + const d = this.#plan.destination + if (!this.#workspaceCreated || d?.kind !== 'new') return + await WorkspaceService.deleteWorkspace({ workspace: d.id }) + // Leave the app pointing somewhere that exists. The run switched into the + // workspace it created; without this the store keeps the deleted id, the + // layout persists it to local/sessionStorage, and the next full page load + // fails `getUserExt` and logs the user out. + switchWorkspace(this.#priorWorkspace) + await refreshWorkspaceList() + this.#workspaceCreated = false + // The id is free again; the next `#ensureWorkspace` asks the instance and finds it + // gone, so a retry creates rather than adopts. + this.#set('create', 'pending') + this.done = false + this.results = [] + } +} + +function itemCount(e: ProjectExport): number { + return e.scripts.length + e.flows.length + e.apps.length + e.resources.length +} diff --git a/frontend/src/lib/importWizard/migrationOutcome.test.ts b/frontend/src/lib/importWizard/migrationOutcome.test.ts new file mode 100644 index 0000000000..87a23cc8ef --- /dev/null +++ b/frontend/src/lib/importWizard/migrationOutcome.test.ts @@ -0,0 +1,94 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +/** + * A run that imported every item but failed a data table migration is not a clean finish. + * `error` is what offers Retry, so this is about whether the user can act on the failure — + * the `migrate` row already says it happened. + */ + +vi.mock('$lib/gen', () => ({ + WorkspaceService: { + createWorkspace: vi.fn(), + listDataTables: vi.fn(async () => []), + listWorkspaces: vi.fn(async () => []) + }, + UserService: { + whoami: vi.fn(async () => ({ username: 'u' })), + globalWhoami: vi.fn(async () => ({ email: 'u@example.com' })) + } +})) +vi.mock('$lib/storeUtils', () => ({ switchWorkspace: vi.fn() })) +vi.mock('./probe', async (orig) => ({ + ...(await orig()), + probeWorkspace: vi.fn(async () => ({ exists: false, ours: false })), + probeImportedPaths: vi.fn(async () => new Set()) +})) +vi.mock('$lib/user', () => ({ getUserExt: vi.fn(async () => ({ username: 'u' })) })) + +/** Whether the migration this run applies succeeds. The item writes always do. */ +const outcome = vi.hoisted(() => ({ migrationOk: true })) + +vi.mock('$lib/components/workspaceSettings/projectInstall', () => ({ + installProject: vi.fn(async (args: any) => { + args.onResult({ path: 'f/calendly/one', ok: true }) + if (args.migrations?.length) { + args.onMigrationsStart?.() + for (const m of args.migrations) { + args.onResult({ + path: `data table: ${m.datatable_name}`, + ok: outcome.migrationOk, + error: outcome.migrationOk ? undefined : 'relation already exists' + }) + } + } + }) +})) + +const EXPORT = { + project: { slug: 'calendly', name: 'Calendly', summary: '', readme: null }, + scripts: [], + flows: [], + apps: [], + resources: [], + triggers: [], + migrations: [] +} +vi.stubGlobal( + 'fetch', + vi.fn(async () => ({ ok: true, status: 200, text: async () => JSON.stringify(EXPORT) })) +) + +import { ImportExecution } from './execution.svelte' + +const PLAN = { slug: 'calendly', destination: { kind: 'existing' as const, workspaceId: 'ws-a' } } +const MIGRATION = { + datatable_name: 'main', + sql: 'CREATE TABLE IF NOT EXISTS "calendly"."config" (id int)', + sql_down: '', + enabled: true +} +const deps = { reviewMigrations: async () => [MIGRATION], hasEeLicense: false } + +describe('a failed migration', () => { + beforeEach(() => { + outcome.migrationOk = true + }) + + it('marks the migrate row failed and leaves the run retryable', async () => { + outcome.migrationOk = false + const run = new ImportExecution(PLAN, deps as any) + await run.run() + expect(run.tasks.find((t) => t.key === 'migrate')?.status).toBe('failed') + // `error` is what the page reads to offer Retry: a `migrate` row saying failed with + // `error` unset sends it down the finished-run path with no way to run it again. + expect(run.error).toBeTruthy() + expect(run.error).toContain('migration') + }) + + it('says nothing went wrong when the migration succeeds', async () => { + const run = new ImportExecution(PLAN, deps as any) + await run.run() + expect(run.tasks.find((t) => t.key === 'migrate')?.status).toBe('done') + expect(run.error).toBeFalsy() + }) +}) diff --git a/frontend/src/lib/importWizard/plan.test.ts b/frontend/src/lib/importWizard/plan.test.ts new file mode 100644 index 0000000000..c81ed700bc --- /dev/null +++ b/frontend/src/lib/importWizard/plan.test.ts @@ -0,0 +1,172 @@ +import { describe, expect, it } from 'vitest' + +import { + planProblem, + planToSearch, + planWorkspaceId, + readPlan, + type ImportPlan, + type WizardStep +} from './plan' + +// The wizard keeps no state of its own — the plan *is* the URL. Back, forward, the +// stepper and a pasted link are all the same operation, which only holds if this +// round trip is lossless. + +const read = (search: string) => readPlan(new URL(`https://x.dev/projects/import${search}`)) +const roundTrip = (plan: ImportPlan, step: WizardStep) => read(planToSearch(plan, step)) + +describe('readPlan / planToSearch round trip', () => { + const cases: Array<[string, ImportPlan, WizardStep]> = [ + ['bare slug', { slug: 'support-automation' }, 1], + ['new destination', { slug: 's', destination: { kind: 'new', id: 'w', name: 'W' } }, 2], + [ + 'new with username', + { slug: 's', destination: { kind: 'new', id: 'w', name: 'W', username: 'ada' } }, + 2 + ], + [ + 'existing destination', + { slug: 's', destination: { kind: 'existing', workspaceId: 'admins' } }, + 3 + ], + [ + 'folder carried', + { slug: 's', destination: { kind: 'existing', workspaceId: 'admins' }, folder: 'finance' }, + 3 + ] + ] + + for (const [name, plan, step] of cases) { + it(name, () => { + expect(roundTrip(plan, step)).toEqual({ plan, step }) + }) + } + + it('survives characters that need encoding', () => { + const plan: ImportPlan = { + slug: 'a b&c=d', + destination: { kind: 'new', id: 'w', name: 'Name & Co = 100%' } + } + expect(roundTrip(plan, 2)).toEqual({ plan, step: 2 }) + }) +}) + +describe('readPlan', () => { + it('reads legacy links that carry only a workspace', () => { + expect(read('?hub=s&workspace=admins').plan.destination).toEqual({ + kind: 'existing', + workspaceId: 'admins' + }) + }) + + it('reads legacy links that carry only new_workspace_id, falling the name back to the id', () => { + expect(read('?hub=s&new_workspace_id=w').plan.destination).toEqual({ + kind: 'new', + id: 'w', + name: 'w', + username: undefined + }) + }) + + it('clamps and rounds the step', () => { + expect(read('?hub=s&step=0').step).toBe(1) + // 4 is the optional setup step, so that is the ceiling. + expect(read('?hub=s&step=9').step).toBe(4) + expect(read('?hub=s&step=nope').step).toBe(1) + // A fractional step would match neither `=== 2` nor `=== 3`. + expect(read('?hub=s&step=2.5').step).toBe(3) + expect(read('?hub=s&step=2.4').step).toBe(2) + }) + + it('has no destination when nothing names one', () => { + expect(read('?hub=s').plan.destination).toBeUndefined() + }) +}) + +/** + * `create_workspace` takes the username it is given and writes it to `usr.username` without + * checking it — `Some("")` passes its only guard — so a blank or malformed one becomes a + * workspace whose owner has no usable name. The wizard is the last thing that can refuse it. + */ +describe('planProblem — the new-workspace username', () => { + const dest = (username?: string) => ({ + slug: 'calendly', + destination: { kind: 'new' as const, id: 'calendly', name: 'Calendly', username } + }) + + it('asks for nothing when the instance derives the username', () => { + expect(planProblem(dest(undefined))).toBeUndefined() + }) + + it('refuses a blank or whitespace-only username', () => { + expect(planProblem(dest(''))).toMatch(/needs a username/i) + expect(planProblem(dest(' '))).toMatch(/needs a username/i) + }) + + it('refuses one the backend would store verbatim but never accept elsewhere', () => { + expect(planProblem(dest('1bad'))).toMatch(/letters and numbers/i) + expect(planProblem(dest('a b'))).toMatch(/letters and numbers/i) + }) + + it('accepts a valid one', () => { + expect(planProblem(dest('guilhem'))).toBeUndefined() + }) +}) + +/** `validate_workspace_name` refuses > 50 chars, and only at creation — two steps later. */ +describe('planProblem — the new-workspace name length', () => { + const named = (name: string) => ({ + slug: 'calendly', + destination: { kind: 'new' as const, id: 'calendly', name } + }) + + it('accepts the longest name the backend takes', () => { + expect(planProblem(named('x'.repeat(50)))).toBeUndefined() + }) + + it('refuses one character more, rather than failing at create', () => { + expect(planProblem(named('x'.repeat(51)))).toMatch(/too long/i) + }) +}) + +describe('planProblem', () => { + it('names what is missing, in the order the wizard asks for it', () => { + expect(planProblem({ slug: '' })).toMatch(/No project/) + expect(planProblem({ slug: 's' })).toMatch(/Pick a destination/) + expect(planProblem({ slug: 's', destination: { kind: 'new', id: 'w', name: '' } })).toMatch( + /needs a name/ + ) + expect(planProblem({ slug: 's', destination: { kind: 'existing' } })).toMatch( + /Pick the workspace/ + ) + }) + + it('validates the id of either destination kind', () => { + expect( + planProblem({ slug: 's', destination: { kind: 'new', id: 'not valid', name: 'W' } }) + ).toMatch(/letters, numbers and dashes/) + // An existing id arrives from the URL just as a new one does. + expect( + planProblem({ slug: 's', destination: { kind: 'existing', workspaceId: '../admins' } }) + ).toMatch(/not a valid workspace id/) + }) + + it('rejects a folder name the import could not create', () => { + const base: ImportPlan = { slug: 's', destination: { kind: 'existing', workspaceId: 'admins' } } + expect(planProblem({ ...base, folder: 'ok_folder-1' })).toBeUndefined() + expect(planProblem({ ...base, folder: 'not ok' })).toMatch(/Folder/) + }) +}) + +describe('planWorkspaceId', () => { + it('is the id either kind of destination will end up in', () => { + expect(planWorkspaceId({ slug: 's', destination: { kind: 'new', id: 'w', name: 'W' } })).toBe( + 'w' + ) + expect( + planWorkspaceId({ slug: 's', destination: { kind: 'existing', workspaceId: 'admins' } }) + ).toBe('admins') + expect(planWorkspaceId({ slug: 's' })).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/importWizard/plan.ts b/frontend/src/lib/importWizard/plan.ts new file mode 100644 index 0000000000..f209041787 --- /dev/null +++ b/frontend/src/lib/importWizard/plan.ts @@ -0,0 +1,131 @@ +/** + * The import wizard's state, and the only thing the first two steps produce. + * + * Nothing is created, switched or written while the wizard is being filled in: + * steps 1 and 2 only ever describe *what should happen*, and the plan travels in + * the URL. That is what makes the back button and the stepper safe — going back + * is a URL change and cannot leave a half-made workspace behind, because there is + * no state anywhere else to unwind. + * + * `execution.svelte.ts` is the only thing that acts on a plan, and only when the + * user asks it to on the last step. + */ + +import { validateUsername } from '$lib/utils' +import { validateWorkspaceId, WORKSPACE_NAME_MAX_LENGTH } from '$lib/utils/workspaceId' + +/** + * `existing` carries no workspace until one is picked: step 1 answers *which kind* + * of destination and step 2 answers *which one*, and the plan has to be able to + * hold the state in between. Encoding that gap as an absent destination would make + * "chose an existing workspace, has not picked it" and "answered nothing yet" the + * same URL. + */ +export type ImportDestination = + | { kind: 'new'; name: string; id: string; username?: string } + | { kind: 'existing'; workspaceId?: string } + +export interface ImportPlan { + /** The hub project being imported. */ + slug: string + /** Undefined until step 1 has been answered. */ + destination?: ImportDestination + /** Folder the items land in; defaults to the project slug at execution time. */ + folder?: string +} + +/** 4 is the optional setup step, reached only when the import leaves work to do. */ +export type WizardStep = 1 | 2 | 3 | 4 + +export const FOLDER_NAME_RE = /^[a-zA-Z_0-9-]+$/ + +export function readPlan(url: URL): { plan: ImportPlan; step: WizardStep } { + const params = url.searchParams + const slug = params.get('hub') ?? '' + + const newId = params.get('new_workspace_id') + const existing = params.get('workspace') + // `destination` records the step 1 answer on its own; the older links that only + // carried `workspace` or `new_workspace_id` still read as the kind they imply. + const kind = params.get('destination') + const destination: ImportDestination | undefined = + newId || kind === 'new' + ? { + kind: 'new', + id: newId ?? '', + name: params.get('new_workspace_name') || newId || '', + username: params.get('username') || undefined + } + : existing || kind === 'existing' + ? { kind: 'existing', workspaceId: existing || undefined } + : undefined + + // Rounded as well as clamped: the steps are compared with `>` and `===`, so a + // fractional `?step=2.5` would clamp to 2.5 and match neither. + const raw = Number(params.get('step') ?? 1) + const step = (Number.isFinite(raw) ? Math.min(4, Math.max(1, Math.round(raw))) : 1) as WizardStep + + return { plan: { slug, destination, folder: params.get('folder') || undefined }, step } +} + +export function planToSearch(plan: ImportPlan, step: WizardStep): string { + const params = new URLSearchParams({ hub: plan.slug }) + if (step !== 1) params.set('step', String(step)) + if (plan.destination) params.set('destination', plan.destination.kind) + if (plan.destination?.kind === 'new') { + params.set('new_workspace_id', plan.destination.id) + params.set('new_workspace_name', plan.destination.name) + if (plan.destination.username) params.set('username', plan.destination.username) + } else if (plan.destination?.workspaceId) { + params.set('workspace', plan.destination.workspaceId) + } + if (plan.folder) params.set('folder', plan.folder) + return `?${params}` +} + +/** + * Whether the plan can be executed. Returned as a reason rather than a boolean so + * the button can say why it is disabled instead of just being grey. + */ +export function planProblem(plan: ImportPlan): string | undefined { + if (!plan.slug) return 'No project to import' + const d = plan.destination + if (!d) return 'Pick a destination first' + if (d.kind === 'new') { + if (!d.name.trim()) return 'The new workspace needs a name' + // The backend refuses a longer one (`validate_workspace_name`), and only at + // creation — by then the wizard has already walked the user through two more steps. + if (d.name.trim().length > WORKSPACE_NAME_MAX_LENGTH) { + return `The name is too long (${d.name.trim().length} chars). Maximum is ${WORKSPACE_NAME_MAX_LENGTH}.` + } + const idProblem = validateWorkspaceId(d.id) + if (idProblem) return idProblem + // Only asked for when the instance does not derive it. `create_workspace` takes + // whatever it is given here — `Some("")` passes its only check — so a blank or + // malformed username is written to `usr.username` verbatim rather than refused. + // The sibling creator validates it; this is the same check. + if (d.username !== undefined) { + if (!d.username.trim()) return 'The new workspace needs a username' + const bad = validateUsername(d.username.trim()) + if (bad) return bad + } + } else if (!d.workspaceId) { + return 'Pick the workspace to import into' + } else if (validateWorkspaceId(d.workspaceId)) { + // The id arrives from the URL exactly as the new-workspace one does, so it gets + // the same check. Downstream it is interpolated into a credentialed same-origin + // API path and pushed into `workspaceStore`; an id that cannot name a workspace + // has no business reaching either. + return 'That is not a valid workspace id' + } + if (plan.folder && !FOLDER_NAME_RE.test(plan.folder)) { + return 'Folder: letters, digits, dashes and underscores' + } + return undefined +} + +/** The workspace the plan lands in, once one has been named or picked. */ +export function planWorkspaceId(plan: ImportPlan): string | undefined { + const d = plan.destination + return (d?.kind === 'new' ? d.id : d?.workspaceId) || undefined +} diff --git a/frontend/src/lib/importWizard/probe.test.ts b/frontend/src/lib/importWizard/probe.test.ts new file mode 100644 index 0000000000..1d66a0e14b --- /dev/null +++ b/frontend/src/lib/importWizard/probe.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from 'vitest' + +import { expectedTables } from './probe' + +/** + * `expectedTables` is inference over what the export happens to say, so it is the part of the + * probe that can be wrong while everything still type-checks. The network reads around it + * either answer or throw. + */ + +describe('expectedTables', () => { + // The shape `datatableSchemaSql.ts` emits, and what every published project carries. + const REAL = `BEGIN; +CREATE SCHEMA IF NOT EXISTS "calendly"; +CREATE TABLE IF NOT EXISTS "calendly"."config" ( + "id" SERIAL NOT NULL, + "host_name" text NOT NULL DEFAULT ''::text +); +CREATE TABLE IF NOT EXISTS "calendly"."bookings" ( "id" SERIAL NOT NULL ); +COMMIT;` + + it('reads every table a migration creates', () => { + expect(expectedTables(REAL)).toEqual(['calendly.config', 'calendly.bookings']) + }) + + it('does not mistake the schema for a table', () => { + expect(expectedTables(REAL)).not.toContain('calendly') + }) + + it('reads the form without IF NOT EXISTS', () => { + expect(expectedTables('CREATE TABLE "bitly"."links" (id int)')).toEqual(['bitly.links']) + }) + + it('is case- and whitespace-insensitive the way SQL is', () => { + expect(expectedTables('create table\n "a" . "b" (x int)')).toEqual(['a.b']) + }) + + it('reports each table once, however many times it is named', () => { + const sql = 'CREATE TABLE "a"."b" (x int); CREATE TABLE IF NOT EXISTS "a"."b" (x int);' + expect(expectedTables(sql)).toEqual(['a.b']) + }) + + /** + * The answer that keeps a caller honest. An unquoted or unqualified `CREATE TABLE` is + * something this cannot resolve — the schema would come from `search_path` at run time — + * so it reads as nothing expected, and the caller treats that as "cannot tell" rather than + * as "no tables, so the migration must have run". + */ + it('claims nothing about SQL it cannot resolve', () => { + expect(expectedTables('CREATE TABLE links (id int)')).toEqual([]) + expect(expectedTables('CREATE TABLE bitly.links (id int)')).toEqual([]) + expect(expectedTables('')).toEqual([]) + }) +}) diff --git a/frontend/src/lib/importWizard/probe.ts b/frontend/src/lib/importWizard/probe.ts new file mode 100644 index 0000000000..0df5173536 --- /dev/null +++ b/frontend/src/lib/importWizard/probe.ts @@ -0,0 +1,181 @@ +/** + * What is already true in the destination, read from the destination itself. + * + * Nothing here is remembered between runs, and nothing may be: a note saying "this run + * created workspace X" outlives the reload it was written for, but it also outlives the + * workspace it names, and the two are indistinguishable when it is read back. The plan in + * the URL says what should exist; these functions ask the instance what does. + * + * The one thing the instance cannot answer is *which tables a migration was supposed to + * create*. That is inferred from the SQL the project ships (`expectedTables`), because the + * export states only what to run, never what running it should produce. + */ + +import { ResourceService, ScriptService, FlowService, AppService, WorkspaceService } from '$lib/gen' +import type { ProjectMigration } from '$lib/components/workspaceSettings/projectBundle' +import { + presenceKey, + type ImportedKind +} from '$lib/components/workspaceSettings/projectInstall' +import { listAllWorkspaceTriggers } from '$lib/components/triggers/workspaceTriggersList' + +/** + * The tables a migration creates, as `schema.table`, read off its `CREATE TABLE` statements. + * + * Inference, not a contract: the export ships SQL and nothing else, so this is the only way + * to check a migration's work without a record of it having run. It deliberately reads only + * the shape this project's generator emits (`datatableSchemaSql.ts` always writes the + * schema-qualified, quoted form) — anything hand-edited into a different shape simply reads + * as no expected tables, which makes the caller fall back to "cannot tell" rather than to a + * confident wrong answer. + */ +export function expectedTables(sql: string): string[] { + const out: string[] = [] + const re = /CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?"([^"]+)"\s*\.\s*"([^"]+)"/gi + let m: RegExpExecArray | null + while ((m = re.exec(sql)) !== null) out.push(`${m[1]}.${m[2]}`) + return [...new Set(out)] +} + +export interface WorkspaceState { + /** The user is a member of a workspace with this id. */ + exists: boolean + /** …and it is one they own, so a run of theirs is what made it. */ + ours: boolean +} + +/** + * Whether the destination workspace is there, and whether it is the caller's. + * + * `listWorkspaces` answers both in one call: it returns only workspaces the caller is a + * member of, each carrying the `owner` email set at creation. Ownership is what makes + * skipping a create safe — an id that exists but belongs to someone else is not a workspace + * this run made, and importing into it would be importing into a stranger's. + */ +export async function probeWorkspace( + id: string, + email: string | undefined +): Promise { + try { + const mine = await WorkspaceService.listWorkspaces() + const found = mine.find((w) => w.id === id) + if (!found) return { exists: false, ours: false } + return { exists: true, ours: !!email && found.owner === email } + } catch { + // Cannot tell. Reported as absent so the caller creates rather than adopts: a failed + // create is a clear error, adopting the wrong workspace is a silent one. + return { exists: false, ours: false } + } +} + +const PROBE_PAGE_SIZE = 100 +/** + * A stop, not a limit on what may be imported: 100 pages is 10,000 items in one folder, far + * past any project, and a paginating endpoint that never returns a short page would otherwise + * loop forever. Hitting it under-reports, which only ever means "still to do". + */ +const MAX_PROBE_PAGES = 100 + +/** + * Which of the items the import would write are already there, as `presenceKey` keys. + * + * Scoped by `pathStart` to the import's own folder, so the four path-bearing kinds are four + * small reads rather than a workspace scan. Presence is not provenance — importing into an + * existing workspace that already held a path reads the same as having imported it — so + * callers use this to decide what is left to do, never to claim credit for what is there. + * + * Triggers are asked for separately and only when the project ships some: they have no + * prefix-filtered list endpoint, so answering for them means one call per trigger kind, and a + * project without triggers should not pay for that. + */ +export async function probeImportedPaths( + workspace: string, + folder: string, + opts?: { triggers?: boolean; hasEeLicense?: boolean } +): Promise> { + const pathStart = `f/${folder}/` + const found = new Set() + /** + * Every page, not the first one. These endpoints paginate and default to 30 rows, so a + * single call answers for a small project and quietly under-reports a large one — leaving + * everything past the first page to be created again, and rejected as already existing. + */ + const collectAll = async ( + kind: ImportedKind, + list: (page: number) => Promise + ): Promise => { + for (let page = 1; page <= MAX_PROBE_PAGES; page++) { + const rows = ((await list(page)) as { path?: string }[] | undefined) ?? [] + for (const r of rows) if (r.path) found.add(presenceKey(kind, r.path)) + if (rows.length < PROBE_PAGE_SIZE) return + } + } + const calls: Promise[] = [ + collectAll('script', (page) => + ScriptService.listScripts({ workspace, pathStart, page, perPage: PROBE_PAGE_SIZE }) + ), + collectAll('flow', (page) => + FlowService.listFlows({ workspace, pathStart, page, perPage: PROBE_PAGE_SIZE }) + ), + collectAll('app', (page) => + AppService.listApps({ workspace, pathStart, page, perPage: PROBE_PAGE_SIZE }) + ), + collectAll('resource', (page) => + ResourceService.listResource({ workspace, pathStart, page, perPage: PROBE_PAGE_SIZE }) + ) + ] + if (opts?.triggers) { + // `failedKinds` is deliberately ignored: a kind that could not be listed leaves its + // triggers out of the set, and a missing key only ever means "still to do". + calls.push( + listAllWorkspaceTriggers(workspace, { + includeEeOnly: opts.hasEeLicense === true + }).then(({ triggers }) => { + for (const t of triggers) { + if (t.path?.startsWith(pathStart)) found.add(presenceKey(`trigger:${t.kind}`, t.path)) + } + }) + ) + } + // One kind failing should narrow the answer, not lose the others: a missing key only ever + // means "still to do", which is the safe direction. + await Promise.allSettled(calls) + return found +} + +/** + * Whether every table a data table's migrations create is in it. + * + * The ground truth for "did this run", and the only one that covers both paths + * `applyOneMigration` takes: it records a migration when the data table has migrations + * enabled, and otherwise runs the SQL once as a job that nothing remembers. The tables + * outlive both. + * + * Every migration for one data table at once: they all target the same schema, so asking + * per migration would introspect the same database N times for one answer. + * + * `undefined` means the question could not be answered — the schema was unreadable, or the + * SQL named no tables this can recognise. Distinct from `false`, because "not there" invites + * a caller to run the migration and "cannot tell" does not. + */ +export async function probeMigrationsApplied( + workspace: string, + datatableName: string, + migrations: ProjectMigration[] +): Promise { + const wanted = [...new Set(migrations.flatMap((m) => expectedTables(m.sql ?? '')))] + if (wanted.length === 0) return undefined + try { + const schema = (await WorkspaceService.getDatatableFullSchema({ + workspace, + requestBody: { source: `datatable://${datatableName}` } + })) as Record> + const present = new Set() + for (const [schemaName, tables] of Object.entries(schema ?? {})) { + for (const table of Object.keys(tables ?? {})) present.add(`${schemaName}.${table}`) + } + return wanted.every((t) => present.has(t)) + } catch { + return undefined + } +} diff --git a/frontend/src/lib/importWizard/probePaging.test.ts b/frontend/src/lib/importWizard/probePaging.test.ts new file mode 100644 index 0000000000..17e7fb9a11 --- /dev/null +++ b/frontend/src/lib/importWizard/probePaging.test.ts @@ -0,0 +1,75 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +/** + * The list endpoints paginate. A probe that reads one page answers correctly for a small + * project and under-reports a large one, which puts every item past the first page back + * through a create call that rejects it. + */ + +const calls = vi.hoisted(() => ({ script: [] as { page?: number; perPage?: number }[] })) +/** What the destination's trigger listing answers with, per test. */ +const triggerRows = vi.hoisted(() => ({ rows: [] as { kind: string; path: string }[] })) + +/** 150 scripts in the folder — more than one page at any page size the probe might pick. */ +const ALL = Array.from({ length: 150 }, (_, i) => ({ path: `f/calendly/s${i}` })) + +vi.mock('$lib/gen', () => ({ + ScriptService: { + listScripts: vi.fn(async (args: any) => { + calls.script.push({ page: args.page, perPage: args.perPage }) + const per = args.perPage ?? 30 + return ALL.slice((args.page - 1) * per, args.page * per) + }) + }, + FlowService: { listFlows: vi.fn(async () => []) }, + AppService: { listApps: vi.fn(async () => []) }, + ResourceService: { listResource: vi.fn(async () => []) }, + WorkspaceService: { listWorkspaces: vi.fn(async () => []), getDatatableFullSchema: vi.fn() } +})) +vi.mock('$lib/components/triggers/workspaceTriggersList', () => ({ + listAllWorkspaceTriggers: vi.fn(async () => ({ triggers: triggerRows.rows, failedKinds: [] })) +})) + +import { probeImportedPaths } from './probe' +import { presenceKey } from '$lib/components/workspaceSettings/projectInstall' + +describe('probeImportedPaths paging', () => { + beforeEach(() => { + calls.script = [] + }) + + it('reads every page, not just the first', async () => { + const found = await probeImportedPaths('w', 'calendly') + expect(found.size).toBe(150) + expect(found.has(presenceKey('script', 'f/calendly/s0'))).toBe(true) + // The one that a single-page probe misses, and would then try to create again. + expect(found.has(presenceKey('script', 'f/calendly/s149'))).toBe(true) + }) + + it('stops on the first short page rather than asking forever', async () => { + await probeImportedPaths('w', 'calendly') + const pages = calls.script.map((c) => c.page) + expect(pages).toEqual([1, 2]) + expect(new Set(calls.script.map((c) => c.perPage))).toEqual(new Set([100])) + }) +}) + +describe('trigger presence keys', () => { + beforeEach(() => { + triggerRows.rows = [] + }) + + /** + * Each trigger kind is its own table keyed on `(path, workspace_id)`, so a workspace can + * hold a schedule and an HTTP trigger both called `f/calendly/sync`. The probe builds the + * keys and `installProject` reads them, so they have to agree that those are two things — + * key on the path alone and the one that exists reports the other as already imported. + */ + it('keys a trigger by its kind, so one kind cannot answer for another', async () => { + triggerRows.rows = [{ kind: 'http', path: 'f/calendly/sync' }] + const found = await probeImportedPaths('w', 'calendly', { triggers: true }) + expect(found.has(presenceKey('trigger:http', 'f/calendly/sync'))).toBe(true) + // The schedule the project also ships at that path has not been imported. + expect(found.has(presenceKey('trigger:schedule', 'f/calendly/sync'))).toBe(false) + }) +}) diff --git a/frontend/src/lib/importWizard/retargetDeployed.test.ts b/frontend/src/lib/importWizard/retargetDeployed.test.ts new file mode 100644 index 0000000000..adf7a4c795 --- /dev/null +++ b/frontend/src/lib/importWizard/retargetDeployed.test.ts @@ -0,0 +1,393 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +/** + * The safety property: the stub survives anything the scan cannot account for. Rewriting an + * item onto the chosen resource is safe on its own, so it always happens; deleting the stub + * while an item still reads it is what breaks the imported project, and that item is exactly + * the one nobody looks at afterwards. + */ + +const state = vi.hoisted(() => ({ + apps: [] as any[], + scripts: [] as any[], + triggers: [] as any[], + scheduleListError: undefined as any, + failingTriggerPath: undefined as string | undefined, + deployedJs: 'COMPILED', + deletedResources: [] as string[], + updatedRawApps: [] as any[], + updatedApps: [] as any[], + updatedFlows: [] as any[], + flows: [] as any[], + updatedTriggers: [] as any[] +})) + +vi.mock('$lib/gen', () => ({ + ScriptService: { + listSearchScript: vi.fn(async () => state.scripts), + getScriptByPath: vi.fn(), + createScript: vi.fn() + }, + FlowService: { + listSearchFlow: vi.fn(async () => state.flows), + getFlowByPath: vi.fn(async ({ path }: any) => state.flows.find((f: any) => f.path === path)), + updateFlow: vi.fn(async (p: any) => state.updatedFlows.push(p)) + }, + AppService: { + listSearchApp: vi.fn(async () => state.apps), + getAppByPath: vi.fn(async ({ path }: any) => state.apps.find((a) => a.path === path)), + getPublicSecretOfLatestVersionOfApp: vi.fn(async () => 'secret'), + updateApp: vi.fn(async (p: any) => state.updatedApps.push(p)), + updateAppRaw: vi.fn(async (p: any) => state.updatedRawApps.push(p)) + }, + ScheduleService: { updateSchedule: vi.fn() }, + ResourceService: { + deleteResource: vi.fn(async ({ path }: any) => state.deletedResources.push(path)) + } +})) + +vi.mock('$lib/components/triggers/workspaceTriggersList', () => ({ + TRIGGER_KINDS: { + schedule: { + badge: 'Schedule', + list: vi.fn(async () => { + if (state.scheduleListError) throw state.scheduleListError + return [] + }) + }, + postgres: { + badge: 'Postgres', + list: vi.fn(async () => state.triggers), + update: vi.fn(async (_w: string, path: string, body: any) => { + if (path === state.failingTriggerPath) throw new Error('the update was rejected') + state.updatedTriggers.push({ path, body }) + }) + } + }, + WORKSPACE_TRIGGER_KINDS: ['schedule', 'postgres'], + createWorkspaceTriggerDisabled: vi.fn(), + triggerHandlerRefs: () => [] +})) + +vi.mock('$lib/components/apps/editor/appPolicy', () => ({ updatePolicy: vi.fn(async () => ({})) })) +vi.mock('$lib/sharedUtils', () => ({ updateRawAppPolicy: vi.fn(async () => ({})) })) + +// The deployed bundle is served by secret, not through the generated client. +vi.stubGlobal( + 'fetch', + vi.fn(async (url: string) => ({ + ok: true, + status: 200, + text: async () => (url.endsWith('.js') ? state.deployedJs : 'STYLES') + })) +) + +import { applyRetarget, seesWholeWorkspace } from './retargetDeployed' + +const FROM = 'f/proj/smtp' +const TO = 'f/shared/company_smtp' + +/** A deployed raw app: sources plus runnables, with the bundle stored out of the value. */ +const rawApp = { + path: 'f/proj/dash', + raw_app: true, + value: { files: { '/App.tsx': 'v1' }, runnables: { send: { fields: { smtp: `$res:${FROM}` } } } } +} + +async function run() { + return applyRetarget({ + workspace: 'w', + folder: 'proj', + from: FROM, + to: TO, + seesWholeWorkspace: true + }) +} + +describe('applyRetarget', () => { + beforeEach(() => { + state.apps = [rawApp] + state.scripts = [] + state.triggers = [] + state.deployedJs = 'COMPILED' + state.scheduleListError = undefined + state.failingTriggerPath = undefined + state.deletedResources = [] + state.updatedRawApps = [] + state.updatedApps = [] + state.flows = [] + state.updatedFlows = [] + state.updatedTriggers = [] + }) + + // `updateAppRaw` refuses without a bundle, and the browser cannot rebuild one. The bundle + // that goes back is the deployed one, read back by secret. + it('sends the deployed bundle back with the rewritten value, then drops the stub', async () => { + const outcome = await run() + expect(outcome.error).toBeUndefined() + expect(outcome.gaps).toEqual([]) + expect(outcome.stubDeleted).toBe(true) + expect(state.deletedResources).toEqual([FROM]) + const sent = state.updatedRawApps[0] + expect(sent.formData.js).toBe('COMPILED') + expect(sent.formData.css).toBe('STYLES') + // Rewritten, and the bundle entries stay out of the value the way the import leaves them. + expect(JSON.stringify(sent.formData.app.value)).toContain(`$res:${TO}`) + expect(Object.keys(sent.formData.app.value.files)).toEqual(['/App.tsx']) + }) + + // The bundle is compiled from the sources, so a `$res:` a source spells out is baked into + // it. `retargetProjectExport` rewrites that copy on import, while /bundle.js is still one + // of `files` — sending the deployed bundle back untouched would undo exactly that. + it("rewrites the deployed bundle's own tokens before sending it back", async () => { + state.deployedJs = `const cfg = "$res:${FROM}"; export default cfg` + const outcome = await run() + expect(outcome.error).toBeUndefined() + expect(outcome.stubDeleted).toBe(true) + const sent = state.updatedRawApps[0] + expect(sent.formData.js).toContain(`$res:${TO}`) + expect(sent.formData.js).not.toContain(`$res:${FROM}`) + }) + + // A path the bundle names any other way is one nothing here can move, so the app is left + // alone and the stub it still reads has to survive. + it('keeps the stub when the bundle names the resource outside a $res: token', async () => { + state.deployedJs = `const cfg = await getResource("${FROM}")` + const outcome = await run() + expect(outcome.error).toBeUndefined() + expect(outcome.stubDeleted).toBe(false) + expect(state.deletedResources).toEqual([]) + expect(state.updatedRawApps).toEqual([]) + }) + + // The path written inside a source file is a reference too, and no rewriter reaches it. + // + it('keeps the stub when a source file names the resource in code', async () => { + state.apps = [ + { + path: 'f/proj/dash', + raw_app: true, + value: { + files: { '/App.tsx': `const c = await getResource("${FROM}")` }, + runnables: {} + } + } + ] + const outcome = await run() + expect(outcome.error).toBeUndefined() + expect(outcome.stubDeleted).toBe(false) + expect(state.deletedResources).toEqual([]) + expect(state.updatedRawApps).toEqual([]) + }) + + // A trigger holds its resource as a bare path in its own column, not as a `$res:` token. + it('finds a trigger that holds the resource as a bare path', async () => { + state.apps = [] + state.triggers = [ + { + path: 'f/proj/ingest', + script_path: 'f/proj/run', + postgres_resource_path: FROM, + permissioned_as: 'u/service_account', + enabled: true + } + ] + const outcome = await run() + expect(outcome.error).toBeUndefined() + expect(state.updatedTriggers[0].body.postgres_resource_path).toBe(TO) + // The trigger keeps its own path even though it was the string being remapped. + expect(state.updatedTriggers[0].path).toBe('f/proj/ingest') + // Pointing a trigger at a credential must not also start it: `enabled` is left out so + // the backend keeps whatever the trigger is set to. + expect(state.updatedTriggers[0].body).not.toHaveProperty('enabled') + // Nor run it as whoever picked the credential: a trigger states its identity as + // `permissioned_as`, which the backend keeps only when told to preserve it. + expect(state.updatedTriggers[0].body.permissioned_as).toBe('u/service_account') + expect(state.updatedTriggers[0].body.preserve_permissioned_as).toBe(true) + expect(state.deletedResources).toEqual([FROM]) + }) + + // The stub is what a reference this run did not move still resolves through, so a write + // that fails partway must not take it: the moved items and the rest both keep working. + it('keeps the stub when a write fails, and reports what had already moved', async () => { + state.apps = [] + state.triggers = [ + { path: 'f/proj/first', postgres_resource_path: FROM }, + { path: 'f/proj/second', postgres_resource_path: FROM } + ] + state.failingTriggerPath = 'f/proj/second' + const outcome = await run() + expect(outcome.error).toContain('the update was rejected') + expect(outcome.rewritten.map((r) => r.path)).toEqual(['f/proj/first']) + expect(outcome.stubDeleted).toBe(false) + expect(state.deletedResources).toEqual([]) + }) + + // Most trigger kinds are cargo features an instance may not compile in, and their routes + // then 404. Reading that as a failed listing keeps the stub on every stock build. + it('drops the stub when a trigger kind is not compiled in, keeps it when one truly fails', async () => { + state.scheduleListError = { status: 404 } + expect((await run()).gaps).toEqual([]) + expect(state.deletedResources).toEqual([FROM]) + + state.deletedResources = [] + state.scheduleListError = { status: 500 } + expect((await run()).gaps.map((g) => g.path)).toContain('Schedule triggers') + expect(state.deletedResources).toEqual([]) + }) + + // The referrer scan matches a bare path anywhere in an app's value, while the rewriter only + // relocates `$res:` tokens and runnable paths. Rewriting such an app would claim a move that + // did not happen, and the stub it still reads would go. + it('leaves an app holding the resource path as a bare string alone, and keeps the stub', async () => { + state.apps = [ + { + path: 'f/proj/page', + value: { grid: [{ data: { input: { type: 'static', value: FROM } } }] } + } + ] + const outcome = await run() + expect(outcome.rewritten).toEqual([]) + expect(outcome.gaps.map((g) => g.path)).toEqual(['f/proj/page']) + expect(state.deletedResources).toEqual([]) + }) + + // Both spellings at once. The token is moved so the item stops depending on the stub for + // what it could, and the mention it spells out still keeps the stub alive. + it('rewrites the token of an app that also names the resource in code, and keeps the stub', async () => { + state.apps = [ + { + path: 'f/proj/dash', + raw_app: true, + value: { + files: { '/App.tsx': `// the credential lives at ${FROM}` }, + runnables: { send: { fields: { smtp: `$res:${FROM}` } } } + } + } + ] + const outcome = await run() + expect(outcome.error).toBeUndefined() + expect(outcome.rewritten.map((r) => r.path)).toEqual(['f/proj/dash']) + expect(outcome.gaps.map((g) => g.path)).toEqual(['f/proj/dash']) + expect(outcome.stubDeleted).toBe(false) + expect(JSON.stringify(state.updatedRawApps[0].formData.app.value)).toContain(`$res:${TO}`) + }) + + // Scripts, flows and resources share a path namespace, and the map holds a resource path. + // The import's rewriters remap a runnable's own `path` on an exact match, which here would + // repoint the step at the credential. + it("moves an app's tokens without repointing a runnable that shares the path", async () => { + state.apps = [ + { + path: 'f/proj/page', + value: { + grid: [{ data: { type: 'runnableByPath', runType: 'script', path: FROM } }], + inline: `$res:${FROM}` + } + } + ] + const outcome = await run() + expect(outcome.error).toBeUndefined() + const sent = state.updatedApps[0] + expect(sent.requestBody.value.inline).toBe(`$res:${TO}`) + expect(sent.requestBody.value.grid[0].data.path).toBe(FROM) + }) + + // A token moves wherever it sits in the value, and a step's own path is left alone. + it('moves a flow token outside the fields the import rewriter reached', async () => { + state.flows = [ + { + path: 'f/proj/pipeline', + value: { + modules: [ + { id: 'a', summary: `reads $res:${FROM}`, value: { type: 'script', path: FROM } } + ] + } + } + ] + const outcome = await run() + expect(outcome.error).toBeUndefined() + const sent = state.updatedFlows[0] + expect(sent.requestBody.value.modules[0].summary).toBe(`reads $res:${TO}`) + expect(sent.requestBody.value.modules[0].value.path).toBe(FROM) + }) + + // The listings run as the caller and row-level security filters them inside the query, so + // an item a member cannot read is invisible rather than counted. The stub has to outlive + // a scan that cannot see the whole workspace. + it('keeps the stub when the caller is not shown the whole workspace', async () => { + const outcome = await applyRetarget({ + workspace: 'w', + folder: 'proj', + from: FROM, + to: TO, + seesWholeWorkspace: false + }) + expect(outcome.error).toBeUndefined() + expect(outcome.stubDeleted).toBe(false) + expect(state.deletedResources).toEqual([]) + expect(outcome.gaps.map((g) => g.path)).toContain('This workspace') + }) + + // A handler field names a runnable, and the map holds a resource path. A script sharing + // that path is not the reference being moved. + it("moves a trigger's resource field without repointing its error handler", async () => { + state.apps = [] + state.triggers = [ + { + path: 'f/proj/ingest', + script_path: 'f/proj/run', + postgres_resource_path: FROM, + on_failure: `script/${FROM}`, + error_handler_path: FROM, + permissioned_as: 'u/service_account' + } + ] + const outcome = await run() + expect(outcome.error).toBeUndefined() + expect(state.updatedTriggers[0].body.postgres_resource_path).toBe(TO) + expect(state.updatedTriggers[0].body.on_failure).toBe(`script/${FROM}`) + // The bare spelling too: a handler path is a path and never a `$res:` token. + expect(state.updatedTriggers[0].body.error_handler_path).toBe(FROM) + }) + + // `listSearchApp` caps at 1000 rows server-side, unordered and unpaginated, so a full page + // may not hold the project's own app — and the stub would go anyway. + it('keeps the stub when the app listing comes back at its server-side cap', async () => { + state.apps = Array.from({ length: 1000 }, (_, i) => ({ path: `f/other/a${i}`, value: {} })) + const outcome = await run() + expect(outcome.gaps.map((g) => g.path)).toContain('Apps') + expect(state.deletedResources).toEqual([]) + }) + + // The listings are workspace-wide. An item outside the project's folder is the user's own, + // so it is not rewritten — and it is exactly why the stub it reads has to stay. + it('rewrites what it owns and keeps the stub for a reference outside the project', async () => { + state.apps = [] + state.scripts = [{ path: 'u/alice/report', content: `$res:${FROM}` }] + state.triggers = [ + { path: 'f/proj/ingest', script_path: 'f/proj/run', postgres_resource_path: FROM } + ] + const outcome = await run() + expect(outcome.error).toBeUndefined() + expect(state.updatedTriggers[0].body.postgres_resource_path).toBe(TO) + expect(outcome.gaps.map((g) => g.path)).toEqual(['u/alice/report']) + expect(outcome.stubDeleted).toBe(false) + expect(state.deletedResources).toEqual([]) + }) +}) + +describe('seesWholeWorkspace', () => { + // The user record is per-workspace and survives a workspace change, so reading `is_admin` + // without checking which workspace it describes answers for the wrong one — and a wrong + // yes here is what lets an RLS-filtered scan clear the stub for deletion. + it.each([ + ['admin of this workspace', { workspace_id: 'w', is_admin: true }, false, true], + ['admin of another workspace', { workspace_id: 'other', is_admin: true }, false, false], + ['member of this workspace', { workspace_id: 'w', is_admin: false }, false, false], + ['superadmin, record stale', { workspace_id: 'other', is_admin: false }, true, true], + ['no user record', undefined, false, false] + ])('%s', (_label, user, isSuperadmin, expected) => { + expect(seesWholeWorkspace(user as any, isSuperadmin, 'w')).toBe(expected) + }) +}) diff --git a/frontend/src/lib/importWizard/retargetDeployed.ts b/frontend/src/lib/importWizard/retargetDeployed.ts new file mode 100644 index 0000000000..b33a69a667 --- /dev/null +++ b/frontend/src/lib/importWizard/retargetDeployed.ts @@ -0,0 +1,650 @@ +/** + * Point an already-imported project at a resource the workspace already has. + * + * The import writes `$res:f//` into every item that uses the project's + * resource — except a trigger, which holds the bare path in its own `*_resource_path` + * field. This rewrites those references to an existing resource, so the project reads the + * workspace's own credential — the same end state the import would have produced, reached + * after the fact. + * + * Two rules make that safe to run over deployed items: + * + * - Only items inside the project's folder are rewritten. The import wrote nothing outside + * it, so a reference from elsewhere is the user's own and not ours to move. + * - The stub is deleted only when the scan can prove it saw every reference to it. + * Listings come back capped, a trigger kind can fail to list, a reference can sit where no + * rewriter reaches, and the listings themselves are row-level-security filtered so a + * caller who is not a workspace admin is not shown every item — each of those is a gap, + * and any gap keeps the stub. + * + * Rewriting is separable from deleting, and only the delete is destructive. An item moved + * onto the chosen resource resolves whether or not the stub survives; an item the scan never + * saw resolves only while the stub is there. So an incomplete scan downgrades the run to + * "rewritten, stub kept" rather than refusing it — the outcome names the gaps so the caller + * can say the placeholder is still around. + */ + +import { AppService, FlowService, ResourceService, ScheduleService, ScriptService } from '$lib/gen' +import { + rewriteContent, + rewriteTriggerConfig, + referencesResourcePath, + holdsResourceToken, + textHoldsBarePath +} from '$lib/components/workspaceSettings/projectBundle' +import { + TRIGGER_KINDS, + WORKSPACE_TRIGGER_KINDS, + type WorkspaceTriggerKind +} from '$lib/components/triggers/workspaceTriggersList' +import { updatePolicy } from '$lib/components/apps/editor/appPolicy' +import { updateRawAppPolicy } from '$lib/sharedUtils' +import type { App } from '$lib/components/apps/types' +import { apiErrorMessage as errorMessage } from '$lib/utils' + +export type ReferrerKind = 'script' | 'flow' | 'app' | 'trigger' + +export interface Referrer { + kind: ReferrerKind + path: string + /** Present for triggers: which kind's table it lives in. */ + triggerKind?: WorkspaceTriggerKind + /** + * The plan already recorded this item as naming the path somewhere no rewrite reaches, so + * the stub survives whatever the write does. Its tokens are still worth moving, and the + * write's own staleness check has nothing left to add. + */ + gapped?: true + /** + * Present for triggers: the row the scan read, which is also what the write sends back. + * Re-reading it costs another listing of the whole kind per trigger — and for schedules a + * listing plus a detail fetch per row, since `list` resolves each one. + */ + row?: Record +} + +/** Something the scan could not account for. `path` names an item, or a listing standing in + * for every item it failed to return. */ +export interface Gap { + path: string + reason: string +} + +export interface RetargetPlan { + /** Items whose reference this run will move. */ + referrers: Referrer[] + /** + * Why the scan cannot claim it saw every reference to the stub. Empty is the only state + * in which deleting the stub is provably safe. + */ + gaps: Gap[] +} + +export interface RetargetOutcome { + /** Items now reading the chosen resource. */ + rewritten: Referrer[] + /** Why the stub was kept, when it was. */ + gaps: Gap[] + stubDeleted: boolean + /** Set when a write failed. The run stopped there and the stub stays, so what was + * rewritten before it and what was not both resolve. */ + error?: string +} + +/** + * Whether this caller's listings are the whole workspace, and so whether a clean scan proves + * anything. Row-level security filters the listings inside the query for everyone else. + * + * `UserExt` is per-workspace and outlives a workspace change, so the role is only this + * workspace's role when the record says it is. On the setup step's reload path nothing + * re-fetches it, and it still describes the workspace the user came from — reading + * `is_admin` alone there answers for the wrong workspace. An instance superadmin bypasses + * the policies everywhere, which is why it is asked separately. + */ +export function seesWholeWorkspace( + user: { workspace_id?: string; is_admin?: boolean } | undefined, + isSuperadmin: boolean, + workspace: string +): boolean { + if (isSuperadmin) return true + return !!user?.is_admin && user.workspace_id === workspace +} + +/** Everything under the project's folder, which is all this rewrites. */ +function inFolder(path: unknown, folder: string): boolean { + return typeof path === 'string' && path.startsWith(`f/${folder}/`) +} + +/** + * Whether the item names the resource path anywhere a rewriter would not reach it. + * + * A script, flow or app spells a reference the rewriters move as a `$res:` token and nothing + * else. The path appearing any other way is either a reference beyond them — a static string + * value in a component, an argument inline code assembles itself — or not the resource at + * all, such as a step running a script that happens to share the path. Neither is rewritable: + * moving the first is beyond the rewriters, and moving the second would repoint a runnable at + * a credential. Both are reasons the stub has to outlive the run. + * + * The whole serialized item is searched, because inline code is a string inside it and a path + * written in code is no less a reference for being surrounded by other characters. Triggers + * are the exception and hold the bare path by design, so they never come here. + */ +function namesPathUnreachably(value: unknown, path: string): boolean { + return textHoldsBarePath(JSON.stringify(value ?? null), path) +} + +/** + * What each `listSearch*` endpoint caps its answer at, server-side. The queries carry no + * `ORDER BY` and the routes take no pagination, so a full page is an arbitrary subset with no + * page two to ask for. + * + * A full page is a sound truncation test only for an unscoped caller, which a wizard session + * is. The server applies its scope-path predicate to the rows the `LIMIT` already returned, + * so a scoped token can be handed a short page cut from a truncated query — reuse this scan + * under one and the cap goes undetected. + */ +const SEARCH_LIMITS = { script: 10000, flow: 1000, app: 1000 } + +/** + * What a trigger listing caps at. The kinds' list routes take pagination this table does not + * pass, so each answers with the server's `DEFAULT_PER_PAGE`. A full page is read the same + * way as a full `listSearch*` page: as a listing that cannot account for the rest. + */ +const TRIGGER_LIST_LIMIT = 1000 + +/** + * Where a trigger names a runnable, taken from what `triggerHandlerRefs` reads. + * + * `rewriteTriggerConfig` remaps a runnable reference on an exact path match, which is right + * for the folder-wide map the import hands it and wrong for a map holding one resource path: + * a script sharing that path is not the reference being moved, and remapping it makes the + * trigger run a resource. So these are put back from the row after the rewrite. + * + * Split by spelling, because only one of the two can also hold a `$res:` token. A prefixed + * field is restored only when it holds the runnable spelling, so a token in it still moves; + * a bare field is a path and nothing else, so it is always restored. + */ +const PREFIXED_RUNNABLE_FIELDS = ['on_failure', 'on_recovery', 'on_success', 'url'] +const PREFIXED_RUNNABLE_RE = /^(?:\$(?:script|flow):|(?:script|flow)\/)/ +const BARE_RUNNABLE_FIELDS = ['dynamic_skip', 'error_handler_path', 'script_path'] + +/** The rewritten config with every runnable reference put back as the row holds it. */ +function restoreRunnableRefs(rewritten: any, row: any): any { + const out = { ...rewritten } + for (const k of PREFIXED_RUNNABLE_FIELDS) { + if (typeof row?.[k] === 'string' && PREFIXED_RUNNABLE_RE.test(row[k])) out[k] = row[k] + } + for (const k of BARE_RUNNABLE_FIELDS) { + if (typeof row?.[k] === 'string') out[k] = row[k] + } + // A websocket's initial messages can each carry a runnable result, whose `path` is bare. + // Only that field is put back; the rest of the message is rewritten like any other value. + if (Array.isArray(out.initial_messages)) { + out.initial_messages = out.initial_messages.map((m: any, i: number) => { + const was = row?.initial_messages?.[i]?.runnable_result?.path + return m?.runnable_result && typeof was === 'string' + ? { ...m, runnable_result: { ...m.runnable_result, path: was } } + : m + }) + } + return out +} + +/** A reference this run will not move, recorded so the stub outlives it. */ +const OUTSIDE_PROJECT = 'reads this resource from outside the project' +const UNSEEN_BY_CALLER = 'holds items this account is not shown' +const UNREACHABLE_REFERENCE = 'names the resource path outside a $res: reference' + +/** + * Which deployed items reference the stub, which of them this run can move, and what it + * could not account for. + * + * The `listSearch*` endpoints return each item's content in one call per kind, so this is + * three calls plus one per trigger kind rather than one per item. + */ +export async function planRetarget( + workspace: string, + folder: string, + from: string, + opts: { seesWholeWorkspace: boolean } +): Promise { + const referrers: Referrer[] = [] + const gaps: Gap[] = [] + + // The listings run as the caller. Row-level security filters the rows out inside the + // query, so an item this account cannot read is not merely absent from the answer — it + // does not count towards the full-page test either, and nothing downstream can notice it. + // A colleague's private script reading this stub is exactly that shape. + if (!opts.seesWholeWorkspace) gaps.push({ path: 'This workspace', reason: UNSEEN_BY_CALLER }) + + const [scripts, flows, apps] = await Promise.all([ + ScriptService.listSearchScript({ workspace }), + FlowService.listSearchFlow({ workspace }), + AppService.listSearchApp({ workspace }) + ]) + + for (const [label, rows, limit] of [ + ['Scripts', scripts, SEARCH_LIMITS.script], + ['Flows', flows, SEARCH_LIMITS.flow], + ['Apps', apps, SEARCH_LIMITS.app] + ] as const) { + if ((rows?.length ?? 0) >= limit) gaps.push({ path: label, reason: 'could not all be listed' }) + } + + // The listings are workspace-wide, so an item outside the folder is seen for free. It is + // the user's own and stays on the stub, which is the whole reason the stub stays too. + for (const s of scripts ?? []) { + const content = String(s.content ?? '') + const reads = referencesResourcePath(content, from) + // `rewriteContent` moves the `$res:` tokens and nothing else, so a path the code spells + // out is a reference this run leaves behind. + const bare = textHoldsBarePath(content, from) + if (!reads && !bare) continue + if (!inFolder(s.path, folder)) { + gaps.push({ path: s.path!, reason: OUTSIDE_PROJECT }) + continue + } + if (bare) { + gaps.push({ path: s.path!, reason: UNREACHABLE_REFERENCE }) + if (!reads) continue + } + referrers.push({ kind: 'script', path: s.path!, ...(bare ? { gapped: true as const } : {}) }) + } + for (const f of flows ?? []) { + const value: any = f.value ?? {} + // A token is the only spelling a rewriter moves. `referencesResourcePath` would also + // count a whole string equal to the path, which is the unreachable case `bare` covers. + const reads = holdsResourceToken(value, from) + const bare = namesPathUnreachably(value, from) + if (!reads && !bare) continue + if (!inFolder(f.path, folder)) { + gaps.push({ path: f.path!, reason: OUTSIDE_PROJECT }) + continue + } + if (bare) { + gaps.push({ path: f.path!, reason: UNREACHABLE_REFERENCE }) + if (!reads) continue + } + const gapped = bare ? ({ gapped: true } as const) : undefined + referrers.push({ kind: 'flow', path: f.path!, ...gapped }) + } + for (const a of apps ?? []) { + const value: any = a.value ?? {} + // A token is the only spelling a rewriter moves. `referencesResourcePath` would also + // count a whole string equal to the path, which is the unreachable case `bare` covers. + const reads = holdsResourceToken(value, from) + const bare = namesPathUnreachably(value, from) + if (!reads && !bare) continue + if (!inFolder(a.path, folder)) { + gaps.push({ path: a.path!, reason: OUTSIDE_PROJECT }) + continue + } + if (bare) { + gaps.push({ path: a.path!, reason: UNREACHABLE_REFERENCE }) + if (!reads) continue + } + const gapped = bare ? ({ gapped: true } as const) : undefined + referrers.push({ kind: 'app', path: a.path!, ...gapped }) + } + + for (const kind of WORKSPACE_TRIGGER_KINDS) { + const def = TRIGGER_KINDS[kind] + // No `eeOnly` skip. That reads a client-side store, which is empty on an EE instance + // whose licence is unset or whose fetch failed — while the rows are still in the + // database and the routes still answer. A kind skipped that way leaves no gap, so the + // stub would go while an EE trigger still points at it. On CE the routes are not + // registered and the 404 below says so, from the server. + let rows: Array> = [] + // A 404 is not an incomplete listing — the instance has that trigger feature compiled + // out, so there is no trigger of the kind to have missed. Anything else means triggers + // of this kind may reference the stub without this ever seeing them. + let incomplete = false + try { + rows = await def.list(workspace, () => (incomplete = true)) + } catch (e: any) { + if (e?.status === 404) continue + incomplete = true + } + if (incomplete) { + gaps.push({ path: `${def.badge} triggers`, reason: 'could not be listed' }) + continue + } + if (rows.length >= TRIGGER_LIST_LIMIT) { + gaps.push({ path: `${def.badge} triggers`, reason: 'could not all be listed' }) + continue + } + for (const t of rows) { + if (!referencesResourcePath(t, from)) continue + if (!inFolder(t.path, folder)) { + gaps.push({ path: String(t.path), reason: OUTSIDE_PROJECT }) + continue + } + // `schedule` has no `update` in the table because its service takes a different body + // shape; `rewriteTrigger` handles it directly, the way the import's create does. + if (kind !== 'schedule' && !def.update) { + gaps.push({ + path: String(t.path), + reason: `${def.badge} triggers cannot be updated from here` + }) + continue + } + referrers.push({ kind: 'trigger', path: String(t.path), triggerKind: kind, row: t }) + } + } + + return { referrers, gaps } +} + +/** + * Rewrite every referrer the plan found, then delete the stub if the plan came back clean. + * + * A gap never stops the rewriting — moving an item onto the chosen resource is safe on its + * own. It stops only the delete, which is the one step that can strand a reference nobody + * looked at. + */ +export async function applyRetarget(args: { + workspace: string + folder: string + from: string + to: string + /** Whether the listings the scan reads are the whole workspace. See `planRetarget`. */ + seesWholeWorkspace: boolean +}): Promise { + const { workspace, folder, from, to, seesWholeWorkspace } = args + const map = new Map([[from, to]]) + const rewritten: Referrer[] = [] + + const plan = await planRetarget(workspace, folder, from, { seesWholeWorkspace }) + const gaps = [...plan.gaps] + + for (const r of plan.referrers) { + let moved: true | string + try { + if (r.kind === 'script') moved = await rewriteScript(workspace, r, map) + else if (r.kind === 'flow') moved = await rewriteFlow(workspace, r, map) + else if (r.kind === 'app') moved = await rewriteApp(workspace, r, map) + else moved = await rewriteTrigger(workspace, r, map) + } catch (e: any) { + return { rewritten, gaps, stubDeleted: false, error: errorMessage(e) } + } + if (moved === true) rewritten.push(r) + else gaps.push({ path: r.path, reason: moved }) + } + + if (gaps.length > 0) return { rewritten, gaps, stubDeleted: false } + + try { + await ResourceService.deleteResource({ workspace, path: from }) + } catch (e: any) { + return { rewritten, gaps, stubDeleted: false, error: errorMessage(e) } + } + return { rewritten, gaps, stubDeleted: true } +} + +/** + * The rewrite this module needs: `$res:` tokens and nothing else. + * + * `rewriteFlowValue` and `rewriteAppValue` also remap a runnable's own `path` on an exact + * match. That is right for the folder-wide map the import hands them, where every path is + * moving. Here the map holds one entry, a resource path — and scripts, flows and resources + * share a namespace, so a project shipping both a script and a resource named `smtp` would + * have the step calling `f//smtp` repointed at the credential. Rewriting the + * serialized value moves the tokens and leaves every path alone. + */ +function rewriteTokens(value: T, map: Map): T { + return JSON.parse(rewriteContent(JSON.stringify(value ?? null), map)) +} + +/** + * Whether the item as just read names a path where no rewrite reaches it. + * + * The plan classifies items from the `listSearch*` rows; every rewriter then re-reads its + * item by path. A value that has gained an unreachable mention in between is one the plan + * cleared and the write must not: rewriting its tokens is fine, but the stub it still names + * has to survive. Checking the read the write is about to send is the only place that can be + * seen. + */ +function readsPathUnreachably(value: unknown, map: Map): boolean { + for (const from of map.keys()) if (namesPathUnreachably(value, from)) return true + return false +} + +/** + * Every write here is an in-place edit of a deployed item, not a redeployment by whoever + * opened the wizard, so each one has to say so. + * + * `preserve_on_behalf_of` is the flag that says it. Without it the backend replaces the + * item's stored run identity with the caller's — `resolve_on_behalf_of` for scripts and + * flows, the `should_preserve` branch of `update_app` for apps — and an imported item that + * ran as a service account would silently start running as the person who picked a + * credential. The backend still gates it on `wm_deployers` membership, so a caller who + * cannot preserve gets what they would have got anyway. + * + * For apps the policy is the other half: it carries the run identity, the execution mode and + * the sandbox rules, so it is read from the deployed app and handed back to the recompute + * rather than rebuilt from nothing. + */ +const PRESERVE_DEPLOYED_IDENTITY = { preserve_on_behalf_of: true } + +/** + * A new script version, the way the editor saves one. Spread rather than field-by-field: + * `Script` and `NewScript` share their names, and listing them here would silently drop + * whichever field someone adds next. + */ +async function rewriteScript( + workspace: string, + r: Referrer, + map: Map +): Promise { + const path = r.path + const s: any = await ScriptService.getScriptByPath({ workspace, path }) + if (!r.gapped) + for (const stub of map.keys()) + if (textHoldsBarePath(String(s.content ?? ''), stub)) return UNREACHABLE_REFERENCE + const content = rewriteContent(s.content ?? '', map) + if (content === s.content) return true + await ScriptService.createScript({ + workspace, + requestBody: { + ...s, + ...PRESERVE_DEPLOYED_IDENTITY, + content, + parent_hash: s.hash, + deployment_message: undefined + } + }) + return true +} + +async function rewriteFlow( + workspace: string, + r: Referrer, + map: Map +): Promise { + const path = r.path + const f: any = await FlowService.getFlowByPath({ workspace, path }) + if (!r.gapped && readsPathUnreachably(f.value ?? {}, map)) return UNREACHABLE_REFERENCE + const value = rewriteTokens(f.value ?? {}, map) + // The tokens the plan saw are gone from the deployed item, so there is nothing to write. + if (JSON.stringify(value) === JSON.stringify(f.value ?? {})) return true + await FlowService.updateFlow({ + workspace, + path, + requestBody: { ...f, ...PRESERVE_DEPLOYED_IDENTITY, path, value } + }) + return true +} + +/** + * The deployed policy is recomputed, not rebuilt. Recomputing is required: `triggerables_v2` + * is keyed by `:rawscript/`, and rewriting an inline + * runnable's content changes that key, so a policy copied verbatim would leave the component + * "forbidden by policy". Handing the deployed policy to the recompute is what keeps the run + * identity, the sandbox rules and everything else it does not touch. + * + * No execution mode is defaulted. The backend keeps the deployed mode when the submitted + * policy states none, and stating one here would put a `viewer` app on the publisher's + * identity. + */ +async function rewriteApp( + workspace: string, + r: Referrer, + map: Map +): Promise { + const path = r.path + const a: any = await AppService.getAppByPath({ workspace, path }) + // The deployed record says which kind this is. `list_search_apps` returns only the path and + // the value, so the scan could only have guessed from the value's shape — and a guess wrong + // in either direction is a deploy the backend refuses for changing an app's kind. + if (a.raw_app) return rewriteRawApp(workspace, r, map, a) + if (!r.gapped && readsPathUnreachably(a.value ?? {}, map)) return UNREACHABLE_REFERENCE + const next = rewriteTokens(a.value ?? {}, map) + // The tokens the plan saw are gone from the deployed item, so there is nothing to write. + if (JSON.stringify(next) === JSON.stringify(a.value ?? {})) return true + const policy = (await updatePolicy(next as App, a.policy)) as any + await AppService.updateApp({ + workspace, + path, + requestBody: { ...PRESERVE_DEPLOYED_IDENTITY, path, value: next, policy } + }) + return true +} + +/** + * One half of a deployed raw app's compiled bundle, read back the way the Hub publish reads + * it: `/apps/get_data/v/{secret}.{ext}` serves it to anyone holding the secret, and the + * secret is minted for the caller against a plain `apps:read:` check. + * + * A missing `.css` is an app that ships no styles. A missing `.js` is a broken deployment, + * and uploading an empty one in its place would break it further. + */ +async function fetchBundlePart( + workspace: string, + secret: string, + ext: 'js' | 'css' +): Promise { + const res = await fetch( + `/api/w/${encodeURIComponent(workspace)}/apps/get_data/v/${secret}.${ext}`, + { credentials: 'include' } + ) + if (res.ok) return await res.text() + if (ext === 'css' && res.status === 404) return '' + throw new Error(`the compiled bundle could not be read (${res.status})`) +} + +/** + * A raw app: its deployed value carries the sources and the runnables, and `updateAppRaw` + * refuses without a bundle. The bundle is the deployed one, read back, rewritten and sent + * again — rebuilding it is not possible from the browser and is not needed, but re-uploading + * it untouched is not an option either. + * + * The bundle is compiled from the sources, so a `$res:` a source file spells out is baked + * into it. The import rewrites that copy — `retargetProjectExport` runs while `/bundle.js` + * is still one of `files`, and only `installProject` splits it out afterwards. Sending the + * deployed bundle back unrewritten would undo on reuse what the import got right, and the + * app would keep reading a resource this run is about to delete. + */ +async function rewriteRawApp( + workspace: string, + r: Referrer, + map: Map, + a: any +): Promise { + const path = r.path + const value: any = a.value ?? {} + // One walk over the whole value: `$res:` tokens live in the runnables and can appear in + // the sources too, and both are plain text inside this JSON. + if (!r.gapped && readsPathUnreachably(value, map)) return UNREACHABLE_REFERENCE + const next = rewriteTokens(value, map) + const runnables = next.runnables ?? {} + const policy = (await updateRawAppPolicy(runnables, a.policy)) as any + const secret = await AppService.getPublicSecretOfLatestVersionOfApp({ workspace, path }) + const [deployedJs, deployedCss] = await Promise.all([ + fetchBundlePart(workspace, secret, 'js'), + fetchBundlePart(workspace, secret, 'css') + ]) + const js = rewriteContent(deployedJs, map) + const css = rewriteContent(deployedCss, map) + // `rewriteContent` moves the `$res:` tokens. A path the bundle spells out any other way + // is one nothing here can move, and uploading it would leave the app reading a resource + // about to be deleted. + for (const stub of map.keys()) if (textHoldsBarePath(js, stub)) return UNREACHABLE_REFERENCE + // The tokens the plan saw are gone from both the value and the bundle, so there is nothing + // to write, and uploading would cut a version differing from the last in nothing. + if (js === deployedJs && css === deployedCss && JSON.stringify(next) === JSON.stringify(value)) + return true + const files = { ...(next.files ?? {}) } + delete files['/bundle.js'] + delete files['/bundle.css'] + await AppService.updateAppRaw({ + workspace, + path, + formData: { + app: { + ...PRESERVE_DEPLOYED_IDENTITY, + path, + summary: a.summary ?? '', + value: { + files, + runnables, + ...(next.data !== undefined ? { data: next.data } : {}), + ...(next.datatables !== undefined ? { datatables: next.datatables } : {}) + }, + policy + }, + js, + css + } + }) + return true +} + +/** + * The trigger's own row, rewritten and written back. `enabled` is deliberately not sent: + * imported triggers are created disabled and re-enabling one is the user's decision, not a + * side effect of pointing it at a credential. + * + * `path` is put back from the row afterwards, and so is every runnable reference — see + * `restoreRunnableRefs`. The rewrite remaps any string equal to the stub's path, so a trigger + * sitting at the path the resource used to hold would otherwise be renamed along with the + * reference. + * + * A trigger states its run identity as `permissioned_as`, not the `on_behalf_of` the other + * kinds use, and `resolve_permissioned_as` keeps the row's value only when + * `preserve_permissioned_as` says so. Without the pair a trigger created under a folder's + * `default_permissioned_as` would start running as whoever picked the credential. + */ +async function rewriteTrigger( + workspace: string, + r: Referrer, + map: Map +): Promise { + const def = TRIGGER_KINDS[r.triggerKind!] + const row: any = r.row ?? {} + const { enabled: _enabled, ...rest } = { + ...restoreRunnableRefs(rewriteTriggerConfig(row, map), row), + path: r.path, + ...(typeof row.permissioned_as === 'string' + ? { permissioned_as: row.permissioned_as, preserve_permissioned_as: true } + : {}) + } + // No unreachable-mention check: a trigger holds the bare path by design, in its own + // resource field, and `rewriteTriggerConfig` remaps every bare match at every depth. What + // can still read `from` afterwards is a restored identity field, not a reference. + if (r.triggerKind === 'schedule') { + // `EditSchedule` needs these three; everything else on the row carries over by name. + await ScheduleService.updateSchedule({ + workspace, + path: r.path, + requestBody: { + ...rest, + schedule: rest.schedule ?? '0 0 * * * *', + timezone: rest.timezone ?? 'UTC', + args: rest.args ?? {} + } + }) + return true + } + await def.update!(workspace, r.path, rest) + return true +} diff --git a/frontend/src/lib/mcpEndpointTools.ts b/frontend/src/lib/mcpEndpointTools.ts index 3e8ae4cba0..ce4adf82be 100644 --- a/frontend/src/lib/mcpEndpointTools.ts +++ b/frontend/src/lib/mcpEndpointTools.ts @@ -885,7 +885,7 @@ export const mcpEndpointTools: EndpointTool[] = [ { name: "runScriptByPath", description: "run script by path", - instructions: "You should first use getScriptByPath to retrieve the script's schema and understand what arguments are expected.", + instructions: "You should first use getScriptByPath to retrieve the script's schema and understand what arguments are expected. A resource argument (schema format `resource-`) is the bare string `$res:` as that whole argument value -- not a wrapper object, not a plain path. A variable is `$var:`.", path: "/w/{workspace}/jobs/run/p/{path}", method: "POST", pathParamsSchema: { @@ -1426,7 +1426,7 @@ export const mcpEndpointTools: EndpointTool[] = [ { name: "runFlowByPath", description: "run flow by path", - instructions: "You should first use getFlowByPath to retrieve the flow's schema and understand what arguments are expected.", + instructions: "You should first use getFlowByPath to retrieve the flow's schema and understand what arguments are expected. A resource argument (schema format `resource-`) is the bare string `$res:` as that whole argument value -- not a wrapper object, not a plain path. A variable is `$var:`.", path: "/w/{workspace}/jobs/run/f/{path}", method: "POST", pathParamsSchema: { diff --git a/frontend/src/lib/schema.test.ts b/frontend/src/lib/schema.test.ts new file mode 100644 index 0000000000..195a7b9478 --- /dev/null +++ b/frontend/src/lib/schema.test.ts @@ -0,0 +1,62 @@ +import { describe, it, expect } from 'vitest' +import { argsToJsonPayload } from './schema' +import type { Schema } from './common' + +const schemaOf = (...names: string[]): Schema => + ({ + $schema: undefined, + type: 'object', + properties: Object.fromEntries(names.map((n) => [n, { type: 'string' }])), + required: [] + }) as Schema + +describe('argsToJsonPayload', () => { + it('spells out every schema property in schema order, unset ones as null', () => { + // `0`, `false` and `''` are values, not gaps: only a missing arg becomes `null`. + expect(argsToJsonPayload(schemaOf('a', 'b', 'c'), { c: 0, a: false })).toBe( + JSON.stringify({ a: false, b: null, c: 0 }, null, '\t') + ) + }) + + it('keeps args the schema does not declare, after the declared ones', () => { + expect(argsToJsonPayload(schemaOf('a'), { z: 9, a: 1 })).toBe( + JSON.stringify({ a: 1, z: 9 }, null, '\t') + ) + }) + + it('falls back to the schema default for an absent arg, but not over an explicit null', () => { + // `args` only carries defaults once a `SchemaForm` has mounted for that schema, and the + // JSON view alone never mounts one — seeding `null` there would commit `null` over the + // argument's default on the first keystroke. + const schema = schemaOf('a', 'b') + schema.properties.a.default = 'hi' + schema.properties.b.default = 42 + expect(argsToJsonPayload(schema, {})).toBe(JSON.stringify({ a: 'hi', b: 42 }, null, '\t')) + expect(argsToJsonPayload(schema, { a: null })).toBe( + JSON.stringify({ a: null, b: 42 }, null, '\t') + ) + }) + + it('keeps declared args named after Object.prototype members', () => { + // A plain `nargs[key]` read returns the inherited function for an unset `constructor`, + // and `JSON.stringify` drops function-valued properties — the argument would vanish. + expect(argsToJsonPayload(schemaOf('constructor', 'toString', 'ok'), {})).toBe( + JSON.stringify({ constructor: null, toString: null, ok: null }, null, '\t') + ) + }) + + it('keeps undeclared args named after Object.prototype members', () => { + // On a plain `{}` accumulator, `'constructor' in payload` is true before anything is + // assigned to it. + expect(argsToJsonPayload(undefined, { constructor: 'x', toString: 'y', ok: 1 })).toBe( + JSON.stringify({ constructor: 'x', toString: 'y', ok: 1 }, null, '\t') + ) + }) + + it('handles a missing schema or missing args', () => { + expect(argsToJsonPayload(undefined, undefined)).toBe('{}') + expect(argsToJsonPayload(schemaOf('a'), undefined)).toBe( + JSON.stringify({ a: null }, null, '\t') + ) + }) +}) diff --git a/frontend/src/lib/schema.ts b/frontend/src/lib/schema.ts index c96b635202..36faa8a590 100644 --- a/frontend/src/lib/schema.ts +++ b/frontend/src/lib/schema.ts @@ -52,3 +52,33 @@ export function schemaToObject(schema: Schema, args: Record): Objec }) return object } + +/** Args as the JSON payload the JSON editor starts from. Every schema property is spelled out, + * so an argument with no value yet still shows its name; args the schema does not declare are + * kept, since what the editor holds replaces the args wholesale on the next keystroke. */ +export function argsToJsonPayload( + schema: Schema | undefined, + args: Record | undefined +): string { + const nargs = args ?? {} + // Null prototype: an arg named after an `Object.prototype` member (`constructor`, + // `toString`) has to be an own key here, or the `in` check below reads it as already + // present and its value never reaches the payload. + const payload: Record = Object.create(null) + const props = schema?.properties ?? {} + // Schema order first, so the payload reads like the form it replaces. + for (const key of Object.keys(props)) { + // Own-property read: an arg named after an `Object.prototype` member (`constructor`, + // `toString`) would otherwise come back as the inherited function, which `JSON.stringify` + // drops. An arg that is merely absent falls back to the schema default — `args` only + // carries defaults once a `SchemaForm` has mounted, which the JSON view alone never does. + payload[key] = + (Object.prototype.hasOwnProperty.call(nargs, key) ? nargs[key] : props[key]?.default) ?? null + } + for (const [key, value] of Object.entries(nargs)) { + if (!(key in payload)) { + payload[key] = value + } + } + return JSON.stringify(payload, null, '\t') +} diff --git a/frontend/src/lib/utils/templateLiteral.test.ts b/frontend/src/lib/utils/templateLiteral.test.ts new file mode 100644 index 0000000000..3aa16ecf38 --- /dev/null +++ b/frontend/src/lib/utils/templateLiteral.test.ts @@ -0,0 +1,133 @@ +import { describe, it, expect } from 'vitest' +import { escapeTemplateBackticks, unescapeTemplateBackticks } from './templateLiteral' + +// Template mode stores its value as a JS template literal, so backticks in the text have to be +// escaped. Escaping them inside `${...}` too is what broke nested template literals: a backslash +// is a syntax error in expression position. +describe('escapeTemplateBackticks', () => { + const nested = + '--input ${flow_input.iter.value}${results.config ? ` --config ${results.config}` : ""} --host ${flow_input.hostname}' + + it('leaves a nested template literal inside an interpolation intact', () => { + const expr = '`' + escapeTemplateBackticks(nested) + '`' + expect(expr).not.toContain('\\`') + expect( + new Function('flow_input', 'results', 'return ' + expr)( + { iter: { value: 'data.csv' }, hostname: 'host1' }, + { config: '/tmp/cfg.json' } + ) + ).toBe('--input data.csv --config /tmp/cfg.json --host host1') + }) + + it('still escapes a backtick in the literal text', () => { + expect(escapeTemplateBackticks('a ` b')).toBe('a \\` b') + expect(escapeTemplateBackticks('a ` ${x} ` b')).toBe('a \\` ${x} \\` b') + }) + + it('leaves an escaped interpolation as literal text', () => { + // `\\${...}` is escaped in the template source, so the backticks inside it are literal + // text and still need escaping. + expect(escapeTemplateBackticks('\\${foo `bar`}')).toBe('\\${foo \\`bar\\`}') + expect(unescapeTemplateBackticks('\\${foo \\`bar\\`}')).toBe('\\${foo `bar`}') + expect( + () => new Function('return `' + escapeTemplateBackticks('\\${foo `bar`}') + '`') + ).not.toThrow() + }) + + // Braces, quotes, regex literals and comments all hide backticks and braces from anything + // short of a real lexer, which is why the parser decides. + it('handles text only a lexer can read correctly', () => { + const inputs = [ + '${ x["}"] } `', + "${ f({ a: '`' }) } `", + "${ x.replace(/'/g, '') } `", + '${ /* ` */ x } `', + '{"match": ${/{/.test(flow_input.x)}, "literal": "`x`"}' + ] + for (const v of inputs) { + expect(() => new Function('return `' + escapeTemplateBackticks(v) + '`')).not.toThrow() + expect(unescapeTemplateBackticks(escapeTemplateBackticks(v))).toBe(v) + } + }) + + // The failure that matters most is not a syntax error but literal text quietly becoming code: + // here the author's `+ flow_input.y +` must stay text rather than being evaluated. + it('never lets literal text escape into the expression', () => { + const v = '{"match": ${/{/.test(flow_input.x)}, "literal": "` + flow_input.y + `"}' + const evaluated = new Function('flow_input', 'return `' + escapeTemplateBackticks(v) + '`')({ + x: 'x', + y: 'LEAKED' + }) + expect(evaluated).not.toContain('LEAKED') + expect(evaluated).toContain('` + flow_input.y + `') + }) + + // An expression the old blanket rule broke — it escaped backticks inside `${...}`, which + // does not parse — comes back as the author typed it, instead of showing the backslashes + // and escaping them one deeper on every save. + it('heals an expression the old rule broke', () => { + const broken = '-p ${a}${b ? \\` --x ${c}\\` : ""}' + const clean = '-p ${a}${b ? ` --x ${c}` : ""}' + expect(unescapeTemplateBackticks(broken)).toBe(clean) + expect(escapeTemplateBackticks(clean)).toBe(clean) + }) + + // A text that already parses is left alone: an over-escaped legacy value and a backslash the + // author wrote are the same bytes, so healing on looks alone would drop a real character. + it('leaves an expression that already parses alone', () => { + const run = (body: string) => new Function('return `' + body + '`')() + for (const stored of ['${"\\`"}', '${"a\\\\`"}']) { + expect(unescapeTemplateBackticks(stored)).toBe(stored) + expect(run(escapeTemplateBackticks(unescapeTemplateBackticks(stored)))).toBe(run(stored)) + } + }) + + // ...but a backtick escaped inside a nested template belongs there and must survive. + it('leaves an escaped backtick that is inside a nested template alone', () => { + const stored = '${cond ? `a\\`b` : ""}' + expect(unescapeTemplateBackticks(stored)).toBe(stored) + }) + + // Escapes the author wrote inside a nested template are not the old rule's doing, and + // stripping them changes what the expression means — here into chained tagged templates, + // which throw. Only a text whose backticks are *all* escaped came from the old rule. + it('leaves escapes that belong to a nested template alone', () => { + const run = (body: string) => new Function('flag', 'value', 'return `' + body + '`')(true, 'X') + for (const stored of ['${flag ? `\\`\\`${value}\\`\\`` : ""}', '${flag ? `a\\`b` : ""}']) { + expect(unescapeTemplateBackticks(stored)).toBe(stored) + expect(escapeTemplateBackticks(unescapeTemplateBackticks(stored))).toBe(stored) + expect(run(stored)).toBe(run(escapeTemplateBackticks(unescapeTemplateBackticks(stored)))) + } + }) + + it('round-trips through unescapeTemplateBackticks', () => { + for (const v of [ + nested, + 'a ` b', + '${ x["}"] } `', + 'plain', + '${a}${b}', + '\\${a}', + '${cond ? `a\\`b` : ""}' + ]) { + expect(unescapeTemplateBackticks(escapeTemplateBackticks(v))).toBe(v) + } + }) + // The guarantee that matters: opening a flow in the editor and saving it back must not change + // the stored expression, including for a value that only the all-or-nothing fallback can + // handle. + it('never rewrites the stored expression on a view/save cycle', () => { + const inputs = [ + '{"match": ${/{/.test(flow_input.x)}, "literal": "`x`"}', + '${cond ? `a\\`b` : ""}', + "${ x.replace(/'/g, '') } `", + 'a ` b', + '${ /* ` */ x } `' + ] + for (const v of inputs) { + const stored = escapeTemplateBackticks(v) + expect(() => new Function('return `' + stored + '`')).not.toThrow() + expect(escapeTemplateBackticks(unescapeTemplateBackticks(stored))).toBe(stored) + } + }) +}) diff --git a/frontend/src/lib/utils/templateLiteral.ts b/frontend/src/lib/utils/templateLiteral.ts new file mode 100644 index 0000000000..388169ea95 --- /dev/null +++ b/frontend/src/lib/utils/templateLiteral.ts @@ -0,0 +1,59 @@ +import { parseExpressionAt } from 'acorn' + +/** + * Template mode stores what the author typed as a JS template literal, so the text is spliced + * between backticks. A backtick in the literal part has to be escaped or it ends the literal + * early — but one inside a `${...}` must not be, since a backslash is a syntax error in + * expression position and a nested template literal there is legitimate. + * + * Telling those apart means knowing where each `${...}` ends, which needs a real JS lexer: + * regex literals, comments and nested templates all hide braces and backticks from anything + * simpler. So rather than escaping selectively, ask the parser whether the text already reads as + * one template literal. If it does, it needs no escaping at all; if it does not, escape every + * backtick, which is what this did before nested templates were supported. + * + * Known limitation: a value mixing a bare literal backtick with a nested template cannot be + * expressed either way, and gets the all-or-nothing fallback. Escaping the literal one by hand + * makes the whole value parse and it is then kept verbatim. + */ +function isCompleteTemplateBody(text: string): boolean { + const source = '`' + text + '`' + try { + const node = parseExpressionAt(source, 0, { ecmaVersion: 'latest' }) + // The type is what rejects a body that closes its own literal early: `` ` + evil() + ` `` + // parses, but as a concatenation, and would evaluate the author's literal text. The span + // rejects a body that stops short, like `` `a` x ``. + return node.type === 'TemplateLiteral' && node.start === 0 && node.end === source.length + } catch { + return false + } +} + +/** Escape `text` so it can be wrapped in backticks and mean what the author typed. */ +export function escapeTemplateBackticks(text: string): string { + // No backtick means nothing to escape and nothing to decide, which is every ordinary value. + if (!text.includes('`')) { + return text + } + return isCompleteTemplateBody(text) ? text : text.replaceAll('`', '\\`') +} + +/** Inverse of {@link escapeTemplateBackticks}, for turning an expression back into a template. */ +export function unescapeTemplateBackticks(text: string): string { + if (!text.includes('`')) { + return text + } + const unescaped = text.replaceAll('\\`', '`') + if (escapeTemplateBackticks(unescaped) === text) { + return unescaped + } + // Only an expression the old blanket rule *broke* is healed: it escaped backticks inside + // `${...}` too, which does not parse, while the unescaped form does. A text that already + // parses is left alone even if it looks over-escaped, because the two are indistinguishable + // from the text alone and guessing changes what the expression means — `${"a\\\\`"}` is a + // backslash the author wrote, not one the old rule added. + if (!isCompleteTemplateBody(text) && isCompleteTemplateBody(unescaped)) { + return unescaped + } + return text +} diff --git a/frontend/src/lib/utils/workspaceId.test.ts b/frontend/src/lib/utils/workspaceId.test.ts new file mode 100644 index 0000000000..8972498138 --- /dev/null +++ b/frontend/src/lib/utils/workspaceId.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, it } from 'vitest' + +import { WORKSPACE_ID_MAX_LENGTH, toWorkspaceId, validateWorkspaceId } from './workspaceId' + +// This module was extracted so the import wizard and the create-workspace form +// could not drift on what a workspace id is. These pin the contract they share. + +describe('validateWorkspaceId', () => { + it('accepts dash-separated groups of word characters', () => { + for (const id of ['a', 'prod', 'my_ws', 'a-b-c', 'a1-2b', 'wm-fork-x']) { + expect(validateWorkspaceId(id), id).toBeUndefined() + } + }) + + it('rejects leading, trailing and doubled dashes, and anything outside \\w', () => { + for (const id of ['-a', 'a-', 'a--b', '', 'a b', 'a.b', 'a/b', 'é']) { + expect(validateWorkspaceId(id), id).toMatch(/letters, numbers and dashes/) + } + }) + + it('measures length against effectiveId, but the character rule against what was typed', () => { + const typed = 'a'.repeat(45) + // The typed id is fine on its own... + expect(validateWorkspaceId(typed)).toBeUndefined() + // ...but a fork submits it prefixed, and that is what the backend stores. + const prefixed = `wm-fork-${typed}` + expect(prefixed.length).toBeGreaterThan(WORKSPACE_ID_MAX_LENGTH) + expect(validateWorkspaceId(typed, prefixed)).toMatch(/too long/) + }) + + it('reports the character problem before the length one', () => { + expect(validateWorkspaceId('a b'.padEnd(80, 'c'))).toMatch(/letters, numbers and dashes/) + }) + + it('allows exactly the maximum length', () => { + const id = 'a'.repeat(WORKSPACE_ID_MAX_LENGTH) + expect(validateWorkspaceId(id)).toBeUndefined() + expect(validateWorkspaceId(id + 'a')).toMatch(/too long/) + }) +}) + +describe('toWorkspaceId', () => { + it('produces something validateWorkspaceId accepts', () => { + for (const raw of [ + 'Support automation', + ' GitHub Release Dashboard ', + 'a//b__c', + 'Ünïcødé nåme', + '---leading and trailing---', + 'MiXeD CaSe' + ]) { + const id = toWorkspaceId(raw) + expect(validateWorkspaceId(id), `${raw} -> ${id}`).toBeUndefined() + } + }) + + it('lowercases, collapses runs into single dashes, and trims them', () => { + expect(toWorkspaceId('Support automation')).toBe('support-automation') + expect(toWorkspaceId('a//b')).toBe('a-b') + expect(toWorkspaceId(' spaced out ')).toBe('spaced-out') + }) + + it('clips to the maximum length without leaving a trailing dash', () => { + // Slicing at 50 lands mid-separator here; the result must still be valid. + const raw = `${'a'.repeat(WORKSPACE_ID_MAX_LENGTH - 1)} tail` + const id = toWorkspaceId(raw) + expect(id.length).toBeLessThanOrEqual(WORKSPACE_ID_MAX_LENGTH) + expect(id.endsWith('-')).toBe(false) + expect(validateWorkspaceId(id)).toBeUndefined() + }) +}) + +describe('validateWorkspaceId — the reserved id', () => { + // `check_w_id_conflict` refuses it, and `existsWorkspace` reports it free, so without + // this the wizard walks the user to the last step before the create fails. + it('refuses `global`', () => { + expect(validateWorkspaceId('global')).toMatch(/not allowed/i) + }) + + it('refuses it as the effective id too', () => { + expect(validateWorkspaceId('wm-fork-x', 'global')).toMatch(/not allowed/i) + }) + + it('allows a fork named `global`, which reaches the backend as `wm-fork-global`', () => { + expect(validateWorkspaceId('global', 'wm-fork-global')).toBeUndefined() + }) + + it('still accepts ids that merely contain it', () => { + expect(validateWorkspaceId('global-ops')).toBeUndefined() + expect(validateWorkspaceId('my-global')).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/utils/workspaceId.ts b/frontend/src/lib/utils/workspaceId.ts new file mode 100644 index 0000000000..492a00d391 --- /dev/null +++ b/frontend/src/lib/utils/workspaceId.ts @@ -0,0 +1,53 @@ +/** + * What the backend accepts as a workspace id, in one place. Every screen that lets + * someone name a workspace validates against this — a laxer copy elsewhere only + * moves the rejection from the form to the create call, after the user has + * finished the whole flow. + */ + +/** Letters, digits and underscores in dash-separated groups: no leading, trailing or doubled dash. */ +export const WORKSPACE_ID_RE = /^\w+(-\w+)*$/ +/** The DB column and the git branch name derived from it both stop here. */ +export const WORKSPACE_ID_MAX_LENGTH = 50 + +/** `validate_workspace_name` (windmill-common/src/workspaces.rs:246) refuses a longer name. */ +export const WORKSPACE_NAME_MAX_LENGTH = 50 + +/** `check_w_id_conflict` (windmill-api-workspaces/src/workspaces.rs:5111) rejects this id. */ +const RESERVED_WORKSPACE_ID = 'global' + +/** + * The reason `id` is not a usable workspace id, or undefined when it is. + * + * `effectiveId` is what actually reaches the backend: a fork's id is submitted + * with a `wm-fork-` prefix, so the length limit applies to the prefixed form while + * the character rule still applies to what the user typed. + */ +export function validateWorkspaceId(id: string, effectiveId: string = id): string | undefined { + if (!WORKSPACE_ID_RE.test(id)) { + return 'ID can only contain letters, numbers and dashes and must not finish by a dash' + } + // `check_w_id_conflict` refuses it outright, and `existsWorkspace` reports it free — + // so without this the wizard walks the user to the last step before the create fails. + // Only the effective id, which is what the backend receives: it defaults to the raw one, + // so a plain `global` is still caught, while a fork named `global` — submitted as + // `wm-fork-global`, which the backend accepts — is not. + if (effectiveId === RESERVED_WORKSPACE_ID) { + return `'${RESERVED_WORKSPACE_ID}' is not allowed as a workspace ID` + } + if (effectiveId.length > WORKSPACE_ID_MAX_LENGTH) { + return `ID '${effectiveId}' is too long (${effectiveId.length} chars). Maximum is ${WORKSPACE_ID_MAX_LENGTH}.` + } + return undefined +} + +/** Slugifies free text into something `validateWorkspaceId` accepts, for a prefill. */ +export function toWorkspaceId(raw: string): string { + return raw + .toLowerCase() + .replace(/[^a-z0-9-]+/g, '-') + .replace(/-{2,}/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, WORKSPACE_ID_MAX_LENGTH) + .replace(/-+$/, '') +} diff --git a/frontend/src/lib/workspaceCreation.ts b/frontend/src/lib/workspaceCreation.ts new file mode 100644 index 0000000000..49e4bdbea3 --- /dev/null +++ b/frontend/src/lib/workspaceCreation.ts @@ -0,0 +1,80 @@ +/** + * The instance policy around creating a workspace, and what the client has to do + * once one exists. Shared by every screen that creates workspaces — the workspace + * settings creator and the hub import wizard — so the two cannot drift apart on + * who is allowed to create, what username the backend expects, or what state the + * client is left in afterwards. + * + * Id validation lives next door in `$lib/utils/workspaceId`, which has no imports + * and so can be used from anywhere. + */ + +import { SettingService, UserService, WorkspaceService } from '$lib/gen' +import { usersWorkspaceStore } from '$lib/stores' +import { switchWorkspace } from '$lib/storeUtils' +import { isCloudHosted } from '$lib/cloud' +import { base } from '$lib/base' + +/** + * Whether this user may create a workspace at all. Self-hosted instances default + * `CREATE_WORKSPACE_REQUIRE_SUPERADMIN` to true, so offering the choice to + * everyone ends in a 403 at the last step. + * + * Superadmin arrives asynchronously in most callers, so pass the current value and + * call again when it flips. When the gate is on, the server is asked directly + * rather than trusting that value: `refreshSuperadmin` skips its fetch once the + * store holds anything, so a page loaded logged out leaves it `false` for the rest + * of the session — including right after signing in. + */ +export async function canCreateWorkspace(isSuperadmin: boolean): Promise { + if (isSuperadmin || isCloudHosted()) return true + try { + const r = await fetch(base + '/api/workspaces/create_workspace_require_superadmin') + if ((await r.text()) != 'true') return true + return !!(await UserService.globalWhoami()).super_admin + } catch { + return false + } +} + +export interface UsernamePolicy { + /** When true the backend derives the username and rejects one sent explicitly. */ + automate: boolean + /** A username to prefill with, when the caller has to ask for one. */ + suggested?: string +} + +/** + * `createWorkspace` rejects a username when the instance automates them and + * requires one when it does not, so the field only exists in the second case. + */ +export async function loadUsernamePolicy(): Promise { + const automate = + ((await SettingService.getGlobal({ + key: 'automate_username_creation' + })) as boolean | null) ?? true + if (automate) return { automate: true } + try { + const me = await UserService.globalWhoami() + const from = me.name ? me.name.split(' ')[0] : me.email.split('@')[0] + return { automate: false, suggested: from.replace(/\./g, '').toLowerCase() } + } catch { + return { automate: false } + } +} + +/** + * Re-reads the workspaces this user belongs to. Anything that creates or deletes a + * workspace owes the client this call: `usersWorkspaceStore` is what the picker and + * every derived workspace list read from, and nothing else refreshes it until a + * full page load. + */ +export async function refreshWorkspaceList(): Promise { + usersWorkspaceStore.set(await WorkspaceService.listUserWorkspaces()) +} + +/** Enters a workspace that was just created, with the list refreshed to match. */ +export async function enterNewWorkspace(id: string): Promise { + await refreshWorkspaceList() + switchWorkspace(id) +} diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index 257af0fbbd..a20c61f2ff 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -83,6 +83,7 @@ import SessionPicker from '$lib/components/sessions/SessionPicker.svelte' import SessionModeSwitch from '$lib/components/sessions/SessionModeSwitch.svelte' import { isGlobalAiEnabled } from '$lib/components/copilot/chat/global/gate' + import { copilotInfo } from '$lib/aiStore' import { parsePreviewItemRoute } from '$lib/components/sessions/previewPaths' import { rememberNavRoute } from '$lib/components/sessions/sessionSwitch.svelte' import { sessionState } from '$lib/components/sessions/sessionState.svelte' @@ -250,6 +251,10 @@ // so it follows the gate; opted-out users get the legacy Ask-AI pane instead. // The /sessions page has its own gate for direct navigation. const globalAiEnabled = isGlobalAiEnabled() + // A workspace that hid the assistant (`ai_config.copilot_disabled`) loses both entry + // points: the Workspace ⇄ Sessions switch and the legacy Ask-AI button. + const sessionsSwitchShown = $derived(globalAiEnabled && !$copilotInfo.workspaceDisabled) + const askAiShown = $derived(!globalAiEnabled && !$copilotInfo.workspaceDisabled) if (page.status == 404) { goto('/user/login') @@ -999,7 +1004,7 @@
    - {#if !embedded && globalAiEnabled} + {#if !embedded && sessionsSwitchShown}
    -
    +
    {/if} @@ -1047,7 +1052,7 @@ class="!text-xs" shortcut={`${getModifierKey()}k`} /> - {#if !globalAiEnabled} + {#if askAiShown}
    - {#if !embedded && globalAiEnabled} + {#if !embedded && sessionsSwitchShown}
    @@ -1145,7 +1150,7 @@ -
    +
    {/if} @@ -1184,7 +1189,7 @@ class="!text-xs" shortcut={`${getModifierKey()}k`} /> - {#if !globalAiEnabled} + {#if askAiShown} - aiChatManager.toggleOpen()} - {isCollapsed} - icon={WandSparkles} - iconProps={{ - forceDarkMode: true - }} - label="Ask AI" - class="!text-xs" - iconClasses="!text-ai" - shortcut={`${getModifierKey()}L`} - /> + {#if !$copilotInfo.workspaceDisabled} + aiChatManager.toggleOpen()} + {isCollapsed} + icon={WandSparkles} + iconProps={{ + forceDarkMode: true + }} + label="Ask AI" + class="!text-xs" + iconClasses="!text-ai" + shortcut={`${getModifierKey()}L`} + /> + {/if}
    -
    - {#if $workspaceStore == 'admins'} -
    +
    + + + {#if isGlobalAiEnabled()} + + {/if} + + {#if $workspaceStore == 'admins'} The Admins workspace is for admins only and contains scripts whose purpose is to manage your Windmill instance, such as keeping resource types up to date. +
    {/if} -
    -

    - Home -

    -
    - {#if !$userStore?.operator && HOME_SHOW_HUB} - - {/if} - - {#if !$userStore?.operator && showCreateButtons} -
    - -
    - {/if} -
    -
    - - (showCreateButtons = v)} /> @@ -396,9 +359,8 @@
    {#if tab == 'workspace'} - + {/if}
    - diff --git a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte index 96f0c4f2f6..6d2a0a5ca4 100644 --- a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte @@ -732,9 +732,6 @@ rightTooltip: 'Fill args from JSON' }} lightMode - on:change={(e) => { - runForm?.setCode(JSON.stringify(args ?? {}, null, '\t')) - }} /> {/if}
    @@ -817,7 +814,7 @@ const nargs = JSON.parse(JSON.stringify(e.detail)) args = nargs if (jsonView) { - runForm?.setCode(JSON.stringify(args ?? {}, null, '\t')) + runForm?.syncJsonEditor() } }} /> diff --git a/frontend/src/routes/(root)/(logged)/folders/+page.svelte b/frontend/src/routes/(root)/(logged)/folders/+page.svelte index e35c4a7381..b184e4aa4a 100644 --- a/frontend/src/routes/(root)/(logged)/folders/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/folders/+page.svelte @@ -4,11 +4,10 @@ import CenteredPage from '$lib/components/CenteredPage.svelte' import Dropdown from '$lib/components/DropdownV2.svelte' - import FolderEditor from '$lib/components/FolderEditor.svelte' + import FolderEditorDrawer from '$lib/components/FolderEditorDrawer.svelte' import PageHeader from '$lib/components/PageHeader.svelte' import { userStore, workspaceStore, userWorkspaces } from '$lib/stores' import { Button, Drawer, DrawerContent, EmptyState, Skeleton } from '$lib/components/common' - import Popover from '$lib/components/meltComponents/Popover.svelte' import FolderInfo from '$lib/components/FolderInfo.svelte' import FolderUsageInfo from '$lib/components/FolderUsageInfo.svelte' import { sendUserToast } from '$lib/utils' @@ -28,9 +27,8 @@ isDemoWorkspaceRestricted($workspaceStore, $userStore?.is_admin, $userStore?.is_super_admin) ) - let newFolderName: string = $state('') let folders: FolderW[] | undefined = $state(undefined) - let folderDrawer: Drawer | undefined = $state() + let folderEditorDrawer: FolderEditorDrawer | undefined = $state() let hubDrawer: Drawer | undefined = $state() let publishFolderName: string = $state('') @@ -47,23 +45,11 @@ }) } - function handleKeyUp(event: KeyboardEvent, close: () => void) { - const key = event.key - if (key === 'Enter') { - event.preventDefault() - addFolder() - close() - } - } - async function addFolder() { - await FolderService.createFolder({ - workspace: $workspaceStore ?? '', - requestBody: { name: newFolderName } - }) - $userStore?.folders.push(newFolderName) - loadFolders() - editFolderName = newFolderName - folderDrawer?.openDrawer() + function onFolderSaved(name: string, created: boolean) { + if (created) $userStore?.folders.push(name) + // Returned, not fired: the drawer reports a failed reload, and it can only see one + // through the promise this hands back. + return loadFolders() } $effect(() => { @@ -74,8 +60,6 @@ } }) - let editFolderName: string = $state('') - function computeMembers(owners: string[], extra_perms: Record) { const members = new Set(owners) for (const [user, _] of Object.entries(extra_perms)) { @@ -85,48 +69,18 @@ } -{#snippet newFolderPopover( - label: string, - placement: 'bottom' | 'bottom-end', - variant: 'accent' | 'default' -)} - folderEditorDrawer?.initNew()} > - {#snippet trigger()} - - {/snippet} - {#snippet content({ close })} - handleKeyUp(e, () => close())} - placeholder="New folder name" - bind:value={newFolderName} - /> - -
    - -
    - {/snippet} -     + {label} + {/snippet} - - - - - + {:else} - {@render newFolderPopover('New folder', 'bottom-end', 'accent')} + {@render newFolderButton('New folder', 'accent')} {/if}
    @@ -181,7 +135,7 @@ description="Folders are how you grant permissions: make a user or group viewer, writer or admin on a folder and that access applies to every script, flow, app, resource and schedule inside it." > {#if !restricted} - {@render newFolderPopover('Add a folder', 'bottom', 'default')} + {@render newFolderButton('Add a folder', 'default')} {/if} {:else} @@ -196,8 +150,8 @@ Schedules Variables Resources - Participants - + Members + Actions @@ -211,13 +165,7 @@ {/each} {:else} {#each folders as { name, extra_perms, owners, canWrite, summary, labels } (name)} - { - editFolderName = name - folderDrawer?.openDrawer() - }} - > + folderEditorDrawer?.initEdit(name)}> {name} {#if summary} @@ -250,17 +198,14 @@ - + { - editFolderName = name - folderDrawer?.openDrawer() - } + action: () => folderEditorDrawer?.initEdit(name) }, { displayName: 'Publish to Hub', diff --git a/frontend/src/routes/(root)/(logged)/groups/+page.svelte b/frontend/src/routes/(root)/(logged)/groups/+page.svelte index 85b4b536fe..ce8c6a8db0 100644 --- a/frontend/src/routes/(root)/(logged)/groups/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/groups/+page.svelte @@ -5,9 +5,8 @@ import CenteredPage from '$lib/components/CenteredPage.svelte' import { Button, Drawer, DrawerContent, Skeleton } from '$lib/components/common' - import Popover from '$lib/components/meltComponents/Popover.svelte' import Dropdown from '$lib/components/DropdownV2.svelte' - import GroupEditor from '$lib/components/GroupEditor.svelte' + import GroupEditorDrawer from '$lib/components/GroupEditorDrawer.svelte' import InstanceGroupEditor from '$lib/components/InstanceGroupEditor.svelte' import GroupInfo from '$lib/components/GroupInfo.svelte' import PageHeader from '$lib/components/PageHeader.svelte' @@ -20,7 +19,6 @@ import Cell from '$lib/components/table/Cell.svelte' import Row from '$lib/components/table/Row.svelte' import { untrack } from 'svelte' - import TextInput from '$lib/components/text_input/TextInput.svelte' import { Tooltip } from '$lib/components/meltComponents' import { DEMO_RESTRICTION_HINT, isDemoWorkspaceRestricted } from '$lib/cloud' @@ -30,10 +28,9 @@ isDemoWorkspaceRestricted($workspaceStore, $userStore?.is_admin, $userStore?.is_super_admin) ) - let newGroupName: string = $state('') let groups: GroupW[] | undefined = $state(undefined) let instanceGroups: InstanceGroupWithWorkspaces[] | undefined = $state(undefined) - let groupDrawer: Drawer | undefined = $state() + let groupEditorDrawer: GroupEditorDrawer | undefined = $state() async function loadGroups(): Promise { groups = (await GroupService.listGroups({ workspace: $workspaceStore! })).map((x) => { @@ -49,24 +46,6 @@ } } - function handleKeyUp(event: KeyboardEvent, close: () => void) { - const key = event.key - if (key === 'Enter') { - event.preventDefault() - addGroup() - close() - } - } - async function addGroup() { - await GroupService.createGroup({ - workspace: $workspaceStore ?? '', - requestBody: { name: newGroupName } - }) - loadGroups() - editGroupName = newGroupName - groupDrawer?.openDrawer() - } - $effect(() => { untrack(() => loadInstanceGroups()) if ($workspaceStore && $userStore) { @@ -74,16 +53,11 @@ } }) - let editGroupName: string = $state('') let instanceGroupDrawer: Drawer | undefined = $state() let editInstanceGroupName: string = $state('') - - - - - + {:else} - - {#snippet trigger()} - - {/snippet} - {#snippet content({ close })} -
    - handleKeyUp(e, close) - }} - bind:value={newGroupName} - /> - -
    - {/snippet} -     + {/if}
    @@ -161,7 +112,7 @@ Name Members - + Actions @@ -175,13 +126,7 @@ {/each} {:else} {#each groups as { name, summary, extra_perms, canWrite } (name)} - { - editGroupName = name - groupDrawer?.openDrawer() - }} - > + groupEditorDrawer?.initEdit(name)}>
    @@ -197,7 +142,7 @@ - + { e?.stopPropagation() - editGroupName = name - groupDrawer?.openDrawer() + groupEditorDrawer?.initEdit(name) } }, { diff --git a/frontend/src/routes/(root)/(logged)/projects/import/+page@(root).svelte b/frontend/src/routes/(root)/(logged)/projects/import/+page@(root).svelte new file mode 100644 index 0000000000..a6f4632526 --- /dev/null +++ b/frontend/src/routes/(root)/(logged)/projects/import/+page@(root).svelte @@ -0,0 +1,551 @@ + + +{#if leaving} + +
    + + Taking you to {planWorkspaceId(plan) ?? 'your workspace'}… +
    +     +{:else if !slug} + +

    + This page needs a ?hub=<slug> to know which project to import. + Open it from a project on the hub. +

    +     +{:else} + + + + {#snippet subtitleSnippet()} + + Connected as {$usersWorkspaceStore?.email} + {#if step === 1} + · + + Switch account + + {/if} + + {/snippet} + + + + {#if step === 1} +
    + {#if project} + + {:else if projectError} +

    + Could not read {slug} from the hub. You can still choose a + destination — the project is fetched again before it is imported. +

    + {:else} +
    + Loading {slug}… +
    + {/if} + +
    +

    Where should it go?

    + + +
    + {#if canCreate} + (chosen = 'new')} + > + {#snippet icon()} + + {/snippet} + {#snippet description()} + Creates {name || slug} and imports + {itemsLabel} into it. + {/snippet} + + {/if} + + (chosen = 'existing')} + > + {#snippet icon()} + + {/snippet} + +
    + +
    + +
    +
    +
    + {:else if step === 2} +
    + {#if !choiceIsExisting} +
    +

    Name the new workspace

    +
    + +
    + + +
    + + {#if !automateUsername} + + {/if} + {:else} +
    +

    Pick a workspace

    +

    The project is imported into this one.

    +
    + + {#if workspaceList.loading} +
    + Loading your workspaces… +
    + {:else if workspaceList.error} +

    + Could not list your workspaces. Reload the page, or go back and create a new one. +

    + {:else if workspaces.length === 0} +

    + You are not a member of any workspace yet. Go back and create one, or ask an admin to + invite you. +

    + {:else} + + {#if workspaces.length > 1} +
    +
    + + +
    + {#if hasForks} + + {/if} +
    + {/if} + + + + {/if} + {/if} + +
    + + {#if !choiceIsExisting} + + {/if} +
    +
    + {:else if step === 3} + go({ folder }, 3, { replace: true })} + onFinish={() => + setupNeeded + ? // Replaces rather than pushes: after a reload on step 4 the run is gone, and + // a step-3 entry in history is a browser-Back route to the same fresh import + // the stepper is now blocked from reaching. + go({}, 4, { replace: true }) + : finish()} + onBack={() => go({}, 2)} + onExecution={(e) => (execution = e)} + resume={execution} + /> + {:else} + go({}, 3) : undefined} + /> + {/if} +     +{/if} diff --git a/frontend/src/routes/(root)/(logged)/projects/install/+page.svelte b/frontend/src/routes/(root)/(logged)/projects/install/+page.svelte deleted file mode 100644 index 06f2344356..0000000000 --- a/frontend/src/routes/(root)/(logged)/projects/install/+page.svelte +++ /dev/null @@ -1,374 +0,0 @@ - - -
    - {#if !slug} -

    Missing ?hub=<slug>.

    - {:else if loading} -
    - Loading project… -
    - {:else if loadError} -

    Failed to load project: {loadError}

    - {:else if data} -

    Add “{data.project.name}” to workspace

    -

    {data.project.summary}

    - -
    -

    - Folder in {workspace} -

    - -

    - Items import under f/{folderName.trim() || data.project.slug}/. -

    -
    - -
    - {counts?.scripts} scripts - {counts?.flows} flows - {counts?.apps} apps - {counts?.resources} resources - {counts?.triggers} triggers - {#if counts && counts.migrations > 0} - {counts.migrations} data table migrations - {/if} -
    - -
    - Resources are imported as empty stubs — set their values after import; a resource whose path - already exists is reported as failed (existing values are never overwritten). Trigger kinds - are recreated disabled, except GCP and Azure triggers, which manage cloud subscriptions at - creation and must be re-created manually after filling their resource. Kafka, NATS, SQS, GCP - and Azure triggers all require Enterprise. Triggers that reference a resource depend on stubs - imported empty, so fill in the resource value before re-enabling the trigger. -
    - -
    - - {#if done} - - {/if} -
    - - {#if results.length} -
      - {#each results as r} -
    • - {r.ok ? '✓' : '✗'} - {r.path} - {#if !r.ok}— {r.error}{/if} -
      • - {/each} -
    - {/if} - {/if} -
    - - - - - - closeMigrationReview(false)}> - closeMigrationReview(false)}> -
    -

    - This project ships migrations that recreate the data tables it uses. Review and edit the - SQL, then choose which to run. A migration runs against the data table of the same name in - {workspace}; if that data table has migrations enabled it is - recorded, otherwise it runs once as a preview job. -

    - {#each reviewList as m (m.datatable_name)} -
    -
    - {m.datatable_name} - -
    - {#if m.run} - - {/if} -
    - {/each} -
    - {#snippet actions()} - - - {/snippet} -     -     diff --git a/frontend/src/routes/(root)/(logged)/projects/install/+page.ts b/frontend/src/routes/(root)/(logged)/projects/install/+page.ts new file mode 100644 index 0000000000..718b624bd4 --- /dev/null +++ b/frontend/src/routes/(root)/(logged)/projects/install/+page.ts @@ -0,0 +1,13 @@ +import { redirect } from '@sveltejs/kit' +import { base } from '$app/paths' +import type { PageLoad } from './$types' + +/** + * `/projects/install?hub=` was where the hub's "Add to workspace" button + * pointed before the import wizard existed. Hubs upgrade on their own schedule — + * a self-hosted one may keep sending people here for a long time — so the old + * entry point forwards to the wizard rather than 404ing, query string intact. + */ +export const load: PageLoad = ({ url }) => { + redirect(307, `${base}/projects/import${url.search}`) +} diff --git a/frontend/src/routes/(root)/(logged)/resources/+page.svelte b/frontend/src/routes/(root)/(logged)/resources/+page.svelte index 6f3fd1a6dc..4932355654 100644 --- a/frontend/src/routes/(root)/(logged)/resources/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/resources/+page.svelte @@ -1086,8 +1086,8 @@ Path Resource type Description - - + Status + Actions @@ -1254,8 +1254,8 @@ {/if}
    - -
    + +
    {#if path && assetCanBeExplored({ kind: 'resource', path }, { resource_type }) && !$userStore?.operator} Name Description - + Actions @@ -1418,7 +1418,7 @@
    - + {#if !canWrite} diff --git a/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte b/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte index 6b3b0e5c1f..f0023ddb49 100644 --- a/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte @@ -388,7 +388,7 @@ if (!held || !current || block?.label !== 'retry' || block['dbt_retry_job']) return args = { ...current, command: { ...block, dbt_retry_job: held } } if (jsonView) { - runForm?.setCode(JSON.stringify(args, null, '\t')) + runForm?.syncJsonEditor() } }) .catch(() => {}) @@ -430,7 +430,7 @@ } } if (jsonView) { - runForm?.setCode(JSON.stringify(args, null, '\t')) + runForm?.syncJsonEditor() } } @@ -934,9 +934,6 @@ rightTooltip: 'Fill args from JSON' }} lightMode - on:change={(e) => { - runForm?.setCode(JSON.stringify(args ?? {}, null, '\t')) - }} /> {/if}
    @@ -1051,7 +1048,7 @@ const nargs = JSON.parse(JSON.stringify(e.detail)) args = nargs if (jsonView) { - runForm?.setCode(JSON.stringify(args ?? {}, null, '\t')) + runForm?.syncJsonEditor() } }} /> diff --git a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte index b6f5029616..da8fc31c65 100644 --- a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte @@ -1,5 +1,5 @@ diff --git a/frontend/src/routes/(root)/(logged)/user/(user)/workspaces/+page.svelte b/frontend/src/routes/(root)/(logged)/user/(user)/workspaces/+page.svelte index c4d2168d26..ed9bf2b11e 100644 --- a/frontend/src/routes/(root)/(logged)/user/(user)/workspaces/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/user/(user)/workspaces/+page.svelte @@ -26,6 +26,7 @@ import { switchWorkspace } from '$lib/storeUtils' import { GitFork, Settings, User, Search, ChevronsDownUp, ChevronsUpDown } from 'lucide-svelte' import { isCloudHosted } from '$lib/cloud' + import { canCreateWorkspace } from '$lib/workspaceCreation' import AnimatedButton from '$lib/components/common/button/AnimatedButton.svelte' import { emptyString } from '$lib/utils' import { getUserExt } from '$lib/user' @@ -104,9 +105,7 @@ let onlyAdminsWorkspace = $derived(allWorkspaces.length === 1 && allWorkspaces[0].id === 'admins') async function getCreateWorkspaceRequireSuperadmin() { - const r = await fetch(base + '/api/workspaces/create_workspace_require_superadmin') - const t = await r.text() - createWorkspace = t != 'true' + createWorkspace = await canCreateWorkspace(false) } let createWorkspace = $state($superadmin || isCloudHosted()) diff --git a/frontend/src/routes/(root)/(logged)/variables/+page.svelte b/frontend/src/routes/(root)/(logged)/variables/+page.svelte index 0a23656514..016841f1ae 100644 --- a/frontend/src/routes/(root)/(logged)/variables/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/variables/+page.svelte @@ -391,8 +391,8 @@ Path Value Description - - + Status + Actions @@ -494,7 +494,7 @@ {#if refresh_error} + so it can't paint over anything that scrolls past it -->
    @@ -546,7 +546,7 @@ {/if}
    - + { let owner = isOwner(path, $userStore, $workspaceStore) diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index 782da1bb2c..8dd2937273 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -41,7 +41,7 @@ import { sendUserToast } from '$lib/toast' import { clone, emptyString, encodeState, hasUnsavedChanges } from '$lib/utils' import { downloadViaClient, shouldDownloadViaClient } from '$lib/utils/downloadFile' - import { Slack, Target } from 'lucide-svelte' + import { ExternalLink, Slack, Target } from 'lucide-svelte' import SidebarNavigation from '$lib/components/common/sidebar/SidebarNavigation.svelte' import PremiumInfo from '$lib/components/settings/PremiumInfo.svelte' @@ -1472,9 +1472,30 @@ This workspace is a fork of {currentWorkspace.parent_workspace_id}. It runs on the parent's plan and its executions count toward the parent's usage and bill, - so there is no separate subscription here. Manage billing, seats, and quotas from - the parent workspace's settings. + so it is never invoiced separately. Manage billing, seats, and quotas from the + parent workspace's settings. + {#if plan} +
    + + It is on a paid plan that is billed on its own, so this workspace is paid for + twice. Cancel that subscription in the customer portal to keep only + {currentWorkspace.parent_workspace_id}'s plan. This workspace keeps + running either way, on the parent's plan. + {#if customer_id} +
    + +
    + {/if} +     +
    + {/if} {:else} {/if} diff --git a/frontend/src/routes/(root)/+layout.svelte b/frontend/src/routes/(root)/+layout.svelte index 1f7a813eac..0e404afbcc 100644 --- a/frontend/src/routes/(root)/+layout.svelte +++ b/frontend/src/routes/(root)/+layout.svelte @@ -1,4 +1,5 @@