From 6131f7d2aebbddc26a0e3b06c034b6df5fae18cf Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 17 Jul 2026 00:22:56 +0200 Subject: [PATCH 01/14] chore(main): release 1.761.0 (#10148) * chore(main): release 1.761.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 27 ++ backend/Cargo.lock | 252 +++++++++--------- backend/Cargo.toml | 4 +- .../parsers/windmill-parser-wasm/Cargo.lock | 48 ++-- .../parsers/windmill-parser-wasm/Cargo.toml | 2 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/core/constants.ts | 2 +- frontend/package-lock.json | 54 +++- frontend/package.json | 2 +- lsp/Pipfile | 2 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 17 files changed, 241 insertions(+), 168 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed3e5e116b..7b65e78e8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,32 @@ # Changelog +## [1.761.0](https://github.com/windmill-labs/windmill/compare/v1.760.1...v1.761.0) (2026-07-16) + + +### Features + +* **ai-sessions:** CRUD markdown artifacts in sessions ([#10046](https://github.com/windmill-labs/windmill/issues/10046)) ([0ea5705](https://github.com/windmill-labs/windmill/commit/0ea570570e65abf64b3b171e592a8dd3eea0b105)) +* **alerts:** include disk total and top consumers in low-disk alert ([#10144](https://github.com/windmill-labs/windmill/issues/10144)) ([4e0fd4d](https://github.com/windmill-labs/windmill/commit/4e0fd4db5589f91426ca025184da7b977d5a16d6)) +* automatic git-to-windmill sync (polling, webhooks, in-app PRs + checks) ([#9552](https://github.com/windmill-labs/windmill/issues/9552)) ([51d8db6](https://github.com/windmill-labs/windmill/commit/51d8db6602bc192879e50b54a5fe6b6b8beb63f4)) +* display openai reasoning summaries in ai chat ([#10147](https://github.com/windmill-labs/windmill/issues/10147)) ([4ee1d32](https://github.com/windmill-labs/windmill/commit/4ee1d32101eb527ef707c3c4af456892f0a059ab)) +* **forks:** add "Hide unchanged drafts" toggle to fork deploy-draft tab ([#10022](https://github.com/windmill-labs/windmill/issues/10022)) ([4fc3f30](https://github.com/windmill-labs/windmill/commit/4fc3f304c632eda15876c28cb6f473a799a41f48)) +* **frontend:** flatten workspace pickers, whole-tab picker trigger ([#10145](https://github.com/windmill-labs/windmill/issues/10145)) ([7fda6a0](https://github.com/windmill-labs/windmill/commit/7fda6a05345d14f62dc2124d682650d50eecbe98)) + + +### Bug Fixes + +* **ai:** show the question in askUserQuestion tool-call labels ([#10153](https://github.com/windmill-labs/windmill/issues/10153)) ([fa03984](https://github.com/windmill-labs/windmill/commit/fa03984a14bb71f0ea2a27bfa23339a9d0d8e86b)) +* **alerts:** identify server replica in low-disk alert + per-host dedup tag ([#10143](https://github.com/windmill-labs/windmill/issues/10143)) ([3bd9f05](https://github.com/windmill-labs/windmill/commit/3bd9f0593866740935f79c8b2c1cb92829059105)) +* **frontend:** sanitize job result markup, gate it on unsandboxed public apps ([#10127](https://github.com/windmill-labs/windmill/issues/10127)) ([7b813d1](https://github.com/windmill-labs/windmill/commit/7b813d1f74baef200d9f1387e02925083b79b265)) +* **frontend:** show friendly draft path for draft-only items in pickers ([#10136](https://github.com/windmill-labs/windmill/issues/10136)) ([568dbbe](https://github.com/windmill-labs/windmill/commit/568dbbee852d59986581b0d16f5ac3e06d6bad6a)) +* **frontend:** surface real tool call errors in AI chat ([#10146](https://github.com/windmill-labs/windmill/issues/10146)) ([0694b84](https://github.com/windmill-labs/windmill/commit/0694b84da7bbb9404fd386e3ce109a574a0e9c93)) +* heartbeat job ping during s3object materialization in SQL executors ([#10152](https://github.com/windmill-labs/windmill/issues/10152)) ([7d5009e](https://github.com/windmill-labs/windmill/commit/7d5009e3928bb200150eb998ed0b3016c3395987)) +* **mcp:** push granular scope patterns into SQL so scoped scripts/flows aren't truncated ([#10140](https://github.com/windmill-labs/windmill/issues/10140)) ([91d6606](https://github.com/windmill-labs/windmill/commit/91d6606868d0e9c5f78ab48a592ef95ffaeeca61)) +* **migrations:** grant zombie_job_counter to windmill roles ([#10159](https://github.com/windmill-labs/windmill/issues/10159)) ([0e547ad](https://github.com/windmill-labs/windmill/commit/0e547adf23b615f0caccc88eb9101bb0604bb51b)) +* **raw-apps:** full code ui builder improvements ([c55ac53](https://github.com/windmill-labs/windmill/commit/c55ac5326fbb4532f22b7250a58b69396a4c868a)) +* **raw-apps:** prevent and surface the silent blank screen from an unmounted #root ([#10150](https://github.com/windmill-labs/windmill/issues/10150)) ([24750e6](https://github.com/windmill-labs/windmill/commit/24750e6ef1a1a975e0063df6809774f913b5d01b)) +* **self-host:** unbreak self-hosted Caddy after the caddy-l4 syntax change ([#10156](https://github.com/windmill-labs/windmill/issues/10156)) ([2f6c35b](https://github.com/windmill-labs/windmill/commit/2f6c35b15bf70f17eb8ff08281e0ecf8313e7b35)) + ## [1.760.1](https://github.com/windmill-labs/windmill/compare/v1.760.0...v1.760.1) (2026-07-15) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index f8c671d63e..64427a820c 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1532,7 +1532,7 @@ version = "0.71.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cexpr", "clang-sys", "itertools 0.13.0", @@ -1552,7 +1552,7 @@ version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cexpr", "clang-sys", "itertools 0.13.0", @@ -1595,9 +1595,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.13.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" dependencies = [ "serde_core", ] @@ -1761,9 +1761,9 @@ dependencies = [ [[package]] name = "borsh" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f3f6da4992df95bbcd9af42a6c7dcb994498fc9048230405f3b36ff7cd3f145" +checksum = "a88b7ea17d208c4193f2c1e6de3c35fe71f98c96982d5ced308bdcc749ff6e1f" dependencies = [ "borsh-derive", "bytes", @@ -1772,9 +1772,9 @@ dependencies = [ [[package]] name = "borsh-derive" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae8fb4fb5740e4b2c4884ff95f5f32f5e8479db1e8fd8eb49ddbe09eb09bb7c" +checksum = "d8f347189c62a579b8cd5f80714efa178f52e461dc2e6d701d264f5ff22e566c" dependencies = [ "once_cell", "proc-macro-crate", @@ -2124,9 +2124,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] name = "chacha20" @@ -4674,7 +4674,7 @@ version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "rustc_version 0.4.1", ] @@ -6182,7 +6182,7 @@ version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9080b15e63775b9a2ac7dca720f7050a8b955e092ea0f6020a4a80f69998cdc0" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cfg-if", "libc", ] @@ -6674,7 +6674,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e668df13f2e97f3eed52d9301f6b1c4c1ccfccc30eab9e6628e4a8c1fc3546" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "bytes", "lazy_static", "libgssapi-sys", @@ -6723,7 +6723,7 @@ version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "libc", "plain", "redox_syscall 0.9.0", @@ -7356,7 +7356,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f27695f286b461da077b8c2f72f47feaa04ce3c3f9c0976257410e90e21208a" dependencies = [ "base64 0.22.1", - "bitflags 2.13.0", + "bitflags 2.13.1", "btoi", "byteorder", "bytes", @@ -7415,7 +7415,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cfg-if", "libc", ] @@ -7426,7 +7426,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cfg-if", "cfg_aliases", "libc", @@ -7438,7 +7438,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cfg-if", "cfg_aliases", "libc", @@ -7450,7 +7450,7 @@ version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cfg-if", "cfg_aliases", "libc", @@ -7899,7 +7899,7 @@ version = "6.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "libc", "once_cell", "onig_sys", @@ -7958,7 +7958,7 @@ version = "0.10.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cfg-if", "foreign-types", "libc", @@ -8909,7 +8909,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "chrono", "flate2", "hex", @@ -8923,7 +8923,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "chrono", "hex", ] @@ -9010,7 +9010,7 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "getopts", "memchr", "unicase", @@ -9304,7 +9304,7 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", ] [[package]] @@ -9418,7 +9418,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", ] [[package]] @@ -9427,7 +9427,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c5102a6aaa05aa011a238e178e6bca86d2cb56fc9f586d37cb80f5bca6e07759" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", ] [[package]] @@ -9474,9 +9474,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -9486,9 +9486,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.15" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f388202e4b80542a0921078cc23b6333bcf1409c1e3f86404cae4766a6131db" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", @@ -9994,7 +9994,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys 0.4.15", @@ -10007,7 +10007,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys 0.12.1", @@ -10461,7 +10461,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -10474,7 +10474,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -11210,7 +11210,7 @@ dependencies = [ "atoi", "base64 0.22.1", "bigdecimal", - "bitflags 2.13.0", + "bitflags 2.13.1", "byteorder", "bytes", "chrono", @@ -11255,7 +11255,7 @@ dependencies = [ "atoi", "base64 0.22.1", "bigdecimal", - "bitflags 2.13.0", + "bitflags 2.13.1", "byteorder", "chrono", "crc", @@ -11530,7 +11530,7 @@ version = "15.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c25af97d53cf8aab66a6c68f3418663313fc969ad267fc2a4d19402c329be1" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "is-macro", "num-bigint", "once_cell", @@ -11586,7 +11586,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "017d06ea85008234aa9fb34d805c7dc563f2ea6e03869ed5ac5a2dc27d561e4d" dependencies = [ "arrayvec", - "bitflags 2.13.0", + "bitflags 2.13.1", "either", "num-bigint", "phf 0.11.3", @@ -11908,7 +11908,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "byteorder", "enum-as-inner", "libc", @@ -11936,7 +11936,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -12852,7 +12852,7 @@ checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "async-compression", "base64 0.22.1", - "bitflags 2.13.0", + "bitflags 2.13.1", "bytes", "futures-core", "futures-util", @@ -13475,9 +13475,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.5" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea5fab0d6c3c01ae70085a09cb03d4c7a1d6314e2b3e075392783396d724ca0a" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -13492,7 +13492,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33995a1fee055ff743281cde33a41f0d618ee0bdbe8bdf6859e11864499c2595" dependencies = [ "bindgen 0.71.1", - "bitflags 2.13.0", + "bitflags 2.13.1", "fslock", "gzip-header", "home", @@ -13870,7 +13870,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "async-nats", @@ -13955,7 +13955,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.760.1" +version = "1.761.0" dependencies = [ "async-stream", "async-trait", @@ -13988,7 +13988,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.760.1" +version = "1.761.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14001,7 +14001,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "argon2", @@ -14139,7 +14139,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.760.1" +version = "1.761.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14162,7 +14162,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.760.1" +version = "1.761.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14177,7 +14177,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14203,7 +14203,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.760.1" +version = "1.761.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -14213,7 +14213,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.760.1" +version = "1.761.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14230,7 +14230,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.760.1" +version = "1.761.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -14252,7 +14252,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14275,7 +14275,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.760.1" +version = "1.761.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14291,7 +14291,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.760.1" +version = "1.761.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14312,7 +14312,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.760.1" +version = "1.761.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14333,7 +14333,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.760.1" +version = "1.761.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14347,7 +14347,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "async-nats", @@ -14382,7 +14382,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14407,7 +14407,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.760.1" +version = "1.761.0" dependencies = [ "axum 0.8.9", "flate2", @@ -14425,7 +14425,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14447,7 +14447,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.760.1" +version = "1.761.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14467,7 +14467,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.760.1" +version = "1.761.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14504,7 +14504,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14532,7 +14532,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.760.1" +version = "1.761.0" dependencies = [ "lazy_static", "serde", @@ -14544,7 +14544,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.760.1" +version = "1.761.0" dependencies = [ "argon2", "axum 0.8.9", @@ -14569,7 +14569,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.760.1" +version = "1.761.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14583,7 +14583,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.760.1" +version = "1.761.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14618,7 +14618,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.760.1" +version = "1.761.0" dependencies = [ "chrono", "lazy_static", @@ -14632,7 +14632,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14651,7 +14651,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.760.1" +version = "1.761.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -14668,7 +14668,7 @@ dependencies = [ "axum 0.8.9", "backon", "base64 0.22.1", - "bitflags 2.13.0", + "bitflags 2.13.1", "bytes", "chrono", "chrono-tz", @@ -14753,7 +14753,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.760.1" +version = "1.761.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -14772,7 +14772,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.760.1" +version = "1.761.0" dependencies = [ "regex", "serde", @@ -14787,7 +14787,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -14811,7 +14811,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "futures", @@ -14828,7 +14828,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.760.1" +version = "1.761.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -14844,7 +14844,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "async-trait", @@ -14865,7 +14865,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "async-trait", @@ -14896,7 +14896,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "arc-swap", @@ -14921,7 +14921,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "async-stream", @@ -14955,7 +14955,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "futures", @@ -14973,7 +14973,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.760.1" +version = "1.761.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -14982,7 +14982,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "lazy_static", @@ -14994,7 +14994,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "serde_json", @@ -15006,7 +15006,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "gosyn", @@ -15018,7 +15018,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "lazy_static", @@ -15030,7 +15030,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "serde_json", @@ -15042,7 +15042,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "nu-parser", @@ -15053,7 +15053,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15064,7 +15064,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15076,7 +15076,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15087,7 +15087,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "async-recursion", @@ -15109,7 +15109,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "serde_json", @@ -15121,7 +15121,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "lazy_static", @@ -15135,7 +15135,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15152,7 +15152,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "lazy_static", @@ -15165,7 +15165,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "serde", @@ -15177,7 +15177,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "lazy_static", @@ -15195,7 +15195,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -15211,7 +15211,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15227,7 +15227,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "serde", @@ -15238,7 +15238,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "async-recursion", @@ -15277,7 +15277,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "const_format", @@ -15317,7 +15317,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.760.1" +version = "1.761.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -15328,7 +15328,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "async-recursion", @@ -15362,7 +15362,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "async-trait", @@ -15386,7 +15386,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "async-trait", @@ -15419,7 +15419,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "async-trait", @@ -15452,7 +15452,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "async-trait", @@ -15472,7 +15472,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "async-trait", @@ -15506,7 +15506,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "async-trait", @@ -15542,7 +15542,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "async-trait", @@ -15565,7 +15565,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "async-trait", @@ -15589,7 +15589,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "async-nats", @@ -15613,7 +15613,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "async-trait", @@ -15648,7 +15648,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "async-trait", @@ -15676,7 +15676,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "async-trait", @@ -15701,10 +15701,10 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", - "bitflags 2.13.0", + "bitflags 2.13.1", "chrono", "hex", "itertools 0.14.0", @@ -15720,7 +15720,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "async-once-cell", @@ -15830,7 +15830,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.760.1" +version = "1.761.0" dependencies = [ "bytes", "futures", @@ -16081,7 +16081,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d24d6bcc7f734a4091ecf8d7a64c5f7d7066f45585c1861eba06449909609c8a" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "widestring", "windows-sys 0.52.0", ] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 6d3451afb7..9aa5717bb0 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.760.1" +version = "1.761.0" authors.workspace = true edition.workspace = true @@ -87,7 +87,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.760.1" +version = "1.761.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index f71ebedbe4..74e913c46b 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.760.1" +version = "1.761.0" dependencies = [ "aho-corasick", "anyhow", @@ -6272,7 +6272,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.760.1" +version = "1.761.0" dependencies = [ "proc-macro2", "quote", @@ -6284,7 +6284,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.760.1" +version = "1.761.0" dependencies = [ "convert_case", "serde", @@ -6293,7 +6293,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "lazy_static", @@ -6305,7 +6305,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "serde_json", @@ -6317,7 +6317,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "gosyn", @@ -6329,7 +6329,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "lazy_static", @@ -6341,7 +6341,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "serde_json", @@ -6353,7 +6353,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "nu-parser", @@ -6364,7 +6364,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6375,7 +6375,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6387,7 +6387,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6398,7 +6398,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "async-recursion", @@ -6420,7 +6420,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "serde_json", @@ -6432,7 +6432,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "lazy_static", @@ -6446,7 +6446,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "convert_case", @@ -6463,7 +6463,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "lazy_static", @@ -6476,7 +6476,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "serde", @@ -6488,7 +6488,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "lazy_static", @@ -6506,7 +6506,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6522,7 +6522,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6538,7 +6538,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6570,7 +6570,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "serde", @@ -6581,7 +6581,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.760.1" +version = "1.761.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 24903d6edb..c29d4c8bd4 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.760.1" +version = "1.761.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index a923956f4c..2d976d1df9 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.760.1 + version: 1.761.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 34a898542d..30bc6e4a1d 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.760.1"; +export const VERSION = "v1.761.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 305a1836be..b51700e5a6 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.760.1"; +export const VERSION = "1.761.0"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 6452137d53..3f5484438c 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.760.1", + "version": "1.761.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.760.1", + "version": "1.761.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { @@ -879,6 +879,7 @@ "version": "1.11.2", "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.2.tgz", "integrity": "sha512-TC8MkTuZUtcTSiFeuC0ksCh9QIJ5+F21MvZ4Wn4ORfYaFJ/0dsiudv5tVkejgwZlwQ39jL9WWDe2lz8x0WglOA==", + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -890,6 +891,7 @@ "version": "1.11.2", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.2.tgz", "integrity": "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA==", + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -900,6 +902,7 @@ "version": "1.2.2", "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1415,6 +1418,7 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.4.tgz", "integrity": "sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow==", + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1563,6 +1567,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1579,6 +1584,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1595,6 +1601,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1611,6 +1618,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1627,6 +1635,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1643,6 +1652,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1659,6 +1669,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1675,6 +1686,7 @@ "cpu": [ "ppc64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1691,6 +1703,7 @@ "cpu": [ "s390x" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1707,6 +1720,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1723,6 +1737,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1739,6 +1754,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1755,6 +1771,7 @@ "cpu": [ "wasm32" ], + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1773,6 +1790,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1789,6 +1807,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2094,6 +2113,7 @@ "version": "0.10.2", "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.2.tgz", "integrity": "sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==", + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -7329,7 +7349,7 @@ "version": "1.21.7", "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.7.tgz", "integrity": "sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==", - "devOptional": true, + "dev": true, "license": "MIT", "bin": { "jiti": "bin/jiti.js" @@ -7864,6 +7884,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7884,6 +7905,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7904,6 +7926,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7924,6 +7947,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7944,6 +7968,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7964,6 +7989,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7984,6 +8010,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8004,6 +8031,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8024,6 +8052,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8044,6 +8073,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8064,6 +8094,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -12751,6 +12782,21 @@ } } }, + "node_modules/svelte-check/node_modules/picomatch": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", + "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/svelte-eslint-parser": { "version": "0.43.0", "resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-0.43.0.tgz", @@ -13490,7 +13536,7 @@ "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "devOptional": true, + "dev": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", diff --git a/frontend/package.json b/frontend/package.json index 135116b908..d7541af7bf 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.760.1", + "version": "1.761.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/lsp/Pipfile b/lsp/Pipfile index 98820f3272..017325033b 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.760.1" +wmill = ">=1.761.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 634e6387ea..687cf148ab 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.760.1 + version: 1.761.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 01313ddfea..b1065fc8db 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.760.1' + ModuleVersion = '1.761.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index be4b577ecc..fd1b6c11aa 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.760.1" +version = "1.761.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 9ca1a210d7..329b0408e9 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.760.1", + "version": "1.761.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index b3d040dc46..be72cec08e 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.760.1", + "version": "1.761.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/typescript-client#readme", diff --git a/version.txt b/version.txt index 1a33933b31..5e7b269992 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.760.1 +1.761.0 From be57dd91e40551ff0d6fbbeccd97e2b98589c5b9 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 17 Jul 2026 15:09:24 +0200 Subject: [PATCH 02/14] fix(ansible): keep persistent-connection socket path under the AF_UNIX limit (#10167) * fix(ansible): keep persistent-connection socket path under the AF_UNIX limit Co-Authored-By: Claude Opus 4.8 (1M context) * fix(ansible): root socket dir at WINDMILL_DIR and verify we own it Co-Authored-By: Claude Opus 4.8 (1M context) * fix(ansible): fail closed on an untrusted socket root and honor commented section headers Co-Authored-By: Claude Opus 4.8 (1M context) * fix(ansible): validate the socket root after creating it, not before Co-Authored-By: Claude Opus 4.8 (1M context) * fix(ansible): root socket dir at sticky /tmp so the image's 0777 windmill dir cannot disable it Co-Authored-By: Claude Opus 4.8 (1M context) * fix(ansible): only sweep socket dirs the worker could have created Co-Authored-By: Claude Opus 4.8 (1M context) * fix(ansible): drop the configurable socket root, fixing its ancestor and sweep hazards Co-Authored-By: Claude Opus 4.8 (1M context) * fix(ansible): require the socket root be usable, not just safe Co-Authored-By: Claude Opus 4.8 (1M context) * fix(ansible): do not override a control_path_dir the job set itself Co-Authored-By: Claude Opus 4.8 (1M context) * refactor(ansible): drop the dead parent-creation step for the fixed /tmp root Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- backend/windmill-worker/Cargo.toml | 5 +- .../windmill-worker/src/ansible_executor.rs | 737 +++++++++++++++++- backend/windmill-worker/src/worker.rs | 3 + .../windmill-worker/src/worker_lockfiles.rs | 2 +- 4 files changed, 724 insertions(+), 23 deletions(-) diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 14aeb734ed..5b1047a153 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -115,8 +115,9 @@ hmac.workspace = true pem = { workspace = true, optional = true } rsa = { workspace = true, optional = true } urlencoding.workspace = true -# `fs` adds flock(2) for the cross-process Python install lock (shared cache mounts) -nix = { workspace = true, features = ["fs"] } +# `fs` adds flock(2) for the cross-process Python install lock (shared cache mounts); +# `user` adds geteuid(2) to verify ownership of the ansible socket-dir root +nix = { workspace = true, features = ["fs", "user"] } bytes.workspace = true reqwest.workspace = true reqwest-middleware.workspace = true diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 92bffe0cf0..708250dac6 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -55,6 +55,190 @@ const WINDMILL_ANSIBLE_PASSWORD_FILENAME: &str = ".windmill.ansible_vault_passwo const DELEGATE_GIT_REPO_TARGET: &str = "delegate_git_repository"; +/// Usable bytes in `sockaddr_un.sun_path` (108 minus the NUL). An ABI constant, not a +/// filesystem limit — which is why only the socket breaks while every regular file in the +/// same job dir is fine. +const AF_UNIX_PATH_LIMIT: usize = 107; + +/// Root for the per-job dir in which ansible's persistent-connection plugins +/// (`network_cli`, `httpapi`, `netconf`) bind their unix socket, named after a digest of the +/// connection. `sockaddr_un.sun_path` caps the whole socket path at [`AF_UNIX_PATH_LIMIT`], +/// which the job dir alone already exhausts, so the socket dir must stay short and cannot +/// live under `ANSIBLE_HOME` (which Windmill pins into the job dir). +/// +/// Fixed, and directly under `/tmp`, for two reasons that are easy to undo by accident: +/// `/tmp`'s sticky bit is what stops another uid renaming our root away, the one property +/// [`prepare_socket_root`] needs from a parent; and every component of a fixed path is one +/// nobody can point elsewhere, so trusting the root does not mean trusting an ancestor +/// chain. Notably NOT under `WINDMILL_DIR`: the shipped image chmods that tree to a +/// non-sticky 0777 so any UID can write it (`Dockerfile`, "Make directories +/// world-accessible for any UID"), which is exactly the parent an attacker can swap entries +/// in. +const PERSISTENT_CONTROL_PATH_ROOT: &str = "/tmp/wm-pc"; + +/// Ansible's env var for `[persistent_connection] control_path_dir`. +const ANSIBLE_CONTROL_PATH_DIR_ENV: &str = "ANSIBLE_PERSISTENT_CONTROL_PATH_DIR"; + +/// The budget this whole change exists to protect: root + `/` + a 32-char job uuid + `/` + +/// a socket name, allowing a full 40-char sha1 (ansible truncates it far shorter today, but +/// a custom control path may not). +const _: () = assert!(PERSISTENT_CONTROL_PATH_ROOT.len() + 1 + 32 + 1 + 40 <= AF_UNIX_PATH_LIMIT); + +/// Cleared when the root cannot be trusted (see [`prepare_socket_root`]), which makes jobs +/// stop naming it and fall back to ansible's own `{ANSIBLE_HOME}/pc` default — inside the +/// job dir, so worker-owned. Network playbooks then fail on the path length as they did +/// before this dir existed, which beats handing an attacker the socket a device session +/// runs over. Defaults to trusted: the check runs at worker start, before any job. +static SOCKET_ROOT_TRUSTED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(true); + +/// Socket dir for `job_id`, or `None` when the root is untrusted. Per-job on purpose: +/// socket names hash host+credentials, so concurrent jobs sharing a dir would reuse each +/// other's connection daemon. +fn persistent_control_path_dir(job_id: &Uuid) -> Option { + SOCKET_ROOT_TRUSTED + .load(std::sync::atomic::Ordering::Relaxed) + .then(|| format!("{PERSISTENT_CONTROL_PATH_ROOT}/{}", job_id.simple())) +} + +/// Whether `name` is one this module could have created, i.e. `Uuid::simple` (32 hex, no +/// hyphens). Belt to the root check's braces: nothing else should ever be in there. +fn is_persistent_control_path_dir_name(name: &str) -> bool { + name.len() == 32 && Uuid::try_parse(name).is_ok() +} + +/// Removes the job's socket dir on the way out. It lives outside `job_dir`, so the +/// worker's job-dir sweep does not cover it. +struct PersistentControlPathGuard(String); + +impl Drop for PersistentControlPathGuard { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +/// Claim the socket-dir root at worker start, then reap dirs left behind by workers that +/// died before their guard could run. +#[cfg(unix)] +pub async fn prepare_persistent_control_path_root() { + // A play holds its socket dir for as long as it runs, touching the mtime only when + // connections open, so anything younger than the longest permitted job may still be + // live — including on another worker sharing this host. + let stale_after = std::time::Duration::from_secs( + windmill_common::worker::MAX_TIMEOUT.saturating_add(24 * 60 * 60), + ); + prepare_socket_root(PERSISTENT_CONTROL_PATH_ROOT, stale_after).await +} + +/// Reject a root that another local user could control, and mark it untrusted so jobs stop +/// naming it. Returns without sweeping in that case. +/// +/// SECURITY: the root sits in a world-writable `/tmp`, so a local user who wins the race to +/// create it owns the parent of every job's socket dir — +/// enough to hand ansible a socket of their choosing (a device session, credentials and +/// all, runs over it), or to swap in a symlink and redirect the sweep's path-based +/// `remove_dir_all` onto a target of their choosing, as the worker's uid. Three things must +/// hold: the root is a real directory (`symlink_metadata` reports the link's own type +/// without following it, so `is_dir()` cannot be satisfied by a symlink), we own it and +/// nobody else can write it, and its parent cannot be used to replace it — which needs the +/// parent either not writable by others, or sticky, since the sticky bit is exactly what +/// stops a non-owner renaming an entry out of a shared dir. The root is validated after the +/// create attempt, never before: anything else races whoever creates it first. +#[cfg(unix)] +async fn prepare_socket_root(root: &str, stale_after: std::time::Duration) { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let untrusted = |reason: String| { + tracing::error!( + "Refusing to use the ansible persistent-connection socket root at {root}: {reason}. \ + Ansible network playbooks on this host will keep failing with `AF_UNIX path too \ + long` until this is resolved." + ); + SOCKET_ROOT_TRUSTED.store(false, std::sync::atomic::Ordering::Relaxed); + }; + + if let Some(parent) = std::path::Path::new(root).parent() { + // Resolved, not `symlink_metadata`: what matters is the mode of the directory the + // entries actually live in, and a symlinked parent is normal (macOS `/tmp`). + match tokio::fs::metadata(parent).await { + Ok(meta) => { + let mode = meta.permissions().mode(); + if mode & 0o022 != 0 && mode & 0o1000 == 0 { + return untrusted(format!( + "its parent {} is writable by other users and not sticky (mode={:o}), \ + so they could replace the root", + parent.display(), + mode & 0o7777 + )); + } + } + Err(e) => return untrusted(format!("cannot stat its parent: {e}")), + } + } + + // Non-recursive on purpose: `recursive` reports success for a path that already + // exists, which under a sticky parent (where others may still *create* the + // not-yet-existing `pc`, only not rename ours away) would hand us whatever another uid + // raced into place. Create-or-EEXIST, then validate whatever is actually there. + match tokio::fs::DirBuilder::new().mode(0o700).create(root).await { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(e) => return untrusted(format!("it could not be created: {e}")), + } + + match tokio::fs::symlink_metadata(root).await { + Ok(meta) if meta.is_dir() => { + let mode = meta.permissions().mode(); + // Trusted means usable as well as safe: without owner rwx ansible cannot create + // its per-job dir, and a trusted-but-unusable root would hand every network + // playbook a permission error instead of the working fallback. + if meta.uid() != nix::unistd::Uid::effective().as_raw() + || mode & 0o022 != 0 + || mode & 0o700 != 0o700 + { + return untrusted(format!( + "it is not owned by this worker, is writable by others, or is not \ + writable by us (uid={}, mode={:o})", + meta.uid(), + mode & 0o7777 + )); + } + } + Ok(_) => { + return untrusted( + "it is not a directory (possibly a symlink planted by another local user)" + .to_string(), + ) + } + Err(e) => return untrusted(format!("it could not be stat'd: {e}")), + } + + let Ok(mut entries) = tokio::fs::read_dir(root).await else { + return; + }; + while let Ok(Some(entry)) = entries.next_entry().await { + // Only reap what we could have created. Nothing else should ever be in a root we + // made 0700 ourselves, but this is a recursive delete running as the worker's uid: + // cheap to bound by name, expensive to get wrong. + if !entry + .file_name() + .to_str() + .is_some_and(is_persistent_control_path_dir_name) + { + continue; + } + // `DirEntry::metadata` does not traverse symlinks, so a planted link is never + // followed here either. + let stale = match entry.metadata().await.and_then(|m| m.modified()) { + Ok(modified) => modified.elapsed().is_ok_and(|e| e > stale_after), + Err(_) => false, + }; + if stale { + let _ = tokio::fs::remove_dir_all(entry.path()).await; + } + } +} + lazy_static::lazy_static! { static ref TEMPLATE_RE: regex::Regex = regex::Regex::new(r"\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}").unwrap(); } @@ -903,6 +1087,7 @@ pub fn create_ansible_cfg( reqs: Option<&AnsibleRequirements>, job_dir: &str, vault_password_file_exists: bool, + job_id: &Uuid, ) -> error::Result<()> { let mut passwords_cfg = String::new(); if vault_password_file_exists { @@ -922,6 +1107,9 @@ pub fn create_ansible_cfg( passwords_cfg.push_str(&format!("vault_identity_list = {password_files}\n")); } } + let persistent_cfg = persistent_control_path_dir(job_id) + .map(|dir| format!("[persistent_connection]\ncontrol_path_dir = {dir}\n")) + .unwrap_or_default(); let ansible_cfg_content = format!( r#" [defaults] @@ -931,7 +1119,7 @@ home={job_dir}/.ansible local_tmp={job_dir}/.ansible/tmp remote_tmp={job_dir}/.ansible/tmp {passwords_cfg} -"# +{persistent_cfg}"# ); write_file(job_dir, "ansible.cfg", &ansible_cfg_content)?; @@ -939,6 +1127,15 @@ remote_tmp={job_dir}/.ansible/tmp Ok(()) } +/// The section a header line opens, if it is one. Mirrors configparser's `SECTCRE` +/// (`\[(?P
.+)\]`, matched not fullmatched, `.+` greedy): the name runs to the +/// *last* `]`, and anything after it — an inline comment, say — is ignored. +fn parse_ansible_cfg_section_header(trimmed: &str) -> Option<&str> { + let rest = trimmed.strip_prefix('[')?; + let end = rest.rfind(']')?; + Some(rest[..end].trim()) +} + /// Read a colon-separated path list (e.g. `roles_path`, `collections_path`) from /// the `[defaults]` section of an ansible.cfg. Returns the raw entries as written, /// unresolved. Deliberately minimal: no inline-comment or continuation handling, @@ -949,10 +1146,8 @@ fn parse_ansible_cfg_path_list(content: &str, key: &str) -> Option> let mut in_defaults = false; for line in content.lines() { let trimmed = line.trim(); - if trimmed.starts_with('[') && trimmed.ends_with(']') { - in_defaults = trimmed[1..trimmed.len() - 1] - .trim() - .eq_ignore_ascii_case("defaults"); + if let Some(section) = parse_ansible_cfg_section_header(trimmed) { + in_defaults = section.eq_ignore_ascii_case("defaults"); continue; } if !in_defaults || trimmed.starts_with('#') || trimmed.starts_with(';') { @@ -978,6 +1173,29 @@ fn parse_ansible_cfg_path_list(content: &str, key: &str) -> Option> None } +/// Whether `section` declares `key` in an ansible.cfg. Same deliberately minimal +/// parsing as [`parse_ansible_cfg_path_list`], for a scalar key in a named section. +fn ansible_cfg_declares(content: &str, section: &str, key: &str) -> bool { + let mut in_section = false; + for line in content.lines() { + let trimmed = line.trim(); + if let Some(header) = parse_ansible_cfg_section_header(trimmed) { + in_section = header.eq_ignore_ascii_case(section); + continue; + } + if !in_section || trimmed.starts_with('#') || trimmed.starts_with(';') { + continue; + } + let sep = trimmed.find('=').into_iter().chain(trimmed.find(':')).min(); + if let Some(sep) = sep { + if trimmed[..sep].trim().eq_ignore_ascii_case(key) { + return true; + } + } + } + false +} + /// Prepend Windmill's dependency install dir to the repo cfg's declared path list. /// Relative entries from the repo cfg are resolved against `cfg_dir` to match how /// ansible resolves them relative to the config file's own directory. @@ -1006,6 +1224,8 @@ async fn build_ansible_cfg_override_envs( job_dir: &str, vault_password_file_exists: bool, reqs: Option<&AnsibleRequirements>, + job_id: &Uuid, + job_envs: &HashMap, ) -> error::Result> { let mut envs = vec![ ("ANSIBLE_CONFIG".to_string(), cfg_path.to_string()), @@ -1053,6 +1273,18 @@ async fn build_ansible_cfg_override_envs( )) })?; + // Persistent-connection socket dir: only a default. Unlike ANSIBLE_HOME this value is + // not runtime-bound, so a repo that picks its own dir keeps it — and so does a job that + // sets the env var itself, which these overrides are applied after and would otherwise + // silently outrank. + if !ansible_cfg_declares(&cfg_content, "persistent_connection", "control_path_dir") + && !job_envs.contains_key(ANSIBLE_CONTROL_PATH_DIR_ENV) + { + if let Some(dir) = persistent_control_path_dir(job_id) { + envs.push((ANSIBLE_CONTROL_PATH_DIR_ENV.to_string(), dir)); + } + } + envs.push(( "ANSIBLE_ROLES_PATH".to_string(), resolve_and_prepend_path( @@ -1606,7 +1838,8 @@ pub async fn handle_ansible_job( None => false, }; - create_ansible_cfg(reqs.as_ref(), job_dir, vault_password_file_exists)?; + create_ansible_cfg(reqs.as_ref(), job_dir, vault_password_file_exists, &job.id)?; + let _control_path_guard = persistent_control_path_dir(&job.id).map(PersistentControlPathGuard); // When the run delegates to a git repo that ships its own ansible.cfg, that // file becomes the effective config (ansible loads exactly one config file and @@ -1628,6 +1861,8 @@ pub async fn handle_ansible_job( job_dir, vault_password_file_exists, reqs.as_ref(), + &job.id, + &envs, ) .await? } @@ -1941,6 +2176,10 @@ async fn get_resource_or_variable_content( mod tests { use super::*; + fn no_job_envs() -> HashMap { + HashMap::new() + } + fn args_from_json(v: serde_json::Value) -> HashMap> { let serde_json::Value::Object(map) = v else { panic!("expected object"); @@ -2027,12 +2266,40 @@ mod tests { vault_id: vec!["dev@vault_pass.txt".to_string()], ..Default::default() }; - create_ansible_cfg(Some(&reqs), job_dir, false).unwrap(); + create_ansible_cfg(Some(&reqs), job_dir, false, &Uuid::new_v4()).unwrap(); let cfg = std::fs::read_to_string(dir.path().join("ansible.cfg")).unwrap(); assert!(cfg.contains("vault_identity_list = dev@vault_pass.txt")); assert!(!cfg.contains("library")); } + /// The socket ansible binds under `control_path_dir` must fit `sun_path` (107 + /// usable bytes), which the job dir alone blows past — hence a short dir outside + /// `ANSIBLE_HOME`. + #[test] + fn test_create_ansible_cfg_control_path_dir_fits_af_unix_limit() { + let dir = tempfile::tempdir().unwrap(); + let job_dir = dir.path().to_str().unwrap(); + let job_id = Uuid::new_v4(); + let flag = TrustFlag::lock(); + flag.set(true); + create_ansible_cfg(None, job_dir, false, &job_id).unwrap(); + + let cfg = std::fs::read_to_string(dir.path().join("ansible.cfg")).unwrap(); + let control_path_dir = persistent_control_path_dir(&job_id).unwrap(); + assert!(cfg.contains("[persistent_connection]")); + assert!(cfg.contains(&format!("control_path_dir = {control_path_dir}"))); + // The whole point: the socket dir must escape the job dir, whose length is what + // blows the budget. + assert!(!control_path_dir.starts_with(job_dir)); + + // dir + `/` + socket name, budgeted at a full 40-char sha1 (ansible truncates + // it far shorter today, but a custom control path may not). + assert!( + control_path_dir.len() + 1 + 40 <= AF_UNIX_PATH_LIMIT, + "socket path would exceed sun_path: {control_path_dir}" + ); + } + #[test] fn test_create_ansible_cfg_rejects_vault_id_injection() { let dir = tempfile::tempdir().unwrap(); @@ -2042,7 +2309,7 @@ mod tests { ..Default::default() }; // Defense-in-depth boundary: a poisoned entry must error before any config is written. - assert!(create_ansible_cfg(Some(&reqs), job_dir, false).is_err()); + assert!(create_ansible_cfg(Some(&reqs), job_dir, false, &Uuid::new_v4()).is_err()); assert!(!dir.path().join("ansible.cfg").exists()); } @@ -2161,7 +2428,7 @@ collections_path : a/col:b/col std::fs::write(repo.join("play.yml"), play).unwrap(); // Windmill's own generated cfg (the negative-control config that exists today). - create_ansible_cfg(None, job_dir, false).unwrap(); + create_ansible_cfg(None, job_dir, false, &Uuid::new_v4()).unwrap(); let playbook = format!("{DELEGATE_GIT_REPO_TARGET}/play.yml"); let run = |envs: Vec<(String, String)>| { @@ -2185,10 +2452,16 @@ collections_path : a/col:b/col // With the override: ANSIBLE_CONFIG points at the repo cfg and roles_path // is honored, so the role runs. - let envs = - build_ansible_cfg_override_envs(cfg_path.to_str().unwrap(), job_dir, false, None) - .await - .unwrap(); + let envs = build_ansible_cfg_override_envs( + cfg_path.to_str().unwrap(), + job_dir, + false, + None, + &Uuid::new_v4(), + &no_job_envs(), + ) + .await + .unwrap(); let after = run(envs); let stdout = String::from_utf8_lossy(&after.stdout); assert!( @@ -2213,15 +2486,30 @@ collections_path : a/col:b/col vault_id: vec!["dev@vault_pass.txt".to_string()], ..Default::default() }; - let envs = build_ansible_cfg_override_envs(cfg_path, job_dir, true, Some(&reqs)) - .await - .unwrap(); + let job_id = Uuid::new_v4(); + let flag = TrustFlag::lock(); + flag.set(true); + let envs = build_ansible_cfg_override_envs( + cfg_path, + job_dir, + true, + Some(&reqs), + &job_id, + &no_job_envs(), + ) + .await + .unwrap(); let map: std::collections::HashMap<_, _> = envs.into_iter().collect(); assert_eq!( map.get("ANSIBLE_CONFIG").map(|s| s.as_str()), Some(cfg_path) ); + // The repo cfg declares no control_path_dir, so Windmill's short default applies. + assert_eq!( + map.get("ANSIBLE_PERSISTENT_CONTROL_PATH_DIR"), + persistent_control_path_dir(&job_id).as_ref() + ); assert_eq!( map.get("ANSIBLE_HOME"), Some(&format!("{job_dir}/.ansible")) @@ -2261,14 +2549,423 @@ collections_path : a/col:b/col // are not silently dropped when the env override replaces the cfg value. std::fs::write(&cfg_path, "[defaults]\ncollections_paths = my_cols\n").unwrap(); - let envs = - build_ansible_cfg_override_envs(cfg_path.to_str().unwrap(), job_dir, false, None) - .await - .unwrap(); + let envs = build_ansible_cfg_override_envs( + cfg_path.to_str().unwrap(), + job_dir, + false, + None, + &Uuid::new_v4(), + &no_job_envs(), + ) + .await + .unwrap(); let map: std::collections::HashMap<_, _> = envs.into_iter().collect(); assert_eq!( map.get("ANSIBLE_COLLECTIONS_PATH"), Some(&format!("{job_dir}:{}/my_cols", repo_dir.to_str().unwrap())) ); } + + /// These overrides are applied after the job's own env, so a default that ignores what + /// the job set would silently outrank it. Not runtime-bound, so the job wins. + #[tokio::test] + async fn test_build_ansible_cfg_override_envs_keeps_job_env_control_path_dir() { + let dir = tempfile::tempdir().unwrap(); + let job_dir = dir.path().to_str().unwrap(); + let repo_dir = dir.path().join(DELEGATE_GIT_REPO_TARGET); + std::fs::create_dir_all(&repo_dir).unwrap(); + let cfg_path = repo_dir.join("ansible.cfg"); + // Cfg is silent on control_path_dir; the job env is not. + std::fs::write(&cfg_path, "[defaults]\nroles_path = my_roles\n").unwrap(); + + let job_envs = HashMap::from([( + ANSIBLE_CONTROL_PATH_DIR_ENV.to_string(), + "/tmp/job-picked".to_string(), + )]); + + let flag = TrustFlag::lock(); + flag.set(true); + let envs = build_ansible_cfg_override_envs( + cfg_path.to_str().unwrap(), + job_dir, + false, + None, + &Uuid::new_v4(), + &job_envs, + ) + .await + .unwrap(); + + assert!( + !envs.iter().any(|(k, _)| k == ANSIBLE_CONTROL_PATH_DIR_ENV), + "must not override a control_path_dir the job set itself" + ); + } + + #[tokio::test] + async fn test_build_ansible_cfg_override_envs_keeps_user_control_path_dir() { + let dir = tempfile::tempdir().unwrap(); + let job_dir = dir.path().to_str().unwrap(); + let repo_dir = dir.path().join(DELEGATE_GIT_REPO_TARGET); + std::fs::create_dir_all(&repo_dir).unwrap(); + let cfg_path = repo_dir.join("ansible.cfg"); + std::fs::write( + &cfg_path, + "[defaults]\nroles_path = my_roles\n\n[persistent_connection]\ncontrol_path_dir = /tmp/my_pc\n", + ) + .unwrap(); + + let envs = build_ansible_cfg_override_envs( + cfg_path.to_str().unwrap(), + job_dir, + false, + None, + &Uuid::new_v4(), + &no_job_envs(), + ) + .await + .unwrap(); + let map: std::collections::HashMap<_, _> = envs.into_iter().collect(); + assert_eq!(map.get("ANSIBLE_PERSISTENT_CONTROL_PATH_DIR"), None); + } + + /// `SOCKET_ROOT_TRUSTED` is process-global and cargo runs tests in parallel: hold this + /// while reading or flipping it, and the default is restored on the way out. + struct TrustFlag(#[allow(dead_code)] std::sync::MutexGuard<'static, ()>); + + impl TrustFlag { + fn lock() -> Self { + static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + Self(LOCK.lock().unwrap_or_else(|e| e.into_inner())) + } + fn set(&self, trusted: bool) { + SOCKET_ROOT_TRUSTED.store(trusted, std::sync::atomic::Ordering::Relaxed); + } + fn get(&self) -> bool { + SOCKET_ROOT_TRUSTED.load(std::sync::atomic::Ordering::Relaxed) + } + } + + impl Drop for TrustFlag { + fn drop(&mut self) { + SOCKET_ROOT_TRUSTED.store(true, std::sync::atomic::Ordering::Relaxed); + } + } + + #[cfg(unix)] + fn backdate(path: &std::path::Path, age: std::time::Duration) { + let times = std::fs::FileTimes::new().set_modified(std::time::SystemTime::now() - age); + std::fs::File::open(path).unwrap().set_times(times).unwrap(); + } + + /// The sweep only reaps what no live job can own: a play may hold its socket dir for + /// the whole of MAX_TIMEOUT without touching the mtime again. And it only ever touches + /// names it could have created itself. + #[cfg(unix)] + #[tokio::test] + async fn test_prepare_socket_root_sweeps_only_stale_dirs() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("wm-pc"); + std::fs::create_dir(&root).unwrap(); + + let stale = root.join(Uuid::new_v4().simple().to_string()); + let live = root.join(Uuid::new_v4().simple().to_string()); + let foreign = root.join("someone-elses-data"); + for p in [&stale, &live, &foreign] { + std::fs::create_dir(p).unwrap(); + } + backdate(&stale, std::time::Duration::from_secs(48 * 60 * 60)); + backdate(&live, std::time::Duration::from_secs(12 * 60 * 60)); + backdate(&foreign, std::time::Duration::from_secs(48 * 60 * 60)); + + let _flag = TrustFlag::lock(); + prepare_socket_root( + root.to_str().unwrap(), + std::time::Duration::from_secs(24 * 60 * 60), + ) + .await; + + assert!(!stale.exists(), "dir older than the cutoff must be reaped"); + assert!(live.exists(), "a dir a live job may still own must be kept"); + assert!( + foreign.exists(), + "a stale dir we never created must be left alone" + ); + } + + #[test] + fn test_is_persistent_control_path_dir_name() { + assert!(is_persistent_control_path_dir_name( + &Uuid::new_v4().simple().to_string() + )); + // Hyphenated form is not what we create, so it is not ours to delete. + assert!(!is_persistent_control_path_dir_name( + &Uuid::new_v4().to_string() + )); + assert!(!is_persistent_control_path_dir_name("someone-elses-data")); + assert!(!is_persistent_control_path_dir_name("")); + } + + #[cfg(unix)] + #[tokio::test] + async fn test_prepare_socket_root_creates_root_private() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("wm-pc"); + let flag = TrustFlag::lock(); + flag.set(true); + prepare_socket_root(root.to_str().unwrap(), std::time::Duration::from_secs(1)).await; + + let meta = std::fs::metadata(&root).unwrap(); + assert!(meta.is_dir()); + // Owning the root 0700 is what stops another local user replacing it later. + assert_eq!(meta.permissions().mode() & 0o777, 0o700); + assert!(flag.get(), "a root we created ourselves is trusted"); + } + + /// The root must be validated *after* the create attempt, not before: under a sticky + /// parent another uid may still win the race to create the not-yet-existing `pc` + /// (sticky stops them renaming ours away, not creating it first), and a create that + /// tolerates `AlreadyExists` would otherwise hand us their directory unchecked. + #[cfg(unix)] + #[tokio::test] + async fn test_prepare_socket_root_validates_raced_creation() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let parent = dir.path().join("windmill"); + std::fs::create_dir(&parent).unwrap(); + std::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o1777)).unwrap(); + + // Stand in for the racer's dir: present before we look, and not exclusively ours. + let root = parent.join("pc"); + std::fs::create_dir(&root).unwrap(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o777)).unwrap(); + + let flag = TrustFlag::lock(); + flag.set(true); + prepare_socket_root(root.to_str().unwrap(), std::time::Duration::from_secs(1)).await; + + assert!( + !flag.get(), + "a root raced into place under a sticky parent must not be trusted" + ); + } + + /// Safe but unusable is still not trusted: ansible cannot create its per-job dir under + /// a root we cannot write, and naming it anyway would swap the working fallback for a + /// permission error on every network playbook. + #[cfg(unix)] + #[tokio::test] + async fn test_prepare_socket_root_refuses_unwritable_root() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("wm-pc"); + std::fs::create_dir(&root).unwrap(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o500)).unwrap(); + + let flag = TrustFlag::lock(); + flag.set(true); + prepare_socket_root(root.to_str().unwrap(), std::time::Duration::from_secs(1)).await; + + assert!(!flag.get(), "a root we cannot write must not be trusted"); + // Let the tempdir clean itself up. + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap(); + } + + /// A root we do not exclusively own may have been pre-planted by another local user, + /// who then controls the parent of every job's socket dir — and could swap a symlink + /// in after this check, redirecting the sweep's path-based `remove_dir_all`. + #[cfg(unix)] + #[tokio::test] + async fn test_prepare_socket_root_refuses_world_writable_root() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("wm-pc"); + std::fs::create_dir(&root).unwrap(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o777)).unwrap(); + + // UUID-named, so survival proves the trust check stopped the sweep rather than the + // name filter. + let stale = root.join(Uuid::new_v4().simple().to_string()); + std::fs::create_dir(&stale).unwrap(); + backdate(&stale, std::time::Duration::from_secs(48 * 60 * 60)); + + let _flag = TrustFlag::lock(); + prepare_socket_root( + root.to_str().unwrap(), + std::time::Duration::from_secs(24 * 60 * 60), + ) + .await; + + assert!( + stale.exists(), + "must not sweep a root that others can write to" + ); + } + + /// The root must hang off `/tmp`, whose sticky bit is what protects it. The trap this + /// guards: the shipped image chmods the whole `WINDMILL_DIR` tree to a non-sticky 0777 + /// so any UID can write it, so parenting the root there would make it untrusted and + /// silently disable this fix in the standard image while every local test still passed. + #[test] + fn test_control_path_root_hangs_off_tmp() { + assert_eq!( + std::path::Path::new(PERSISTENT_CONTROL_PATH_ROOT).parent(), + Some(std::path::Path::new("/tmp")) + ); + } + + /// A parent that others can write (and that is not sticky) lets them rename the root + /// away and drop a symlink in its place after the checks — so the root cannot be + /// trusted no matter how it currently looks. + #[cfg(unix)] + #[tokio::test] + async fn test_prepare_socket_root_refuses_writable_parent() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let parent = dir.path().join("windmill"); + let root = parent.join("pc"); + std::fs::create_dir_all(&root).unwrap(); + // UUID-named, so survival proves the trust check stopped the sweep rather than the + // name filter. + let stale = root.join(Uuid::new_v4().simple().to_string()); + std::fs::create_dir(&stale).unwrap(); + backdate(&stale, std::time::Duration::from_secs(48 * 60 * 60)); + std::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o777)).unwrap(); + + let flag = TrustFlag::lock(); + flag.set(true); + prepare_socket_root( + root.to_str().unwrap(), + std::time::Duration::from_secs(24 * 60 * 60), + ) + .await; + + assert!(stale.exists(), "must not sweep under a replaceable parent"); + assert!( + !flag.get(), + "an untrusted root must be marked so jobs stop naming it" + ); + } + + /// A sticky parent (like /tmp itself) is fine: the sticky bit is what stops a + /// non-owner renaming our root out of it. + #[cfg(unix)] + #[tokio::test] + async fn test_prepare_socket_root_accepts_sticky_world_writable_parent() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let parent = dir.path().join("windmill"); + let root = parent.join("pc"); + std::fs::create_dir_all(&parent).unwrap(); + std::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o1777)).unwrap(); + + let flag = TrustFlag::lock(); + flag.set(true); + prepare_socket_root( + root.to_str().unwrap(), + std::time::Duration::from_secs(24 * 60 * 60), + ) + .await; + + assert!(root.is_dir(), "root must be created under a sticky parent"); + assert!(flag.get()); + } + + /// Fail closed: when the root is untrusted the cfg must not name it, so ansible falls + /// back to its own `{ANSIBLE_HOME}/pc` default inside the worker-owned job dir. + #[test] + fn test_create_ansible_cfg_omits_untrusted_control_path_dir() { + let dir = tempfile::tempdir().unwrap(); + let job_dir = dir.path().to_str().unwrap(); + + let flag = TrustFlag::lock(); + flag.set(false); + create_ansible_cfg(None, job_dir, false, &Uuid::new_v4()).unwrap(); + + let cfg = std::fs::read_to_string(dir.path().join("ansible.cfg")).unwrap(); + assert!(!cfg.contains("control_path_dir")); + assert!(!cfg.contains("[persistent_connection]")); + } + + /// A symlinked root must never be swept: `remove_dir_all` through it would delete + /// whatever the link points at, as the worker's uid. + #[cfg(unix)] + #[tokio::test] + async fn test_prepare_socket_root_refuses_symlinked_root() { + let dir = tempfile::tempdir().unwrap(); + let victim = dir.path().join("victim"); + // UUID-named, so survival proves the symlink was not followed rather than the name + // filter sparing it. + let victim_child = victim.join(Uuid::new_v4().simple().to_string()); + std::fs::create_dir_all(&victim_child).unwrap(); + backdate(&victim_child, std::time::Duration::from_secs(48 * 60 * 60)); + + let root = dir.path().join("wm-pc"); + std::os::unix::fs::symlink(&victim, &root).unwrap(); + + let _flag = TrustFlag::lock(); + prepare_socket_root( + root.to_str().unwrap(), + std::time::Duration::from_secs(24 * 60 * 60), + ) + .await; + + assert!( + victim_child.exists(), + "sweep must not follow a symlinked root" + ); + } + + /// configparser matches `\[(?P
.+)\]` without anchoring the end of the line, so + /// a header with anything trailing it is still that section — and missing it here + /// would silently override the user's own control_path_dir. + #[test] + fn test_ansible_cfg_section_header_with_trailing_text() { + assert_eq!( + parse_ansible_cfg_section_header("[persistent_connection] ; note"), + Some("persistent_connection") + ); + assert_eq!(parse_ansible_cfg_section_header("not a header"), None); + // Greedy `.+` runs to the last `]`. + assert_eq!(parse_ansible_cfg_section_header("[a]b]"), Some("a]b")); + + assert!(ansible_cfg_declares( + "[persistent_connection] ; note\ncontrol_path_dir = /tmp/mine\n", + "persistent_connection", + "control_path_dir" + )); + assert_eq!( + parse_ansible_cfg_path_list("[defaults] # note\nroles_path = my_roles\n", "roles_path"), + Some(vec!["my_roles".to_string()]) + ); + } + + #[test] + fn test_ansible_cfg_declares_scoped_to_section() { + let cfg = "\ +[defaults] +control_path_dir = /wrong/section + +[persistent_connection] +# control_path_dir = /commented +connect_timeout = 30 +"; + assert!(!ansible_cfg_declares( + cfg, + "persistent_connection", + "control_path_dir" + )); + assert!(ansible_cfg_declares( + cfg, + "persistent_connection", + "connect_timeout" + )); + } } diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 69642cd4ea..2a4541b095 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -2006,6 +2006,9 @@ pub async fn run_worker( create_directory_async(&worker_dir).await; + #[cfg(all(feature = "python", unix))] + crate::ansible_executor::prepare_persistent_control_path_root().await; + if is_sandboxing_enabled() { let _ = write_file( &worker_dir, diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index d98a2cd2e0..5c695ac64f 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -2468,7 +2468,7 @@ async fn ansible_dep( let ansible_lockfile; - create_ansible_cfg(Some(&reqs), job_dir, false)?; + create_ansible_cfg(Some(&reqs), job_dir, false, job_id)?; if let Some(collections) = reqs.roles_and_collections.as_ref() { install_galaxy_collections( From 97f44770698eb3dd9e296012360b7d9a4492e28d Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 17 Jul 2026 15:13:36 +0200 Subject: [PATCH 03/14] feat(forks): let a fork's creator manage developers on it without being an admin (#10166) * feat(forks): let a fork's creator manage developers on it without being an admin Co-Authored-By: Claude Opus 4.8 (1M context) * fix(forks): bind the fork-creator grant to the member's parent username and lock the delete Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- ...5702eba0b0d674b99f952af94487b3e54211c.json | 23 ++ ...3021ae3e99a5a61ea1c944c1df1368746dfa5.json | 76 ++++++ ...c5aed4273f9f3977144d7c564be07504af9af.json | 23 ++ ...e3f746cc8f1909f94a15bd8cde33ce974caa7.json | 29 +++ .../tests/fixtures/fork_member_grant.sql | 33 +++ .../tests/fork_member_grant.rs | 187 ++++++++++++++ backend/windmill-api-users/src/users.rs | 53 +++- .../windmill-api-workspaces/src/workspaces.rs | 83 +++++- backend/windmill-api/openapi.yaml | 4 +- backend/windmill-common/src/workspaces.rs | 29 +++ .../settings/ForkMemberSettings.svelte | 239 ++++++++++++++++++ frontend/src/lib/stores.ts | 1 + .../(logged)/workspace_settings/+page.svelte | 68 ++++- 13 files changed, 828 insertions(+), 20 deletions(-) create mode 100644 backend/.sqlx/query-67d86d334029276d78bf8fa14b25702eba0b0d674b99f952af94487b3e54211c.json create mode 100644 backend/.sqlx/query-88a134e4ca82d5ce0334977c7713021ae3e99a5a61ea1c944c1df1368746dfa5.json create mode 100644 backend/.sqlx/query-931b2a18250879b0bbd0bec2c88c5aed4273f9f3977144d7c564be07504af9af.json create mode 100644 backend/.sqlx/query-be927c5174436cb9574bacc7cb4e3f746cc8f1909f94a15bd8cde33ce974caa7.json create mode 100644 backend/windmill-api-integration-tests/tests/fixtures/fork_member_grant.sql create mode 100644 backend/windmill-api-integration-tests/tests/fork_member_grant.rs create mode 100644 frontend/src/lib/components/settings/ForkMemberSettings.svelte diff --git a/backend/.sqlx/query-67d86d334029276d78bf8fa14b25702eba0b0d674b99f952af94487b3e54211c.json b/backend/.sqlx/query-67d86d334029276d78bf8fa14b25702eba0b0d674b99f952af94487b3e54211c.json new file mode 100644 index 0000000000..9a3cbd6703 --- /dev/null +++ b/backend/.sqlx/query-67d86d334029276d78bf8fa14b25702eba0b0d674b99f952af94487b3e54211c.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT username FROM usr\n WHERE workspace_id = $1 AND email = $2 AND NOT operator AND NOT disabled", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "username", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "67d86d334029276d78bf8fa14b25702eba0b0d674b99f952af94487b3e54211c" +} diff --git a/backend/.sqlx/query-88a134e4ca82d5ce0334977c7713021ae3e99a5a61ea1c944c1df1368746dfa5.json b/backend/.sqlx/query-88a134e4ca82d5ce0334977c7713021ae3e99a5a61ea1c944c1df1368746dfa5.json new file mode 100644 index 0000000000..7cfe9aa070 --- /dev/null +++ b/backend/.sqlx/query-88a134e4ca82d5ce0334977c7713021ae3e99a5a61ea1c944c1df1368746dfa5.json @@ -0,0 +1,76 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace.id, workspace.name, usr.username, workspace_settings.color, workspace.parent_workspace_id,\n workspace.is_dev_workspace, workspace.dev_workspace_label,\n workspace.owner AS \"created_by?\",\n CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings,\n usr.disabled\n FROM workspace\n JOIN usr ON usr.workspace_id = workspace.id\n JOIN workspace_settings ON workspace_settings.workspace_id = workspace.id\n WHERE usr.email = $1 AND workspace.deleted = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "username", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "color", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "parent_workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "is_dev_workspace", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "dev_workspace_label", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "created_by?", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "operator_settings", + "type_info": "Jsonb" + }, + { + "ordinal": 9, + "name": "disabled", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true, + true, + false, + true, + false, + null, + false + ] + }, + "hash": "88a134e4ca82d5ce0334977c7713021ae3e99a5a61ea1c944c1df1368746dfa5" +} diff --git a/backend/.sqlx/query-931b2a18250879b0bbd0bec2c88c5aed4273f9f3977144d7c564be07504af9af.json b/backend/.sqlx/query-931b2a18250879b0bbd0bec2c88c5aed4273f9f3977144d7c564be07504af9af.json new file mode 100644 index 0000000000..10de699b74 --- /dev/null +++ b/backend/.sqlx/query-931b2a18250879b0bbd0bec2c88c5aed4273f9f3977144d7c564be07504af9af.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT parent_workspace_id FROM workspace\n WHERE id = $1 AND owner = $2 AND parent_workspace_id IS NOT NULL AND NOT deleted", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "parent_workspace_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "931b2a18250879b0bbd0bec2c88c5aed4273f9f3977144d7c564be07504af9af" +} diff --git a/backend/.sqlx/query-be927c5174436cb9574bacc7cb4e3f746cc8f1909f94a15bd8cde33ce974caa7.json b/backend/.sqlx/query-be927c5174436cb9574bacc7cb4e3f746cc8f1909f94a15bd8cde33ce974caa7.json new file mode 100644 index 0000000000..cde9284501 --- /dev/null +++ b/backend/.sqlx/query-be927c5174436cb9574bacc7cb4e3f746cc8f1909f94a15bd8cde33ce974caa7.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT email, is_admin FROM usr where username = $1 AND workspace_id = $2 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "is_admin", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "be927c5174436cb9574bacc7cb4e3f746cc8f1909f94a15bd8cde33ce974caa7" +} diff --git a/backend/windmill-api-integration-tests/tests/fixtures/fork_member_grant.sql b/backend/windmill-api-integration-tests/tests/fixtures/fork_member_grant.sql new file mode 100644 index 0000000000..bd1fd11788 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/fixtures/fork_member_grant.sql @@ -0,0 +1,33 @@ +-- `test2` (a non-admin developer of test-workspace) has forked it. The fork's `usr` row copies the +-- non-admin role they hold in the parent, which is the situation the fork-creator grant exists for. +INSERT INTO workspace (id, name, owner, parent_workspace_id) VALUES + ('wm-fork-test', 'fork of test-workspace', 'test2@windmill.dev', 'test-workspace'); + +INSERT INTO workspace_settings (workspace_id) VALUES ('wm-fork-test'); + +INSERT INTO workspace_key (workspace_id, kind, key) VALUES ('wm-fork-test', 'cloud', 'test-key'); + +INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES + ('wm-fork-test', 'all', 'All users', '{}'); + +INSERT INTO usr (workspace_id, email, username, is_admin, role) VALUES + ('wm-fork-test', 'test2@windmill.dev', 'test-user-2', false, 'User'), + -- An admin of the fork, whom its creator must not be able to remove. + ('wm-fork-test', 'test@windmill.dev', 'test-user', true, 'Admin'); + +-- An operator of the parent: the eligibility bar for being added to the fork is developer-or-above +-- there, so this user must be rejected. +INSERT INTO password (email, password_hash, login_type, super_admin, verified, name, username) + VALUES ('test4@windmill.dev', 'not-a-real-hash', 'password', false, true, 'Test User 4', 'test-user-4'); + +INSERT INTO usr (workspace_id, email, username, is_admin, operator, role) VALUES + ('test-workspace', 'test4@windmill.dev', 'test-user-4', false, true, 'Operator'); + +-- add_user resolves the instance-wide username from `password`. +UPDATE password SET username = 'test-user-2' WHERE email = 'test2@windmill.dev'; +UPDATE password SET username = 'test-user-3' WHERE email = 'test3@windmill.dev'; + +-- With automated username creation off, `add_user` takes the username from the caller. That branch +-- is what the fork creator must not be able to steer, so the tests run against it. +INSERT INTO global_settings (name, value) VALUES ('automate_username_creation', 'false'::jsonb) + ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value; diff --git a/backend/windmill-api-integration-tests/tests/fork_member_grant.rs b/backend/windmill-api-integration-tests/tests/fork_member_grant.rs new file mode 100644 index 0000000000..8789391854 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/fork_member_grant.rs @@ -0,0 +1,187 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +/// `test2` created the fork `wm-fork-test` but is only a developer in it. +const FORK_OWNER_TOKEN: &str = "SECRET_TOKEN_2"; +/// `test3` is a developer of the parent and of the fork, but created neither. +const FORK_MEMBER_TOKEN: &str = "SECRET_TOKEN_3"; + +fn as_user(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {token}")) +} + +async fn add_user( + port: u16, + w_id: &str, + token: &str, + body: serde_json::Value, +) -> reqwest::Response { + as_user( + client().post(format!( + "http://localhost:{port}/api/w/{w_id}/workspaces/add_user" + )), + token, + ) + .json(&body) + .send() + .await + .unwrap() +} + +fn developer(email: &str) -> serde_json::Value { + json!({ "email": email, "is_admin": false, "operator": false }) +} + +/// The creator of a fork may manage developers on it without being an admin of it, and may do +/// nothing beyond that. A fork clones its parent wholesale (secrets included), so each of these +/// bounds is what keeps the grant from becoming a way for any developer to widen access to the +/// parent's data or to mint an admin. +#[sqlx::test(migrations = "../migrations", fixtures("base", "fork_member_grant"))] +async fn test_fork_creator_can_only_manage_developers_on_their_fork( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Seating an eligible member on someone else's username is what would hand them that user's + // cloned private assets. `test-user-4` is the target that matters: a fork clones the parent's + // `u/test-user-4/` scripts, variables and secrets but not their membership, so the username is + // free of the unique constraint on `usr` and the squat would otherwise land. + let resp = add_user( + port, + "wm-fork-test", + FORK_OWNER_TOKEN, + json!({ "email": "test3@windmill.dev", "username": "test-user-4", "is_admin": false, "operator": false }), + ) + .await; + assert_eq!( + resp.status(), + 403, + "fork creator cannot choose the username a member joins under" + ); + + // The creator adds a developer of the parent as a developer of their fork. + let resp = add_user( + port, + "wm-fork-test", + FORK_OWNER_TOKEN, + developer("test3@windmill.dev"), + ) + .await; + assert_eq!(resp.status(), 201, "fork creator can add a developer"); + + // They join under the username they hold in the parent, which is the `u/` namespace the fork + // cloned for them. + let username: String = sqlx::query_scalar( + "SELECT username FROM usr WHERE workspace_id = 'wm-fork-test' AND email = 'test3@windmill.dev'", + ) + .fetch_one(&db) + .await?; + assert_eq!( + username, "test-user-3", + "added member keeps their parent username" + ); + + // ... but never as an admin. + let resp = add_user( + port, + "wm-fork-test", + FORK_OWNER_TOKEN, + json!({ "email": "test4@windmill.dev", "is_admin": true, "operator": false }), + ) + .await; + assert_eq!(resp.status(), 403, "fork creator cannot add an admin"); + + // ... nor anyone who is only an operator of the parent, which would widen their access. + let resp = add_user( + port, + "wm-fork-test", + FORK_OWNER_TOKEN, + developer("test4@windmill.dev"), + ) + .await; + assert_eq!( + resp.status(), + 403, + "fork creator cannot add an operator of the parent" + ); + + // ... nor anyone from outside the parent workspace. + let resp = add_user( + port, + "wm-fork-test", + FORK_OWNER_TOKEN, + developer("outsider@windmill.dev"), + ) + .await; + assert_eq!( + resp.status(), + 403, + "fork creator cannot add a non-member of the parent" + ); + + // The grant covers the fork alone, not the workspace it was forked from. + let resp = add_user( + port, + "test-workspace", + FORK_OWNER_TOKEN, + developer("test4@windmill.dev"), + ) + .await; + assert_eq!( + resp.status(), + 403, + "fork creator gains nothing on the parent workspace" + ); + + // ... and belongs to the creator, not to every member of the fork. + let resp = add_user( + port, + "wm-fork-test", + FORK_MEMBER_TOKEN, + developer("test4@windmill.dev"), + ) + .await; + assert_eq!( + resp.status(), + 403, + "a fork member who did not create it gains nothing" + ); + + // Removing is the counterpart of adding: allowed for the developer they just added... + let resp = as_user( + client().delete(format!( + "http://localhost:{port}/api/w/wm-fork-test/users/delete/test-user-3" + )), + FORK_OWNER_TOKEN, + ) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "fork creator can remove a developer"); + + // ... but not for an admin of the fork. + let resp = as_user( + client().delete(format!( + "http://localhost:{port}/api/w/wm-fork-test/users/delete/test-user" + )), + FORK_OWNER_TOKEN, + ) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 403, + "fork creator cannot remove an admin of the fork" + ); + + Ok(()) +} diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index ff0e8e754e..b98bc6f414 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -1850,6 +1850,39 @@ pub async fn delete_workspace_user_internal( Ok(()) } +/// Non-admin path for `delete_workspace_user`: the creator of a fork may remove non-admin members +/// from the fork they created, so that adding the wrong collaborator is theirs to undo rather than +/// an admin's. Never on a root workspace, and never against an admin of the fork — the counterpart +/// of the add grant, whose bounds are spelled out on `add_user` in `windmill-api-workspaces`. +/// +/// `target_is_admin` must come from a row locked by the caller's deletion transaction: the grant +/// turns on the target not being an admin, so a promotion committing between the check and the +/// delete would remove an admin after all. `None` (no such member) is left to the caller's 404, +/// which is raised only after this returns so that a non-creator cannot probe who exists. +async fn authorize_fork_owner_delete_user( + tx: &mut Transaction<'_, Postgres>, + w_id: &str, + authed: &ApiAuthed, + username_to_delete: &str, + target_is_admin: Option, +) -> Result<()> { + if windmill_common::workspaces::fork_owned_by(&mut **tx, w_id, &authed.email) + .await? + .is_none() + { + return Err(Error::RequireAdmin(authed.username.clone())); + } + + if target_is_admin == Some(true) { + return Err(Error::PermissionDenied(format!( + "as the creator of fork {w_id} you cannot remove {username_to_delete}, who is an admin \ + of it" + ))); + } + + Ok(()) +} + async fn delete_workspace_user( authed: ApiAuthed, Extension(db): Extension, @@ -1857,17 +1890,27 @@ async fn delete_workspace_user( ) -> Result { let mut tx = db.begin().await?; - require_admin(authed.is_admin, &authed.username)?; - - let email_to_delete_o = sqlx::query_scalar!( - "SELECT email FROM usr where username = $1 AND workspace_id = $2", + // Locked so that the authorization below and the delete it guards see the same row. + let target = sqlx::query!( + "SELECT email, is_admin FROM usr where username = $1 AND workspace_id = $2 FOR UPDATE", username_to_delete, &w_id, ) .fetch_optional(&mut *tx) .await?; - let email_to_delete = not_found_if_none(email_to_delete_o, "User", &username_to_delete)?; + if !authed.is_admin { + authorize_fork_owner_delete_user( + &mut tx, + &w_id, + &authed, + &username_to_delete, + target.as_ref().map(|t| t.is_admin), + ) + .await?; + } + + let email_to_delete = not_found_if_none(target, "User", &username_to_delete)?.email; delete_workspace_user_internal( &w_id, diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 8aa1ff680c..74b31c1410 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -514,6 +514,10 @@ struct UserWorkspace { pub parent_workspace_id: Option, pub is_dev_workspace: bool, pub dev_workspace_label: Option, + /// Creator of the workspace (`workspace.owner`). On a fork it identifies the forker, who gets a + /// narrow membership grant over it even without being an admin — the UI keys the fork members + /// screen off this. + pub created_by: Option, pub disabled: bool, } @@ -4298,6 +4302,7 @@ async fn user_workspaces( UserWorkspace, "SELECT workspace.id, workspace.name, usr.username, workspace_settings.color, workspace.parent_workspace_id, workspace.is_dev_workspace, workspace.dev_workspace_label, + workspace.owner AS \"created_by?\", CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings, usr.disabled FROM workspace @@ -7174,6 +7179,72 @@ If you do not have an account on {}, login with SSO or ask an admin to create an )) } +/// Non-admin path for `add_user`: the creator of a fork may bring collaborators into the fork they +/// created, so a team can work on it without an admin of the fork having to step in. The grant is +/// deliberately narrow, because a fork holds a full clone of its parent (secrets included) and the +/// creator may be an ordinary developer: +/// - only on a fork they created, never on a root workspace; +/// - only as a developer, so it can never mint an admin (nor an operator, which would need the +/// workspace's operator settings to be meaningful); +/// - only for someone who is already a developer or admin of the parent, so pulling them into the +/// fork cannot widen who can read the parent's data. +/// +/// Anything outside those bounds stays an admin's call. Returns the username the new member must be +/// given in the fork. +async fn authorize_fork_owner_add_user( + db: &DB, + w_id: &str, + authed: &ApiAuthed, + nu: &NewWorkspaceUser, +) -> Result { + let parent = windmill_common::workspaces::fork_owned_by(db, w_id, &authed.email) + .await? + .ok_or_else(|| Error::RequireAdmin(authed.username.clone()))?; + + if nu.is_admin || nu.operator { + return Err(Error::PermissionDenied(format!( + "as the creator of fork {w_id} you can only add members as developers; ask an admin of \ + {w_id} for any other role" + ))); + } + + let parent_username = sqlx::query_scalar!( + "SELECT username FROM usr + WHERE workspace_id = $1 AND email = $2 AND NOT operator AND NOT disabled", + parent, + nu.email, + ) + .fetch_optional(db) + .await?; + + let Some(parent_username) = parent_username else { + return Err(Error::PermissionDenied(format!( + "as the creator of fork {w_id} you can only add developers or admins of its parent \ + workspace {parent}; {} is not one, so only an admin of {w_id} can add them", + nu.email + ))); + }; + + // Ownership of a `u//` path is decided by the username alone, and the fork holds a + // clone of every such path from the parent. Seating the new member on a username other than + // their own would therefore hand them that parent user's cloned scripts, variables and secrets + // — so their parent username is the only one they may be given here, whatever the caller asked + // for (`add_user` otherwise lets the caller choose it when AUTOMATE_USERNAME_CREATION is off). + if nu + .username + .as_deref() + .is_some_and(|u| !u.is_empty() && u != parent_username) + { + return Err(Error::PermissionDenied(format!( + "as the creator of fork {w_id} you cannot choose the username of a member you add; {} \ + joins as '{parent_username}', the username they already have in {parent}", + nu.email + ))); + } + + Ok(parent_username) +} + async fn add_user( authed: ApiAuthed, Extension(db): Extension, @@ -7181,8 +7252,6 @@ async fn add_user( Path(w_id): Path, Json(mut nu): Json, ) -> Result<(StatusCode, String)> { - require_admin(authed.is_admin, &authed.username)?; - #[cfg(not(feature = "enterprise"))] if w_id == "admins" { return Err(Error::BadRequest( @@ -7192,6 +7261,12 @@ async fn add_user( nu.email = nu.email.to_lowercase(); + let fork_owner_username = if !authed.is_admin { + Some(authorize_fork_owner_add_user(&db, &w_id, &authed, &nu).await?) + } else { + None + }; + #[cfg(feature = "enterprise")] if let Some(msg) = windmill_common::ee_oss::check_seat_cap_for_new_user(&db, &nu.email, nu.operator).await? @@ -7227,7 +7302,9 @@ async fn add_user( .flatten() .unwrap_or(true); - let username = if automate_username_creation { + let username = if let Some(username) = fork_owner_username { + username + } else if automate_username_creation { if nu.username.is_some() && nu.username.unwrap().len() > 0 { return Err(Error::BadRequest( "username is not allowed when username creation is automated".to_string(), diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 2d976d1df9..668cdae8cc 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -847,7 +847,7 @@ paths: /w/{workspace}/users/delete/{username}: delete: - summary: delete user (require admin privilege) + summary: delete user (require admin privilege, except for the creator of a fork removing a non-admin member of the fork) operationId: deleteUser tags: - user @@ -3026,7 +3026,7 @@ paths: /w/{workspace}/workspaces/add_user: post: - summary: add user to workspace + summary: add user to workspace (require admin privilege, except for the creator of a fork adding a developer/admin of its parent workspace as a developer of the fork) operationId: addUser tags: - workspace diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 14dbfdf11d..33c80cea4b 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -652,6 +652,35 @@ pub async fn fork_subtree_height(db: &crate::DB, w_id: &str) -> Result { Ok(height) } +/// Parent id of `w_id` when `email` is the creator of that fork, `None` otherwise (including for a +/// root workspace, which has no creator in this sense). +/// +/// The creator is recorded as `workspace.owner`, but the `usr` row they get in the fork is copied +/// from the parent — so a forker who is not an admin of the parent is not an admin of the fork they +/// just created either, and cannot bring anyone in to work on it. Being the creator therefore grants +/// a narrow membership right over the fork; the callers own the exact bounds of that grant (see +/// `add_user` in `windmill-api-workspaces`). +/// +/// Unauthenticated helper: reads workspace hierarchy for any `w_id`, so callers must already be +/// authorized for that workspace (or run in trusted server-side code). Takes any executor so that a +/// caller can run it inside the transaction whose writes the grant authorizes. +pub async fn fork_owned_by<'e, E: sqlx::Executor<'e, Database = sqlx::Postgres>>( + db: E, + w_id: &str, + email: &str, +) -> Result> { + let parent = sqlx::query_scalar!( + "SELECT parent_workspace_id FROM workspace + WHERE id = $1 AND owner = $2 AND parent_workspace_id IS NOT NULL AND NOT deleted", + w_id, + email + ) + .fetch_optional(db) + .await + .map_err(|e| Error::internal_err(format!("checking fork ownership of {w_id}: {e:#}")))?; + Ok(parent.flatten()) +} + /// Ids of every fork/dev workspace anywhere under `w_id` (excludes `w_id` itself), including live /// descendants beneath a soft-deleted intermediate. Used to invalidate per-workspace caches for a /// whole subtree after its ancestor is reparented. diff --git a/frontend/src/lib/components/settings/ForkMemberSettings.svelte b/frontend/src/lib/components/settings/ForkMemberSettings.svelte new file mode 100644 index 0000000000..6b75018cae --- /dev/null +++ b/frontend/src/lib/components/settings/ForkMemberSettings.svelte @@ -0,0 +1,239 @@ + + + + + + You are not an admin of this workspace, so you can only add developers or admins of + {parentWorkspaceId} as developers of this fork, and remove the ones who are not admins here. + Ask an admin of this workspace for any other membership change. + + +
+ +
+ {#snippet action()} + + {#snippet trigger()} + + {/snippet} + {#snippet content()} +
+ Add a collaborator + + They join as a developer of this fork. Only members of + {parentWorkspaceId} who are developers or admins there can be added. + + +

+ Extra CA certificates added to the proxy's upstream trust store, on top of the + system roots. Use this to trace internal endpoints signed by a private CA while + keeping certificate verification enabled — preferred over the insecure list above + when you have the CA. +

+
{/if} {:else if setting.fieldType == 'object_store_config'} From 1edee8aa34503d2892c1ef7b1deabf99e4b528b4 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Fri, 17 Jul 2026 18:05:38 +0200 Subject: [PATCH 06/14] perf: lazy-load session preview editor views for snappy AI sessions switch (#10172) * perf: lazy-load session preview editor views to speed up AI sessions switch Co-Authored-By: Claude Fable 5 * fix: stop editor prefetch chain on session page destroy and swallow chunk-load failures Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- .../components/sessions/PreviewTabHost.svelte | 83 ++++++++++++------- .../(root)/(logged)/sessions/+page.svelte | 35 ++++++++ 2 files changed, 88 insertions(+), 30 deletions(-) diff --git a/frontend/src/lib/components/sessions/PreviewTabHost.svelte b/frontend/src/lib/components/sessions/PreviewTabHost.svelte index 4972785b78..d41c06c123 100644 --- a/frontend/src/lib/components/sessions/PreviewTabHost.svelte +++ b/frontend/src/lib/components/sessions/PreviewTabHost.svelte @@ -8,12 +8,9 @@ type SessionPreviewTab } from './sessionState.svelte' import type { SessionRuntime } from './sessionRuntime.svelte' + import { Loader2 } from 'lucide-svelte' import { resolvePreviewTab, parsePreviewItemRoute } from './previewRouter' import { withMenuHidden } from './sessionMode.svelte' - import ScriptEditorView from './ScriptEditorView.svelte' - import FlowEditorView from './FlowEditorView.svelte' - import RawAppEditorView from './RawAppEditorView.svelte' - import PipelineEditorView from './PipelineEditorView.svelte' import ArtifactViewer from '../copilot/chat/artifacts/ArtifactViewer.svelte' let { @@ -130,38 +127,64 @@ $effect(() => () => clearTimeout(flashTimer)) +{#snippet editorLoading()} +
+ +
+{/snippet} + {#if slot.kind === 'editor' && mounted && runtime}
+ {#if slot.editorKind === 'flow'} - + {#await import('./FlowEditorView.svelte')} + {@render editorLoading()} + {:then Module} + + {/await} {:else if slot.editorKind === 'script'} - + {#await import('./ScriptEditorView.svelte')} + {@render editorLoading()} + {:then Module} + + {/await} {:else if slot.editorKind === 'pipeline'} - + {#await import('./PipelineEditorView.svelte')} + {@render editorLoading()} + {:then Module} + + {/await} {:else} - + {#await import('./RawAppEditorView.svelte')} + {@render editorLoading()} + {:then Module} + + {/await} {/if}
{:else if slot.kind === 'artifact' && mounted} diff --git a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte index 7062e8fc8d..958ad92fc3 100644 --- a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte @@ -70,6 +70,41 @@ // iframe context and refuse to mount when embedded. const embedded = typeof window !== 'undefined' && window.self !== window.top + // Warm the lazily-loaded editor views (see PreviewTabHost) once the page is + // idle: entering session mode stays instant, and by the time the user opens + // an editor tab its chunk is usually already cached. Sequential so the + // prefetch trickles instead of fanning out four heavy graphs at once. + $effect(() => { + if (embedded || !globalEnabled) return + // Once the chain has started, cancelling the idle handle no longer helps — + // the disposed check between imports is what stops a user who left session + // mode from pulling the remaining graphs on whatever page they went to. + // (An import already in flight can't be aborted; only the tail is skipped.) + let disposed = false + const prefetch = async () => { + const loaders = [ + () => import('$lib/components/sessions/ScriptEditorView.svelte'), + () => import('$lib/components/sessions/FlowEditorView.svelte'), + () => import('$lib/components/sessions/RawAppEditorView.svelte'), + () => import('$lib/components/sessions/PipelineEditorView.svelte') + ] + for (const load of loaders) { + if (disposed) return + await load() + } + } + // Best-effort warming: swallow chunk-load failures — the {#await} on the + // actual open path surfaces (and retries) them. + const run = () => void prefetch().catch(() => {}) + const hasIdle = 'requestIdleCallback' in window + const handle = hasIdle ? window.requestIdleCallback(run) : window.setTimeout(run, 2000) + return () => { + disposed = true + if (hasIdle) window.cancelIdleCallback(handle) + else window.clearTimeout(handle) + } + }) + const sessionName = $derived(page.url.searchParams.get('session_name') ?? '') // Unfiltered resolution by name — drives the "session not found" fallback and From 9762089fcbdb33dbf453d22342661e5857b2ad7e Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 17 Jul 2026 18:08:10 +0200 Subject: [PATCH 07/14] fix(schedules): re-arm enabled schedules left with no queued occurrence (#10174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(schedules): re-arm enabled schedules left with no queued occurrence * fix(schedules): lock schedule row while re-arming and report outcome * fix(schedules): make reconcile lock cancellation-safe, re-check armed under lock Address review feedback on the schedule reconciler: - Use a transaction-scoped advisory lock (pg_try_advisory_xact_lock) instead of a session-scoped one. monitor_db runs under a 600s timeout; on cancellation a session lock on a pooled connection would be stranded, wedging reconciliation on every replica. An xact lock releases when its transaction is dropped. - rearm_schedule re-checks for a queued occurrence under the row lock and returns NoOp if already armed, closing the scan→lock window that could double-push across a cron boundary. Add a regression test. - Make reconcile_unarmed_schedules private (its only caller is in monitor.rs) and document its system-only contract. - Log the disable only after the guarded UPDATE actually disables the schedule. Co-Authored-By: Claude Opus 4.8 * fix(schedules): never disable from reconciliation and cap re-arms per pass --------- Co-authored-by: claude[bot] <41898282+claude[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 --- ...c87be642091a2f8a6973d001e59be892a984d.json | 26 +++ backend/src/monitor.rs | 193 +++++++++++++++++- backend/windmill-queue/src/schedule.rs | 115 +++++++++++ backend/windmill-queue/tests/schedule_push.rs | 121 ++++++++++- 4 files changed, 453 insertions(+), 2 deletions(-) create mode 100644 backend/.sqlx/query-2756fab586489be33a3031fabfdc87be642091a2f8a6973d001e59be892a984d.json diff --git a/backend/.sqlx/query-2756fab586489be33a3031fabfdc87be642091a2f8a6973d001e59be892a984d.json b/backend/.sqlx/query-2756fab586489be33a3031fabfdc87be642091a2f8a6973d001e59be892a984d.json new file mode 100644 index 0000000000..f9cf82a9fd --- /dev/null +++ b/backend/.sqlx/query-2756fab586489be33a3031fabfdc87be642091a2f8a6973d001e59be892a984d.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT s.workspace_id, s.path\n FROM schedule s JOIN workspace w ON w.id = s.workspace_id AND NOT w.deleted\n WHERE s.enabled IS TRUE\n AND NOT EXISTS (\n SELECT 1 FROM v2_job_queue q JOIN v2_job j USING (id)\n WHERE j.workspace_id = s.workspace_id\n AND j.trigger_kind = 'schedule'\n AND j.trigger = s.path\n AND j.runnable_path = s.script_path\n AND j.parent_job IS NULL\n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false + ] + }, + "hash": "2756fab586489be33a3031fabfdc87be642091a2f8a6973d001e59be892a984d" +} diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 0392e128b3..d1c7a78059 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -108,7 +108,11 @@ use windmill_common::{ }; #[cfg(feature = "parquet")] use windmill_object_store::reload_object_store_setting; -use windmill_queue::{cancel_job, get_queued_job_v2, SameWorkerPayload}; +use windmill_queue::{ + cancel_job, get_queued_job_v2, + schedule::{find_unarmed_schedules, rearm_schedule, RearmOutcome}, + SameWorkerPayload, +}; use windmill_worker::{ result_processor::handle_job_error, JobCompletedSender, JobIsolationLevel, OtelTracingProxySettings, SameWorkerSender, WorkspaceRegistryMap, BUNFIG_INSTALL_SCOPES, @@ -3286,6 +3290,15 @@ pub async fn monitor_db( } }; + // run every 30 iterations (~5min at the default LISTEN_NEW_EVENTS_INTERVAL_SEC). + let reconcile_unarmed_schedules_f = async { + if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(30) { + if let Some(db) = conn.as_sql() { + reconcile_unarmed_schedules(&db).await; + } + } + }; + // Poll git-sync repositories for new commits and pull them into the // workspace (repo → Windmill auto-pull). Runs every 2 iterations. let git_auto_pull_f = async { @@ -3349,9 +3362,144 @@ pub async fn monitor_db( cleanup_scheduled_job_deletions_f, git_auto_pull_f, pipeline_freshness_watchdog_f, + reconcile_unarmed_schedules_f, ); } +/// Advisory lock id ensuring only one server replica reconciles schedules at a +/// time (adjacent to GIT_AUTO_PULL_LOCK_ID). +const SCHEDULE_RECONCILE_LOCK_ID: i64 = 737_483_922; + +/// Consecutive reconciliation passes an enabled schedule must be observed with no +/// queued occurrence before it is re-armed. The next occurrence is pushed in the +/// same transaction that completes the previous one (or, for flows, on entry to +/// step 0), so an unarmed schedule is normally only ever a mid-flight push or a +/// push being retried. Requiring two passes keeps the reconciler from racing +/// those and double-pushing an occurrence. +const SCHEDULE_RECONCILE_STRIKES: u8 = 2; + +/// Most schedules re-armed in one pass, so a large first-pass backlog is drained +/// over several passes instead of enqueuing every occurrence at once. +const SCHEDULE_RECONCILE_MAX_PER_PASS: usize = 50; + +lazy_static::lazy_static! { + /// `(workspace_id, path)` -> consecutive passes seen with no queued occurrence. + /// Bounded by the number of enabled schedules; entries drop as soon as a + /// schedule is seen armed again. + static ref UNARMED_SCHEDULES: Mutex> = + Mutex::new(std::collections::HashMap::new()); +} + +/// Re-arm enabled schedules that have no queued occurrence. +/// +/// Every path that completes a scheduled job is supposed to push the next +/// occurrence atomically, but a run that dies through an abnormal path (a flow +/// whose status update fails and is later force-completed by zombie detection, +/// say) can skip that push and leave the schedule enabled yet dead forever. This +/// is the backstop: without it the only recovery is a manual disable/enable. +/// +/// Replicas each run their own passes (staggered by `rd_shift`) and each keep +/// their own strike tally, so the scan cost is per-replica. That is deliberate: +/// scanning inside the advisory lock is what makes a double-push impossible — +/// whoever holds it re-reads the unarmed set, so a schedule another replica just +/// re-armed is seen armed and its tally dropped, rather than pushed twice. +/// +/// Not an authorization boundary: it re-arms schedules across every workspace, so +/// this is a system caller (the monitor loop) only. +async fn reconcile_unarmed_schedules(db: &Pool) { + // Transaction-scoped advisory lock, not session-scoped: monitor_db runs under a + // 600s timeout, and if it fires the whole future is dropped mid-pass. A session + // lock taken on a pooled connection would then ride that connection back into the + // pool still held, wedging reconciliation on every replica until the process + // restarts. An xact lock is released when its transaction ends — including the + // rollback a dropped `Transaction` performs — so cancellation can't strand it. + // The tx is held open only to own the lock; the scan and re-arm run on separate + // pool connections. + let mut lock_tx = match db.begin().await { + Ok(tx) => tx, + Err(e) => { + tracing::error!("schedule reconcile: failed to begin lock tx: {e:#}"); + return; + } + }; + let locked: bool = match sqlx::query_scalar("SELECT pg_try_advisory_xact_lock($1)") + .bind(SCHEDULE_RECONCILE_LOCK_ID) + .fetch_one(&mut *lock_tx) + .await + { + Ok(v) => v, + Err(e) => { + tracing::error!("schedule reconcile: advisory lock failed: {e:#}"); + return; + } + }; + if !locked { + // Another replica is already reconciling this tick. + return; + } + + if let Err(e) = reconcile_unarmed_schedules_inner(db).await { + tracing::error!("schedule reconcile: {e:#}"); + } + + // Ends the transaction and releases the xact lock; a plain drop would too. + if let Err(e) = lock_tx.rollback().await { + tracing::error!("schedule reconcile: releasing lock failed: {e:#}"); + } +} + +/// Record this pass's unarmed schedules against `seen` and return those that have +/// now struck out. An armed observation drops the schedule's tally entirely, so +/// the strikes a re-arm rests on are always consecutive. +fn strike_unarmed( + seen: &mut std::collections::HashMap<(String, String), u8>, + current: std::collections::HashSet<(String, String)>, +) -> Vec<(String, String)> { + seen.retain(|k, _| current.contains(k)); + current + .into_iter() + .filter(|k| { + let strikes = seen.entry(k.clone()).or_insert(0); + *strikes = strikes.saturating_add(1); + *strikes >= SCHEDULE_RECONCILE_STRIKES + }) + .collect() +} + +async fn reconcile_unarmed_schedules_inner(db: &Pool) -> error::Result<()> { + let current = find_unarmed_schedules(db).await?.into_iter().collect(); + let mut to_rearm = strike_unarmed(&mut UNARMED_SCHEDULES.lock().unwrap(), current); + + // The first pass on an instance that has never been swept can find a large + // backlog; re-arming it all at once would enqueue that whole backlog in one + // go. The overflow keeps its tally and is picked up next pass. + if to_rearm.len() > SCHEDULE_RECONCILE_MAX_PER_PASS { + tracing::warn!( + "schedule reconcile: {} schedules have no queued occurrence, re-arming {} this pass and the rest on later passes", + to_rearm.len(), + SCHEDULE_RECONCILE_MAX_PER_PASS + ); + to_rearm.truncate(SCHEDULE_RECONCILE_MAX_PER_PASS); + } + + for (w_id, path) in to_rearm { + match rearm_schedule(db, &w_id, &path).await { + Ok(outcome) => { + if outcome == RearmOutcome::Rearmed { + tracing::warn!( + "schedule reconcile: re-armed enabled schedule {path} in {w_id}, which had no queued occurrence" + ); + } + UNARMED_SCHEDULES.lock().unwrap().remove(&(w_id, path)); + } + Err(e) => tracing::error!( + "schedule reconcile: could not re-arm schedule {path} in {w_id}: {e:#}" + ), + } + } + Ok(()) +} + /// Advisory lock id ensuring only one server replica runs the git auto-pull /// poll at a time (adjacent to RESTART_LOCK_ID used for restart coordination). #[cfg(feature = "private")] @@ -5390,3 +5538,46 @@ mod retention_overrides_tests { assert!(parse_retention_overrides(over_cap).is_err()); } } + +#[cfg(test)] +mod strike_unarmed_tests { + use super::strike_unarmed; + use std::collections::{HashMap, HashSet}; + + fn key(path: &str) -> (String, String) { + ("ws".to_string(), path.to_string()) + } + + fn set(paths: &[&str]) -> HashSet<(String, String)> { + paths.iter().map(|p| key(p)).collect() + } + + /// The strike threshold is the only thing keeping the reconciler from racing + /// an in-flight push: `push_scheduled_job`'s own `already_exists` guard keys + /// on the same columns as the scan, so it is false by construction whenever a + /// schedule is found unarmed. + #[test] + fn rearms_only_after_consecutive_unarmed_passes() { + let mut seen = HashMap::new(); + assert!(strike_unarmed(&mut seen, set(&["a"])).is_empty()); + assert_eq!(strike_unarmed(&mut seen, set(&["a"])), vec![key("a")]); + } + + #[test] + fn armed_observation_resets_the_tally() { + let mut seen = HashMap::new(); + assert!(strike_unarmed(&mut seen, set(&["a"])).is_empty()); + // `a` is armed again on this pass, so its strike must not carry over. + assert!(strike_unarmed(&mut seen, set(&[])).is_empty()); + assert!(strike_unarmed(&mut seen, set(&["a"])).is_empty()); + assert_eq!(strike_unarmed(&mut seen, set(&["a"])), vec![key("a")]); + } + + #[test] + fn tallies_are_per_schedule() { + let mut seen = HashMap::new(); + assert!(strike_unarmed(&mut seen, set(&["a"])).is_empty()); + assert_eq!(strike_unarmed(&mut seen, set(&["a", "b"])), vec![key("a")]); + assert_eq!(strike_unarmed(&mut seen, set(&["b"])), vec![key("b")]); + } +} diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index 2b5aef96a5..55944b08ca 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -597,6 +597,121 @@ pub async fn push_scheduled_job<'c>( Ok(tx) // TODO: Bubble up pushed UUID from here } +/// Enabled schedules with no occurrence in the queue, as `(workspace_id, path)`. +/// +/// Every path that completes a scheduled job pushes the next occurrence in the +/// same transaction (for flows, on entry to step 0), so an enabled schedule +/// always has a queued occurrence — a run in progress is itself one. A run that +/// dies through an abnormal path can skip that push though, leaving the schedule +/// enabled yet dead until it is manually disabled and re-enabled. This is how the +/// monitor spots that state; see `rearm_schedule` for the recovery. +/// +/// Not an authorization boundary: it reports schedules across every workspace, so +/// this is for system callers (the monitor's reconciliation pass) only and its +/// result must never be returned to a user unfiltered. +pub async fn find_unarmed_schedules(db: &DB) -> Result> { + let rows = sqlx::query!( + // Query plan: the anti-join builds from `v2_job_queue` (only pending and + // running jobs) rather than probing `v2_job` once per schedule. + "SELECT s.workspace_id, s.path + FROM schedule s JOIN workspace w ON w.id = s.workspace_id AND NOT w.deleted + WHERE s.enabled IS TRUE + AND NOT EXISTS ( + SELECT 1 FROM v2_job_queue q JOIN v2_job j USING (id) + WHERE j.workspace_id = s.workspace_id + AND j.trigger_kind = 'schedule' + AND j.trigger = s.path + AND j.runnable_path = s.script_path + AND j.parent_job IS NULL + )" + ) + .fetch_all(db) + .await?; + Ok(rows.into_iter().map(|r| (r.workspace_id, r.path)).collect()) +} + +#[derive(Debug, PartialEq, Eq)] +pub enum RearmOutcome { + /// The next occurrence was pushed. + Rearmed, + /// Nothing to do: the schedule was deleted or disabled since it was found. + NoOp, +} + +/// Push the next occurrence of a schedule that has none queued. +/// +/// Only ever starts a schedule, never stops one: re-arming something that did not +/// need it costs one extra run, whereas wrongly disabling one is the silent +/// permanent stoppage this whole mechanism exists to prevent. So an occurrence +/// that cannot be pushed is logged and left alone — the schedule is already not +/// running, and `try_schedule_next_job` still disables on the completion path, +/// where the population is limited to actively-cycling schedules. Keep it that +/// way: this sweeps *every* enabled schedule, including ones broken long before +/// this code existed and never swept before. +/// +/// Not an authorization boundary: it pushes under the schedule's own +/// `permissioned_as` identity for any `(w_id, path)`, so this is for system +/// callers (the monitor's reconciliation pass) only. A caller acting for a user +/// MUST already have enforced their permissions on `w_id` and `path`. +pub async fn rearm_schedule(db: &DB, w_id: &str, path: &str) -> Result { + let mut tx = db.begin().await?; + // Lock the row for the whole push: an edit or a disable committing between the + // read and the push would otherwise leave a queued occurrence for a schedule + // that is disabled, or one built from superseded settings. + let schedule = sqlx::query_as::<_, Schedule>( + "SELECT workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, args, extra_perms, email, permissioned_as, error, on_failure, on_failure_times, on_failure_exact, on_failure_extra_args, on_recovery, on_recovery_times, on_recovery_extra_args, on_success, on_success_extra_args, ws_error_handler_muted, retry, no_flow_overlap, summary, description, tag, paused_until, cron_version, dynamic_skip, labels FROM schedule WHERE path = $1 AND workspace_id = $2 FOR UPDATE", + ) + .bind(path) + .bind(w_id) + .fetch_optional(&mut *tx) + .await?; + let Some(schedule) = schedule else { + return Ok(RearmOutcome::NoOp); + }; + if !schedule.enabled { + return Ok(RearmOutcome::NoOp); + } + // Re-check for a queued occurrence now that the row is locked: a normal + // completion, an edit, or a re-enable could have pushed one between the unarmed + // scan and this lock. push_scheduled_job only dedups the exact computed + // scheduled_for, so re-arming a schedule that has since become armed and crossed a + // cron boundary would queue a second root occurrence. Mirrors the anti-join in + // find_unarmed_schedules. + let already_armed: bool = sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 FROM v2_job_queue q JOIN v2_job j USING (id) + WHERE j.workspace_id = $1 + AND j.trigger_kind = 'schedule' + AND j.trigger = $2 + AND j.runnable_path = $3 + AND j.parent_job IS NULL + )", + ) + .bind(w_id) + .bind(path) + .bind(&schedule.script_path) + .fetch_one(&mut *tx) + .await?; + if already_armed { + return Ok(RearmOutcome::NoOp); + } + match push_scheduled_job(db, tx, &schedule, None, None).await { + Ok(tx) => { + tx.commit().await?; + Ok(RearmOutcome::Rearmed) + } + // An occurrence that can never be pushed (runnable gone, quota blown) is + // reported, not acted on — see the note above on why this never disables. + Err(err @ (error::Error::NotFound(_) | error::Error::QuotaExceeded(_))) => { + tracing::error!( + "Could not re-arm schedule {path} in {w_id}: {err}. Leaving it enabled; it will not run until the cause is fixed." + ); + Ok(RearmOutcome::NoOp) + } + Err(err) => Err(err), + } +} + pub async fn get_schedule_opt<'c>( e: impl PgExecutor<'c>, w_id: &str, diff --git a/backend/windmill-queue/tests/schedule_push.rs b/backend/windmill-queue/tests/schedule_push.rs index e11ced23bc..9ac2e3a143 100644 --- a/backend/windmill-queue/tests/schedule_push.rs +++ b/backend/windmill-queue/tests/schedule_push.rs @@ -10,7 +10,9 @@ mod schedule_push { use windmill_common::scripts::ScriptHash; use windmill_common::users::username_to_permissioned_as; use windmill_queue::jobs::{try_schedule_next_job, MiniCompletedJob}; - use windmill_queue::schedule::push_scheduled_job; + use windmill_queue::schedule::{ + find_unarmed_schedules, push_scheduled_job, rearm_schedule, RearmOutcome, + }; fn make_schedule(overrides: impl FnOnce(&mut Schedule)) -> Schedule { let mut s = Schedule { @@ -1762,4 +1764,121 @@ mod schedule_push { assert!(!row_exists, "managed schedule row must be deleted"); Ok(()) } + + // ----------------------------------------------------------------------- + // find_unarmed_schedules / rearm_schedule: recovery for a schedule left + // enabled with no queued occurrence (a run that died on an abnormal path + // skipped its next-occurrence push). Without this the chain stays dead + // until the schedule is manually disabled and re-enabled. + // ----------------------------------------------------------------------- + + async fn insert_schedule(db: &Pool, path: &str, script_path: &str, enabled: bool) { + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, email, extra_perms, ws_error_handler_muted, no_flow_overlap, permissioned_as) + VALUES ('test-workspace', $1, 'test-user', now(), '0 0 */5 * * *', 'UTC', $3, $2, false, 'test@windmill.dev', '{}', false, true, 'u/test-user')", + ) + .bind(path) + .bind(script_path) + .bind(enabled) + .execute(db) + .await + .unwrap(); + } + + #[sqlx::test(migrations = "../migrations", fixtures("base", "schedule_push"))] + async fn test_find_unarmed_schedules(db: Pool) -> anyhow::Result<()> { + insert_schedule(&db, "f/system/test_schedule", "f/system/test_script", true).await; + insert_schedule(&db, "f/system/disabled", "f/system/test_script", false).await; + + // No occurrence queued yet: the enabled schedule is unarmed, the disabled one is ignored. + assert_eq!( + find_unarmed_schedules(&db).await?, + vec![( + "test-workspace".to_string(), + "f/system/test_schedule".to_string() + )] + ); + + // Once an occurrence is queued it is armed and must not be reported — + // re-arming it would double-push the occurrence. + let tx = db.begin().await?; + let tx = push_scheduled_job(&db, tx, &make_schedule(|_| {}), None, None).await?; + tx.commit().await?; + assert_eq!(count_queued_jobs(&db).await, 1); + assert!(find_unarmed_schedules(&db).await?.is_empty()); + Ok(()) + } + + #[sqlx::test(migrations = "../migrations", fixtures("base", "schedule_push"))] + async fn test_rearm_schedule_pushes_next_occurrence(db: Pool) -> anyhow::Result<()> { + insert_schedule(&db, "f/system/test_schedule", "f/system/test_script", true).await; + + assert_eq!( + rearm_schedule(&db, "test-workspace", "f/system/test_schedule").await?, + RearmOutcome::Rearmed + ); + + assert_eq!(count_queued_jobs(&db).await, 1); + assert!(find_unarmed_schedules(&db).await?.is_empty()); + Ok(()) + } + + #[sqlx::test(migrations = "../migrations", fixtures("base", "schedule_push"))] + async fn test_rearm_schedule_skips_disabled(db: Pool) -> anyhow::Result<()> { + // A disable that lands between the scan and the re-arm must win: pushing an + // occurrence for a disabled schedule would resurrect a schedule the user + // just turned off. + insert_schedule(&db, "f/system/test_schedule", "f/system/test_script", false).await; + + assert_eq!( + rearm_schedule(&db, "test-workspace", "f/system/test_schedule").await?, + RearmOutcome::NoOp + ); + assert_eq!(count_queued_jobs(&db).await, 0); + Ok(()) + } + + #[sqlx::test(migrations = "../migrations", fixtures("base", "schedule_push"))] + async fn test_rearm_schedule_never_disables(db: Pool) -> anyhow::Result<()> { + insert_schedule(&db, "f/system/bad_schedule", "f/system/nonexistent", true).await; + + // Reconciliation only ever starts a schedule. An unpushable occurrence is + // reported and left alone: this sweeps every enabled schedule in the + // instance, so disabling here would turn a wrong invariant into the exact + // silent stoppage the reconciler exists to undo. + assert_eq!( + rearm_schedule(&db, "test-workspace", "f/system/bad_schedule").await?, + RearmOutcome::NoOp + ); + + assert_eq!(count_queued_jobs(&db).await, 0); + let (enabled, error): (bool, Option) = sqlx::query_as( + "SELECT enabled, error FROM schedule WHERE workspace_id = 'test-workspace' AND path = 'f/system/bad_schedule'", + ) + .fetch_one(&db) + .await?; + assert!(enabled, "reconciliation must never disable a schedule"); + assert!(error.is_none()); + Ok(()) + } + + #[sqlx::test(migrations = "../migrations", fixtures("base", "schedule_push"))] + async fn test_rearm_schedule_skips_already_armed(db: Pool) -> anyhow::Result<()> { + // An occurrence can be queued (a normal completion, an edit, a re-enable) + // between the unarmed scan and rearm_schedule acquiring the row lock. Re-arming + // then would double-push, since push_scheduled_job only dedups the exact + // computed scheduled_for. + insert_schedule(&db, "f/system/test_schedule", "f/system/test_script", true).await; + let tx = db.begin().await?; + let tx = push_scheduled_job(&db, tx, &make_schedule(|_| {}), None, None).await?; + tx.commit().await?; + assert_eq!(count_queued_jobs(&db).await, 1); + + assert_eq!( + rearm_schedule(&db, "test-workspace", "f/system/test_schedule").await?, + RearmOutcome::NoOp + ); + assert_eq!(count_queued_jobs(&db).await, 1); + Ok(()) + } } From 2ff5a918d54074bc340b6e92222e9f5bb4369ba0 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 17 Jul 2026 18:18:54 +0200 Subject: [PATCH 08/14] feat(worker-tags): add `*` fork marker to workspace-scoped custom tags (#10177) * feat(worker-tags): add `*` fork marker to workspace-scoped custom tags Co-Authored-By: Claude Opus 4.8 (1M context) * test(worker-tags): pin fork-marker tag admission through real lineage Also render WorkspaceMatcher in its authored form in the operator-facing "not in the allowed CUSTOM_TAGS" error, and correct the authorization note on workspace_with_fork_ancestors. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(worker-tags): gate fork-lineage tag lookup on workspace membership exists_workers_with_tags is a global route whose workspace is a query param. Resolving its fork lineage for a caller who is not a member disclosed whether an arbitrary workspace descends from one named by a tag(parent*) rule. Require membership before the lineage lookup, and sharpen the workspace_with_fork_ancestors contract accordingly. Co-Authored-By: Claude Opus 4.8 (1M context) * chore(sqlx): add offline cache for fork-marker test INSERT query The new test_fork_marker_tag_admission_through_lineage integration test inserts a fork workspace with a bind-parameterized query that had no cached entry, failing the SQLX_OFFLINE=true CI build. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- ...27d1e30b63c946a4eb640e55850100c3edd3c.json | 14 ++ backend/tests/worker.rs | 63 +++++- backend/windmill-api-workers/src/lib.rs | 23 ++- backend/windmill-common/src/jobs.rs | 14 +- backend/windmill-common/src/worker.rs | 195 ++++++++++++++++-- backend/windmill-common/src/workspaces.rs | 15 ++ .../lib/components/AssignableTagsInner.svelte | 23 ++- 7 files changed, 312 insertions(+), 35 deletions(-) create mode 100644 backend/.sqlx/query-be21088e8b88e01b50a544f220e27d1e30b63c946a4eb640e55850100c3edd3c.json diff --git a/backend/.sqlx/query-be21088e8b88e01b50a544f220e27d1e30b63c946a4eb640e55850100c3edd3c.json b/backend/.sqlx/query-be21088e8b88e01b50a544f220e27d1e30b63c946a4eb640e55850100c3edd3c.json new file mode 100644 index 0000000000..a0697321be --- /dev/null +++ b/backend/.sqlx/query-be21088e8b88e01b50a544f220e27d1e30b63c946a4eb640e55850100c3edd3c.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace (id, name, owner, parent_workspace_id)\n VALUES ($1, $1, 'test-user', 'test-workspace')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "be21088e8b88e01b50a544f220e27d1e30b63c946a4eb640e55850100c3edd3c" +} diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 2349d476b7..3140cd2d7f 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -5338,9 +5338,10 @@ async fn test_duckdb_ffi(db: Pool) -> anyhow::Result<()> { /// This validates that `check_tag_available_for_workspace_internal` is properly called /// when pushing jobs from worker_flow. #[sqlx::test(fixtures("base"))] +#[serial] async fn test_flow_substep_tag_availability_check(db: Pool) -> anyhow::Result<()> { use windmill_common::worker::{ - CustomTags, SpecificTagData, SpecificTagType, CUSTOM_TAGS_PER_WORKSPACE, + CustomTags, SpecificTagData, SpecificTagType, WorkspaceMatcher, CUSTOM_TAGS_PER_WORKSPACE, }; initialize_tracing().await; @@ -5354,7 +5355,10 @@ async fn test_flow_substep_tag_availability_check(db: Pool) -> anyhow: "restricted-tag".to_string(), SpecificTagData { tag_type: SpecificTagType::NoneExcept, - workspaces: vec!["other-workspace".to_string()], + workspaces: vec![WorkspaceMatcher { + id: "other-workspace".to_string(), + include_forks: false, + }], }, )]), })); @@ -5404,6 +5408,61 @@ async fn test_flow_substep_tag_availability_check(db: Pool) -> anyhow: Ok(()) } +/// The `*` fork marker only grants through a real `parent_workspace_id` lineage lookup, which the +/// parse-level unit tests cannot reach: they hand `applies_to_workspace` a synthetic chain, so a +/// regression in the lookup or in the `is_fork_scoped()` gate that skips it would pass them. +#[sqlx::test(fixtures("base"))] +#[serial] +async fn test_fork_marker_tag_admission_through_lineage(db: Pool) -> anyhow::Result<()> { + use windmill_common::jobs::check_tag_available_for_workspace_internal; + use windmill_common::worker::{CustomTags, CUSTOM_TAGS_PER_WORKSPACE}; + + initialize_tracing().await; + + // The ancestor chain is cached process-wide by workspace id, so use one no other test takes. + let fork = "wm-fork-tagmarker"; + sqlx::query!( + "INSERT INTO workspace (id, name, owner, parent_workspace_id) + VALUES ($1, $1, 'test-user', 'test-workspace')", + fork + ) + .execute(&db) + .await?; + + CUSTOM_TAGS_PER_WORKSPACE.store(std::sync::Arc::new(CustomTags::from(vec![ + "forky(test-workspace*)".to_string(), + "bare(test-workspace)".to_string(), + ]))); + + // test2 is not a superadmin, who would bypass the scope check entirely. + let email = "test2@windmill.dev"; + + for (w_id, tag) in [("test-workspace", "bare"), ("test-workspace", "forky")] { + assert!( + check_tag_available_for_workspace_internal(&db, w_id, tag, email, None) + .await + .is_ok(), + "{tag} should be available in the workspace it names" + ); + } + assert!( + check_tag_available_for_workspace_internal(&db, fork, "forky", email, None) + .await + .is_ok(), + "a `*` tag must be granted to a fork through its parent lineage" + ); + assert!( + check_tag_available_for_workspace_internal(&db, fork, "bare", email, None) + .await + .is_err(), + "an unmarked tag must not reach a fork of the workspace it names" + ); + + CUSTOM_TAGS_PER_WORKSPACE.store(std::sync::Arc::new(CustomTags::default())); + + Ok(()) +} + #[cfg(all(feature = "quickjs", feature = "python"))] #[sqlx::test(fixtures("base"))] async fn test_whileloop_propagates_inner_iterator_eval_failure( diff --git a/backend/windmill-api-workers/src/lib.rs b/backend/windmill-api-workers/src/lib.rs index d65316555e..f33e12470f 100644 --- a/backend/windmill-api-workers/src/lib.rs +++ b/backend/windmill-api-workers/src/lib.rs @@ -21,6 +21,7 @@ use windmill_common::{ jobs::{HIDE_WORKERS_FOR_NON_ADMINS, TAGS_ARE_SENSITIVE}, utils::{paginate, Pagination}, worker::{ALL_TAGS, CUSTOM_TAGS_PER_WORKSPACE, DEFAULT_TAGS, DEFAULT_TAGS_PER_WORKSPACE}, + workspaces::workspace_with_fork_ancestors, DB, }; @@ -161,9 +162,25 @@ async fn exists_workers_with_tags( let has_devops_role = require_devops_role(&db, &authed.email).await.is_ok(); if !has_devops_role { if let Some(ref workspace) = tags_query.workspace { + // This route is global, so the workspace is an unauthorized query param: check + // membership before reading its lineage, which would otherwise disclose whether + // an arbitrary workspace descends from one named by a `tag(parent*)` rule. + let is_member = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND email = $2 AND NOT disabled)", + workspace, + &authed.email + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + if !is_member { + return Ok(Json(std::collections::HashMap::new())); + } + // Filter to only tags visible in this workspace + let chain = workspace_with_fork_ancestors(&db, workspace).await?; let custom_tags = CUSTOM_TAGS_PER_WORKSPACE.load(); - let allowed_tags = custom_tags.to_string_vec(Some(workspace.clone())); + let allowed_tags = custom_tags.to_string_vec(Some(&chain)); tags.retain(|t| allowed_tags.contains(t)); } else { // No workspace provided and not superadmin - return empty @@ -222,10 +239,12 @@ async fn get_custom_tags( async fn get_custom_tags_for_workspace( _authed: ApiAuthed, + Extension(db): Extension, Path(w_id): Path, ) -> JsonResult> { + let chain = workspace_with_fork_ancestors(&db, &w_id).await?; let tags_o = CUSTOM_TAGS_PER_WORKSPACE.load(); - let all_tags = tags_o.to_string_vec(Some(w_id)); + let all_tags = tags_o.to_string_vec(Some(&chain)); Ok(Json(all_tags)) } diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 2178d9a023..d73637026a 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -20,6 +20,7 @@ use crate::{ users::username_to_permissioned_as, utils::{StripPath, HTTP_CLIENT}, worker::{to_raw_value, CUSTOM_TAGS_PER_WORKSPACE, WINDMILL_DIR}, + workspaces::workspace_with_fork_ancestors, FlowVersionInfo, ScriptHashInfo, Tag, }; @@ -343,8 +344,8 @@ lazy_static::lazy_static! { ).unwrap_or(false); } -pub async fn check_tag_available_for_workspace_internal<'c>( - db: impl sqlx::PgExecutor<'c>, +pub async fn check_tag_available_for_workspace_internal( + db: &DB, w_id: &str, tag: &str, email: &str, @@ -361,7 +362,14 @@ pub async fn check_tag_available_for_workspace_internal<'c>( if custom_tags_per_w.global.contains(&tag.to_string()) { is_tag_in_workspace_custom_tags = true; } else if let Some(specific_tag) = custom_tags_per_w.specific.get(tag) { - is_tag_in_workspace_custom_tags = specific_tag.applies_to_workspace(w_id); + // Only a fork-scoped tag can match through the lineage, so every other tag keeps the + // ancestor lookup off the push path entirely. + let chain = if specific_tag.is_fork_scoped() { + workspace_with_fork_ancestors(db, w_id).await? + } else { + vec![w_id.to_string()] + }; + is_tag_in_workspace_custom_tags = specific_tag.applies_to_workspace(&chain); } match is_tag_in_scope_tags { diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index ea2b5455f3..eb339b56f8 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -52,10 +52,10 @@ impl CustomTags { let tag_name = cap.get(1).unwrap().as_str().to_string(); let workspace_str = cap.get(2).unwrap().as_str(); let tag_type = SpecificTagType::from_regex_string(workspace_str); - let workspaces: Vec = workspace_str + let workspaces: Vec = workspace_str .split(tag_type.corresponding_separator()) .filter(|s| !s.is_empty()) - .map(str::to_string) + .map(WorkspaceMatcher::parse) .collect(); if workspaces.is_empty() { tracing::warn!("Ignoring tag `{}` with empty exclusion/inclusion list", e); @@ -70,11 +70,13 @@ impl CustomTags { Self { global, specific } } - pub fn to_string_vec(&self, filter_with_workspace: Option) -> Vec { - let specific = if let Some(workspace) = filter_with_workspace { + /// `filter_with_workspace` is the workspace's id chain (see [`SpecificTagData::applies_to_workspace`]); + /// `None` re-emits the authored `tag(ws1+ws2)` strings for the settings editor. + pub fn to_string_vec(&self, filter_with_workspace: Option<&[String]>) -> Vec { + let specific = if let Some(chain) = filter_with_workspace { self.specific .iter() - .filter(|(_, tag_data)| tag_data.applies_to_workspace(&workspace)) + .filter(|(_, tag_data)| tag_data.applies_to_workspace(chain)) .map(|(tag, _)| tag.clone()) .collect::>() } else { @@ -82,7 +84,12 @@ impl CustomTags { .iter() .map(|(tag, tag_data)| { let separator = tag_data.tag_type.corresponding_separator(); - let mut workspaces = tag_data.workspaces.join(&*separator.to_string()); + let mut workspaces = tag_data + .workspaces + .iter() + .map(|w| w.to_string()) + .collect::>() + .join(&*separator.to_string()); if tag_data.tag_type == SpecificTagType::AllExcluding { // the AllExcluding tag syntax has a leading separator workspaces.insert(0, separator); @@ -95,19 +102,86 @@ impl CustomTags { all_tags.into_iter().chain(specific.into_iter()).collect() } } + +/// Marker suffixed to a workspace id inside a custom tag's scope (`mytag(prod*)`) to extend the +/// entry to that workspace's forks. `*` cannot appear in a workspace id (the `proper_id` check +/// constraint restricts them to `^\w+(-\w+)*$`), so it can never collide with a real id. +pub const FORK_SCOPE_MARKER: char = '*'; + +/// One workspace entry in a custom tag's scope. Bare (`prod`) matches that workspace only; +/// with the [`FORK_SCOPE_MARKER`] (`prod*`) it also matches its forks, transitively. +/// +/// The marker is opt-in in BOTH scope forms so that no existing tag string changes meaning: +/// `sensitive(^prod)` keeps excluding only `prod` itself, and `sensitive(^prod*)` is how you +/// exclude its forks too. +#[derive(Clone, Serialize, Deserialize, PartialEq)] +pub struct WorkspaceMatcher { + pub id: String, + pub include_forks: bool, +} + +impl WorkspaceMatcher { + fn parse(entry: &str) -> Self { + match entry.strip_suffix(FORK_SCOPE_MARKER) { + Some(id) => Self { id: id.to_string(), include_forks: true }, + None => Self { id: entry.to_string(), include_forks: false }, + } + } + + fn matches(&self, workspace_id: &str, fork_ancestors: &[String]) -> bool { + workspace_id == self.id + || (self.include_forks && fork_ancestors.iter().any(|a| *a == self.id)) + } +} + +impl std::fmt::Display for WorkspaceMatcher { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.id)?; + if self.include_forks { + f.write_str(FORK_SCOPE_MARKER.encode_utf8(&mut [0u8; 4]))?; + } + Ok(()) + } +} + +/// Renders the authored `prod` / `prod*` form rather than the struct fields: `CustomTags` is +/// `{:?}`-dumped into the "tag is not in the allowed CUSTOM_TAGS" error operators see. +impl std::fmt::Debug for WorkspaceMatcher { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + std::fmt::Debug::fmt(&self.to_string(), f) + } +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct SpecificTagData { pub tag_type: SpecificTagType, - pub workspaces: Vec, + pub workspaces: Vec, } impl SpecificTagData { - pub fn applies_to_workspace(&self, workspace_id: &str) -> bool { + /// `chain` is the workspace itself followed by its fork ancestors, nearest-first, as built by + /// `workspaces::workspace_with_fork_ancestors`. Pass a single-element slice when + /// [`Self::is_fork_scoped`] is false: the ancestors cannot affect the outcome then. + pub fn applies_to_workspace(&self, chain: &[String]) -> bool { + let Some((workspace_id, fork_ancestors)) = chain.split_first() else { + return false; + }; + let matched = self + .workspaces + .iter() + .any(|w| w.matches(workspace_id, fork_ancestors)); match self.tag_type { - SpecificTagType::AllExcluding => !self.workspaces.contains(&workspace_id.to_string()), - SpecificTagType::NoneExcept => self.workspaces.contains(&workspace_id.to_string()), + SpecificTagType::AllExcluding => !matched, + SpecificTagType::NoneExcept => matched, } } + + /// Whether any entry carries the fork marker, i.e. whether resolving the workspace's fork + /// lineage can change what [`Self::applies_to_workspace`] returns. Lets hot callers skip the + /// lineage lookup for the (overwhelmingly common) fork-agnostic tag. + pub fn is_fork_scoped(&self) -> bool { + self.workspaces.iter().any(|w| w.include_forks) + } } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub enum SpecificTagType { @@ -295,12 +369,14 @@ lazy_static::lazy_static! { // ^([\w-]+) # Group 1: tag name // \( # Literal '(' // ( # Group 2: the full workspace list - // (?:[\w-]+\+)*[\w-]+ # NoneExcept pattern: ws1+ws2 - // | # OR - // (?:\^[\w-]+)+ # AllExcluding pattern: ^ws1^ws2 + // (?:[\w-]+\*?\+)*[\w-]+\*? # NoneExcept pattern: ws1+ws2* + // | # OR + // (?:\^[\w-]+\*?)+ # AllExcluding pattern: ^ws1^ws2* // ) // \)$ # Closing ')' - static ref CUSTOM_TAG_REGEX: Regex = Regex::new(r"^([\w-]+)\(((?:[\w-]+\+)*[\w-]+|(?:\^[\w-]+)+)\)$").unwrap(); + // + // The optional `*` after each workspace id is the fork marker, see [`WorkspaceMatcher`]. + static ref CUSTOM_TAG_REGEX: Regex = Regex::new(r"^([\w-]+)\(((?:[\w-]+\*?\+)*[\w-]+\*?|(?:\^[\w-]+\*?)+)\)$").unwrap(); pub static ref DISABLE_BUNDLING: bool = std::env::var("DISABLE_BUNDLING") .ok() @@ -2332,6 +2408,19 @@ mod tests { use super::*; use std::collections::HashMap; + fn matcher(id: &str) -> WorkspaceMatcher { + WorkspaceMatcher { id: id.to_string(), include_forks: false } + } + + fn fork_matcher(id: &str) -> WorkspaceMatcher { + WorkspaceMatcher { id: id.to_string(), include_forks: true } + } + + /// A workspace id chain: the workspace itself, then its fork ancestors nearest-first. + fn chain(ids: &[&str]) -> Vec { + ids.iter().map(|s| s.to_string()).collect() + } + #[test] fn test_bash_sandbox_image_annotation() { // `# sandbox ` selects the container runtime and returns the image. @@ -2430,14 +2519,14 @@ mod tests { "feat".to_string(), SpecificTagData { tag_type: SpecificTagType::NoneExcept, - workspaces: vec!["ws1".to_string(), "ws2".to_string()], + workspaces: vec![matcher("ws1"), matcher("ws2")], }, ); expected.insert( "hotfix".to_string(), SpecificTagData { tag_type: SpecificTagType::AllExcluding, - workspaces: vec!["ws3".to_string(), "ws4".to_string()], + workspaces: vec![matcher("ws3"), matcher("ws4")], }, ); @@ -2480,7 +2569,7 @@ mod tests { let data = tags.specific.get("urgent").unwrap(); assert_eq!(data.tag_type, SpecificTagType::NoneExcept); - assert_eq!(data.workspaces, vec!["ws1", "ws2"]); + assert_eq!(data.workspaces, vec![matcher("ws1"), matcher("ws2")]); } #[test] @@ -2493,7 +2582,7 @@ mod tests { let data = tags.specific.get("legacy").unwrap(); assert_eq!(data.tag_type, SpecificTagType::AllExcluding); - assert_eq!(data.workspaces, vec!["ws1", "ws2"]); + assert_eq!(data.workspaces, vec![matcher("ws1"), matcher("ws2")]); } #[test] @@ -2510,10 +2599,10 @@ mod tests { let input = vec!["urgent(ws1+ws2)".to_string()]; let tags = CustomTags::from(input); - let output = tags.to_string_vec(Some("ws1".to_string())); + let output = tags.to_string_vec(Some(&chain(&["ws1"]))); assert_eq!(output, vec!["urgent"]); - let output_none = tags.to_string_vec(Some("ws3".to_string())); + let output_none = tags.to_string_vec(Some(&chain(&["ws3"]))); assert!(output_none.is_empty()); } @@ -2522,10 +2611,10 @@ mod tests { let input = vec!["legacy(^ws1^ws2)".to_string()]; let tags = CustomTags::from(input); - let output = tags.to_string_vec(Some("ws3".to_string())); + let output = tags.to_string_vec(Some(&chain(&["ws3"]))); assert_eq!(output, vec!["legacy"]); - let output_excluded = tags.to_string_vec(Some("ws1".to_string())); + let output_excluded = tags.to_string_vec(Some(&chain(&["ws1"]))); assert!(output_excluded.is_empty()); } @@ -2543,6 +2632,68 @@ mod tests { assert_eq!(result, vec!["foo", "legacy(^ws1^ws2)", "urgent(ws1+ws2)"]); } + #[test] + fn test_fork_marker_parses_and_round_trips() { + let tags = CustomTags::from(vec![ + "urgent(prod*+ws2)".to_string(), + "legacy(^prod*)".to_string(), + ]); + + let urgent = tags.specific.get("urgent").unwrap(); + assert_eq!( + urgent.workspaces, + vec![fork_matcher("prod"), matcher("ws2")] + ); + assert!(urgent.is_fork_scoped()); + + let legacy = tags.specific.get("legacy").unwrap(); + assert_eq!(legacy.workspaces, vec![fork_matcher("prod")]); + + // The settings editor re-emits what it parsed; dropping `*` here would silently widen + // an excluding tag / narrow an including one on every save. + let mut result = tags.to_string_vec(None); + result.sort(); + assert_eq!(result, vec!["legacy(^prod*)", "urgent(prod*+ws2)"]); + } + + #[test] + fn test_fork_marker_extends_none_except_to_forks_only_when_present() { + let fork = chain(&["wm-fork-x", "prod"]); + let nested = chain(&["wm-fork-y", "wm-fork-x", "prod"]); + + let marked = CustomTags::from(vec!["urgent(prod*)".to_string()]); + let marked = marked.specific.get("urgent").unwrap(); + assert!(marked.applies_to_workspace(&chain(&["prod"]))); + assert!(marked.applies_to_workspace(&fork)); + assert!(marked.applies_to_workspace(&nested)); + assert!(!marked.applies_to_workspace(&chain(&["wm-fork-z", "other"]))); + + // Without the marker a fork must NOT inherit the parent's tag. + let unmarked = CustomTags::from(vec!["urgent(prod)".to_string()]); + let unmarked = unmarked.specific.get("urgent").unwrap(); + assert!(unmarked.applies_to_workspace(&chain(&["prod"]))); + assert!(!unmarked.applies_to_workspace(&fork)); + // Gates the ancestor lookup, so a wrong answer here silently disables the marker. + assert!(!unmarked.is_fork_scoped()); + } + + #[test] + fn test_fork_marker_extends_all_excluding_to_forks_only_when_present() { + let fork = chain(&["wm-fork-x", "prod"]); + + // Pre-existing exclusions keep their exact meaning: only `prod` itself is excluded. + let unmarked = CustomTags::from(vec!["legacy(^prod)".to_string()]); + let unmarked = unmarked.specific.get("legacy").unwrap(); + assert!(!unmarked.applies_to_workspace(&chain(&["prod"]))); + assert!(unmarked.applies_to_workspace(&fork)); + + let marked = CustomTags::from(vec!["legacy(^prod*)".to_string()]); + let marked = marked.specific.get("legacy").unwrap(); + assert!(!marked.applies_to_workspace(&chain(&["prod"]))); + assert!(!marked.applies_to_workspace(&fork)); + assert!(marked.applies_to_workspace(&chain(&["other"]))); + } + #[test] fn test_dedicated_worker_tag_short() { let tag = dedicated_worker_tag("demo", "u/alice/script"); diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 33c80cea4b..972cc0a028 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -1420,6 +1420,21 @@ pub async fn fork_ancestor_chain(db: &crate::DB, w_id: &str) -> Result Result> { + let mut chain = Vec::with_capacity(4); + chain.push(w_id.to_string()); + chain.extend(fork_ancestor_chain(db, w_id).await?); + Ok(chain) +} + pub async fn get_ducklake_from_db_unchecked( name: &str, w_id: &str, diff --git a/frontend/src/lib/components/AssignableTagsInner.svelte b/frontend/src/lib/components/AssignableTagsInner.svelte index cfca54bc99..cda34487af 100644 --- a/frontend/src/lib/components/AssignableTagsInner.svelte +++ b/frontend/src/lib/components/AssignableTagsInner.svelte @@ -37,9 +37,14 @@ const dispatch = createEventDispatcher() - const customTagRegex = /^([\w-]+)\(((?:[\w-]+\+)*[\w-]+|(?:\^[\w-]+)+)\)$/ + // Mirrors CUSTOM_TAG_REGEX in backend/windmill-common/src/worker.rs — keep both in sync. + const customTagRegex = /^([\w-]+)\(((?:[\w-]+\*?\+)*[\w-]+\*?|(?:\^[\w-]+\*?)+)\)$/ const dynamicTagRegex = /\$args\[((?:\w+\.)*\w+)\]/ + function formatWorkspace(w: { id: string; includeForks: boolean }) { + return w.includeForks ? `${w.id} (and its forks)` : w.id + } + let dynamicTag = $derived.by(() => { let r = newTag.trim() if (r == '') return undefined @@ -51,14 +56,16 @@ let r = newTag.trim() if (r == '') return undefined let matched = r.match(customTagRegex) - console.log(matched) let tag = matched?.[1] let workspaces_raw = matched?.[2] let tag_type = workspaces_raw?.includes('^') ? 'exclude' : 'include' if (tag_type == 'exclude') { workspaces_raw = workspaces_raw?.slice(1) } - let workspaces = workspaces_raw?.split(tag_type == 'include' ? '+' : '^') + let workspaces = workspaces_raw?.split(tag_type == 'include' ? '+' : '^').map((w) => { + const includeForks = w.endsWith('*') + return { id: includeForks ? w.slice(0, -1) : w, includeForks } + }) if (!workspaces_raw || workspaces_raw?.length == 0) { return undefined } @@ -163,14 +170,14 @@
Workspaces: {#if extractedCustomTag.tag_type == 'include'} - {extractedCustomTag.workspaces?.join(', ')} + {extractedCustomTag.workspaces?.map(formatWorkspace).join(', ')} {:else} - All workspaces except {extractedCustomTag.workspaces?.join(', ')} + All workspaces except {extractedCustomTag.workspaces?.map(formatWorkspace).join(', ')} {/if}
{:else if newTag.trim()} - {#if newTag.includes('(') || newTag.includes(')') || newTag.includes('+') || newTag.includes('^') || ((newTag.includes('.') || newTag.includes('$args[')) && !dynamicTag)} + {#if newTag.includes('(') || newTag.includes(')') || newTag.includes('+') || newTag.includes('^') || newTag.includes('*') || ((newTag.includes('.') || newTag.includes('$args[')) && !dynamicTag)}
Invalid tag
@@ -219,6 +226,10 @@ To exclude 'workspace1' and 'workspace2' from a tag, use 
tag(^workspace1^workspace2)

{#if variant !== 'drawer'}
{/if} + Forks of a workspace do not get its tags. Suffix a workspace with + 
*
+ to also cover its forks, e.g. 
tag(workspace1*)
+
{#if variant !== 'drawer'}
{/if} For Date: Fri, 17 Jul 2026 19:22:31 +0200 Subject: [PATCH 09/14] fix(flows): make updateFlow body path optional so AI can update flows (#10176) * fix(mcp): default a body field to its same-named path param so updateFlow works Co-Authored-By: Claude Opus 4.8 * docs: trim mcp path-param fallback helper comment Co-Authored-By: Claude Opus 4.8 * refactor(mcp): keep path params un-mangled so update tools take plain `path` Co-Authored-By: Claude Opus 4.8 * fix(flows): default update_flow body path from URL via EditFlow Harmonizes updateFlow with the EditVariable/EditResource/EditApp convention: the flow to update is identified by the URL, so the body path is optional and only needed to rename. Fixes the 422 at the API layer for every client (MCP, the in-app AI chat, raw HTTP), not just the MCP tool schema. Co-Authored-By: Claude Opus 4.8 * refactor(mcp): drop redundant body-path fallback now that the server defaults it Co-Authored-By: Claude Opus 4.8 * docs: fix stale generator comment after removing mcp body-path fallback Co-Authored-By: Claude Opus 4.8 * fix(flows): mark updateFlow body path optional in the openapi contract Adds an `EditFlow` schema (path optional) for the update route so the public contract matches the server; createFlow keeps `OpenFlowWPath` (path required). Also trims two test comments to record constraints rather than history. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- .../generate_mcp_tools.py | 31 ++++--- backend/windmill-api-flows/src/flows.rs | 6 +- backend/windmill-api/openapi.yaml | 33 ++++++- .../src/mcp/auto_generated_endpoints.rs | 85 ++++-------------- backend/windmill-api/src/mcp/core.rs | 1 - backend/windmill-api/src/mcp/utils.rs | 65 ++++++++++++-- backend/windmill-mcp/src/server/endpoints.rs | 2 - backend/windmill-types/src/flows.rs | 65 ++++++++++++++ frontend/src/lib/mcpEndpointTools.ts | 86 ++++--------------- 9 files changed, 213 insertions(+), 161 deletions(-) diff --git a/backend/generate_mcp_endpoints_tools/generate_mcp_tools.py b/backend/generate_mcp_endpoints_tools/generate_mcp_tools.py index b120a00d39..20ea1a0a6c 100644 --- a/backend/generate_mcp_endpoints_tools/generate_mcp_tools.py +++ b/backend/generate_mcp_endpoints_tools/generate_mcp_tools.py @@ -165,13 +165,14 @@ def extract_separate_schemas(parameters: List[Dict[str, Any]], request_body: Opt conflicts = (path_keys & query_keys) | (path_keys & body_keys) | (query_keys & body_keys) - path_field_renames = {} query_field_renames = {} body_field_renames = {} for field in conflicts: + # The path parameter keeps the plain name: it identifies the item, is what a + # caller reaches for first, and matches the non-colliding endpoints (`getFlowByPath` + # takes `path`). Only the other locations carry a suffix. schemas_and_renames = [ - (path_params_schema, path_keys, '__path', path_field_renames), (query_params_schema, query_keys, '__query', query_field_renames), (body_schema, body_keys, '__body', body_field_renames), ] @@ -197,11 +198,26 @@ def extract_separate_schemas(parameters: List[Dict[str, Any]], request_body: Opt # Store the reverse mapping: renamed -> original renames_map[new_name] = field + # A body field colliding with a same-named path parameter (`path` on the update + # endpoints) holds the new value and differs only when moving the item, so it must + # stay optional; the server defaults it from the URL path when the caller omits it. + if field in path_keys and field in body_keys and body_schema: + body_name = field + '__body' + if 'required' in body_schema: + body_schema['required'] = [r for r in body_schema['required'] if r != body_name] + prop = body_schema['properties'].get(body_name) + if isinstance(prop, dict): + existing_desc = prop.get('description', '').rstrip('. ') + prop['description'] = ( + f"{existing_desc}. Defaults to `{field}` when omitted; " + f"set it only to change the {field}." + ).lstrip('. ') + # Return None for empty schemas path_params_schema = path_params_schema if path_params_schema and path_params_schema.get('properties') else None query_params_schema = query_params_schema if query_params_schema and query_params_schema.get('properties') else None - return (path_params_schema, query_params_schema, body_schema, path_field_renames, query_field_renames, body_field_renames) + return (path_params_schema, query_params_schema, body_schema, query_field_renames, body_field_renames) # Cache for loaded external files _external_file_cache: Dict[str, Dict[str, Any]] = {} @@ -453,7 +469,7 @@ export const mcpEndpointTools: EndpointTool[] = []; method = tool['method'].upper() # Generate separate schemas - path_params_schema, query_params_schema, body_schema, path_field_renames, query_field_renames, body_field_renames = extract_separate_schemas( + path_params_schema, query_params_schema, body_schema, query_field_renames, body_field_renames = extract_separate_schemas( tool['parameters'], tool['requestBody'], spec, tool['required_fields'], base_path, tool.get('include_fields'), tool.get('opaque_fields'), tool.get('include_query_params') ) @@ -462,7 +478,6 @@ export const mcpEndpointTools: EndpointTool[] = []; path_params_ts = json.dumps(path_params_schema, indent=8) if path_params_schema else "undefined" query_params_ts = json.dumps(query_params_schema, indent=8) if query_params_schema else "undefined" body_schema_ts = json.dumps(body_schema, indent=8) if body_schema else "undefined" - path_field_renames_ts = json.dumps(path_field_renames, indent=8) if path_field_renames else "undefined" query_field_renames_ts = json.dumps(query_field_renames, indent=8) if query_field_renames else "undefined" body_field_renames_ts = json.dumps(body_field_renames, indent=8) if body_field_renames else "undefined" @@ -476,7 +491,6 @@ export const mcpEndpointTools: EndpointTool[] = []; pathParamsSchema: {path_params_ts}, queryParamsSchema: {query_params_ts}, bodySchema: {body_schema_ts}, - pathFieldRenames: {path_field_renames_ts}, queryFieldRenames: {query_field_renames_ts}, bodyFieldRenames: {body_field_renames_ts} }}""" @@ -497,7 +511,6 @@ export interface EndpointTool {{ pathParamsSchema?: object; queryParamsSchema?: object; bodySchema?: object; - pathFieldRenames?: Record; queryFieldRenames?: Record; bodyFieldRenames?: Record; }} @@ -529,7 +542,7 @@ pub fn all_tools() -> Vec {{ method = tool['method'].upper() # Generate separate schemas - path_params_schema, query_params_schema, body_schema, path_field_renames, query_field_renames, body_field_renames = extract_separate_schemas( + path_params_schema, query_params_schema, body_schema, query_field_renames, body_field_renames = extract_separate_schemas( tool['parameters'], tool['requestBody'], spec, tool['required_fields'], base_path, tool.get('include_fields'), tool.get('opaque_fields'), tool.get('include_query_params') ) @@ -537,7 +550,6 @@ pub fn all_tools() -> Vec {{ path_params_rust = schema_to_rust_value(path_params_schema) query_params_rust = schema_to_rust_value(query_params_schema) body_schema_rust = schema_to_rust_value(body_schema) - path_field_renames_rust = schema_to_rust_value(path_field_renames if path_field_renames else None) query_field_renames_rust = schema_to_rust_value(query_field_renames if query_field_renames else None) body_field_renames_rust = schema_to_rust_value(body_field_renames if body_field_renames else None) @@ -551,7 +563,6 @@ pub fn all_tools() -> Vec {{ path_params_schema: {path_params_rust}, query_params_schema: {query_params_rust}, body_schema: {body_schema_rust}, - path_field_renames: {path_field_renames_rust}, query_field_renames: {query_field_renames_rust}, body_field_renames: {body_field_renames_rust}, }}""" diff --git a/backend/windmill-api-flows/src/flows.rs b/backend/windmill-api-flows/src/flows.rs index a9376d02ed..ff555113c3 100644 --- a/backend/windmill-api-flows/src/flows.rs +++ b/backend/windmill-api-flows/src/flows.rs @@ -47,7 +47,7 @@ use windmill_common::HUB_BASE_URL; use windmill_common::{ db::UserDB, error::{self, to_anyhow, Error, JsonResult, Result}, - flows::{Flow, FlowWithStarred, ListFlowQuery, ListableFlow, NewFlow}, + flows::{EditFlow, Flow, FlowWithStarred, ListFlowQuery, ListableFlow, NewFlow}, jobs::JobPayload, schedule::Schedule, utils::{http_get_from_hub, not_found_if_none, paginate, Pagination, RunnableKind, StripPath}, @@ -1007,7 +1007,7 @@ async fn update_flow( Extension(db): Extension, Extension(webhook): Extension, Path((w_id, flow_path)): Path<(String, StripPath)>, - Json(nf): Json, + Json(ef): Json, ) -> Result { if authed.is_operator { return Err(Error::NotAuthorized( @@ -1015,6 +1015,8 @@ async fn update_flow( )); } let flow_path = flow_path.to_path(); + // The URL identifies the flow being updated; the body path is only needed to rename. + let nf = ef.into_new_flow(flow_path); check_scopes(&authed, || format!("flows:write:{}", flow_path))?; if let RuleCheckResult::Blocked(msg) = check_deploy_rules( diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 668cdae8cc..18a9120356 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -10775,7 +10775,7 @@ paths: application/json: schema: allOf: - - $ref: "#/components/schemas/OpenFlowWPath" + - $ref: "#/components/schemas/EditFlow" - type: object properties: deployment_message: @@ -29097,6 +29097,37 @@ components: type: string required: - path + # Like OpenFlowWPath but `path` is optional: on update the flow is identified by + # the URL, so the body path is only needed to rename it. Kept as a separate schema + # (rather than making OpenFlowWPath.path optional) so createFlow still requires path. + EditFlow: + allOf: + - $ref: "../../openflow.openapi.yaml#/components/schemas/OpenFlow" + - type: object + properties: + path: + type: string + tag: + type: string + ws_error_handler_muted: + type: boolean + priority: + type: integer + dedicated_worker: + type: boolean + timeout: + type: number + visible_to_runner_only: + type: boolean + on_behalf_of_email: + type: string + preserve_on_behalf_of: + type: boolean + description: "When true and the caller is a member of the 'wm_deployers' group, preserves the original on_behalf_of_email value instead of overwriting it." + labels: + type: array + items: + type: string FlowPreview: type: object diff --git a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs index d44484f9db..fda5966e30 100644 --- a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs +++ b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs @@ -26,7 +26,6 @@ pub fn all_tools() -> Vec { ] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -54,7 +53,6 @@ pub fn all_tools() -> Vec { ] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -124,7 +122,6 @@ pub fn all_tools() -> Vec { "description" ] })), - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -147,7 +144,6 @@ pub fn all_tools() -> Vec { })), query_params_schema: None, body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -160,13 +156,12 @@ pub fn all_tools() -> Vec { path_params_schema: Some(serde_json::json!({ "type": "object", "properties": { - "path__path": { - "type": "string", - "description": "(path parameter)" + "path": { + "type": "string" } }, "required": [ - "path__path" + "path" ] })), query_params_schema: Some(serde_json::json!({ @@ -205,12 +200,9 @@ pub fn all_tools() -> Vec { }, "path__body": { "type": "string", - "description": "The path to the variable (body parameter)" + "description": "The path to the variable (body parameter). Defaults to `path` when omitted; set it only to change the path." } } -})), - path_field_renames: Some(serde_json::json!({ - "path__path": "path" })), query_field_renames: None, body_field_renames: Some(serde_json::json!({ @@ -253,7 +245,6 @@ pub fn all_tools() -> Vec { "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -307,7 +298,6 @@ pub fn all_tools() -> Vec { "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -362,7 +352,6 @@ pub fn all_tools() -> Vec { "resource_type" ] })), - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -385,7 +374,6 @@ pub fn all_tools() -> Vec { })), query_params_schema: None, body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -398,13 +386,12 @@ pub fn all_tools() -> Vec { path_params_schema: Some(serde_json::json!({ "type": "object", "properties": { - "path__path": { - "type": "string", - "description": "(path parameter)" + "path": { + "type": "string" } }, "required": [ - "path__path" + "path" ] })), query_params_schema: None, @@ -433,12 +420,9 @@ pub fn all_tools() -> Vec { }, "path__body": { "type": "string", - "description": "The path to the resource (body parameter)" + "description": "The path to the resource (body parameter). Defaults to `path` when omitted; set it only to change the path." } } -})), - path_field_renames: Some(serde_json::json!({ - "path__path": "path" })), query_field_renames: None, body_field_renames: Some(serde_json::json!({ @@ -473,7 +457,6 @@ pub fn all_tools() -> Vec { "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -535,7 +518,6 @@ pub fn all_tools() -> Vec { "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -548,7 +530,6 @@ pub fn all_tools() -> Vec { path_params_schema: None, query_params_schema: None, body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -646,7 +627,6 @@ pub fn all_tools() -> Vec { "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -699,7 +679,6 @@ Creates a new version of an existing script when called with the same path and t "language" ] })), - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -722,7 +701,6 @@ Creates a new version of an existing script when called with the same path and t })), query_params_schema: None, body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -754,7 +732,6 @@ Creates a new version of an existing script when called with the same path and t "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -789,7 +766,6 @@ Creates a new version of an existing script when called with the same path and t "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -816,7 +792,6 @@ Creates a new version of an existing script when called with the same path and t "description": "The arguments to pass to the script or flow", "additionalProperties": true })), - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -886,7 +861,6 @@ Creates a new version of an existing script when called with the same path and t "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -921,7 +895,6 @@ Creates a new version of an existing script when called with the same path and t "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -967,7 +940,6 @@ Creates a new version of an existing script when called with the same path and t ], "description": "Top-level flow definition containing metadata, configuration, and the flow structure" })), - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -980,13 +952,12 @@ Creates a new version of an existing script when called with the same path and t path_params_schema: Some(serde_json::json!({ "type": "object", "properties": { - "path__path": { - "type": "string", - "description": "(path parameter)" + "path": { + "type": "string" } }, "required": [ - "path__path" + "path" ] })), query_params_schema: None, @@ -1015,18 +986,14 @@ Creates a new version of an existing script when called with the same path and t }, "path__body": { "type": "string", - "description": "(body parameter)" + "description": "(body parameter). Defaults to `path` when omitted; set it only to change the path." } }, "required": [ "summary", - "value", - "path__body" + "value" ], "description": "Top-level flow definition containing metadata, configuration, and the flow structure" -})), - path_field_renames: Some(serde_json::json!({ - "path__path": "path" })), query_field_renames: None, body_field_renames: Some(serde_json::json!({ @@ -1061,7 +1028,6 @@ Creates a new version of an existing script when called with the same path and t "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -1099,7 +1065,6 @@ Creates a new version of an existing script when called with the same path and t "policy" ] })), - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -1112,13 +1077,12 @@ Creates a new version of an existing script when called with the same path and t path_params_schema: Some(serde_json::json!({ "type": "object", "properties": { - "path__path": { - "type": "string", - "description": "(path parameter)" + "path": { + "type": "string" } }, "required": [ - "path__path" + "path" ] })), query_params_schema: None, @@ -1139,12 +1103,9 @@ Creates a new version of an existing script when called with the same path and t }, "path__body": { "type": "string", - "description": "(body parameter)" + "description": "(body parameter). Defaults to `path` when omitted; set it only to change the path." } } -})), - path_field_renames: Some(serde_json::json!({ - "path__path": "path" })), query_field_renames: None, body_field_renames: Some(serde_json::json!({ @@ -1174,7 +1135,6 @@ Creates a new version of an existing script when called with the same path and t "description": "The arguments to pass to the script or flow", "additionalProperties": true })), - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -1269,7 +1229,6 @@ Creates a new version of an existing script when called with the same path and t "language" ] })), - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -1390,7 +1349,6 @@ Creates a new version of an existing script when called with the same path and t "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -1557,7 +1515,6 @@ Creates a new version of an existing script when called with the same path and t "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -1596,7 +1553,6 @@ Creates a new version of an existing script when called with the same path and t "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -1628,7 +1584,6 @@ Creates a new version of an existing script when called with the same path and t "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -1841,7 +1796,6 @@ You should get the schema of the script or flow before creating the schedule to "args" ] })), - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -2047,7 +2001,6 @@ You should get the schema of the script or flow before updating the schedule to "args" ] })), - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -2070,7 +2023,6 @@ You should get the schema of the script or flow before updating the schedule to })), query_params_schema: None, body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -2102,7 +2054,6 @@ You should get the schema of the script or flow before updating the schedule to "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -2168,7 +2119,6 @@ You should get the schema of the script or flow before updating the schedule to "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -2198,7 +2148,6 @@ You should get the schema of the script or flow before updating the schedule to "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, } diff --git a/backend/windmill-api/src/mcp/core.rs b/backend/windmill-api/src/mcp/core.rs index cdf1f0b3a3..5d78536d86 100644 --- a/backend/windmill-api/src/mcp/core.rs +++ b/backend/windmill-api/src/mcp/core.rs @@ -295,7 +295,6 @@ impl McpBackend for WindmillBackend { workspace_id, args_map, &endpoint_tool.path_params_schema, - &endpoint_tool.path_field_renames, )?; let query_string = build_query_string( args_map, diff --git a/backend/windmill-api/src/mcp/utils.rs b/backend/windmill-api/src/mcp/utils.rs index 70a4c6c1f4..7bca92dd57 100644 --- a/backend/windmill-api/src/mcp/utils.rs +++ b/backend/windmill-api/src/mcp/utils.rs @@ -295,7 +295,7 @@ pub async fn get_hub_script_schema(path: &str, db: &DB) -> Result // ============================================================================ /// Look up the original field name from a field_renames map. -/// field_renames maps renamed_key -> original_key (e.g. {"path__path": "path"}). +/// field_renames maps renamed_key -> original_key (e.g. {"path__body": "path"}). fn get_original_name(renamed_key: &str, field_renames: &Option) -> String { field_renames .as_ref() @@ -373,20 +373,17 @@ pub fn substitute_path_params( workspace_id: &str, args_map: &serde_json::Map, path_schema: &Option, - path_field_renames: &Option, ) -> BackendResult { let mut path_template = path.replace("{workspace}", workspace_id); if let Some(schema) = path_schema { if let Some(props) = schema.get("properties").and_then(|p| p.as_object()) { for (param_name, _) in props { - // param_name may be renamed (e.g. "path__path"), get original for URL placeholder - let original_name = get_original_name(param_name, path_field_renames); - let placeholder = format!("{{{}}}", original_name); + let placeholder = format!("{{{}}}", param_name); match args_map.get(param_name) { Some(param_value) => { if let Some(str_val) = param_value.as_str() { - validate_path_param_value(&original_name, str_val)?; + validate_path_param_value(param_name, str_val)?; path_template = path_template.replace(&placeholder, str_val); } } @@ -675,6 +672,60 @@ mod tests { ); } + // updateFlow-shaped: `path` is both a path parameter and a body field. The path + // parameter keeps the plain name; only the body side is mangled. + fn update_flow_schemas() -> (Option, Option, Option) { + ( + Some(json!({ + "type": "object", + "properties": { "path": { "type": "string" } }, + "required": ["path"] + })), + Some(json!({ + "type": "object", + "properties": { + "summary": { "type": "string" }, + "value": { "type": "object" }, + "path__body": { "type": "string" } + }, + "required": ["summary", "value"] + })), + Some(json!({ "path__body": "path" })), + ) + } + + #[test] + fn build_request_body_maps_body_path_alias_for_rename() { + // The mangled body field carries the *new* path when renaming; it must reach the + // API under its original name `path`. (An omitted `path__body` is intentionally + // absent from the body; the server defaults it from the URL path parameter.) + let (path_schema, body_schema, body_renames) = update_flow_schemas(); + let args: serde_json::Map = json!({ + "path": "f/team/my_flow", + "path__body": "f/team/renamed_flow", + "summary": "s", + "value": {} + }) + .as_object() + .unwrap() + .clone(); + + let body = build_request_body( + "POST", + &args, + &body_schema, + &body_renames, + &path_schema, + &None, + ) + .expect("body should be built"); + assert_eq!( + body.as_object().unwrap().get("path"), + Some(&json!("f/team/renamed_flow")), + "path__body must be sent as `path` so a rename takes effect" + ); + } + #[test] fn build_request_body_get_has_no_body() { let body_schema = Some(json!({ "type": "object", "additionalProperties": true })); @@ -745,7 +796,6 @@ mod tests { "dev", &args, &path_schema, - &None, ); assert!( result.is_err(), @@ -767,7 +817,6 @@ mod tests { "dev", &args, &path_schema, - &None, ) .expect("legitimate path should substitute"); assert_eq!(result, "/w/dev/scripts/get/p/u/alice/my_script"); diff --git a/backend/windmill-mcp/src/server/endpoints.rs b/backend/windmill-mcp/src/server/endpoints.rs index c3a50f9715..9e3adffc0b 100644 --- a/backend/windmill-mcp/src/server/endpoints.rs +++ b/backend/windmill-mcp/src/server/endpoints.rs @@ -21,7 +21,6 @@ pub struct EndpointTool { pub path_params_schema: Option, pub query_params_schema: Option, pub body_schema: Option, - pub path_field_renames: Option, pub query_field_renames: Option, pub body_field_renames: Option, } @@ -215,7 +214,6 @@ mod tests { "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, } diff --git a/backend/windmill-types/src/flows.rs b/backend/windmill-types/src/flows.rs index 0c95b9612b..7e70dc5acd 100644 --- a/backend/windmill-types/src/flows.rs +++ b/backend/windmill-types/src/flows.rs @@ -145,6 +145,57 @@ impl NewFlow { } } +/// Body for updating an existing flow. Mirrors `NewFlow`, but `path` is optional: the +/// flow to update is identified by the URL, so the body only needs `path` to rename it. +/// This matches the `EditVariable` / `EditResource` / `EditApp` convention and lets a +/// caller update in place without restating the path. +#[derive(Debug, Deserialize)] +pub struct EditFlow { + #[serde(default)] + pub path: Option, + pub summary: String, + pub description: Option, + #[serde(deserialize_with = "validate_flow_value")] + pub value: Box, + pub schema: Option, + pub tag: Option, + pub dedicated_worker: Option, + pub timeout: Option, + pub deployment_message: Option, + pub visible_to_runner_only: Option, + pub on_behalf_of_email: Option, + pub preserve_on_behalf_of: Option, + pub ws_error_handler_muted: Option, + #[serde(default)] + pub labels: Option>, + #[serde(default)] + pub skip_draft_deletion: Option, +} + +impl EditFlow { + /// Resolve into a `NewFlow`, defaulting the target path to `current_path` (the flow's + /// URL path) when the body omits it. A body `path` that differs renames the flow. + pub fn into_new_flow(self, current_path: &str) -> NewFlow { + NewFlow { + path: self.path.unwrap_or_else(|| current_path.to_string()), + summary: self.summary, + description: self.description, + value: self.value, + schema: self.schema, + tag: self.tag, + dedicated_worker: self.dedicated_worker, + timeout: self.timeout, + deployment_message: self.deployment_message, + visible_to_runner_only: self.visible_to_runner_only, + on_behalf_of_email: self.on_behalf_of_email, + preserve_on_behalf_of: self.preserve_on_behalf_of, + ws_error_handler_muted: self.ws_error_handler_muted, + labels: self.labels, + skip_draft_deletion: self.skip_draft_deletion, + } + } +} + fn validate_retry(retry: &Retry, module_id: &str) -> anyhow::Result<()> { if retry.exponential.attempts > 0 && retry.exponential.seconds == 0 { return Err(anyhow::anyhow!( @@ -1233,6 +1284,20 @@ mod tests { use super::*; use serde_json::json; + #[test] + fn edit_flow_defaults_path_from_url_and_renames_when_given() { + // An omitted body path resolves to the URL path; an explicit body path renames. + let ef: EditFlow = + serde_json::from_value(json!({ "summary": "s", "value": { "modules": [] } })).unwrap(); + assert_eq!(ef.into_new_flow("f/team/my_flow").path, "f/team/my_flow"); + + let ef: EditFlow = serde_json::from_value( + json!({ "path": "f/team/renamed", "summary": "s", "value": { "modules": [] } }), + ) + .unwrap(); + assert_eq!(ef.into_new_flow("f/team/my_flow").path, "f/team/renamed"); + } + #[test] fn flow_value_ignores_notes_and_groups() { // FlowValue should parse successfully even when notes/groups are present — diff --git a/frontend/src/lib/mcpEndpointTools.ts b/frontend/src/lib/mcpEndpointTools.ts index 8a46163009..336c24cefc 100644 --- a/frontend/src/lib/mcpEndpointTools.ts +++ b/frontend/src/lib/mcpEndpointTools.ts @@ -10,7 +10,6 @@ export interface EndpointTool { pathParamsSchema?: object; queryParamsSchema?: object; bodySchema?: object; - pathFieldRenames?: Record; queryFieldRenames?: Record; bodyFieldRenames?: Record; } @@ -36,7 +35,6 @@ export const mcpEndpointTools: EndpointTool[] = [ ] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -64,7 +62,6 @@ export const mcpEndpointTools: EndpointTool[] = [ ] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -134,7 +131,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "description" ] }, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -157,7 +153,6 @@ export const mcpEndpointTools: EndpointTool[] = [ }, queryParamsSchema: undefined, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -170,13 +165,12 @@ export const mcpEndpointTools: EndpointTool[] = [ pathParamsSchema: { "type": "object", "properties": { - "path__path": { - "type": "string", - "description": "(path parameter)" + "path": { + "type": "string" } }, "required": [ - "path__path" + "path" ] }, queryParamsSchema: { @@ -215,12 +209,9 @@ export const mcpEndpointTools: EndpointTool[] = [ }, "path__body": { "type": "string", - "description": "The path to the variable (body parameter)" + "description": "The path to the variable (body parameter). Defaults to `path` when omitted; set it only to change the path." } } -}, - pathFieldRenames: { - "path__path": "path" }, queryFieldRenames: undefined, bodyFieldRenames: { @@ -263,7 +254,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -317,7 +307,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -372,7 +361,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "resource_type" ] }, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -395,7 +383,6 @@ export const mcpEndpointTools: EndpointTool[] = [ }, queryParamsSchema: undefined, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -408,13 +395,12 @@ export const mcpEndpointTools: EndpointTool[] = [ pathParamsSchema: { "type": "object", "properties": { - "path__path": { - "type": "string", - "description": "(path parameter)" + "path": { + "type": "string" } }, "required": [ - "path__path" + "path" ] }, queryParamsSchema: undefined, @@ -443,12 +429,9 @@ export const mcpEndpointTools: EndpointTool[] = [ }, "path__body": { "type": "string", - "description": "The path to the resource (body parameter)" + "description": "The path to the resource (body parameter). Defaults to `path` when omitted; set it only to change the path." } } -}, - pathFieldRenames: { - "path__path": "path" }, queryFieldRenames: undefined, bodyFieldRenames: { @@ -483,7 +466,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -545,7 +527,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -558,7 +539,6 @@ export const mcpEndpointTools: EndpointTool[] = [ pathParamsSchema: undefined, queryParamsSchema: undefined, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -656,7 +636,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -708,7 +687,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "language" ] }, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -731,7 +709,6 @@ export const mcpEndpointTools: EndpointTool[] = [ }, queryParamsSchema: undefined, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -763,7 +740,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -798,7 +774,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -825,7 +800,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "description": "The arguments to pass to the script or flow", "additionalProperties": true }, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -895,7 +869,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -930,7 +903,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -976,7 +948,6 @@ export const mcpEndpointTools: EndpointTool[] = [ ], "description": "Top-level flow definition containing metadata, configuration, and the flow structure" }, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -989,13 +960,12 @@ export const mcpEndpointTools: EndpointTool[] = [ pathParamsSchema: { "type": "object", "properties": { - "path__path": { - "type": "string", - "description": "(path parameter)" + "path": { + "type": "string" } }, "required": [ - "path__path" + "path" ] }, queryParamsSchema: undefined, @@ -1024,18 +994,14 @@ export const mcpEndpointTools: EndpointTool[] = [ }, "path__body": { "type": "string", - "description": "(body parameter)" + "description": "(body parameter). Defaults to `path` when omitted; set it only to change the path." } }, "required": [ "summary", - "value", - "path__body" + "value" ], "description": "Top-level flow definition containing metadata, configuration, and the flow structure" -}, - pathFieldRenames: { - "path__path": "path" }, queryFieldRenames: undefined, bodyFieldRenames: { @@ -1070,7 +1036,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -1108,7 +1073,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "policy" ] }, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -1121,13 +1085,12 @@ export const mcpEndpointTools: EndpointTool[] = [ pathParamsSchema: { "type": "object", "properties": { - "path__path": { - "type": "string", - "description": "(path parameter)" + "path": { + "type": "string" } }, "required": [ - "path__path" + "path" ] }, queryParamsSchema: undefined, @@ -1148,12 +1111,9 @@ export const mcpEndpointTools: EndpointTool[] = [ }, "path__body": { "type": "string", - "description": "(body parameter)" + "description": "(body parameter). Defaults to `path` when omitted; set it only to change the path." } } -}, - pathFieldRenames: { - "path__path": "path" }, queryFieldRenames: undefined, bodyFieldRenames: { @@ -1183,7 +1143,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "description": "The arguments to pass to the script or flow", "additionalProperties": true }, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -1278,7 +1237,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "language" ] }, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -1399,7 +1357,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -1566,7 +1523,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -1605,7 +1561,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -1637,7 +1592,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -1847,7 +1801,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "args" ] }, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -2050,7 +2003,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "args" ] }, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -2073,7 +2025,6 @@ export const mcpEndpointTools: EndpointTool[] = [ }, queryParamsSchema: undefined, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -2105,7 +2056,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -2171,7 +2121,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -2201,7 +2150,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined } From a9fc9f74b2c5b280cc087f0bfbac50112081c5e8 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Fri, 17 Jul 2026 19:23:31 +0200 Subject: [PATCH 10/14] fix(ai): unbreak session chat compaction for Anthropic models (#10171) * fix(ai): cap chat compaction summary output so Anthropic non-streaming calls succeed Co-Authored-By: Claude Fable 5 * test: pin the compaction summarizer's maxTokensCap Co-Authored-By: Claude Fable 5 * fix(ai): cap testKey completion so Anthropic key tests pass the SDK pre-flight Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- .../lib/components/copilot/chat/AIChatManager.svelte.ts | 7 ++++++- .../src/lib/components/copilot/chat/AIChatManager.test.ts | 3 +++ frontend/src/lib/components/copilot/lib.ts | 7 +++++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index ebc925588b..2853c392f5 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -1002,12 +1002,17 @@ export class AIChatManager { ): Promise<'ok' | 'empty' | 'aborted' | 'error'> => { this.compacting = true try { + // Cap the summarizer's output at the budget already reserved for the + // summary. Without a cap the model's default max_tokens applies, and the + // Anthropic SDK rejects non-streaming requests whose max_tokens implies + // >10 minutes of generation (~21k tokens) before anything is sent. const raw = await getNonStreamingCompletion( [ ...sanitizeToolCallArguments(prefix), { role: 'user', content: getCompactionSummaryPrompt() } ], - abortController + abortController, + { maxTokensCap: SUMMARY_OUTPUT_RESERVE_TOKENS } ) const formatted = formatCompactSummary(raw ?? '') if (!formatted) { diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts index 94270d0e6a..3ca57f948d 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts @@ -1143,6 +1143,9 @@ describe('AIChatManager manual compaction', () => { expect(summaryReq[0].content).toBe('q1') expect(summaryReq[3].content).toBe('a2') expect(summaryReq[4].content).toContain('detailed summary') + // The summarizer's output must stay capped: without it the model default + // applies and the Anthropic SDK rejects the non-streaming call pre-flight. + expect(mocks.getNonStreamingCompletion.mock.calls[0][2]).toEqual({ maxTokensCap: 8000 }) // Nothing kept verbatim: messages collapse to just the summary user message. expect(manager.messages).toHaveLength(1) diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index 2b261a21e0..6670789404 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -487,7 +487,9 @@ export async function testKey({ // getNonStreamingCompletion routes Anthropic-Messages-API models (native // Anthropic and Claude on Azure Foundry) through the Anthropic SDK and // everything else through OpenAI chat completions, so the test exercises the - // same request shape the feature actually sends. + // same request shape the feature actually sends. The cap keeps max_tokens + // under the Anthropic SDK's non-streaming pre-flight limit (~21k tokens), + // which would otherwise reject the request before it is sent. await getNonStreamingCompletion(messages, abortController, { apiKey, workspace, @@ -495,7 +497,8 @@ export async function testKey({ forceModelProvider: { model: modelToTest, provider: aiProvider - } + }, + maxTokensCap: METADATA_MAX_TOKENS }) } From d0aa7dca13aedfec90b7e023954cc824dafa3d34 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Fri, 17 Jul 2026 19:24:07 +0200 Subject: [PATCH 11/14] fix: parse all names in grouped go param declarations (#10165) * fix: parse all names in grouped go param declarations Co-Authored-By: Claude Fable 5 * chore: bump windmill-parser-wasm-go to 1.761.0 Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- backend/parsers/windmill-parser-go/src/lib.rs | 69 +++++++++++++++++-- cli/bun.lock | 4 +- cli/package-lock.json | 8 +-- cli/package.json | 2 +- frontend/package-lock.json | 58 ++-------------- frontend/package.json | 2 +- 6 files changed, 77 insertions(+), 66 deletions(-) diff --git a/backend/parsers/windmill-parser-go/src/lib.rs b/backend/parsers/windmill-parser-go/src/lib.rs index 0f340943aa..2a2d8c310d 100644 --- a/backend/parsers/windmill-parser-go/src/lib.rs +++ b/backend/parsers/windmill-parser-go/src/lib.rs @@ -25,17 +25,23 @@ pub fn parse_go_sig(code: &str) -> anyhow::Result { .params .list .iter() - .map(|param| { + .flat_map(|param| { let (otyp, typ) = parse_go_typ(¶m.typ); - Arg { - name: get_name(param), - otyp, - typ, + // a single field can declare several like-typed params: `func main(a, b string)` + let names: Vec = if param.name.is_empty() { + vec!["".to_string()] + } else { + param.name.iter().map(|y| y.name.to_string()).collect() + }; + names.into_iter().map(move |name| Arg { + name, + otyp: otyp.clone(), + typ: typ.clone(), default: None, has_default: false, oidx: None, otyp_inferred: false, - } + }) }) .collect_vec(); Ok(MainArgSignature { @@ -267,6 +273,57 @@ func main(x int, y string, z bool, l []string, o struct { Name string `json:"nam Ok(()) } + + #[test] + fn test_parse_go_sig_grouped_params() -> anyhow::Result<()> { + let code = r#" +package main + +func main(a, b string, c int) { +} +"#; + assert_eq!( + parse_go_sig(code)?, + MainArgSignature { + star_args: false, + star_kwargs: false, + args: vec![ + Arg { + otyp: Some("string".to_string()), + name: "a".to_string(), + typ: Typ::Str(None), + default: None, + has_default: false, + oidx: None, + otyp_inferred: false, + }, + Arg { + otyp: Some("string".to_string()), + name: "b".to_string(), + typ: Typ::Str(None), + default: None, + has_default: false, + oidx: None, + otyp_inferred: false, + }, + Arg { + otyp: Some("int".to_string()), + name: "c".to_string(), + typ: Typ::Int, + default: None, + has_default: false, + oidx: None, + otyp_inferred: false, + }, + ], + auto_kind: None, + has_preprocessor: None, + ..Default::default() + } + ); + + Ok(()) + } } #[test] diff --git a/cli/bun.lock b/cli/bun.lock index a745acbcba..7c76732a90 100644 --- a/cli/bun.lock +++ b/cli/bun.lock @@ -21,7 +21,7 @@ "tar-stream": "^3.1.7", "windmill-parser-wasm-asset": "1.749.0", "windmill-parser-wasm-csharp": "1.510.1", - "windmill-parser-wasm-go": "1.510.1", + "windmill-parser-wasm-go": "1.761.0", "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.647.1", @@ -294,7 +294,7 @@ "windmill-parser-wasm-csharp": ["windmill-parser-wasm-csharp@1.510.1", "", {}, "sha512-qm09YmnbeYHLwYn1jUnObVzPhYO9NZKMlIO7nlo7zPJBXqksgG5fK/KCtwGw9rChrnz+DsvM9wP5FhrwRLMtwQ=="], - "windmill-parser-wasm-go": ["windmill-parser-wasm-go@1.510.1", "", {}, "sha512-HOkk6LXK0wrwvkn+zjm3Gxo90HmyL6TYqmLo2yp8fZuppy7GOngT27zwYeBtwONiPyvDKskzoqPQoEfd8VuUsQ=="], + "windmill-parser-wasm-go": ["windmill-parser-wasm-go@1.761.0", "", {}, "sha512-jNM0kh0U5uSetwBVSmueFs91GTdbn/tffny2XB1LrrSUElP5i+PK7i12zrWNT2q9JUK2L215CbmyfNmkacbo1Q=="], "windmill-parser-wasm-java": ["windmill-parser-wasm-java@1.510.1", "", {}, "sha512-Zle+JZT/ZwUArUVacUudYlS+CaHp2lSnkqD/IhWaRUG+gcv26VbERnrrHPonqXbVMS+eA9ElfXrFM5j0ukaXUw=="], diff --git a/cli/package-lock.json b/cli/package-lock.json index a9d72696b2..328cfa2d64 100644 --- a/cli/package-lock.json +++ b/cli/package-lock.json @@ -22,7 +22,7 @@ "tar-stream": "^3.1.7", "windmill-parser-wasm-asset": "1.749.0", "windmill-parser-wasm-csharp": "1.510.1", - "windmill-parser-wasm-go": "1.510.1", + "windmill-parser-wasm-go": "1.761.0", "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.647.1", @@ -1422,9 +1422,9 @@ "integrity": "sha512-qm09YmnbeYHLwYn1jUnObVzPhYO9NZKMlIO7nlo7zPJBXqksgG5fK/KCtwGw9rChrnz+DsvM9wP5FhrwRLMtwQ==" }, "node_modules/windmill-parser-wasm-go": { - "version": "1.510.1", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-go/-/windmill-parser-wasm-go-1.510.1.tgz", - "integrity": "sha512-HOkk6LXK0wrwvkn+zjm3Gxo90HmyL6TYqmLo2yp8fZuppy7GOngT27zwYeBtwONiPyvDKskzoqPQoEfd8VuUsQ==" + "version": "1.761.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-go/-/windmill-parser-wasm-go-1.761.0.tgz", + "integrity": "sha512-jNM0kh0U5uSetwBVSmueFs91GTdbn/tffny2XB1LrrSUElP5i+PK7i12zrWNT2q9JUK2L215CbmyfNmkacbo1Q==" }, "node_modules/windmill-parser-wasm-java": { "version": "1.510.1", diff --git a/cli/package.json b/cli/package.json index 589bc52271..ba8a702526 100644 --- a/cli/package.json +++ b/cli/package.json @@ -30,7 +30,7 @@ "tar-stream": "^3.1.7", "windmill-parser-wasm-asset": "1.749.0", "windmill-parser-wasm-csharp": "1.510.1", - "windmill-parser-wasm-go": "1.510.1", + "windmill-parser-wasm-go": "1.761.0", "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.647.1", diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 3f5484438c..555aba4ba1 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -82,7 +82,7 @@ "vscode-ws-jsonrpc": "~3.5.0", "windmill-parser-wasm-asset": "1.749.0", "windmill-parser-wasm-csharp": "1.510.1", - "windmill-parser-wasm-go": "1.510.1", + "windmill-parser-wasm-go": "1.761.0", "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.647.1", @@ -879,7 +879,6 @@ "version": "1.11.2", "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.2.tgz", "integrity": "sha512-TC8MkTuZUtcTSiFeuC0ksCh9QIJ5+F21MvZ4Wn4ORfYaFJ/0dsiudv5tVkejgwZlwQ39jL9WWDe2lz8x0WglOA==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -891,7 +890,6 @@ "version": "1.11.2", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.2.tgz", "integrity": "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -902,7 +900,6 @@ "version": "1.2.2", "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1418,7 +1415,6 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.4.tgz", "integrity": "sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1567,7 +1563,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1584,7 +1579,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1601,7 +1595,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1618,7 +1611,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1635,7 +1627,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1652,7 +1643,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1669,7 +1659,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1686,7 +1675,6 @@ "cpu": [ "ppc64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1703,7 +1691,6 @@ "cpu": [ "s390x" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1720,7 +1707,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1737,7 +1723,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1754,7 +1739,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1771,7 +1755,6 @@ "cpu": [ "wasm32" ], - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1790,7 +1773,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1807,7 +1789,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2113,7 +2094,6 @@ "version": "0.10.2", "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.2.tgz", "integrity": "sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -7349,7 +7329,7 @@ "version": "1.21.7", "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.7.tgz", "integrity": "sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==", - "dev": true, + "devOptional": true, "license": "MIT", "bin": { "jiti": "bin/jiti.js" @@ -7884,7 +7864,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7905,7 +7884,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7926,7 +7904,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7947,7 +7924,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7968,7 +7944,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7989,7 +7964,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8010,7 +7984,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8031,7 +8004,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8052,7 +8024,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8073,7 +8044,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -8094,7 +8064,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -12782,21 +12751,6 @@ } } }, - "node_modules/svelte-check/node_modules/picomatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", - "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/svelte-eslint-parser": { "version": "0.43.0", "resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-0.43.0.tgz", @@ -13536,7 +13490,7 @@ "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", @@ -14332,9 +14286,9 @@ "integrity": "sha512-qm09YmnbeYHLwYn1jUnObVzPhYO9NZKMlIO7nlo7zPJBXqksgG5fK/KCtwGw9rChrnz+DsvM9wP5FhrwRLMtwQ==" }, "node_modules/windmill-parser-wasm-go": { - "version": "1.510.1", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-go/-/windmill-parser-wasm-go-1.510.1.tgz", - "integrity": "sha512-HOkk6LXK0wrwvkn+zjm3Gxo90HmyL6TYqmLo2yp8fZuppy7GOngT27zwYeBtwONiPyvDKskzoqPQoEfd8VuUsQ==" + "version": "1.761.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-go/-/windmill-parser-wasm-go-1.761.0.tgz", + "integrity": "sha512-jNM0kh0U5uSetwBVSmueFs91GTdbn/tffny2XB1LrrSUElP5i+PK7i12zrWNT2q9JUK2L215CbmyfNmkacbo1Q==" }, "node_modules/windmill-parser-wasm-java": { "version": "1.510.1", diff --git a/frontend/package.json b/frontend/package.json index d7541af7bf..795304bc1e 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -157,7 +157,7 @@ "vscode-ws-jsonrpc": "~3.5.0", "windmill-parser-wasm-asset": "1.749.0", "windmill-parser-wasm-csharp": "1.510.1", - "windmill-parser-wasm-go": "1.510.1", + "windmill-parser-wasm-go": "1.761.0", "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.647.1", From ae3d9ce2c031beeafc4e8156e0c85f163647af0f Mon Sep 17 00:00:00 2001 From: hugocasa Date: Fri, 17 Jul 2026 19:34:17 +0200 Subject: [PATCH 12/14] fix(mcp): apply token scopes consistently across mcp endpoint tools (#10162) * fix(mcp): apply token scopes consistently across mcp endpoint tools Endpoint-tool authorization is now shared between single- and multi-workspace modes, honors the token's script/flow path patterns for every path-taking tool, and the JWT minted for proxied endpoint calls carries scopes derived from the caller's own. Co-Authored-By: Claude Fable 5 * chore(mcp): review nits: restore create_http_request doc, reword comments Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- backend/windmill-api/src/mcp/utils.rs | 120 ++++- backend/windmill-mcp/src/server/runner.rs | 546 ++++++++++++++++++---- 2 files changed, 545 insertions(+), 121 deletions(-) diff --git a/backend/windmill-api/src/mcp/utils.rs b/backend/windmill-api/src/mcp/utils.rs index 7bca92dd57..d68afe826d 100644 --- a/backend/windmill-api/src/mcp/utils.rs +++ b/backend/windmill-api/src/mcp/utils.rs @@ -518,6 +518,42 @@ pub fn build_request_body( } } +/// Scopes to embed in the JWT minted for a proxied MCP endpoint request. The MCP +/// runner already authorized *which* endpoint may be called; this bounds *what +/// the resulting internal request can do*. +/// +/// - Unscoped caller (cookie / full-privilege token): unscoped JWT. +/// - Scope-restricted caller whose own scopes already authorize the route: keep +/// those scopes verbatim, so the target handler's per-path `check_scopes` still +/// enforces the caller's path caps (e.g. a `variables:read:u/admin/safe/*` +/// token can't read `u/admin/secret` via the getVariable proxy). +/// - Otherwise the caller has no route scope for this domain (the common +/// `mcp:`-only token): mint a least-privilege scope for exactly this route, +/// failing closed if the route can't be resolved. +fn jwt_scopes_for_proxied_route( + caller_scopes: Option<&[String]>, + method: &str, + route_path: &str, +) -> BackendResult>> { + let caller_restricted = + caller_scopes.is_some_and(|s| s.iter().any(|x| !x.starts_with("if_jobs:filter_tags:"))); + if !caller_restricted { + return Ok(None); + } + if windmill_api_auth::scopes::check_scopes_for_route(caller_scopes, route_path, method).is_ok() + { + return Ok(caller_scopes.map(|s| s.to_vec())); + } + let scope = + windmill_api_auth::scopes::scope_for_route(method, route_path).ok_or_else(|| { + ErrorData::internal_error( + "Could not derive route scope for proxied MCP endpoint".to_string(), + None, + ) + })?; + Ok(Some(vec![scope])) +} + /// Create HTTP request with authentication pub async fn create_http_request( method: &str, @@ -541,31 +577,12 @@ pub async fn create_http_request( } }; - // Bound the minted JWT to exactly this proxied route so a scope-restricted - // MCP token can't be widened into a full-privilege blank check. The - // endpoint-name gate (in the MCP runner) already authorized *which* endpoint - // may be called; this constrains what the resulting request can do. Unscoped - // callers (cookie / full-privilege tokens) keep an unscoped JWT to preserve - // existing behavior. A scope-restricted caller whose route can't be resolved - // fails closed. - let caller_restricted = api_authed - .scopes - .as_deref() - .is_some_and(|s| s.iter().any(|x| !x.starts_with("if_jobs:filter_tags:"))); - let scopes = if caller_restricted { - let parsed = reqwest::Url::parse(url) - .map_err(|e| ErrorData::internal_error(format!("Invalid proxied URL: {}", e), None))?; - let scope = - windmill_api_auth::scopes::scope_for_route(method, parsed.path()).ok_or_else(|| { - ErrorData::internal_error( - "Could not derive route scope for proxied MCP endpoint".to_string(), - None, - ) - })?; - Some(vec![scope]) - } else { - None - }; + // Scope the minted JWT to the proxied route so a scope-restricted MCP token + // can't be widened into a full-privilege blank check. See + // `jwt_scopes_for_proxied_route`. + let parsed = reqwest::Url::parse(url) + .map_err(|e| ErrorData::internal_error(format!("Invalid proxied URL: {}", e), None))?; + let scopes = jwt_scopes_for_proxied_route(api_authed.scopes.as_deref(), method, parsed.path())?; // Add authorization header let authed = Authed::from(api_authed.clone()); @@ -620,6 +637,59 @@ mod tests { use super::*; use serde_json::json; + fn scopes(list: &[&str]) -> Vec { + list.iter().map(|s| s.to_string()).collect() + } + + #[test] + fn proxy_jwt_unscoped_caller_keeps_none() { + // No scopes, or filter-tags-only, is treated as unscoped -> unscoped JWT. + assert_eq!( + jwt_scopes_for_proxied_route(None, "GET", "/api/w/ws/variables/get/u/a/b").unwrap(), + None + ); + let ft = scopes(&["if_jobs:filter_tags:foo"]); + assert_eq!( + jwt_scopes_for_proxied_route(Some(&ft), "GET", "/api/w/ws/variables/get/u/a/b") + .unwrap(), + None + ); + } + + #[test] + fn proxy_jwt_bare_mcp_token_falls_back_to_route_scope() { + // A token whose only authority is its mcp: scope has no variables route + // scope, so the JWT gets a least-privilege route scope for this request. + let s = scopes(&["mcp:endpoints:getVariable"]); + assert_eq!( + jwt_scopes_for_proxied_route(Some(&s), "GET", "/api/w/ws/variables/get/u/admin/secret") + .unwrap(), + Some(scopes(&["variables:read"])) + ); + } + + #[test] + fn proxy_jwt_mixed_token_passes_through_caller_route_scope() { + // The caller's route scope is preserved so the target handler's per-path + // check_scopes enforces the cap; the coarse route match here is path-blind. + let s = scopes(&["mcp:endpoints:getVariable", "variables:read:u/admin/safe/*"]); + assert_eq!( + jwt_scopes_for_proxied_route(Some(&s), "GET", "/api/w/ws/variables/get/u/admin/secret") + .unwrap(), + Some(s.clone()) + ); + } + + #[test] + fn proxy_jwt_run_script_bare_mcp_falls_back_to_jobs_run_scripts() { + let s = scopes(&["mcp:scripts:f/team/*", "mcp:endpoints:*"]); + assert_eq!( + jwt_scopes_for_proxied_route(Some(&s), "POST", "/api/w/ws/jobs/run/p/f/team/deploy") + .unwrap(), + Some(scopes(&["jobs:run:scripts"])) + ); + } + #[test] fn build_request_body_passthrough_forwards_script_args_minus_path() { // runScriptByPath-shaped body: additionalProperties, no declared props. diff --git a/backend/windmill-mcp/src/server/runner.rs b/backend/windmill-mcp/src/server/runner.rs index 5db32faedb..ee6540520e 100644 --- a/backend/windmill-mcp/src/server/runner.rs +++ b/backend/windmill-mcp/src/server/runner.rs @@ -12,7 +12,7 @@ use crate::common::transform::{ use crate::common::types::{McpToken, MultiWorkspaceMcp, ResourceInfo, ToolableItem, WorkspaceId}; use crate::server::backend::{McpAuth, McpBackend, PathFilter}; use crate::server::endpoints::{ - endpoint_tool_to_mcp_tool, endpoint_tool_to_mcp_tool_multi, list_workspaces_tool, + endpoint_tool_to_mcp_tool, endpoint_tool_to_mcp_tool_multi, list_workspaces_tool, EndpointTool, }; use crate::server::tools::create_tool_from_item; use rmcp::handler::server::ServerHandler; @@ -126,20 +126,176 @@ impl Runner { } } -/// The run-by-path endpoint tools execute an arbitrary script/flow named by a -/// `path` argument. In multi-workspace mode they are the only way to run -/// scripts/flows, so their authorization must honor the `mcp:scripts:` / -/// `mcp:flows:` path scopes (not the generic endpoint scope) — otherwise a -/// granular token could run items outside its allowed paths. Returns the scope -/// resource type ("script"/"flow") for these endpoints, `None` otherwise. -fn run_by_path_scope_kind(endpoint_name: &str) -> Option<&'static str> { +/// How an endpoint tool interacts with the token's `mcp:scripts:` / `mcp:flows:` +/// path scopes. +enum EndpointPathPolicy { + /// Executes the script/flow named by the `path` argument. Gated by the + /// script/flow scope alone (an endpoint scope is not enough to run things): + /// in multi-workspace mode these are the only way to run scripts/flows, so a + /// granular token must not run items outside its allowed paths. + RunByPath(&'static str), + /// Reads/writes the script/flow named by the listed path arguments. The + /// endpoint scope grants the capability; when the token also carries path + /// patterns for `kind`, every listed argument must match them. + PathArgs { kind: &'static str, fields: &'static [&'static str] }, + /// Affects scripts without taking a checkable path (delete-by-hash) or + /// executes arbitrary code (preview). Unavailable to path-confined tokens — + /// allowing these would bypass the path patterns entirely. + Unconfinable(&'static str), +} + +fn endpoint_path_policy(endpoint_name: &str) -> Option { + use EndpointPathPolicy::*; match endpoint_name { - "runScriptByPath" => Some("script"), - "runFlowByPath" => Some("flow"), + "runScriptByPath" => Some(RunByPath("script")), + "runFlowByPath" => Some(RunByPath("flow")), + "getScriptByPath" | "deleteScriptByPath" | "createScript" => { + Some(PathArgs { kind: "script", fields: &["path"] }) + } + "getFlowByPath" | "deleteFlowByPath" | "createFlow" => { + Some(PathArgs { kind: "flow", fields: &["path"] }) + } + // updateFlow addresses the flow via the URL path and can move it to the + // path given in the body — both must stay within scope. + "updateFlow" => Some(PathArgs { kind: "flow", fields: &["path__path", "path__body"] }), + "deleteScriptByHash" | "runScriptPreviewAndWaitResult" => Some(Unconfinable("script")), _ => None, } } +/// Whether the token restricts `kind` ("script"/"flow") to specific paths. A +/// `*` pattern grants every path (see `is_resource_allowed`), so it does not +/// count as confinement. +fn path_confined(scope_config: &crate::common::scope::McpScopeConfig, kind: &str) -> bool { + if scope_config.all { + return false; + } + let patterns = match kind { + "script" => &scope_config.scripts, + "flow" => &scope_config.flows, + _ => return false, + }; + !patterns.is_empty() && !patterns.iter().any(|p| p == "*") +} + +fn require_path_arg<'a>( + endpoint_tool: &EndpointTool, + args: &'a Value, + field: &str, +) -> Result<&'a str, ErrorData> { + args.get(field) + .and_then(|v| v.as_str()) + .filter(|s| !s.is_empty()) + .ok_or_else(|| { + ErrorData::invalid_params( + format!( + "Missing required '{}' argument for tool '{}'.", + field, endpoint_tool.name + ), + None, + ) + }) +} + +/// Whether an endpoint tool is in scope for *listing*. Run-by-path endpoints are +/// gated by the script/flow scope (`has_any` — the token can run at least one +/// path of that kind); unconfinable endpoints are hidden from path-confined +/// tokens (their calls would always be denied); every other endpoint by the +/// endpoint-name scope. +fn endpoint_tool_in_scope( + scope_config: &crate::common::scope::McpScopeConfig, + endpoint_tool: &EndpointTool, +) -> bool { + let endpoint_allowed = + !scope_config.granular || scope_config.is_allowed("endpoint", &endpoint_tool.name); + match endpoint_path_policy(&endpoint_tool.name) { + Some(EndpointPathPolicy::RunByPath(kind)) => scope_config.has_any(kind), + Some(EndpointPathPolicy::Unconfinable(kind)) => { + endpoint_allowed && !path_confined(scope_config, kind) + } + _ => endpoint_allowed, + } +} + +/// Authorize an endpoint-tool *call* against the token's MCP scopes and +/// read-only flag. Shared by single- and multi-workspace modes so both enforce +/// the same rules — otherwise a granular token could run items outside its +/// allowed paths through the single-workspace path. Run-by-path endpoints +/// (runScriptByPath/runFlowByPath) are gated by the script/flow scope for the +/// requested `path` (the endpoint-name scope alone is insufficient); other +/// endpoints by the endpoint-name scope, with script/flow path arguments +/// additionally confined to the token's path patterns when it has any; and +/// non-GET endpoints are refused for read-only tokens. +fn authorize_endpoint_call( + scope_config: &crate::common::scope::McpScopeConfig, + endpoint_tool: &EndpointTool, + args: &Value, + read_only: bool, +) -> Result<(), ErrorData> { + match endpoint_path_policy(&endpoint_tool.name) { + Some(EndpointPathPolicy::RunByPath(kind)) => { + let path = require_path_arg(endpoint_tool, args, "path")?; + // No `granular` gate: is_allowed already encodes every mode — true for + // mcp:all, pattern-matched for granular scopes, and false for + // mcp:favorites (a favorites token can't run an arbitrary path). + if !scope_config.is_allowed(kind, path) { + return Err(ErrorData::internal_error( + format!("Access denied: {} '{}' not in token scope", kind, path), + None, + )); + } + } + policy => { + if scope_config.granular && !scope_config.is_allowed("endpoint", &endpoint_tool.name) { + return Err(ErrorData::internal_error( + format!( + "Access denied: endpoint '{}' not in token scope", + endpoint_tool.name + ), + None, + )); + } + match policy { + Some(EndpointPathPolicy::PathArgs { kind, fields }) + if path_confined(scope_config, kind) => + { + for field in fields { + let path = require_path_arg(endpoint_tool, args, field)?; + if !scope_config.is_allowed(kind, path) { + return Err(ErrorData::internal_error( + format!("Access denied: {} '{}' not in token scope", kind, path), + None, + )); + } + } + } + Some(EndpointPathPolicy::Unconfinable(kind)) + if path_confined(scope_config, kind) => + { + return Err(ErrorData::internal_error( + format!( + "Access denied: endpoint '{}' is not available to a token restricted to specific {} paths", + endpoint_tool.name, kind + ), + None, + )); + } + _ => {} + } + } + } + if read_only && !crate::server::is_endpoint_read_only(endpoint_tool) { + return Err(ErrorData::internal_error( + format!( + "Access denied: endpoint '{}' is not read-only and this token is restricted to read-only operations", + endpoint_tool.name + ), + None, + )); + } + Ok(()) +} + fn find_matching_path(candidates: Vec, request_name: &str) -> Option { candidates .into_iter() @@ -373,10 +529,13 @@ impl Runner { } } - // Add endpoint tools from the generated MCP tools, filtered by scope + // Add endpoint tools from the generated MCP tools, filtered by scope. + // Uses the same run-by-path-aware gate as multi-workspace mode so a + // granular token only sees runScriptByPath / runFlowByPath when it can + // actually run scripts / flows. let endpoint_tools = self.backend.all_endpoint_tools(); for endpoint_tool in endpoint_tools { - if scope_config.granular && !scope_config.is_allowed("endpoint", &endpoint_tool.name) { + if !endpoint_tool_in_scope(scope_config, &endpoint_tool) { continue; } if read_only && !crate::server::is_endpoint_read_only(&endpoint_tool) { @@ -403,27 +562,9 @@ impl Runner { let endpoint_tools = self.backend.all_endpoint_tools(); for endpoint_tool in &endpoint_tools { if endpoint_tool.name.as_ref() == name.as_ref() { - // Validate endpoint scope - if scope_config.granular - && !scope_config.is_allowed("endpoint", &endpoint_tool.name) - { - return Err(ErrorData::internal_error( - format!( - "Access denied: endpoint '{}' not in token scope", - endpoint_tool.name - ), - None, - )); - } - if read_only && !crate::server::is_endpoint_read_only(endpoint_tool) { - return Err(ErrorData::internal_error( - format!( - "Access denied: endpoint '{}' is not read-only and this token is restricted to read-only operations", - endpoint_tool.name - ), - None, - )); - } + // Authorize against the token's MCP scopes and read-only flag, + // including the run-by-path path check (shared with multi mode). + authorize_endpoint_call(scope_config, endpoint_tool, &args, read_only)?; // This is an endpoint tool, call via backend let result = self @@ -580,16 +721,7 @@ impl Runner { let endpoint_tools = self.backend.all_endpoint_tools(); for endpoint_tool in endpoint_tools { - // Run-by-path tools are gated by script/flow scope (they run an - // arbitrary path); every other endpoint by the endpoint scope. - let allowed = match run_by_path_scope_kind(&endpoint_tool.name) { - Some(kind) => scope_config.has_any(kind), - None => { - !scope_config.granular - || scope_config.is_allowed("endpoint", &endpoint_tool.name) - } - }; - if !allowed { + if !endpoint_tool_in_scope(scope_config, &endpoint_tool) { continue; } if read_only && !crate::server::is_endpoint_read_only(&endpoint_tool) { @@ -641,59 +773,13 @@ impl Runner { ) })?; - // Authorize the tool. Run-by-path endpoints (runScriptByPath / - // runFlowByPath) run an arbitrary `path` and must be checked against the - // script/flow scope for that path — the endpoint scope alone would let a - // granular token run items outside its allowed paths. - match run_by_path_scope_kind(&endpoint_tool.name) { - Some(kind) => { - let path = args - .get("path") - .and_then(|v| v.as_str()) - .filter(|s| !s.is_empty()) - .ok_or_else(|| { - ErrorData::invalid_params( - format!( - "Missing required 'path' argument for tool '{}'.", - endpoint_tool.name - ), - None, - ) - })?; - // No `granular` gate: is_allowed already encodes every mode — - // true for mcp:all, pattern-matched for granular scopes, and - // false for mcp:favorites (a favorites token can't run an - // arbitrary path, only its enumerated favorites). - if !scope_config.is_allowed(kind, path) { - return Err(ErrorData::internal_error( - format!("Access denied: {} '{}' not in token scope", kind, path), - None, - )); - } - } - None => { - if scope_config.granular - && !scope_config.is_allowed("endpoint", &endpoint_tool.name) - { - return Err(ErrorData::internal_error( - format!( - "Access denied: endpoint '{}' not in token scope", - endpoint_tool.name - ), - None, - )); - } - } - } - if read_only && !crate::server::is_endpoint_read_only(endpoint_tool) { - return Err(ErrorData::internal_error( - format!( - "Access denied: endpoint '{}' is not read-only and this token is restricted to read-only operations", - endpoint_tool.name - ), - None, - )); - } + // Authorize the tool against the token's MCP scopes and read-only flag + // (shared with single-workspace mode). Run-by-path endpoints + // (runScriptByPath / runFlowByPath) run an arbitrary `path` and are + // checked against the script/flow scope for that path — the endpoint + // scope alone would let a granular token run items outside its allowed + // paths. + authorize_endpoint_call(scope_config, endpoint_tool, &args, read_only)?; // Workspace-scoped endpoints need an explicit target workspace and a // per-workspace auth; global endpoints (e.g. docs) use the base identity. @@ -746,3 +832,271 @@ impl Runner { )])) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::common::scope::{parse_mcp_scopes, McpScopeConfig}; + use serde_json::json; + use std::borrow::Cow; + + fn cfg(scopes: &[&str]) -> McpScopeConfig { + parse_mcp_scopes(&scopes.iter().map(|s| s.to_string()).collect::>()).unwrap() + } + + fn ep(name: &'static str, method: &'static str) -> EndpointTool { + EndpointTool { + name: Cow::Borrowed(name), + description: Cow::Borrowed(""), + instructions: Cow::Borrowed(""), + path: Cow::Borrowed("/w/{workspace}/jobs/run/p/{path}"), + method: Cow::Borrowed(method), + path_params_schema: None, + query_params_schema: None, + body_schema: None, + path_field_renames: None, + query_field_renames: None, + body_field_renames: None, + } + } + + // The core invariant: a folder-scoped token (mcp:endpoints:* + + // mcp:scripts:f/team/*, as the folder-scope UI emits) must not run a script + // outside its allowed folders via runScriptByPath — mcp:endpoints:* alone + // must never authorize an arbitrary path. + #[test] + fn run_by_path_call_enforces_script_scope() { + let config = cfg(&[ + "mcp:scripts:f/team/*", + "mcp:flows:f/team/*", + "mcp:endpoints:*", + ]); + let tool = ep("runScriptByPath", "POST"); + + assert!( + authorize_endpoint_call(&config, &tool, &json!({"path": "f/team/deploy"}), false) + .is_ok() + ); + assert!( + authorize_endpoint_call(&config, &tool, &json!({"path": "f/secret/admin"}), false) + .is_err() + ); + } + + #[test] + fn run_by_path_call_requires_path_arg() { + let tool = ep("runFlowByPath", "POST"); + assert!(authorize_endpoint_call(&cfg(&["mcp:all"]), &tool, &json!({}), false).is_err()); + } + + #[test] + fn run_by_path_flow_scope_independent_of_script_scope() { + // A flow-only token can run flows by path but not scripts by path. + let config = cfg(&["mcp:flows:f/team/*", "mcp:endpoints:*"]); + assert!(authorize_endpoint_call( + &config, + &ep("runFlowByPath", "POST"), + &json!({"path": "f/team/x"}), + false + ) + .is_ok()); + assert!(authorize_endpoint_call( + &config, + &ep("runScriptByPath", "POST"), + &json!({"path": "f/team/x"}), + false + ) + .is_err()); + } + + #[test] + fn non_run_by_path_gated_by_endpoint_scope() { + let get_var = ep("getVariable", "GET"); + assert!(authorize_endpoint_call( + &cfg(&["mcp:endpoints:getVariable"]), + &get_var, + &json!({"path": "u/a/b"}), + false + ) + .is_ok()); + // A granular token without the endpoint scope is denied. + assert!(authorize_endpoint_call( + &cfg(&["mcp:scripts:f/team/*"]), + &get_var, + &json!({"path": "u/a/b"}), + false + ) + .is_err()); + // mcp:all (non-granular) allows any endpoint. + assert!(authorize_endpoint_call(&cfg(&["mcp:all"]), &get_var, &json!({}), false).is_ok()); + } + + #[test] + fn read_only_refuses_non_get_endpoint() { + // Reaches the read-only check via mcp:all so scope isn't the blocker. + assert!(authorize_endpoint_call( + &cfg(&["mcp:all"]), + &ep("createResource", "POST"), + &json!({}), + true + ) + .is_err()); + assert!(authorize_endpoint_call( + &cfg(&["mcp:all"]), + &ep("getVariable", "GET"), + &json!({"path": "u/a/b"}), + true + ) + .is_ok()); + } + + #[test] + fn listing_run_by_path_needs_runnable_scope() { + let tool = ep("runScriptByPath", "POST"); + // An endpoint-only token cannot run any script, so the tool isn't listed. + assert!(!endpoint_tool_in_scope(&cfg(&["mcp:endpoints:*"]), &tool)); + // A script-scoped token can, so it is listed. + assert!(endpoint_tool_in_scope( + &cfg(&["mcp:scripts:f/team/*"]), + &tool + )); + assert!(endpoint_tool_in_scope(&cfg(&["mcp:all"]), &tool)); + // Non-run-by-path endpoints are governed by the endpoint scope. + let get_var = ep("getVariable", "GET"); + assert!(endpoint_tool_in_scope( + &cfg(&["mcp:endpoints:getVariable"]), + &get_var + )); + assert!(!endpoint_tool_in_scope( + &cfg(&["mcp:scripts:f/team/*"]), + &get_var + )); + } + + // A folder-scoped token must not read/write/delete scripts or flows outside + // its allowed paths through the non-run endpoint tools either. + #[test] + fn path_arg_tools_confined_by_path_patterns() { + let config = cfg(&[ + "mcp:scripts:f/team/*", + "mcp:flows:f/team/*", + "mcp:endpoints:*", + ]); + for name in ["getScriptByPath", "deleteScriptByPath", "createScript"] { + let tool = ep(name, "POST"); + assert!( + authorize_endpoint_call(&config, &tool, &json!({"path": "f/team/x"}), false) + .is_ok(), + "{name} should allow in-scope path" + ); + assert!( + authorize_endpoint_call(&config, &tool, &json!({"path": "f/secret/x"}), false) + .is_err(), + "{name} should deny out-of-scope path" + ); + // Confinement can't be verified without the path argument. + assert!( + authorize_endpoint_call(&config, &tool, &json!({}), false).is_err(), + "{name} should require the path argument when confined" + ); + } + for name in ["getFlowByPath", "deleteFlowByPath", "createFlow"] { + let tool = ep(name, "POST"); + assert!( + authorize_endpoint_call(&config, &tool, &json!({"path": "f/team/x"}), false) + .is_ok(), + "{name} should allow in-scope path" + ); + assert!( + authorize_endpoint_call(&config, &tool, &json!({"path": "f/secret/x"}), false) + .is_err(), + "{name} should deny out-of-scope path" + ); + } + } + + // A token that never expressed path patterns (endpoints-only) is not + // confined: the endpoint scope alone authorizes any path. + #[test] + fn path_arg_tools_unconfined_without_path_patterns() { + let config = cfg(&["mcp:endpoints:*"]); + for name in ["getScriptByPath", "createScript", "deleteFlowByPath"] { + assert!(authorize_endpoint_call( + &config, + &ep(name, "POST"), + &json!({"path": "f/anywhere/x"}), + false + ) + .is_ok()); + } + // A `*` pattern grants every path, so it doesn't confine either. + let star = cfg(&["mcp:scripts:*", "mcp:endpoints:*"]); + assert!(authorize_endpoint_call( + &star, + &ep("createScript", "POST"), + &json!({"path": "f/anywhere/x"}), + false + ) + .is_ok()); + } + + // updateFlow both addresses a flow (URL path) and can move it (body path): + // a confined token must have both within scope. + #[test] + fn update_flow_checks_target_and_destination_paths() { + let config = cfg(&["mcp:flows:f/team/*", "mcp:endpoints:*"]); + let tool = ep("updateFlow", "POST"); + assert!(authorize_endpoint_call( + &config, + &tool, + &json!({"path__path": "f/team/a", "path__body": "f/team/b"}), + false + ) + .is_ok()); + // Moving a flow out of the allowed folder is denied. + assert!(authorize_endpoint_call( + &config, + &tool, + &json!({"path__path": "f/team/a", "path__body": "f/secret/a"}), + false + ) + .is_err()); + // Touching a flow outside the allowed folder is denied. + assert!(authorize_endpoint_call( + &config, + &tool, + &json!({"path__path": "f/secret/a", "path__body": "f/team/a"}), + false + ) + .is_err()); + } + + // Tools that can't be path-checked (delete-by-hash) or execute arbitrary + // code (preview) would bypass path confinement, so a path-confined token is + // denied them entirely — and doesn't see them listed. + #[test] + fn unconfinable_tools_denied_for_path_confined_token() { + let confined = cfg(&["mcp:scripts:f/team/*", "mcp:endpoints:*"]); + for name in ["deleteScriptByHash", "runScriptPreviewAndWaitResult"] { + let tool = ep(name, "POST"); + assert!(authorize_endpoint_call(&confined, &tool, &json!({}), false).is_err()); + assert!(!endpoint_tool_in_scope(&confined, &tool)); + // Without script path patterns the tools stay available. + assert!( + authorize_endpoint_call(&cfg(&["mcp:endpoints:*"]), &tool, &json!({}), false) + .is_ok() + ); + assert!(authorize_endpoint_call(&cfg(&["mcp:all"]), &tool, &json!({}), false).is_ok()); + assert!(endpoint_tool_in_scope(&cfg(&["mcp:endpoints:*"]), &tool)); + } + // Flow-only confinement doesn't affect script-kind unconfinable tools. + let flow_confined = cfg(&["mcp:flows:f/team/*", "mcp:endpoints:*"]); + assert!(authorize_endpoint_call( + &flow_confined, + &ep("deleteScriptByHash", "POST"), + &json!({}), + false + ) + .is_ok()); + } +} From 7a139ab23e034321554d0c85540d54251e61cc72 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Fri, 17 Jul 2026 19:36:51 +0200 Subject: [PATCH 13/14] feat(ai-chat): image attachments and agent raw-app screenshots (#10130) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(ai-chat): add image attachments and agent raw-app screenshots Co-Authored-By: Claude Opus 4.8 * docs(ai-chat): generalise take_screenshot fidelity caveat Co-Authored-By: Claude Opus 4.8 * fix(ai-chat): keep compaction boundary on a displayed user message Co-Authored-By: Claude Opus 4.8 * fix(raw-apps): count line boxes by vertical overlap, not rect count Co-Authored-By: Claude Opus 4.8 * fix(ai-chat): enforce vision gating and bound image attachments Refuse images on known text-only models instead of warning and sending them anyway; cap input bytes before decode; keep clipboard text when it accompanies a bitmap; don't queue a message whose images can't ride the plain-text queue. Co-Authored-By: Claude Opus 4.8 * perf(ai-chat): trim take_screenshot schema and shrink the card's copy Move the fidelity caveat from the tool def onto the tool result: the def is re-sent every global iteration (~258 tok), while the caveat only matters once a capture exists. Keep a downscaled copy in displayMessages when it is actually smaller — those are never compacted and are re-cloned on every saveChat. Co-Authored-By: Claude Opus 4.8 * fix(ai-chat): carry attached images through the message queue Enter during a streaming turn queued the text and silently dropped the images, so the auto-send was not the message the user submitted. The queue now holds both, moved together via takeQueue/clearQueue/restoreQueue so none of the three flush sites, the dequeue-to-composer path, or the two conversation-switch drops can leak one without the other. Co-Authored-By: Claude Opus 4.8 * fix(ai-chat): gate screenshots on vision, narrow when the tool fires take_screenshot buffered an image unconditionally, so a text-only model got an image_url and rejected the turn; the attach-time check never covered it, nor a model switched after attaching. Gate before capture and again at send. Only reach for the tool when the user raises how the app looks, rather than after every UI edit. A collapsed preview keeps the iframe mounted at zero width, passing the ready checks and then failing inside the rasteriser as '[object Event]'. Name it. Co-Authored-By: Claude Opus 4.8 * fix(ai-chat): hold sending while attachments decode addImages read the free-slot count before its await and appended after it, so a send during the ~50-800ms decode cleared images while the closure still wrote to them, landing the picture on the following message; two drops also claimed the same slots and could pass the cap. Reserve slots up front, block sending until they resolve, and show a placeholder so the held send is explained. Keep only a bounded copy in the transcript: displayMessages are never compacted and are re-cloned on every save. Measured 6.1x smaller per attachment. Co-Authored-By: Claude Opus 4.8 * fix(ai-chat): route screenshots to the visible tab, resend full-res on retry Every mounted raw-app editor claimed the runtime's single screenshot slot, so take_screenshot could capture a background tab's app; ownership now follows the visible tab and only the owner releases it. restartGeneration resent displayMessages' images, which became a 384px thumbnail when the transcript copy was bounded — retries downgraded the model's own input. Recover the sent parts from the API message instead. Move modelSupportsVision to modelConfig: it was untestable behind lib.ts's monaco import chain, and the denylist missed bundled text-only defaults (Groq/Together Llama 3.3, Foundry Phi-4 and Mistral-Large). Llama 3.2 and Phi-4 split by variant, so both are matched narrowly. Pinned against the shipped defaultModels. Decode attachments one at a time and derive the preview from the bounded copy: a 12MP bitmap is ~48MB and the batch was held live at once, decoded twice each. The attach tooltip claimed nothing is uploaded, which is untrue for images. Co-Authored-By: Claude Opus 4.8 * fix(ai-chat): keep images out of text-only turns and bound the queue The vision gate only dropped the current turn's images, so history's image parts still went out after a switch to a text-only model and failed the request; strip the outbound copy instead, leaving history intact for a switch back. queueMessage had no cap, and each queued send clears the composer for another eight, so repeated sends stacked an unbounded batch into one message. Editing a message resent displayMessages' bounded copy, downgrading the model's own input; retries recovered the full-size one but then re-persisted it at full resolution. storedImages pairs the API message with its transcript entry so both paths resend the original and re-persist the bounded copy. Reserve image slots before awaiting text attachments: the gap left sending enabled with an image pending, measured ~90ms for a 40-file drop, now ~8ms regardless of batch size. Co-Authored-By: Claude Opus 4.8 * fix(ai-chat): treat deepseek-v4 as text-only deepseek-v4-pro ships as a bundled default and the gate let images through to it, so an attachment would fail the turn. DeepSeek's vision line is deepseek-vl. Co-Authored-By: Claude Opus 4.8 * fix(ai-chat): drop a rejected image instead of wedging the conversation A provider that refuses an image leaves it in history, so every later turn resends it and fails identically: the chat is stuck until the user edits the message or starts over, and Retry re-sends the same image. The vision gate only knows the models we ship, so this is the net for the rest. Strip the parts on an image-related rejection and say so; unrelated failures keep the image. Verified at the wire that no provider rejects a base64 data URL: anthropic (source.base64), openai/gpt-4o (input_image), googleai and aws_bedrock/claude (image_url passthrough) all 200 and read the image. Co-Authored-By: Claude Opus 4.8 * fix(ai-chat): match text-only models exactly, from provider API docs The gate guessed by substring over model names, which answers the wrong question. What matters is whether a provider's API accepts image parts, not whether the model can see: DeepSeek V4 ships vision in its chat product that its API has no content type for, and o3-mini gained vision in ChatGPT the API never exposed. Neither is inferable from a name. Substrings also block working models. 'mistral-large' matches Mistral Large 3, which takes images; 'phi-4' matches Phi-4-multimodal, which does too. A wrong entry blocks with no override, while a missing one costs a turn and recovers via the rejection path, so the list is now exact ids only, each backed by a provider doc. Verdicts verified against provider API docs rather than recall. Live-checked where a doc was contradicted: Bedrock's compatibility matrix claims no Anthropic model is served over chat completions, but it serves images fine. Co-Authored-By: Claude Opus 4.8 * fix(ai-chat): stop retry resurrecting a rejected image The rejection fallback strips the image from history but leaves the bubble's thumbnail so the user can still see what they sent. storedImages fell back to that thumbnail when the API message had no parts, so Retry re-attached the very image the provider had just refused and failed identically — the conversation stayed wedged through the one control offered to escape it. Found by retrying in the UI; unit tests, wire tests and four review passes all missed it, since it only exists between two separate fixes. Co-Authored-By: Claude Opus 4.8 * fix(ai-chat): harden image rejection recovery and drop-path attachment * fix(ai-chat): fix image drop race, mid-turn vision gate, retry aliasing * fix(ai-chat): key vision denylist by provider, flatten alpha before jpeg * feat(ai-chat): offer take_screenshot on chromium only, ask for one elsewhere * feat(ai-chat): image-only sends and click-to-expand image previews * fix(ai-chat): capture screenshots at 2x and expand tool images full-res * feat(frontend): expandable image previews in composer and result views * fix(ai-chat): image-only send edge cases from review round * fix(ai-chat): keep image-only drafts on rollback, track failing model id * fix(ai-chat): gate rejection recovery on the failing iteration's model * refactor(ai-chat): record iteration model via onBeforeIteration, trim tests Co-Authored-By: Claude Fable 5 * fix(ai-chat): restore composer draft when beforeSend preflight fails Co-Authored-By: Claude Fable 5 * fix(ai-chat): bound cumulative outbound image bytes per request Co-Authored-By: Claude Fable 5 * fix(ai-chat): make the image byte bound part-granular so over-cap turns keep a subset Co-Authored-By: Claude Fable 5 * fix(ai-chat): evict newest-first within a message in the image byte bound Co-Authored-By: Claude Fable 5 * fix(ai-chat): prune over-cap images from stored history, not just requests Co-Authored-By: Claude Fable 5 * fix(ai-chat): bound history at every save boundary, keep thumbnail pairing across eviction Co-Authored-By: Claude Fable 5 * fix(ai-chat): slot-align storedImages so the bubble expands the right image after eviction Co-Authored-By: Claude Fable 5 * fix(ai-chat): match rejection keywords as whole words so provisioning errors keep images Co-Authored-By: Claude Fable 5 * fix(ai-chat): match input_image rejections, restore images refused by non-GLOBAL modes Co-Authored-By: Claude Fable 5 * fix(ai-chat): defer non-GLOBAL image refusal restore past the composer clear Co-Authored-By: Claude Fable 5 * fix(ai-chat): persist full tool screenshots for post-reload expansion Co-Authored-By: Claude Fable 5 * refactor(ai-chat): persist chat images out-of-band via blob-store refs Co-Authored-By: Claude Fable 5 * fix(ai-chat): scope image blobs per chat and stop cap-eviction rotation Co-Authored-By: Claude Fable 5 * fix(ai-chat): keep blob-cap chronology across drop-oldest compaction Co-Authored-By: Claude Fable 5 * refactor(ai-chat): derive blob eviction from the saved record, not write times Co-Authored-By: Claude Fable 5 * fix(ai-chat): serialize chat history DB writes per manager Co-Authored-By: Claude Fable 5 * fix(ai-chat): pin queued history writes to the enqueue-time user database Co-Authored-By: Claude Fable 5 * fix(ai-chat): delete stale image blobs only after the chat record commits Co-Authored-By: Claude Fable 5 * fix(ai-chat): don't double-restore a queued image-only draft on vision refusal Co-Authored-By: Claude Fable 5 * fix(ai-chat): label image-only chats and evicted image-only bubbles Co-Authored-By: Claude Fable 5 * fix(ai-chat): keep the in-memory chat mirror hydrated for DB-less sessions Co-Authored-By: Claude Fable 5 * fix(ai-chat): converge the chat mirror to refs after a successful DB commit Co-Authored-By: Claude Fable 5 * fix(ai-chat): guard mirror convergence against rewinding newer saves Co-Authored-By: Claude Fable 5 * fix(ai-chat): invalidate pending convergences on identity re-init, keep retry image names Co-Authored-By: Claude Fable 5 * fix(ai-chat): bound the screenshot raster before rasterization Co-Authored-By: Claude Fable 5 * refactor(ai-chat): drop the no-IndexedDB in-memory image fallback Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Opus 4.8 --- frontend/package-lock.json | 7 + frontend/package.json | 1 + .../src/lib/components/DisplayResult.svelte | 17 +- .../src/lib/components/S3FilePreview.svelte | 4 +- .../common/image/ExpandableImage.svelte | 81 ++ .../copilot/chat/AIChatDisplay.svelte | 83 ++- .../copilot/chat/AIChatInput.svelte | 182 ++++- .../copilot/chat/AIChatManager.svelte.ts | 377 ++++++++-- .../copilot/chat/AIChatManager.test.ts | 690 +++++++++++++++++- .../copilot/chat/AIChatMessage.svelte | 50 +- .../copilot/chat/ContextTextarea.svelte | 19 + .../copilot/chat/HistoryManager.svelte.ts | 337 ++++++++- .../copilot/chat/HistoryManager.test.ts | 589 +++++++++++++++ .../copilot/chat/QueuedMessageChip.svelte | 23 +- .../copilot/chat/ToolExecutionDisplay.svelte | 12 + .../components/copilot/chat/anthropic.test.ts | 39 +- .../lib/components/copilot/chat/anthropic.ts | 42 +- .../components/copilot/chat/chatLoop.test.ts | 105 +++ .../lib/components/copilot/chat/chatLoop.ts | 24 +- .../copilot/chat/global/core.test.ts | 23 +- .../components/copilot/chat/global/core.ts | 107 ++- .../copilot/chat/imageUtils.test.ts | 173 +++++ .../lib/components/copilot/chat/imageUtils.ts | 273 +++++++ .../copilot/chat/openai-responses.test.ts | 39 + .../copilot/chat/openai-responses.ts | 22 +- .../components/copilot/chat/shared.test.ts | 36 + .../src/lib/components/copilot/chat/shared.ts | 42 ++ .../src/lib/components/copilot/lib.test.ts | 38 + frontend/src/lib/components/copilot/lib.ts | 8 +- .../src/lib/components/copilot/modelConfig.ts | 66 ++ .../components/raw_apps/RawAppEditor.svelte | 115 ++- frontend/src/lib/components/raw_apps/utils.ts | 5 + .../sessions/RawAppEditorView.svelte | 19 +- .../sessions/sessionRuntime.svelte.ts | 50 +- frontend/src/lib/isChromiumBrowser.test.ts | 45 ++ frontend/src/lib/utils.ts | 15 + 36 files changed, 3595 insertions(+), 163 deletions(-) create mode 100644 frontend/src/lib/components/common/image/ExpandableImage.svelte create mode 100644 frontend/src/lib/components/copilot/chat/imageUtils.test.ts create mode 100644 frontend/src/lib/components/copilot/chat/imageUtils.ts create mode 100644 frontend/src/lib/components/copilot/chat/openai-responses.test.ts create mode 100644 frontend/src/lib/isChromiumBrowser.test.ts diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 555aba4ba1..32bf5ec54a 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -55,6 +55,7 @@ "mdast-util-find-and-replace": "^3.0.2", "mermaid": "^11.15.0", "minimatch": "^10.0.1", + "modern-screenshot": "^4.7.0", "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@=25.0.0", "monaco-languageclient": "10.6.0", "monaco-vim": "^0.4.1", @@ -9360,6 +9361,12 @@ "dev": true, "license": "MIT" }, + "node_modules/modern-screenshot": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/modern-screenshot/-/modern-screenshot-4.7.0.tgz", + "integrity": "sha512-9YxN+ddPSMMlhylOv25VHzXrl9u67QRxoh7+SEewGtgUw7t6hHTrjptSDJUSne9oG4Xk/h2cwG15nIt4Hc9ujg==", + "license": "MIT" + }, "node_modules/monaco-editor": { "name": "@codingame/monaco-vscode-editor-api", "version": "25.0.0", diff --git a/frontend/package.json b/frontend/package.json index 795304bc1e..6b6d7d01ee 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -130,6 +130,7 @@ "mdast-util-find-and-replace": "^3.0.2", "mermaid": "^11.15.0", "minimatch": "^10.0.1", + "modern-screenshot": "^4.7.0", "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@=25.0.0", "monaco-languageclient": "10.6.0", "monaco-vim": "^0.4.1", diff --git a/frontend/src/lib/components/DisplayResult.svelte b/frontend/src/lib/components/DisplayResult.svelte index 49a72e0e13..9505bbe210 100644 --- a/frontend/src/lib/components/DisplayResult.svelte +++ b/frontend/src/lib/components/DisplayResult.svelte @@ -4,6 +4,7 @@ import { Highlight } from 'svelte-highlight' import { json } from 'svelte-highlight/languages' import { copyToClipboard, parseS3Object, roughSizeOfObject } from '$lib/utils' + import ExpandableImage from '$lib/components/common/image/ExpandableImage.svelte' import { base } from '$lib/base' import { downloadViaClient, shouldDownloadViaClient } from '$lib/utils/downloadFile' import { appendViewToken } from '$lib/viewToken' @@ -827,7 +828,7 @@
{:else if !forceJson && resultKind === 'png'}
- png rendered {:else if !forceJson && resultKind === 'jpeg'}
- jpeg rendered {:else if !forceJson && resultKind === 'gif'}
- gif rendered - preview rendered +
{:else if s3object?.s3?.endsWith('.pdf')}
@@ -1145,8 +1151,9 @@ {:else if s3object?.s3?.endsWith('.png') || s3object?.s3?.endsWith('.jpeg') || s3object?.s3?.endsWith('.jpg') || s3object?.s3?.endsWith('.webp')} {#if seeS3PreviewFileFromList == s3object?.s3}
- preview rendered diff --git a/frontend/src/lib/components/S3FilePreview.svelte b/frontend/src/lib/components/S3FilePreview.svelte index 2a7c3977cd..cbad09f879 100644 --- a/frontend/src/lib/components/S3FilePreview.svelte +++ b/frontend/src/lib/components/S3FilePreview.svelte @@ -19,6 +19,7 @@ } from '$lib/gen' import { displayDate, displaySize, emptyString } from '$lib/utils' import { twMerge } from 'tailwind-merge' + import ExpandableImage from '$lib/components/common/image/ExpandableImage.svelte' interface Props { fileKey: string | undefined @@ -243,11 +244,12 @@
{:else if fileMetadata?.fileKey.endsWith('.png') || fileMetadata?.fileKey.endsWith('.jpg') || fileMetadata?.fileKey.endsWith('.jpeg') || fileMetadata?.fileKey.endsWith('.webp')}
- S3 preview
{:else if fileMetadata?.fileKey.endsWith('.pdf')} diff --git a/frontend/src/lib/components/common/image/ExpandableImage.svelte b/frontend/src/lib/components/common/image/ExpandableImage.svelte new file mode 100644 index 0000000000..8fbe7774e8 --- /dev/null +++ b/frontend/src/lib/components/common/image/ExpandableImage.svelte @@ -0,0 +1,81 @@ + + + + + + + { + if (e.key === 'Enter' || e.key === ' ') { + e.preventDefault() + expand(e) + } + }} +/> + + + + + diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index cefe798165..6aec1f3ba8 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -45,10 +45,13 @@ import type { SelectedContext } from './app/core' import AttachedFilesBar from './files/AttachedFilesBar.svelte' import { type FileToAttach } from './files/attachedFiles.svelte' + import { isImageFile } from './imageUtils' import { hasFileSystemAccess, pickDirectory, handlesFromDataTransfer, + isDirectoryHandle, + isFileHandle, readDroppedEntries } from './files/fsAccess' import { sendUserToast } from '$lib/toast' @@ -270,9 +273,10 @@ // File attachment is GLOBAL-mode only. const canAttachFiles = $derived(aiChatManager.mode === AIMode.GLOBAL && !disabled) - // Steers the OS file picker toward text formats (soft hint; content sniff is authoritative). + // Steers the OS file picker toward text + image formats (soft hint; images attach to + // the message, other files link as text context after a content sniff). const TEXT_FILE_ACCEPT = - 'text/*,.txt,.csv,.tsv,.json,.jsonl,.ndjson,.md,.markdown,.log,.yaml,.yml,.toml,.ini,.cfg,.conf,.env,.xml,.html,.htm,.css,.js,.mjs,.cjs,.ts,.tsx,.jsx,.py,.rb,.rs,.go,.java,.kt,.c,.h,.cpp,.cc,.cs,.php,.sh,.bash,.zsh,.sql,.svelte,.vue,.dockerfile' + 'image/*,text/*,.txt,.csv,.tsv,.json,.jsonl,.ndjson,.md,.markdown,.log,.yaml,.yml,.toml,.ini,.cfg,.conf,.env,.xml,.html,.htm,.css,.js,.mjs,.cjs,.ts,.tsx,.jsx,.py,.rb,.rs,.go,.java,.kt,.c,.h,.cpp,.cc,.cs,.php,.sh,.bash,.zsh,.sql,.svelte,.vue,.dockerfile' let fileInputEl = $state(null) let folderInputEl = $state(null) let dragDepth = $state(0) @@ -358,17 +362,32 @@ e.preventDefault() const dt = e.dataTransfer if (!dt) return + // Images attach to the message; other files link as text context. Images are + // reserved from dt.files BEFORE any await (a send mid-ingestion would land + // them on the next message), and dt.files is the only place a disk-less drag + // exists — a cross-tab image resolves every getAsFileSystemHandle() to null. + const flatFiles = Array.from(dt.files ?? []) + const topLevelImages = flatFiles.filter(isImageFile) + const imageWork: Promise[] = [] + if (topLevelImages.length > 0) { + imageWork.push(aiChatInput?.addImages(topLevelImages) ?? Promise.resolve()) + } if (canUseFsAccess) { // getAsFileSystemHandle calls are kicked off synchronously inside this call. const handles = await handlesFromDataTransfer(dt) - for (const h of handles) { - if (h.kind === 'directory') { - // Folders link as a live handle. - await addDirHandle(h as FileSystemDirectoryHandle) - } else { - // Files are always snapshotted (handle discarded). - await handleAddFiles([{ file: await (h as FileSystemFileHandle).getFile() }]) - } + // No handles → nothing beyond dt.files exists; its text files are all there is. + // Handle-backed files are top-level by definition, so their images are + // already reserved above — only text files remain to route. + const looseFiles = + handles.length === 0 + ? flatFiles + : await Promise.all(handles.filter(isFileHandle).map((h) => h.getFile())) + // Files are always snapshotted (handle discarded). + const textFiles = looseFiles.filter((f) => !isImageFile(f)) + if (textFiles.length > 0) await handleAddFiles(textFiles) + // Folders link as a live handle. + for (const h of handles.filter(isDirectoryHandle)) { + await addDirHandle(h) } } else { // Fallback (no File System Access API): snapshot dropped files AND folders by walking @@ -376,14 +395,33 @@ // (they're only valid during this event) before its first await; if it yields nothing // (no entry API), fall back to the flat dt.files. const entries = await readDroppedEntries(Array.from(dt.items ?? [])) - if (entries.length > 0) await handleAddFiles(entries) - else if (dt.files.length > 0) await handleAddFiles(dt.files) + const source: FileToAttach[] = entries.length > 0 ? entries : flatFiles + // Only top-level images attach to the message, and those were already + // reserved from dt.files before the walk — drop them here so they aren't + // re-reported as skipped non-text. Folder-nested images are deliberately + // NOT attached (the FSA path never extracts folder contents either); they + // ride the text ingestion and are summarized as skipped. + const textEntries = source.filter((entry) => { + const file = entry instanceof File ? entry : entry.file + const nested = !(entry instanceof File) && entry.path?.includes('/') + return !isImageFile(file) || !!nested + }) + if (textEntries.length > 0) await handleAddFiles(textEntries) } + await Promise.all(imageWork) } - function onFileInputChange(e: Event) { + async function onFileInputChange(e: Event) { const input = e.currentTarget as HTMLInputElement - if (input.files && input.files.length > 0) void handleAddFiles(input.files) + if (input.files && input.files.length > 0) { + const picked = Array.from(input.files) + const imageFiles = picked.filter(isImageFile) + const textFiles = picked.filter((f) => !isImageFile(f)) + // Reserved before the text work is awaited — see onPanelDrop. + const imageWork = imageFiles.length > 0 ? aiChatInput?.addImages(imageFiles) : undefined + if (textFiles.length > 0) await handleAddFiles(textFiles) + await imageWork + } input.value = '' // allow re-selecting the same file } @@ -504,7 +542,7 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> >
- Drop files to attach + Drop files or images to attach
{/if} @@ -797,7 +835,11 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> {#if canAttachFiles} [ - { displayName: 'Attach file', icon: FileText, action: () => linkFiles() }, + { + displayName: 'Attach file or image', + icon: FileText, + action: () => linkFiles() + }, { // A real (live) link needs the File System Access API; without it the // folder is only snapshotted, so call it "Add folder", not "Link folder". @@ -825,9 +867,12 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. -->

Attach files or link a folder

- Nothing is uploaded. Files are kept locally in your browser; a folder is - linked live from disk. The assistant lists, searches, and reads them on - demand — their contents aren't sent unless it reads them. + Text files stay in your browser, and a folder is linked live from disk. + The assistant lists, searches, and reads them on demand, so their contents + are sent only when it reads one. +

+

+ Images are sent with your next message, so the assistant can see them.

{/snippet} diff --git a/frontend/src/lib/components/copilot/chat/AIChatInput.svelte b/frontend/src/lib/components/copilot/chat/AIChatInput.svelte index 8f39b78a27..07d6b03b36 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatInput.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatInput.svelte @@ -11,11 +11,23 @@ import { tick, untrack, type Snippet } from 'svelte' import Portal from '$lib/components/Portal.svelte' import { zIndexes } from '$lib/zIndexes' - import { ArrowUp, Square } from 'lucide-svelte' + import { ArrowUp, Loader2, Square, X } from 'lucide-svelte' import { Button } from '$lib/components/common' import { sendUserToast } from '$lib/toast' import { type PasteAttachment } from './pasteTokens' import { chatDraft, expanded } from './chatDraft' + import { + fileToAttachedImage, + isImageFile, + MAX_ATTACHED_IMAGES, + MAX_IMAGE_BYTES, + type AttachedImage + } from './imageUtils' + import { modelSupportsVision } from '../modelConfig' + import { tryGetCurrentModel } from '$lib/aiStore' + import ExpandableImage, { + isImageViewerOpen + } from '$lib/components/common/image/ExpandableImage.svelte' const aiChatManager = getAiChatManager() @@ -27,6 +39,7 @@ placeholder?: string initialInstructions?: string initialPastes?: PasteAttachment[] + initialImages?: AttachedImage[] editingMessageIndex?: number | null onEditEnd?: () => void className?: string @@ -56,6 +69,7 @@ placeholder, initialInstructions = '', initialPastes = undefined, + initialImages = undefined, editingMessageIndex = null, onEditEnd = () => {}, className = '', @@ -129,6 +143,77 @@ }) // Collapsed big-paste blobs referenced by tokens in `instructions`. let pastes = $state(untrack(() => initialPastes ?? [])) + // Per-message image attachments (drag/drop/paste), GLOBAL mode only. One-shot: + // they attach to the next send and clear, unlike the persistent attached-files store. + let images = $state(untrack(() => initialImages ?? [])) + // Images being decoded right now. Holds off sending so a message can never go + // out without an attachment the user already dropped, and reserves cap slots + // against a concurrent drop. + let pendingImages = $state(0) + + /** Attach dropped/pasted image files (downscaled + bounded). GLOBAL mode only. */ + export async function addImages(files: (File | Blob)[]) { + if (aiChatManager.mode !== AIMode.GLOBAL) return + const imageFiles = files.filter(isImageFile) + if (imageFiles.length === 0) return + // tryGetCurrentModel returns undefined instead of throwing: this runs from a + // drop/paste handler that can't surface a rejection. + const model = tryGetCurrentModel() + // Only known text-only models fail this, so attaching would certainly 400 the + // next turn — refuse rather than warn and send it anyway. + if (model && !modelSupportsVision(model.provider, model.model)) { + sendUserToast(`${model.model} can't read images. Switch to a vision model first.`, true) + return + } + // Count decodes already in flight: two drops that both read `images.length` + // before either resolves would each claim the same free slots and overshoot + // the cap. + const remaining = MAX_ATTACHED_IMAGES - images.length - pendingImages + if (remaining <= 0) { + sendUserToast(`You can attach up to ${MAX_ATTACHED_IMAGES} images.`, true) + return + } + const oversized = imageFiles.filter((f) => f.size > MAX_IMAGE_BYTES) + if (oversized.length > 0) { + const mb = Math.round(MAX_IMAGE_BYTES / 1_000_000) + sendUserToast(`${oversized.length} image(s) over ${mb}MB were skipped.`, true) + } + const usable = imageFiles.filter((f) => f.size <= MAX_IMAGE_BYTES) + if (usable.length === 0) return + const batch = usable.slice(0, remaining) + if (batch.length < usable.length) { + sendUserToast( + `You can attach up to ${MAX_ATTACHED_IMAGES} images; ${usable.length - batch.length} were skipped.`, + true + ) + } + // Claim the slots before awaiting, and hold sending until they resolve: + // decoding takes ~50-800ms, and a send during it would clear `images` while + // this closure still appends to it, landing the picture on the next message. + pendingImages += batch.length + try { + // One at a time: a decoded bitmap costs ~4 bytes per pixel (a 12MP photo is + // ~48MB), so decoding the whole batch at once would hold every one of them + // live simultaneously. + const added: AttachedImage[] = [] + let failed = 0 + for (const file of batch) { + try { + added.push(await fileToAttachedImage(file)) + } catch { + failed++ + } + } + if (added.length > 0) images = [...images, ...added] + if (failed > 0) sendUserToast(`Could not attach ${failed} image(s).`, true) + } finally { + pendingImages -= batch.length + } + } + + function removeImage(index: number) { + images = images.filter((_, i) => i !== index) + } // App mode @ mention state let showAppContextTooltip = $state(false) @@ -190,19 +275,40 @@ } // Restore composer contents after a rolled-back turn. No-op when the user - // already typed a new draft — restoring would clobber it. - export function restoreInstructions(value: string, restoredPastes: PasteAttachment[] = []) { - if (instructions.trim()) return + // already drafted something new — typed text or attached images (including + // ones still decoding) — restoring would clobber it. + export function restoreInstructions( + value: string, + restoredPastes: PasteAttachment[] = [], + restoredImages: AttachedImage[] = [] + ) { + if (instructions.trim() || images.length > 0 || pendingImages > 0) return instructions = value pastes = restoredPastes + images = restoredImages focusInput() } /** Put text back into the textarea (queued-message delete, or restore * after a cancelled/errored turn), prepended to any draft so nothing - * the user typed is lost. */ - export function prependText(text: string) { - instructions = instructions.trim() ? `${text}\n\n${instructions}` : text + * the user typed is lost. Restored images join whatever is already + * attached, up to the cap — dropping them would lose the attachment + * silently, which is the whole reason the queue carries them. */ + export function prependText(text: string, restoredImages: AttachedImage[] = []) { + // An image-only restore has empty text; prepending it would only add blank lines. + if (text) { + instructions = instructions.trim() ? `${text}\n\n${instructions}` : text + } + if (restoredImages.length > 0) { + const merged = [...images, ...restoredImages] + if (merged.length > MAX_ATTACHED_IMAGES) { + sendUserToast( + `You can attach up to ${MAX_ATTACHED_IMAGES} images; ${merged.length - MAX_ATTACHED_IMAGES} restored image(s) were dropped.`, + true + ) + } + images = merged.slice(0, MAX_ATTACHED_IMAGES) + } focusInput() } @@ -215,6 +321,10 @@ function clickOutside(node: HTMLElement) { function handleClick(event: MouseEvent) { + // An expanded image chip renders in a portal, so clicks in it land outside + // this node without being outside the composer. Dismissing on them would + // discard the edit the user opened the image from. + if (isImageViewerOpen()) return if (node && !node.contains(event.target as Node)) { onClickOutside() } @@ -296,25 +406,31 @@ } function sendRequest() { + // The send button is disabled while decoding, but Enter reaches here directly. + // Sending now would drop the in-flight images onto the following message. + if (pendingImages > 0) { + return + } if (aiChatManager.loading) { // Queue the message instead of silently discarding it — it is // auto-sent when the streaming turn completes successfully. // Editing-while-loading keeps the old discard behavior. Paste // tokens are expanded into the queued text (the queue is plain // strings), so the full content survives the auto-send. - if (editingMessageIndex === null && instructions.trim()) { - aiChatManager.queueMessage(expanded(chatDraft(instructions, pastes))) + if (editingMessageIndex === null && (instructions.trim() || images.length > 0)) { + aiChatManager.queueMessage(expanded(chatDraft(instructions, pastes)), images) contextTextareaComponent?.clearForSend() instructions = '' pastes = [] + images = [] } return } if (editingMessageIndex !== null) { - aiChatManager.restartGeneration(editingMessageIndex, instructions, pastes) + aiChatManager.restartGeneration(editingMessageIndex, instructions, pastes, images) onEditEnd() } else { - aiChatManager.sendRequest({ instructions, pastes }) + aiChatManager.sendRequest({ instructions, pastes, images }) // clearForSend() pre-zaps the textarea's mention-sync so the wipe // doesn't drop `selectedContext` before `AIChatManager.beforeSend` // snapshots it. Only mounted in SCRIPT/FLOW/GLOBAL — APP and the @@ -323,6 +439,7 @@ contextTextareaComponent?.clearForSend() instructions = '' pastes = [] + images = [] } } @@ -538,7 +655,8 @@ {#snippet sendStopButton()} {@const isLoading = loading ?? aiChatManager.loading} - {@const sendDisabled = disabled || instructions.trim().length === 0} + {@const sendDisabled = + disabled || (instructions.trim().length === 0 && images.length === 0) || pendingImages > 0} +
+ {/each} + + {#each { length: pendingImages } as _, i (i)} +
+ +
+ {/each} +
+ {/if} +{/snippet} +
void addImages(files) + : undefined} {availableContext} {selectedContext} placeholder={modePlaceholder} diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 2853c392f5..f86db724d8 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -45,6 +45,7 @@ import { prepareScriptUserMessage } from './script/core' import { prepareNavigatorUserMessage } from './navigator/core' import { sendUserToast } from '$lib/toast' import { workspaceAIClients, getNonStreamingCompletion } from '../lib' +import { modelSupportsVision } from '../modelConfig' import { getKnownModelContextWindow } from '../modelConfig' import { getCompactionSummaryPrompt, @@ -57,6 +58,13 @@ import type { UserDraftItemKind } from '$lib/gen' import { maskKey } from '$lib/components/sessions/modifiedItemsMask' import { getStringError } from './utils' import { type PasteAttachment } from './pasteTokens' +import { + type AttachedImage, + imagesFromContent, + MAX_ATTACHED_IMAGES, + messagesHaveImageParts, + stripImagePartsFromMessages +} from './imageUtils' import { chatDraft, expanded } from './chatDraft' import type { FlowModuleState, FlowState } from '$lib/components/flows/flowState' import type { CurrentEditor, ExtendedOpenFlow } from '$lib/components/flows/types' @@ -128,6 +136,9 @@ function prefersInstantReveal(): boolean { // schema changes from mode switches, and the estimate's chars/4 error. const COMPACTION_TRIGGER_RATIO = 0.8 const COMPACTION_TARGET_RATIO = 0.7 +// Flat per-image token estimate for a downscaled (≤1568px) vision image. Used instead +// of chars/4 on the base64 data URL, which would overcount by ~50x. +const IMAGE_TOKEN_ESTIMATE = 1200 // Headroom reserved within the target budget for the summary message itself, so // the summary + kept tail + overhead land under the target ratio. const SUMMARY_OUTPUT_RESERVE_TOKENS = 8000 @@ -274,6 +285,32 @@ function appendWebSearchErrorHint(message: string, shouldAppend: boolean): strin return `${message}${separator}${WEB_SEARCH_ERROR_HINT}` } +/** + * Whether a provider rejected the request over an image it could not take. The + * vision gate only knows the models we ship, so this is the net for the rest: + * every provider words it differently, hence matching on the subject rather than + * a code. Only consulted when the outbound request actually carried an image, so + * an unrelated error mentioning "image" cannot trigger it on its own. + */ +function isImageRejection(err: unknown, models: (string | undefined)[] = []): boolean { + let message = (err instanceof Error ? err.message : String(err)).toLowerCase() + // Vision-capable model ids often contain the subject words themselves + // (llama-3.2-90b-vision-instruct, Phi-4-multimodal-instruct) and providers echo + // the id in unrelated errors (rate limits, capacity). A match inside the id + // would treat those as rejections and destroy good images, so drop the ids + // before matching — only the error's own wording counts. Callers pass every + // model the turn may have used: the error can come from the model selected at + // send time OR the one currently selected (switchable mid-flight). + for (const model of models) { + if (model) message = message.replaceAll(model.toLowerCase(), '') + } + // Whole words only: "provisioning"/"provisioned" contain "vision", and a + // transient capacity error must not destroy good images. image_url and + // input_image are the content-part names providers echo in schema errors + // ('_' is a word char, so \bimage\b alone would miss them). + return /\bimages?(_url)?\b|\binput_image\b|\bvision\b|\bmultimodal\b/.test(message) +} + function getSendRequestErrorMessage(err: unknown, webSearchUnavailable: boolean): string { const errorMessage = err instanceof Error ? err.message : typeof err === 'string' ? err : undefined @@ -306,6 +343,11 @@ export class AIChatManager { // the turn finishes (clean completion or user cancel). Ephemeral — never // saved to displayMessages or history. queuedMessage = $state('') + // Images attached to that message. Kept beside the text rather than inside it + // because the queued chip renders `queuedMessage` as a plain string. Always + // move the two together — #takeQueue/#clearQueue/#restoreQueue exist so no + // call site can drop one and auto-send a message the user never wrote. + queuedImages = $state([]) // Jobs the chat started that detached into the background (global/sessions // chat only). Rendered in the jobs tray, persisted with the chat, and advanced // by a single background poller. See registerJob / #pollBackgroundJobs. @@ -380,6 +422,16 @@ export class AIChatManager { }) displayMessages = $state([]) messages = $state([]) + /** Images buffered by tools (e.g. take_screenshot) during the current tool batch, + * keyed by toolId. Drained by appendPendingToolImages into a follow-up user message + * after the batch. Cleared at each turn start so an aborted batch can't leak. */ + private pendingToolImages = new Map() + /** Model of the most recent loop iteration, recorded via onBeforeIteration. + * The selector stays switchable mid-flight, so when a request fails neither + * the send-time nor the currently-selected model necessarily names the one + * whose request is being classified (A→B→C switches). Reset at each turn + * start, consumed by image-rejection recovery. */ + private lastIterationModel: ReturnType | undefined = undefined /** Provider-reported context size of the last committed turn (prompt + * completion of its latest completion — exact, includes system prompt and * tools), or undefined whenever no report describes the current history @@ -901,6 +953,15 @@ export class AIChatManager { const tokenPerCharacter = 4 if (typeof message.content === 'string') { acc += message.content.length / tokenPerCharacter + } else if (Array.isArray(message.content)) { + // Multimodal content: chars/4 for the text parts, a flat estimate per image + // (a base64 data URL is huge as text but only ~1.1-1.6k tokens as vision input, + // so JSON.stringify here would overcount by orders of magnitude). + for (const part of message.content as any[]) { + if (part?.type === 'text') acc += (part.text?.length ?? 0) / tokenPerCharacter + else if (part?.type === 'image_url') acc += IMAGE_TOKEN_ESTIMATE + else acc += JSON.stringify(part).length / tokenPerCharacter + } } else if (message.content) { acc += JSON.stringify(message.content).length / tokenPerCharacter } @@ -970,12 +1031,15 @@ export class AIChatManager { } this.messages = this.messages.slice(drop) // User display messages carry the index of their API message so restart - // can rewind to it; re-base them on the compacted history. A message - // whose API counterpart was dropped clamps to 0: everything before it - // was dropped too (compaction only removes prefixes), so restarting - // from it restarts from an empty history. + // can rewind to it; re-base them on the compacted history. A message whose + // API counterpart was dropped goes negative — deliberately NOT clamped to + // 0, which would alias it to the first surviving message and let + // storedImages hand a retry that message's images. Negative reads as + // "counterpart gone": storedImages finds nothing there, and restart maps + // it to an empty history (everything before it was dropped too, since + // compaction only removes prefixes). this.displayMessages = this.displayMessages.map((m) => - m.role === 'user' ? { ...m, index: Math.max(0, m.index - drop) } : m + m.role === 'user' ? { ...m, index: m.index - drop } : m ) return freed } @@ -1008,7 +1072,9 @@ export class AIChatManager { // >10 minutes of generation (~21k tokens) before anything is sent. const raw = await getNonStreamingCompletion( [ - ...sanitizeToolCallArguments(prefix), + // Strip image blobs from the summarizer input — the summary text stands in + // for them, so re-sending base64 to the summarizer only wastes tokens. + ...stripImagePartsFromMessages(sanitizeToolCallArguments(prefix)), { role: 'user', content: getCompactionSummaryPrompt() } ], abortController, @@ -1097,9 +1163,16 @@ export class AIChatManager { tailTokens += t keepFrom = i } - // The tail must start on a user message — move the boundary forward over - // any leading tool/assistant messages, folding them into the prefix. - while (keepFrom < last && this.messages[keepFrom].role !== 'user') { + // The tail must start on a user message the transcript also shows — move the + // boundary forward over leading tool/assistant messages, and over synthetic + // user messages that carry no display entry (the image follow-ups + // appendPendingToolImages injects). Landing on one would slice `messages` + // and `displayMessages` at different turns, silently dropping the cards in + // between from the visible history. + const shownUserIndices = new Set( + this.displayMessages.filter((m) => m.role === 'user').map((m) => m.index) + ) + while (keepFrom < last && !shownUserIndices.has(keepFrom)) { keepFrom++ } @@ -1109,11 +1182,10 @@ export class AIChatManager { return false } - // The user message at the boundary has a display counterpart with the same - // index; resolve it before any mutation so a corrupt transcript can never - // result from an unexpected miss. + // Exact index, never >=: a miss must fail the compaction, because resolving + // to a later turn would slice the transcript short of the kept API tail. const displayKeepFrom = this.displayMessages.findIndex( - (m) => m.role === 'user' && m.index >= keepFrom + (m) => m.role === 'user' && m.index === keepFrom ) if (displayKeepFrom === -1) { this.consecutiveCompactionFailures++ @@ -1199,12 +1271,15 @@ export class AIChatManager { // epilogue (loading gated its capture): auto-send after a successful // compaction or a deliberate user cancel — the user is ready to move on — // while a failed/empty compaction or a programmatic cancel leaves it queued. - if ((result === 'ok' || this.wasCancelledByUser()) && this.queuedMessage) { - const next = this.queuedMessage - this.queuedMessage = '' - const accepted = await this.sendRequest({ instructions: next }) + if ((result === 'ok' || this.wasCancelledByUser()) && this.#hasQueuedMessage()) { + const next = this.#takeQueue() + const accepted = await this.sendRequest({ + instructions: next.text, + images: next.images, + queued: true + }) if (accepted === false) { - this.queuedMessage = next + this.#restoreQueue(next) } } } @@ -1338,33 +1413,67 @@ export class AIChatManager { /** Queue the message typed while a turn is streaming. There is only ever * one queued message; pressing Enter again appends the new text as another - * line so it all goes out as a single message. */ - queueMessage(text: string) { + * line so it all goes out as a single message, and its images accumulate + * alongside it. */ + queueMessage(text: string, images: AttachedImage[] = []) { const trimmed = text.trim() - if (!trimmed) { + // An image with no text is still a message; only a fully empty send is ignored. + if (!trimmed && images.length === 0) { return } - this.queuedMessage = this.queuedMessage ? `${this.queuedMessage}\n${trimmed}` : trimmed + if (trimmed) { + this.queuedMessage = this.queuedMessage ? `${this.queuedMessage}\n${trimmed}` : trimmed + } + if (images.length > 0) { + const merged = [...this.queuedImages, ...images] + if (merged.length > MAX_ATTACHED_IMAGES) { + sendUserToast(`Only the first ${MAX_ATTACHED_IMAGES} images are kept.`, true) + } + this.queuedImages = merged.slice(0, MAX_ATTACHED_IMAGES) + } } - /** Remove the queued message and put its text back into the input. */ - dequeueMessage() { - if (!this.queuedMessage) { - return - } - const message = this.queuedMessage + /** Whether anything is waiting in the queue — an image-only message has empty text. */ + #hasQueuedMessage(): boolean { + return this.queuedMessage !== '' || this.queuedImages.length > 0 + } + + /** Detach the queue for sending. Text and images always leave together. */ + #takeQueue(): { text: string; images: AttachedImage[] } { + const taken = { text: this.queuedMessage, images: this.queuedImages } + this.#clearQueue() + return taken + } + + #clearQueue() { this.queuedMessage = '' - this.restoreToInput(message) + this.queuedImages = [] } - /** Put text the user typed back where they can see it: into the input + /** Put a taken queue back after an auto-send bailed before becoming a turn. */ + #restoreQueue(queued: { text: string; images: AttachedImage[] }) { + this.queuedMessage = queued.text + this.queuedImages = queued.images + } + + /** Remove the queued message and put it back into the input, images included. */ + dequeueMessage() { + if (!this.#hasQueuedMessage()) { + return + } + const queued = this.#takeQueue() + this.restoreToInput(queued.text, queued.images) + } + + /** Put what the user typed back where they can see it: into the input * when it's mounted, otherwise back into the queue so it reappears with * the chat panel instead of being silently dropped. */ - private restoreToInput(text: string) { + private restoreToInput(text: string, images: AttachedImage[] = []) { if (this.aiChatInput) { - this.aiChatInput.prependText(text) + this.aiChatInput.prependText(text, images) } else { this.queuedMessage = text + this.queuedImages = images } } @@ -1751,12 +1860,13 @@ export class AIChatManager { modelLenAfterUser: number, instructions: string, pastes: PasteAttachment[], - restoreToInput: boolean = true + restoreToInput: boolean = true, + images: AttachedImage[] = [] ) => { this.displayMessages = this.displayMessages.slice(0, displayLenAfterUser - 1) this.messages = this.messages.slice(0, modelLenAfterUser - 1) if (restoreToInput) { - this.aiChatInput?.restoreInstructions(instructions, pastes) + this.aiChatInput?.restoreInstructions(instructions, pastes, images) } } @@ -1791,6 +1901,10 @@ export class AIChatManager { systemMessage?: ChatCompletionSystemMessageParam onWebSearchUnavailable?: () => void }) => { + // Fresh batch for this turn — drop any images an aborted prior turn left buffered. + this.pendingToolImages.clear() + // Stale from a prior turn it would misattribute a pre-first-iteration failure. + this.lastIterationModel = undefined const onReasoningSummaryUnavailable = () => this.notifyReasoningSummaryUnavailable() try { // Use JS getters so runChatLoop re-reads tools/helpers/systemMessage/modelProvider @@ -1866,7 +1980,8 @@ export class AIChatManager { } return undefined }, - onBeforeIteration: async (tools) => { + onBeforeIteration: async (tools, _helpers, modelProvider) => { + this.lastIterationModel = modelProvider for (const tool of tools) { if (tool.setSchema) { await tool.setSchema(this.helpers) @@ -1985,9 +2100,14 @@ export class AIChatManager { addBackCode?: boolean instructions?: string pastes?: PasteAttachment[] + images?: AttachedImage[] mode?: AIMode lang?: ScriptLang | 'bunnative' isPreprocessor?: boolean + /** Auto-send of a queued draft: on preflight failure the caller re-queues + * it, so the composer restore must not also fire (the draft would exist + * twice — queue chip and composer). */ + queued?: boolean } = {} ) => { // Returns whether the input was consumed: true when it was sent as a chat @@ -2003,10 +2123,17 @@ export class AIChatManager { lang: options.lang, isPreprocessor: options.isPreprocessor }) - if (options.instructions) { + // Explicitly-passed instructions win even when empty: an image-only send + // carries '' and must not inherit stale text a failed or cancelled earlier + // turn left in this.instructions. + if (options.instructions !== undefined) { this.instructions = options.instructions } - if (!this.instructions.trim()) { + // Only a truly empty draft is dropped here. An image with no text is a + // valid GLOBAL-mode message; outside GLOBAL an image-bearing draft must + // still get past this guard to reach the refusal below, which puts it + // back in the composer instead of silently losing it. + if (!this.instructions.trim() && (options.images?.length ?? 0) === 0) { return false } // Built-in session commands run locally instead of becoming a chat turn. @@ -2049,6 +2176,42 @@ export class AIChatManager { // Context elements and the snapshot are attached after beforeSend (see below). const isFirstUserTurn = !this.displayMessages.some((message) => message.role === 'user') const pastes = options.pastes ?? [] + // Images ride only on GLOBAL turns, but the composer stays mounted across + // a mode switch, so chips attached in GLOBAL can arrive with a send in any + // mode. Refuse and restore rather than silently dropping attachments the + // user can see. This sits past the awaits above on purpose: the composer + // clears itself synchronously right after calling sendRequest, so an + // earlier restore would be wiped. Queued drafts are the caller's to + // restore (it re-queues on false). + if ((options.images?.length ?? 0) > 0 && this.mode !== AIMode.GLOBAL) { + sendUserToast('Switch back to the chat mode to send images. Your message was kept.', true) + if (!options.queued) { + this.aiChatInput?.restoreInstructions(this.instructions, pastes, options.images ?? []) + } + return false + } + // Non-GLOBAL sends with images were refused above. The vision check is + // repeated here, not just at attach time: the model can be switched to a + // text-only one after attaching, and sending the image then fails the turn. + const requestedImages = options.images ?? [] + const sendModel = tryGetCurrentModel() + const modelIsBlind = !!sendModel && !modelSupportsVision(sendModel.provider, sendModel.model) + if (requestedImages.length > 0 && modelIsBlind) { + // An image-only message has nothing left once the images are dropped — + // put them back in the composer instead of silently discarding them + // (the input already cleared itself optimistically on send). Queued + // drafts are the caller's to restore (it re-queues on false). + if (!this.instructions.trim()) { + sendUserToast(`${sendModel.model} can't read images. Switch to a vision model first.`, true) + if (!options.queued) this.restoreToInput('', requestedImages) + return false + } + sendUserToast( + `${sendModel.model} can't read images; sending without the ${requestedImages.length} attached image(s).`, + true + ) + } + const images = modelIsBlind ? [] : requestedImages const optimisticIndex = this.displayMessages.length this.loading = true // Create the abort controller before the (possibly slow) beforeSend pre-flight, @@ -2063,11 +2226,14 @@ export class AIChatManager { role: 'user', content: this.instructions, pastes: pastes.length > 0 ? pastes : undefined, + // Same objects as the API message's parts: sharing the exact data URL + // lets the history's blob store persist one copy for both. + images: images.length > 0 ? images : undefined, index: this.messages.length // matching with actual messages index. not -1 because it's not yet added to the messages array } ] // Undo the optimistic bubble + loading/label. Shared by the beforeSend-failure and - // pre-flight-cancel paths below; the input keeps the message text either way. + // pre-flight-cancel paths below; callers put the message back in the composer. const rollbackOptimisticSend = () => { this.displayMessages = this.displayMessages.filter((_, i) => i !== optimisticIndex) this.loading = false @@ -2080,9 +2246,13 @@ export class AIChatManager { // beforeSend commits the session's workspace before the first // message hits the backend. If it throws, sending anyway would // silently target the wrong workspace (typically the parent), so - // abort and tell the user — their message text stays in the input. + // abort and put the message back in the composer (which cleared + // itself optimistically on send). console.error('AIChatManager beforeSend hook failed', e) rollbackOptimisticSend() + if (!options.queued) { + this.aiChatInput?.restoreInstructions(this.instructions, pastes, images) + } sendUserToast( `Could not prepare the session before sending: ${ e instanceof Error ? e.message : String(e) @@ -2104,13 +2274,16 @@ export class AIChatManager { // message auto-sends it) or restore this prompt to the composer so it isn't lost. if (this.abortController.signal.aborted) { rollbackOptimisticSend() - if (this.wasCancelledByUser() && this.queuedMessage) { - const next = this.queuedMessage - this.queuedMessage = '' - const accepted = await this.sendRequest({ instructions: next }) - if (accepted === false) this.queuedMessage = next + if (this.wasCancelledByUser() && this.#hasQueuedMessage()) { + const next = this.#takeQueue() + const accepted = await this.sendRequest({ + instructions: next.text, + images: next.images, + queued: true + }) + if (accepted === false) this.#restoreQueue(next) } else { - this.aiChatInput?.restoreInstructions(this.instructions, pastes) + this.aiChatInput?.restoreInstructions(this.instructions, pastes, images) } return true } @@ -2183,6 +2356,7 @@ export class AIChatManager { // not the expanded LLM text, plus the rollback anchor after the user turn. const sentInstructions = this.instructions const sentPastes = pastes + const sentImages = images // The LLM gets the full pasted content; the display message above keeps // the compact tokens + registry so the bubble can render/expand chips. const oldInstructions = expanded(chatDraft(this.instructions, pastes)) @@ -2232,7 +2406,8 @@ export class AIChatManager { break case AIMode.GLOBAL: userMessage = prepareGlobalUserMessage(modelInstructions, oldSelectedContext, { - workspace: this.operatingWorkspace + workspace: this.operatingWorkspace, + images: sentImages }) break case AIMode.APP: @@ -2318,6 +2493,11 @@ export class AIChatManager { onMessageEnd: () => void } } = { + // The full history goes to the loop, image parts included, even on a + // known text-only model: runChatLoop strips them per iteration for + // whatever model that iteration runs on, so a mid-loop switch in either + // direction (vision→text or text→vision) sees the right view. A copy + // stripped here instead could never be un-stripped by a later iteration. messages: [...this.messages], abortController: this.abortController, callbacks: { @@ -2381,7 +2561,16 @@ export class AIChatManager { requestUserQuestion: this.requestUserQuestion, onItemModified: (kind, path) => this.recordModifiedItem(kind, path), onItemDeployed: (kind, from, to) => void this.renameModifiedItem(kind, from, to), - onItemDiscarded: (kind, path) => void this.removeModifiedItem(kind, path) + onItemDiscarded: (kind, path) => void this.removeModifiedItem(kind, path), + attachToolImage: (toolId, image) => { + const existing = this.pendingToolImages.get(toolId) ?? [] + this.pendingToolImages.set(toolId, [...existing, image]) + }, + takePendingToolImages: () => { + const images = [...this.pendingToolImages.values()].flat() + this.pendingToolImages.clear() + return images + } } } @@ -2438,13 +2627,14 @@ export class AIChatManager { // When the user cancelled with a message queued, that message is // about to auto-send (see the flush below) — drop the rolled-back // prompt instead of restoring it to the input so the handoff is clean. - const willAutoSendQueued = this.wasCancelledByUser() && !!this.queuedMessage + const willAutoSendQueued = this.wasCancelledByUser() && this.#hasQueuedMessage() this.restoreUnsentTurn( displayLenAfterUser, modelLenAfterUser, sentInstructions, sentPastes, - !willAutoSendQueued + !willAutoSendQueued, + sentImages ) if (this.displayMessages.length === 0) { // saveChat no-ops on an empty transcript; the chat persisted earlier @@ -2499,6 +2689,38 @@ export class AIChatManager { // re-committing would duplicate the turn's messages. if (!turnOutcomeHandled) { this.commitInterruptedTurn(collectedMessages, partialReply) + // The turn is kept as context, images and all — but a provider that just + // refused an image would refuse it again on every later turn, wedging the + // conversation with no way out but editing the message or starting over. + // Drop the parts so the text still gets an answer; the bubbles keep their + // thumbnails, so the user can still see what they sent. Gated on the + // history, not this turn's attachments: the refused image can also be a + // screenshot follow-up or an earlier turn's upload (an unlisted text-only + // model gets the full history). + // The failing request is the last iteration's — the loop strips image + // parts per iteration, so that request carried them only if ITS model + // passed the vision gate. The send-time flag is only the fallback for a + // failure before the first iteration read the model (a turn can start on + // a known text-only model and switch mid-loop to an unlisted blind one). + const failingModel = this.lastIterationModel + const requestCarriedImages = failingModel + ? modelSupportsVision(failingModel.provider, failingModel.model) + : !modelIsBlind + if ( + requestCarriedImages && + messagesHaveImageParts(this.messages) && + isImageRejection(err, [ + sendModel?.model, + tryGetCurrentModel()?.model, + failingModel?.model + ]) + ) { + this.messages = stripImagePartsFromMessages(this.messages) + sendUserToast( + `${tryGetCurrentModel()?.model ?? 'The model'} could not read the attached image(s), so they were removed from the conversation. Your message was kept.`, + true + ) + } // Any prior report no longer describes the history (a partial turn // was just committed); clear it so readers estimate instead. When // the failure WAS a context-length error, that high estimate forces @@ -2531,14 +2753,17 @@ export class AIChatManager { // empty-response rollback, or a programmatic cancel (panel teardown, // save-and-clear) leaves it in place as a card so it isn't fired into a // failed or torn-down turn. - if ((turnCommittedCleanly || this.wasCancelledByUser()) && this.queuedMessage) { - const next = this.queuedMessage - this.queuedMessage = '' - const accepted = await this.sendRequest({ instructions: next }) + if ((turnCommittedCleanly || this.wasCancelledByUser()) && this.#hasQueuedMessage()) { + const next = this.#takeQueue() + const accepted = await this.sendRequest({ + instructions: next.text, + images: next.images, + queued: true + }) if (accepted === false) { // The auto-send bailed before becoming a turn (e.g. beforeSend // failed); keep it as the queued message instead of losing it. - this.queuedMessage = next + this.#restoreQueue(next) } } // A background job may have finished mid-turn: its note missed this turn's @@ -2584,10 +2809,29 @@ export class AIChatManager { this.inlineAbortController?.abort(cancelReason) } + /** + * The images of a stored user turn as the model saw them. Anything resending + * a turn (retry, edit) must read them from here, never from the transcript + * bubble: a provider rejection strips them from history while the bubble + * keeps its copy so the user can still see what they sent — resending that + * copy would re-attach the image the provider just refused. + */ + storedImages(displayMessageIndex: number): AttachedImage[] | undefined { + const shown = this.displayMessages[displayMessageIndex] + if (!shown || shown.role !== 'user') return undefined + // The wire format has no filename; recover it from the bubble's entry + // (same attachment order) so a retried/edited image keeps its name — the + // history title of an image-only chat derives from it. + return imagesFromContent(this.messages[shown.index]?.content)?.map((image, i) => + shown.images?.[i]?.name ? { ...image, name: shown.images[i].name } : image + ) + } + restartGeneration = ( displayMessageIndex: number, newContent?: string, - pastes?: PasteAttachment[] + pastes?: PasteAttachment[], + images?: AttachedImage[] ) => { const userMessage = this.displayMessages[displayMessageIndex] @@ -2595,11 +2839,19 @@ export class AIChatManager { throw new Error('No user message found at the specified index') } + // Read while both arrays are intact: storedImages pairs the API message with + // its transcript entry, and the truncations below drop them. + const sentImages = this.storedImages(displayMessageIndex) + // Remove all messages including and after the specified user message this.displayMessages = this.displayMessages.slice(0, displayMessageIndex) - // Find corresponding message in actual messages and remove it and everything after it - let actualMessageIndex = this.messages.findIndex((_, i) => i === userMessage.index) + // Find corresponding message in actual messages and remove it and everything + // after it. A negative index marks a message whose API counterpart was + // removed by drop-oldest compaction — everything before it went too, so + // restarting from it restarts from an empty history. + let actualMessageIndex = + userMessage.index < 0 ? 0 : this.messages.findIndex((_, i) => i === userMessage.index) if (actualMessageIndex === -1) { throw new Error('No actual user message found to restart from') @@ -2615,7 +2867,10 @@ export class AIChatManager { // Resend the request with the same instructions this.instructions = newContent ?? userMessage.content - this.sendRequest({ pastes: pastes ?? userMessage.pastes }) + this.sendRequest({ + pastes: pastes ?? userMessage.pastes, + images: images ?? sentImages + }) } fix = () => { @@ -2648,7 +2903,7 @@ export class AIChatManager { this.cancel('saveAndClear') // Drop any message queued in this conversation so it can't auto-send into // the fresh chat or linger as a card across the switch. - this.queuedMessage = '' + this.#clearQueue() // The tray + poller belong to the conversation being left; the just-saved // chat keeps its persisted jobs (save() omits the arg → fallback preserves). this.clearBackgroundJobs() @@ -2675,11 +2930,11 @@ export class AIChatManager { } loadPastChat = async (id: string) => { - const chat = this.historyManager.loadPastChat(id) + const chat = await this.historyManager.loadPastChat(id) if (chat) { // Drop any message queued in the current conversation so it doesn't // auto-send into the loaded one or linger as a card across the switch. - this.queuedMessage = '' + this.#clearQueue() // Stop the poller for the conversation being left before swapping in the // loaded chat's jobs below. this.clearBackgroundJobs() diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts index 3ca57f948d..f4379b6e90 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts @@ -3,6 +3,7 @@ import type { FlowAIChatHelpers } from './flow/core' import type { CurrentEditor } from '$lib/components/flows/types' import type { ReviewChangesOpts } from './monaco-adapter' import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions.mjs' +import type { AttachedImage } from './imageUtils' import { AIChatManager, AIMode, AIAutonomyMode } from './AIChatManager.svelte' import { runChatLoop } from './chatLoop' @@ -493,7 +494,611 @@ describe('AIChatManager queued messages', () => { manager.dequeueMessage() expect(manager.queuedMessage).toBe('') - expect(input.prependText).toHaveBeenCalledWith('line one\nline two') + expect(input.prependText).toHaveBeenCalledWith('line one\nline two', []) + }) + + const img = (n: string): AttachedImage => ({ + dataUrl: `data:image/png;base64,${n}`, + mediaType: 'image/png', + name: n + }) + + it('carries queued images through to the auto-send', async () => { + replyWith('done') + const manager = createManager(createInputMock()) + manager.mode = AIMode.GLOBAL // sendRequest only assembles images in GLOBAL + manager.queueMessage('look at this', [img('a')]) + + await manager.sendRequest({ instructions: 'first' }) + + // The auto-sent turn must carry the whole submitted message — queueing must + // not send the text alone and drop its images. + expect(mocks.runChatLoop).toHaveBeenCalledTimes(2) + const autoSent = manager.displayMessages.find( + (m) => m.role === 'user' && m.content === 'look at this' + ) + expect(autoSent && 'images' in autoSent ? autoSent.images : undefined).toEqual([img('a')]) + expect(manager.queuedImages).toEqual([]) + }) + + // Attaching is refused on a text-only model, but the model can be switched + // after attaching (or a screenshot buffered), and sending the image then fails + // the whole turn. The send path re-checks rather than trusting the attach gate. + it('drops images when the model in use cannot read them', async () => { + replyWith('done') + const manager = createManager(createInputMock()) + manager.mode = AIMode.GLOBAL + // a real bundled default, so this exercises the actual gate rather than a mock + mocks.tryGetCurrentModel.mockReturnValue({ provider: 'groq', model: 'llama-3.3-70b-versatile' }) + + await manager.sendRequest({ instructions: 'look', images: [img('a')] }) + + const bubble = manager.displayMessages.find((m) => m.role === 'user') + expect(bubble && 'images' in bubble ? bubble.images : undefined).toBeUndefined() + expect(mocks.sendUserToast).toHaveBeenCalledWith( + expect.stringContaining("can't read images"), + true + ) + mocks.tryGetCurrentModel.mockReturnValue(model) + }) + + // displayMessages hold a 384px transcript copy; retrying must resend the + // model's own 1568px image, not a thumbnail of its previous input. + it('resends the model-resolution image on retry, not the transcript thumbnail', async () => { + replyWith('done') + const manager = createManager(createInputMock()) + manager.mode = AIMode.GLOBAL + manager.messages = [ + { + role: 'user', + content: [ + { type: 'text', text: 'look' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,FULLRES' } } + ] as any + }, + { role: 'assistant', content: 'bad answer' } + ] + manager.displayMessages = [ + { + role: 'user', + content: 'look', + index: 0, + images: [{ dataUrl: 'data:image/png;base64,THUMB', mediaType: 'image/png' }] + }, + { role: 'assistant', content: 'bad answer' } + ] + + manager.restartGeneration(0) + await vi.waitFor(() => expect(mocks.runChatLoop).toHaveBeenCalled()) + + const resent = mocks.runChatLoop.mock.calls[0][0].messages.at(-1) + const urls = (resent.content as any[]) + .filter((p) => p.type === 'image_url') + .map((p) => p.image_url.url) + expect(urls).toEqual(['data:image/png;base64,FULLRES']) + }) + + // The loop, not the send, owns the vision strip: it re-applies it per iteration + // for whatever model that iteration runs on, so a mid-loop switch in either + // direction sees the right view. A copy stripped at send time could never be + // un-stripped when the user switches text-only → vision during the turn. + it('passes the full history to the loop even on a text-only model', async () => { + replyWith('done') + const manager = createManager(createInputMock()) + manager.mode = AIMode.GLOBAL + manager.messages = [ + { + role: 'user', + content: [ + { type: 'text', text: 'earlier turn' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,OLD' } } + ] as any + }, + { role: 'assistant', content: 'ok' } + ] + mocks.tryGetCurrentModel.mockReturnValue({ provider: 'groq', model: 'llama-3.3-70b-versatile' }) + + await manager.sendRequest({ instructions: 'plain text follow-up' }) + + const sent = mocks.runChatLoop.mock.calls[0][0].messages + const anyImage = sent.some( + (m: any) => Array.isArray(m.content) && m.content.some((p: any) => p.type === 'image_url') + ) + expect(anyImage).toBe(true) + mocks.tryGetCurrentModel.mockReturnValue(model) + }) + + // Empty instructions are a valid image-only send; they must override, not + // keep, text a failed or cancelled earlier turn left in this.instructions. + it('does not attach stale instructions to an image-only send', async () => { + replyWith('done') + const manager = createManager(createInputMock()) + manager.mode = AIMode.GLOBAL + manager.instructions = 'text from a failed earlier turn' + + await manager.sendRequest({ instructions: '', images: [img('a')] }) + + const sent = mocks.runChatLoop.mock.calls[0][0].messages.at(-1) + const text = Array.isArray(sent.content) + ? sent.content + .filter((p: any) => p.type === 'text') + .map((p: any) => p.text) + .join('\n') + : sent.content + expect(text).not.toContain('text from a failed earlier turn') + }) + + // The failing request may have used the model selected at send time, not the + // currently selected one — a mid-flight switch must not stop its id being + // excluded from the rejection match. + it('does not strip images when the error echoes the send-time model after a mid-flight switch', async () => { + const manager = createManager(createInputMock()) + manager.mode = AIMode.GLOBAL + mocks.tryGetCurrentModel.mockReturnValue({ + provider: 'openrouter', + model: 'meta-llama/llama-3.2-90b-vision-instruct' + }) + mocks.runChatLoop.mockImplementation(async () => { + // the user switches models while the request is in flight... + mocks.tryGetCurrentModel.mockReturnValue({ provider: 'openai', model: 'gpt-4o' }) + // ...and the in-flight model fails with an unrelated error echoing its id + throw new Error('429 Rate limit reached for meta-llama/llama-3.2-90b-vision-instruct') + }) + + await manager.sendRequest({ instructions: 'look at this', images: [img('a')] }) + + const stillThere = manager.messages.some( + (m: any) => Array.isArray(m.content) && m.content.some((p: any) => p.type === 'image_url') + ) + expect(stillThere).toBe(true) + mocks.tryGetCurrentModel.mockReturnValue(model) + }) + + // Queuing clears the composer, so its own counter resets; the cap has to hold + // on the queue or repeated sends stack an unbounded batch into one message. + it('caps images accumulated across repeated queued sends', () => { + const manager = createManager() + for (let i = 0; i < 4; i++) { + manager.queueMessage(`msg ${i}`, [img(`a${i}`), img(`b${i}`), img(`c${i}`)]) + } + expect(manager.queuedImages.length).toBe(8) + }) + + // A rejected image stays in history, so every later turn resends it and fails + // the same way — the conversation wedges with no way out but editing or /clear. + it('removes the image from history when the provider rejects it', async () => { + const manager = createManager(createInputMock()) + manager.mode = AIMode.GLOBAL + mocks.runChatLoop.mockImplementation(async () => { + throw new Error('400 Invalid image content') + }) + + await manager.sendRequest({ instructions: 'look at this', images: [img('a')] }) + + const stillThere = manager.messages.some( + (m: any) => Array.isArray(m.content) && m.content.some((p: any) => p.type === 'image_url') + ) + expect(stillThere).toBe(false) + // the prompt itself survives, so a follow-up still has the text as context + expect(manager.messages.length).toBeGreaterThan(0) + expect(mocks.sendUserToast).toHaveBeenCalledWith( + expect.stringContaining('could not read the attached image'), + true + ) + }) + + // An unrelated failure must not strip a perfectly good image. + it('keeps the image when the failure is unrelated', async () => { + const manager = createManager(createInputMock()) + manager.mode = AIMode.GLOBAL + mocks.runChatLoop.mockImplementation(async () => { + throw new Error('429 rate limit exceeded') + }) + + await manager.sendRequest({ instructions: 'look at this', images: [img('a')] }) + + const stillThere = manager.messages.some( + (m: any) => Array.isArray(m.content) && m.content.some((p: any) => p.type === 'image_url') + ) + expect(stillThere).toBe(true) + }) + + // Vision model ids often contain the rejection subject words themselves, and + // providers echo the id in unrelated errors. A rate limit must not read as an + // image rejection just because the model is called "...-vision-instruct" — + // the strip it would trigger is permanent (retry refuses the transcript copy). + it('keeps the image when a transient error merely echoes a vision model id', async () => { + const manager = createManager(createInputMock()) + manager.mode = AIMode.GLOBAL + mocks.tryGetCurrentModel.mockReturnValue({ + provider: 'openrouter', + model: 'meta-llama/llama-3.2-90b-vision-instruct' + }) + mocks.runChatLoop.mockImplementation(async () => { + throw new Error('429 Rate limit reached for model meta-llama/llama-3.2-90b-vision-instruct') + }) + + await manager.sendRequest({ instructions: 'look at this', images: [img('a')] }) + + const stillThere = manager.messages.some( + (m: any) => Array.isArray(m.content) && m.content.some((p: any) => p.type === 'image_url') + ) + expect(stillThere).toBe(true) + }) + + // The refused image is not always this turn's attachment: an unlisted + // text-only model receives the full history, so a screenshot follow-up or an + // earlier upload can be the part it chokes on. Without the strip, every later + // send resubmits it and fails identically — a wedge with no self-correction. + it('removes historical images from history when the provider rejects them on a text turn', async () => { + const manager = createManager(createInputMock()) + manager.mode = AIMode.GLOBAL + // e.g. a take_screenshot follow-up from an earlier turn + manager.messages = [ + { + role: 'user', + content: [ + { type: 'text', text: 'Screenshot of the app preview:' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,SHOT' } } + ] as any + }, + { role: 'assistant', content: 'looks good' } + ] + mocks.runChatLoop.mockImplementation(async () => { + throw new Error('400 this model does not support image input') + }) + + await manager.sendRequest({ instructions: 'plain text follow-up' }) + + const stillThere = manager.messages.some( + (m: any) => Array.isArray(m.content) && m.content.some((p: any) => p.type === 'image_url') + ) + expect(stillThere).toBe(false) + expect(mocks.sendUserToast).toHaveBeenCalledWith( + expect.stringContaining('could not read the attached image'), + true + ) + }) + + // The rejection fallback strips the image from history but leaves the bubble's + // thumbnail. Retry must not resurrect it, or the retried turn fails identically + // and the conversation is wedged after all. + it('does not resend an image the fallback already stripped', async () => { + replyWith('done') + const manager = createManager(createInputMock()) + manager.mode = AIMode.GLOBAL + // post-rejection shape: history stripped to text, transcript still shows it + manager.messages = [{ role: 'user', content: 'look at this\n[image omitted]' }] + manager.displayMessages = [ + { role: 'user', content: 'look at this', index: 0, images: [img('thumb')] } + ] + + manager.restartGeneration(0) + await vi.waitFor(() => expect(mocks.runChatLoop).toHaveBeenCalled()) + + const resent = mocks.runChatLoop.mock.calls[0][0].messages.at(-1) + const hasImage = + Array.isArray(resent.content) && resent.content.some((p: any) => p.type === 'image_url') + expect(hasImage).toBe(false) + }) + + // The wire format has no filename; a retried/edited image must recover it + // from the bubble's entry — an unnamed resend would downgrade an image-only + // chat's filename-derived history title to the generic fallback. + it('storedImages recovers attachment names from the transcript bubble', () => { + const manager = createManager() + manager.messages = [ + { + role: 'user', + content: [ + { type: 'text', text: 'look' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,FULL' } } + ] as any + } + ] + manager.displayMessages = [ + { + role: 'user', + content: 'look', + index: 0, + images: [ + { dataUrl: 'data:image/png;base64,FULL', mediaType: 'image/png', name: 'mockup.png' } + ] + } as any + ] + + expect(manager.storedImages(0)).toEqual([ + { dataUrl: 'data:image/png;base64,FULL', mediaType: 'image/png', name: 'mockup.png' } + ]) + }) + + // Drop-oldest removes the API counterpart but the transcript keeps the bubble. + // Its restart index must not alias to a surviving message, or retrying/editing + // the dropped prompt would silently attach that other turn's images. + it("does not serve another turn's images for a message dropped by drop-oldest compaction", () => { + const manager = createManager() + manager.messages = [ + { role: 'user', content: 'old prompt' }, + { role: 'assistant', content: 'old answer' }, + { + role: 'user', + content: [ + { type: 'text', text: 'new prompt' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,NEW' } } + ] as any + }, + { role: 'assistant', content: 'new answer' } + ] + manager.displayMessages = [ + { role: 'user', content: 'old prompt', index: 0 }, + { role: 'assistant', content: 'old answer' }, + { role: 'user', content: 'new prompt', index: 2, images: [img('thumb')] }, + { role: 'assistant', content: 'new answer' } + ] + + // frees the first turn (user + assistant), keeps the image-bearing one + manager.compactOldestMessages(1) + + expect(manager.messages.length).toBe(2) + // the dropped message resolves no images... + expect(manager.storedImages(0)).toBeUndefined() + // ...while the surviving one still resolves its own + expect(manager.storedImages(2)?.[0]?.dataUrl).toBe('data:image/png;base64,NEW') + }) + + // Enter with an image but no text must send, not silently discard the image + // (the input clears itself optimistically, so a bail on empty text loses it). + it('sends an image-only message in GLOBAL mode', async () => { + replyWith('done') + const manager = createManager(createInputMock()) + manager.mode = AIMode.GLOBAL + + await manager.sendRequest({ instructions: '', images: [img('a')] }) + + expect(mocks.runChatLoop).toHaveBeenCalled() + const sent = mocks.runChatLoop.mock.calls[0][0].messages.at(-1) + const hasImage = + Array.isArray(sent.content) && sent.content.some((p: any) => p.type === 'image_url') + expect(hasImage).toBe(true) + }) + + it('still ignores a send with no text and no images', async () => { + replyWith('done') + const manager = createManager(createInputMock()) + manager.mode = AIMode.GLOBAL + + await manager.sendRequest({ instructions: '' }) + + expect(mocks.runChatLoop).not.toHaveBeenCalled() + }) + + // With no text, dropping the images leaves nothing to send — they must go back + // to the composer (which already cleared itself optimistically), not vanish. + it('restores the images when an image-only send meets a text-only model', async () => { + replyWith('done') + const input = createInputMock() + const manager = createManager(input) + manager.mode = AIMode.GLOBAL + mocks.tryGetCurrentModel.mockReturnValue({ + provider: 'groq', + model: 'llama-3.3-70b-versatile' + }) + + await manager.sendRequest({ instructions: '', images: [img('a')] }) + + expect(mocks.runChatLoop).not.toHaveBeenCalled() + expect(input.prependText).toHaveBeenCalledWith('', [img('a')]) + mocks.tryGetCurrentModel.mockReturnValue(model) + }) + + // A refused queued draft is the caller's to restore (it re-queues on false) — + // a composer restore on top would leave the same attachment in both places. + it('does not double-restore a queued image-only draft refused by a text-only model', async () => { + const input = createInputMock() + const manager = createManager(input) + manager.mode = AIMode.GLOBAL + mocks.tryGetCurrentModel.mockReturnValue({ + provider: 'groq', + model: 'llama-3.3-70b-versatile' + }) + + const accepted = await manager.sendRequest({ + instructions: '', + images: [img('a')], + queued: true + }) + + expect(accepted).toBe(false) + expect(mocks.runChatLoop).not.toHaveBeenCalled() + expect(input.prependText).not.toHaveBeenCalled() + expect(input.restoreInstructions).not.toHaveBeenCalled() + mocks.tryGetCurrentModel.mockReturnValue(model) + }) + + // A→B→C: the loop can run an iteration on a model that is neither the + // send-time one (A) nor the currently-selected one (C) by the time the + // failure is classified. The failing iteration's id (B) must be excluded + // from the rejection match too. + it('does not strip images when the error echoes an intermediate model (A→B→C)', async () => { + const manager = createManager(createInputMock()) + manager.mode = AIMode.GLOBAL + const a = { provider: 'openai', model: 'gpt-4o' } + const b = { provider: 'openrouter', model: 'meta-llama/llama-3.2-90b-vision-instruct' } + const c = { provider: 'anthropic', model: 'claude-sonnet-4-6' } + mocks.getCurrentModel.mockReturnValue(a) + mocks.tryGetCurrentModel.mockReturnValue(a) + mocks.runChatLoop.mockImplementation(async (config: any) => { + // an iteration starts on B... + await config.onBeforeIteration?.([], config.helpers, b) + // ...the user switches to C while B's request is in flight... + mocks.getCurrentModel.mockReturnValue(c) + mocks.tryGetCurrentModel.mockReturnValue(c) + // ...and B fails with an unrelated error echoing its id + throw new Error('429 Rate limit reached for meta-llama/llama-3.2-90b-vision-instruct') + }) + + await manager.sendRequest({ instructions: 'look at this', images: [img('a')] }) + + const stillThere = manager.messages.some( + (m: any) => Array.isArray(m.content) && m.content.some((p: any) => p.type === 'image_url') + ) + expect(stillThere).toBe(true) + mocks.getCurrentModel.mockReturnValue(model) + mocks.tryGetCurrentModel.mockReturnValue(model) + }) + + // The Responses converter sends images as input_image parts, and '_' is a + // word character — the whole-word regex must still catch that spelling. + it('recovers when the provider rejects the input_image content part', async () => { + const manager = createManager(createInputMock()) + manager.mode = AIMode.GLOBAL + mocks.runChatLoop.mockRejectedValue( + new Error("400 Invalid value: content part type 'input_image' is not supported") + ) + + await manager.sendRequest({ instructions: 'look at this', images: [img('a')] }) + + const stillThere = manager.messages.some( + (m: any) => Array.isArray(m.content) && m.content.some((p: any) => p.type === 'image_url') + ) + expect(stillThere).toBe(false) + }) + + // The composer stays mounted across a mode switch, so chips attached in + // GLOBAL can ride a send in any mode — they must be restored, not dropped. + it('refuses and restores an image-bearing send outside GLOBAL mode', async () => { + const input = createInputMock() + const manager = createManager(input) + manager.mode = AIMode.NAVIGATOR + + const pending = manager.sendRequest({ instructions: 'find it', images: [img('a')] }) + // The composer clears itself synchronously right after calling sendRequest: + // a restore issued before that point would be wiped by the clear. + expect(input.restoreInstructions).not.toHaveBeenCalled() + const accepted = await pending + + expect(accepted).toBe(false) + expect(mocks.runChatLoop).not.toHaveBeenCalled() + expect(input.restoreInstructions).toHaveBeenCalledWith('find it', [], [img('a')]) + }) + + // A refused queued draft is the caller's to restore (it re-queues on false) — + // a composer restore on top would duplicate it. + it('does not double-restore a queued image draft refused outside GLOBAL mode', async () => { + const input = createInputMock() + const manager = createManager(input) + manager.mode = AIMode.NAVIGATOR + + const accepted = await manager.sendRequest({ + instructions: 'queued one', + images: [img('a')], + queued: true + }) + + expect(accepted).toBe(false) + expect(input.restoreInstructions).not.toHaveBeenCalled() + expect(input.prependText).not.toHaveBeenCalled() + }) + + // "provisioning"/"provisioned" contain the word "vision" — a transient + // capacity error must not be classified as an image rejection. + it('does not strip images on a provisioning error', async () => { + const manager = createManager(createInputMock()) + manager.mode = AIMode.GLOBAL + mocks.runChatLoop.mockRejectedValue( + new Error('503 model provisioning failed, please retry later') + ) + + await manager.sendRequest({ instructions: 'look at this', images: [img('a')] }) + + const stillThere = manager.messages.some( + (m: any) => Array.isArray(m.content) && m.content.some((p: any) => p.type === 'image_url') + ) + expect(stillThere).toBe(true) + }) + + // A turn can start on a known text-only model (send-time flag says "no images + // go out") and switch mid-loop to an UNLISTED blind model whose iteration does + // carry the history's images. When that model rejects them, recovery must fire + // — the send-time flag alone would skip it and wedge every later send. + it('recovers when a turn starts text-only but an unlisted blind model rejects mid-loop', async () => { + const manager = createManager(createInputMock()) + manager.mode = AIMode.GLOBAL + manager.messages = [ + { + role: 'user', + content: [ + { type: 'text', text: 'earlier' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,OLD' } } + ] as any + }, + { role: 'assistant', content: 'ok' } + ] + const knownBlind = { provider: 'groq', model: 'llama-3.3-70b-versatile' } + const unlistedBlind = { provider: 'customai', model: 'my-internal-llm' } + mocks.getCurrentModel.mockReturnValue(knownBlind) + mocks.tryGetCurrentModel.mockReturnValue(knownBlind) + mocks.runChatLoop.mockImplementation(async (config: any) => { + // mid-loop switch to a model the deny-list doesn't know... + mocks.getCurrentModel.mockReturnValue(unlistedBlind) + mocks.tryGetCurrentModel.mockReturnValue(unlistedBlind) + await config.onBeforeIteration?.([], config.helpers, unlistedBlind) + // ...its request carries the images and the provider rejects them + throw new Error('400 this model does not support image input') + }) + + await manager.sendRequest({ instructions: 'plain follow-up' }) + + const stillThere = manager.messages.some( + (m: any) => Array.isArray(m.content) && m.content.some((p: any) => p.type === 'image_url') + ) + expect(stillThere).toBe(false) + mocks.getCurrentModel.mockReturnValue(model) + mocks.tryGetCurrentModel.mockReturnValue(model) + }) + + // Images evicted from requests by the byte bound must not keep their full + // data URLs in stored history: provider-reported usage excludes them, so + // compaction would never prune them and every save re-clones the payload. + // The bubble and the API message must share the exact same data URL — the + // history's blob store dedups them to a single record on save, so a + // transcript-side copy (e.g. a downscale) would double the stored bytes. + it('sends and displays the same image copy', async () => { + replyWith('done') + const manager = createManager(createInputMock()) + manager.mode = AIMode.GLOBAL + + await manager.sendRequest({ instructions: 'look', images: [img('a')] }) + + const bubble = manager.displayMessages.find((m) => m.role === 'user') as any + const sent = mocks.runChatLoop.mock.calls[0][0].messages.at(-1) + const sentUrl = sent.content.find((p: any) => p.type === 'image_url').image_url.url + expect(bubble.images[0].dataUrl).toBe(sentUrl) + }) + + it('queues an image-only message and restores it on dequeue', () => { + const input = createInputMock() + const manager = createManager(input) + manager.queueMessage('', [img('a')]) + + expect(manager.queuedMessage).toBe('') + expect(manager.queuedImages).toEqual([img('a')]) + + manager.dequeueMessage() + + expect(manager.queuedImages).toEqual([]) + expect(input.prependText).toHaveBeenCalledWith('', [img('a')]) + }) + + it('drops queued images when the conversation is switched away', async () => { + const manager = createManager(createInputMock()) + manager.queueMessage('stale', [img('a')]) + + await manager.saveAndClear() + + // images must not survive into the next conversation + expect(manager.queuedMessage).toBe('') + expect(manager.queuedImages).toEqual([]) }) it('re-queues instead of dropping when the input is unmounted', () => { @@ -610,6 +1215,23 @@ describe('AIChatManager queued messages', () => { expect(input.prependText).not.toHaveBeenCalled() }) + // The composer clears itself optimistically on send, so a preflight failure + // must put the whole draft back — images can't just be re-dropped from memory. + it('restores text and images to the composer when beforeSend rejects a direct send', async () => { + const input = createInputMock() + const manager = createManager(input) + manager.mode = AIMode.GLOBAL + manager.beforeSend = vi.fn().mockRejectedValue(new Error('workspace fork failed')) + + const accepted = await manager.sendRequest({ instructions: 'look', images: [img('a')] }) + + expect(accepted).toBe(false) + expect(mocks.runChatLoop).not.toHaveBeenCalled() + expect(input.restoreInstructions).toHaveBeenCalledWith('look', [], [img('a')]) + // the optimistic bubble is rolled back + expect(manager.displayMessages).toHaveLength(0) + }) + it('drops the queued message when switching conversations (no cross-chat leak)', async () => { const manager = createManager(createInputMock()) @@ -904,7 +1526,7 @@ describe('AIChatManager context compaction', () => { expect(manager.messages.map((m) => m.role)).toEqual(['user', 'user']) }) - it('re-bases display message indices and clamps fully-compacted ones to 0', () => { + it('re-bases display message indices, marking fully-compacted ones negative', () => { const manager = new AIChatManager() manager.messages = [ { role: 'user', content: 'a'.repeat(400) }, // ~100 estimated tokens @@ -920,8 +1542,11 @@ describe('AIChatManager context compaction', () => { ] manager.compactOldestMessages(150) expect(manager.messages.map((m) => m.content)).toEqual(['c', 'd']) + // A dropped message's index goes negative rather than clamping to 0: + // 0 would alias it to the first surviving message, and storedImages + // would serve that message's images to a retry of this one. expect(manager.displayMessages.map((m) => ('index' in m ? m.index : undefined))).toEqual([ - 0, + -2, undefined, 0, 1 @@ -1031,6 +1656,59 @@ describe('AIChatManager context compaction', () => { expect(manager.contextUsage).toBeUndefined() }) + // A take_screenshot follow-up is a `user` message with no display counterpart + // (appendPendingToolImages injects it). It must never become the tail + // boundary: `messages` and `displayMessages` would then be sliced at + // different turns and the cards in between would vanish from the transcript + // while the model still sees them. + it('never lands the tail boundary on a screenshot follow-up that has no display counterpart', async () => { + mocks.getCurrentModel.mockReturnValue(gpt4oModel) + mocks.tryGetCurrentModel.mockReturnValue(gpt4oModel) + mocks.getNonStreamingCompletion.mockResolvedValue('SUMMARY TEXT') + const manager = new AIChatManager() + manager.messages = [ + { role: 'user', content: 'OLD1' + 'a'.repeat(100_000) }, + { role: 'assistant', content: 'OLD2' + 'b'.repeat(100_000) }, + { role: 'user', content: 'OLD3' + 'c'.repeat(100_000) }, + { role: 'assistant', content: 'toolTurn', tool_calls: [] as any }, + { role: 'tool', content: 'Screenshot captured', tool_call_id: 't1' } as any, + // the synthetic follow-up: user role, image parts, NO display entry + { + role: 'user', + content: [ + { type: 'text', text: 'Screenshot(s) of the app preview:' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,AAAA' } } + ] as any + }, + // sized so the tail budget breaks just above OLD3: the backward walk + // stops at index 3, and the forward snap then lands on the synthetic + // user message at 5 — the case this test exists for. + { role: 'assistant', content: 'afterShot' + 'g'.repeat(120_000) }, + { role: 'user', content: 'recentQ' + 'h'.repeat(120_000) } + ] + manager.displayMessages = [ + { role: 'user', content: 'old1', index: 0 }, + { role: 'assistant', content: 'old2' }, + { role: 'user', content: 'old3', index: 2 }, + { role: 'assistant', content: 'afterShot' }, + { role: 'user', content: 'recentQ', index: 7 } + ] + manager.contextUsage = 110_000 // over the 0.8 * 128k trigger + manager.instructions = 'next question' + + await manager.sendRequest() + + // Whatever survived summarization, the two views must agree: any assistant + // turn the model can still see must still be visible to the user. + const keptAfterShot = manager.messages.some( + (m) => typeof m.content === 'string' && m.content.includes('afterShot') + ) + const shownAfterShot = manager.displayMessages.some( + (m) => m.role === 'assistant' && m.content.includes('afterShot') + ) + expect(shownAfterShot).toBe(keptAfterShot) + }) + it('falls back to drop-oldest when summarization fails', async () => { mocks.getCurrentModel.mockReturnValue(gpt4oModel) mocks.tryGetCurrentModel.mockReturnValue(gpt4oModel) @@ -1382,7 +2060,7 @@ describe('AIChatManager sendRequest lifecycle', () => { expect(manager.displayMessages.some((m) => m.role === 'user')).toBe(false) expect(manager.messages.some((m) => m.role === 'user')).toBe(false) // ...and its text is handed back to the composer. - expect(restoreInstructions).toHaveBeenCalledWith('do a thing', []) + expect(restoreInstructions).toHaveBeenCalledWith('do a thing', [], []) expect(manager.loading).toBe(false) }) @@ -1411,7 +2089,7 @@ describe('AIChatManager sendRequest lifecycle', () => { expect(manager.displayMessages).toHaveLength(0) expect(manager.messages.some((m) => m.role === 'user')).toBe(false) - expect(restoreInstructions).toHaveBeenCalledWith('do a thing', []) + expect(restoreInstructions).toHaveBeenCalledWith('do a thing', [], []) expect(manager.loading).toBe(false) }) @@ -1513,7 +2191,7 @@ describe('AIChatManager sendRequest lifecycle', () => { expect(manager.messages.some((m) => m.role === 'assistant')).toBe(false) expect(manager.displayMessages.some((m) => m.role === 'assistant')).toBe(false) expect(manager.displayMessages.some((m) => m.role === 'user')).toBe(false) - expect(restoreInstructions).toHaveBeenCalledWith('think hard', []) + expect(restoreInstructions).toHaveBeenCalledWith('think hard', [], []) expect(manager.loading).toBe(false) }) diff --git a/frontend/src/lib/components/copilot/chat/AIChatMessage.svelte b/frontend/src/lib/components/copilot/chat/AIChatMessage.svelte index 4e2df28a50..57be8d9628 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatMessage.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatMessage.svelte @@ -12,6 +12,7 @@ import CompactionBoundary from './CompactionBoundary.svelte' import { messageDraft, segments } from './chatDraft' import { lineCountLabel } from './pasteTokens' + import ExpandableImage from '$lib/components/common/image/ExpandableImage.svelte' const aiChatManager = getAiChatManager() @@ -81,6 +82,7 @@ bind:selectedContext initialInstructions={message.content} initialPastes={message.pastes} + initialImages={aiChatManager.storedImages(messageIndex)} {editingMessageIndex} onClickOutside={() => (editingMessageIndex = null)} onKeyDown={(e) => { @@ -100,23 +102,37 @@ >
{:else} -
- {#each segments(messageDraft(message)) as seg}{#if seg.type === 'text'}{seg.value}{:else if expandedPastes.has(seg.att.id)}{seg.att.content}{:else}{/if}{/each} -
+ {#if message.role === 'user' && message.images && message.images.length > 0} +
+ {#each message.images as image, i (i)} + + {/each} +
+ {/if} + {#if message.content.trim() !== '' || !(message.images && message.images.length > 0)} +
+ {#each segments(messageDraft(message)) as seg}{#if seg.type === 'text'}{seg.value}{:else if expandedPastes.has(seg.att.id)}{seg.att.content}{:else}{/if}{/each} +
+ {/if} {/if}
{/if} diff --git a/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte b/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte index c6f3268e50..777331e808 100644 --- a/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte +++ b/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte @@ -35,6 +35,8 @@ * from the textarea. The host should drop the matching entry from * selectedContext (only items with `deletable !== false` are reported). */ onRemoveContext?: (contextElement: ContextElement) => void + /** Called with image files found in a paste, so the host can attach them. */ + onImageFiles?: (files: File[]) => void className?: string onKeyDown?: (e: KeyboardEvent) => void } @@ -49,6 +51,7 @@ onSendRequest, onAddContext, onRemoveContext, + onImageFiles, className = '', onKeyDown = undefined }: Props = $props() @@ -300,6 +303,22 @@ // widened over overlapped tokens so pasting onto a chip replaces it whole. function handlePaste(e: ClipboardEvent) { const text = e.clipboardData?.getData('text/plain') ?? '' + // Image paste (screenshots, copied images) → hand off to the host to attach. + // Only when the clipboard carries no text: spreadsheet and browser copies put a + // bitmap alongside the text, and pasting a cell range must paste the cells, not + // a picture of them. An OS screenshot carries the image alone, so it still lands + // here. `onImageFiles` is unset outside GLOBAL, where attaching is unsupported — + // the paste must then fall through to text rather than be swallowed. + if (!text.trim() && onImageFiles) { + const imageFiles = Array.from(e.clipboardData?.files ?? []).filter((f) => + f.type.startsWith('image/') + ) + if (imageFiles.length > 0) { + e.preventDefault() + onImageFiles(imageFiles) + return + } + } if (!text || !shouldCollapsePaste(text)) return e.preventDefault() const ta = e.currentTarget as HTMLTextAreaElement diff --git a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts index d24246b711..beb3424998 100644 --- a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts @@ -3,14 +3,23 @@ import type { ChatJob, DisplayMessage } from './shared' import { expanded, messageDraft } from './chatDraft' import { createLongHash } from '$lib/editorLangUtils' import { userScopedDb, type UserScopedDbMigrateDeps } from '$lib/userScopedDb' +import { scopedKey } from '$lib/userScopedStorage' import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs' import type { PersistedContextUsage } from './tokenUsage' +import { IMAGE_OMITTED_PLACEHOLDER, type AttachedImage } from './imageUtils' +import { randomUUID } from '$lib/utils/uuid' // Base IndexedDB name; userScopedDb namespaces the effective DB by the logged-in // user's email so chat messages are never physically shared across users on a // shared browser. The bare name is also the legacy (pre-namespacing) DB, claimed // once on first login. const DB_NAME = 'copilot-chat-history' +// v3 adds the images blob store (replacing v2's short-lived toolImages store). +const DB_VERSION = 3 +/** Newest image blobs kept per chat; each is a bounded (≤1568px) data URL. */ +const MAX_IMAGES_PER_CHAT = 30 +/** Marks a persisted image whose bytes live in the `images` store. */ +const IMAGE_REF_PREFIX = 'wm-image:' interface ChatSchema extends IDBSchema { chats: { @@ -37,12 +46,46 @@ interface ChatSchema extends IDBSchema { backgroundJobs?: ChatJob[] } } + // Image bytes, out-of-band from the chat record on purpose: the record is + // re-cloned into IndexedDB on every saveChat, while a blob is written once + // and read again only when its chat is reloaded. The persisted message + // arrays carry `wm-image:` refs in place of the data URLs; swapping + // happens entirely inside this class (dehydrate on save, hydrate on load), + // so live chat state never sees a ref. + images: { + key: string + value: { + id: string + chatId: string + dataUrl: string + savedAt: number + } + indexes: { 'by-chat': [string, number] } + } } function createChatStore(db: IDBPDatabase): void { if (!db.objectStoreNames.contains('chats')) { db.createObjectStore('chats', { keyPath: 'id' }) } + // v2 briefly kept full-resolution tool screenshots in their own store; the + // general blob store below covers them now. + if ((db.objectStoreNames as DOMStringList).contains('toolImages')) { + db.deleteObjectStore('toolImages' as never) + } + if (!db.objectStoreNames.contains('images')) { + const store = db.createObjectStore('images', { keyPath: 'id' }) + store.createIndex('by-chat', ['chatId', 'savedAt']) + } +} + +/** All image-blob primary keys owned by a chat (via the [chatId, savedAt] index). */ +function imageKeysForChat(db: IDBPDatabase, chatId: string) { + return db.getAllKeysFromIndex( + 'images', + 'by-chat', + IDBKeyRange.bound([chatId, -Infinity], [chatId, Infinity]) + ) } // Shared across all HistoryManager instances. Each instance owns its own @@ -97,7 +140,7 @@ export function __resetLegacyChatClaimForTesting(): void { // the `get` is O(1) on the `id` keyPath. export async function readChatModifiedItems(chatId: string): Promise { const dbh = userScopedDb(DB_NAME, { - version: 1, + version: DB_VERSION, upgrade: createChatStore, migrate: migrateLegacyChatDb }) @@ -118,7 +161,7 @@ export default class HistoryManager { // HistoryManager per AIChatManager (the singleton + one per session runtime), // so the handle must be per-instance — not a module singleton. private dbh = userScopedDb(DB_NAME, { - version: 1, + version: DB_VERSION, upgrade: createChatStore, migrate: migrateLegacyChatDb }) @@ -145,6 +188,55 @@ export default class HistoryManager { // session-tagged chats are excluded from history. private sessionId: string | undefined = $state(undefined) + // chatId+dataUrl → stable blob id, so every save of the same conversation + // maps an image to the record written the first time (write-once) instead of + // minting a new one per save. Hydration seeds it back, so a reloaded chat + // re-saves under its original ids too. Scoped by chat: each blob record has + // exactly one owning chat, so the same image pasted into two chats becomes + // two records — sharing one would let chat A's deletion or cap eviction + // destroy bytes chat B still references. + private imageIdByUrl = new Map() + + private imageIdKey(chatId: string, dataUrl: string): string { + return chatId + '\n' + dataUrl + } + + // Blob writes, stale-blob deletes, and the record put span several IndexedDB + // transactions, and saveChat has concurrent callers (turn saves, the + // modified-items and background-jobs writers). Interleaved, an older save's + // delete pass can remove a blob a newer save just verified, landing the + // newer record with a dangling ref — so every DB write runs through this + // per-manager queue. A failed write is rethrown to its caller without + // wedging the queue. + private dbWriteQueue: Promise = Promise.resolve() + + private enqueueDbWrite(op: (db: IDBPDatabase) => Promise): Promise { + // A write belongs to the user who initiated it: capture the scoped DB name + // now and skip execution if the logged-in user changed while queued — + // resolving the handle only at execution time would write this user's chat + // into the NEXT user's database on an in-place account switch. + const name = scopedKey(DB_NAME) + const exec = async () => { + if (!name || scopedKey(DB_NAME) !== name) return + const db = await this.dbh.whenReady() + if (!db || db.name !== name) return + return op(db) + } + const run = this.dbWriteQueue.then(exec, exec) + this.dbWriteQueue = run.catch(() => {}) + return run + } + + /** Drop cached blob ids of every chat but the given one, so the map doesn't + * pin past chats' data URL strings in memory for the whole session (a + * reopened chat re-seeds its ids through hydration). */ + private pruneImageIds(keepChatId: string) { + const prefix = keepChatId + '\n' + for (const key of this.imageIdByUrl.keys()) { + if (!key.startsWith(prefix)) this.imageIdByUrl.delete(key) + } + } + private pastChats = $derived( Object.values(this.savedChats) .filter((c) => c.id !== this.currentChatId) @@ -153,6 +245,9 @@ export default class HistoryManager { ) async init() { + // (Re)initializing adopts a new identity's history: drop the previous + // identity's cached blob ids with it. + this.imageIdByUrl.clear() // whenReady() is email-gated (returns undefined before the user is known — // all callers run post-login, and the singleton re-inits via onUserChange), // runs the legacy migration once, and reopens automatically on user change. @@ -194,10 +289,7 @@ export default class HistoryManager { const snapshot = $state.snapshot(existing) const updated = { ...snapshot, sessionId } this.savedChats = { ...this.savedChats, [chatId]: updated } - // Resolve the DB via the handle (not a cached ref) so a write always lands - // in the current user's DB, even after an in-place user switch. - const db = await this.dbh.whenReady() - if (db) await db.put('chats', updated) + await this.enqueueDbWrite((db) => db.put('chats', updated)) } getPastChats() { @@ -216,6 +308,168 @@ export default class HistoryManager { return this.savedChats[id]?.backgroundJobs } + /** + * Swap every inline image (data URL) in the given message arrays for a + * `wm-image:` ref, mutating them in place — callers pass a clone bound + * for IndexedDB, never live chat state or the in-memory savedChats mirror. + * Returns the id → dataUrl map of every image the arrays reference, plus + * every reference (pre-existing refs included) in walk order — + * persistImageBlobs ranks ids by their newest reference, so the transcript + * walks FIRST: it is always whole and chronological, while drop-oldest + * compaction removes old API messages, which would misorder a dropped + * message's still-displayed image. + */ + private dehydrateImages( + chatId: string, + actualMessages: ChatCompletionMessageParam[], + displayMessages: DisplayMessage[] + ): { blobs: Map; refs: string[] } { + const blobs = new Map() + const refs: string[] = [] + const refFor = (url: string): string => { + // Already a ref (a record that was never rehydrated): it still counts + // as a reference — omitting it would let the stale-delete pass reclaim + // its blob — but there are no bytes to (re)write. + if (url.startsWith(IMAGE_REF_PREFIX)) { + refs.push(url.slice(IMAGE_REF_PREFIX.length)) + return url + } + const key = this.imageIdKey(chatId, url) + let id = this.imageIdByUrl.get(key) + if (!id) { + id = randomUUID() + this.imageIdByUrl.set(key, id) + } + blobs.set(id, url) + refs.push(id) + return IMAGE_REF_PREFIX + id + } + for (const message of displayMessages) { + if (message.role === 'user' && message.images) { + for (const image of message.images) { + if (image.dataUrl.startsWith('data:') || image.dataUrl.startsWith(IMAGE_REF_PREFIX)) { + image.dataUrl = refFor(image.dataUrl) + } + } + } else if ( + message.role === 'tool' && + (message.imageUrl?.startsWith('data:') || message.imageUrl?.startsWith(IMAGE_REF_PREFIX)) + ) { + message.imageUrl = refFor(message.imageUrl) + } + } + for (const message of actualMessages) { + if (!Array.isArray(message.content)) continue + for (const part of message.content as any[]) { + if ( + part?.type === 'image_url' && + (part.image_url?.url?.startsWith('data:') || + part.image_url?.url?.startsWith(IMAGE_REF_PREFIX)) + ) { + part.image_url.url = refFor(part.image_url.url) + } + } + } + return { blobs, refs } + } + + /** + * The record's newest MAX_IMAGES_PER_CHAT distinct images, ranked by their + * LAST reference — the exact set of blobs the chat should own once the + * record is committed. The saved record is the single source of truth: + * deriving the set from it (rather than from persisted write times) keeps + * eviction deterministic and idempotent when turns are truncated, identical + * bytes are re-attached, or compaction rewrites the arrays. A ref outside + * the kept set hydrates to the omitted-image placeholder. + */ + private keptImageIds(refs: string[]): Set { + const keep = new Set() + for (let i = refs.length - 1; i >= 0 && keep.size < MAX_IMAGES_PER_CHAT; i--) { + keep.add(refs[i]) + } + return keep + } + + private async writeKeptImageBlobs( + db: IDBPDatabase, + chatId: string, + blobs: Map, + keep: Set + ) { + for (const id of keep) { + const dataUrl = blobs.get(id) + if (dataUrl !== undefined && (await db.getKey('images', id)) === undefined) { + await db.put('images', { id, chatId, dataUrl, savedAt: Date.now() }) + } + } + } + + private async deleteStaleImageBlobs( + db: IDBPDatabase, + chatId: string, + keep: Set + ) { + for (const key of await imageKeysForChat(db, chatId)) { + if (!keep.has(key)) await db.delete('images', key) + } + } + + /** + * Resolve every `wm-image:` ref in the chat clone back to its data URL, + * in place. A missing blob (evicted by the per-chat cap, or IndexedDB + * unavailable altogether) degrades the API part to the omitted-image + * placeholder and drops the transcript copy — a ref must never leak into + * bubbles or outgoing requests. Inline data URLs (records persisted before + * the blob store) pass through untouched. + */ + private async hydrateImages( + db: IDBPDatabase | undefined, + chatId: string, + actualMessages: ChatCompletionMessageParam[], + displayMessages: DisplayMessage[] + ) { + const load = async (ref: string): Promise => { + const id = ref.slice(IMAGE_REF_PREFIX.length) + const dataUrl = (await db?.get('images', id))?.dataUrl + if (dataUrl) this.imageIdByUrl.set(this.imageIdKey(chatId, dataUrl), id) + return dataUrl + } + for (const message of actualMessages) { + if (!Array.isArray(message.content)) continue + const content = message.content as any[] + for (let i = 0; i < content.length; i++) { + const part = content[i] + if (part?.type === 'image_url' && part.image_url?.url?.startsWith(IMAGE_REF_PREFIX)) { + const dataUrl = await load(part.image_url.url) + content[i] = dataUrl + ? { ...part, image_url: { ...part.image_url, url: dataUrl } } + : { type: 'text', text: IMAGE_OMITTED_PLACEHOLDER } + } + } + } + for (const message of displayMessages) { + if (message.role === 'user' && message.images) { + const images: AttachedImage[] = [] + for (const image of message.images) { + if (!image.dataUrl.startsWith(IMAGE_REF_PREFIX)) { + images.push(image) + continue + } + const dataUrl = await load(image.dataUrl) + if (dataUrl) images.push({ ...image, dataUrl }) + } + message.images = images.length > 0 ? images : undefined + // An image-only bubble that lost every image would render empty — + // say what happened instead. + if (!message.images && !message.content.trim()) { + message.content = IMAGE_OMITTED_PLACEHOLDER + } + } else if (message.role === 'tool' && message.imageUrl?.startsWith(IMAGE_REF_PREFIX)) { + message.imageUrl = await load(message.imageUrl) + } + } + } + async saveChat( displayMessages: DisplayMessage[], messages: ChatCompletionMessageParam[], @@ -231,12 +485,22 @@ export default class HistoryManager { // expanding collapsed-paste tokens so it reads as text rather than the // chip label + its zero-width id chars. const existingTitle = this.savedChats[this.currentChatId]?.title + const titleSource = displayMessages.find((m) => m.role !== 'summary') ?? displayMessages[0] + const derivedTitle = expanded(messageDraft(titleSource)).slice(0, 50) + // An image-only first turn has no text to derive from — fall back to the + // attachment's filename so the History menu entry isn't blank. + const imageFallback = + titleSource.role === 'user' && titleSource.images?.length + ? (titleSource.images[0].name ?? 'Image attachment') + : '' + // A hydrated omission marker is not user text — deriving from it would + // overwrite the filename title an evicted image-only chat was given. const title = displayMessages[0].role === 'summary' && existingTitle !== undefined ? existingTitle - : expanded( - messageDraft(displayMessages.find((m) => m.role !== 'summary') ?? displayMessages[0]) - ).slice(0, 50) + : derivedTitle.trim() && derivedTitle !== IMAGE_OMITTED_PLACEHOLDER + ? derivedTitle + : imageFallback || existingTitle || '' // we don't want to save the snapshot in the history const updatedChat = { actualMessages: $state.snapshot(messages), @@ -273,13 +537,38 @@ export default class HistoryManager { } : {}) } + // The mirror mirrors what the DB holds (refs — the snapshot is + // dehydrated below before either sees it): a reopened chat hydrates + // through the store, reseeding stable blob ids. When IndexedDB is + // unavailable the writes no-op and hydration degrades the refs to + // omitted-image placeholders — like every other userScopedDb consumer, + // history simply doesn't persist there. + const { blobs, refs } = this.dehydrateImages( + updatedChat.id, + updatedChat.actualMessages, + updatedChat.displayMessages + ) this.savedChats = { ...this.savedChats, [updatedChat.id]: updatedChat } - - const db = await this.dbh.whenReady() - if (db) await db.put('chats', updatedChat) + await this.enqueueDbWrite(async (db) => { + // Write order is the crash-safety story: kept blobs land before the + // record that references them, and stale blobs are deleted only after + // the new record is committed. A failure at any step leaves the last + // committed record fully hydratable — at worst orphan blobs linger + // until the next successful save's delete pass reclaims them. + const keep = this.keptImageIds(refs) + await this.writeKeptImageBlobs(db, updatedChat.id, blobs, keep) + await db.put('chats', updatedChat) + // Best-effort: the record is already committed, so a failed cleanup + // (e.g. a user switch closed this handle mid-op) must not turn a + // successful save into a rejection — the orphans are reclaimed by + // the next successful save's pass. + await this.deleteStaleImageBlobs(db, updatedChat.id, keep).catch((err) => + console.error('Could not prune stale image blobs', err) + ) + }) } } @@ -292,20 +581,32 @@ export default class HistoryManager { ) { await this.saveChat(displayMessages, messages, contextUsage, modifiedItems, backgroundJobs) this.currentChatId = createLongHash() + this.pruneImageIds(this.currentChatId) } deletePastChat(id: string) { this.savedChats = Object.fromEntries( Object.entries(this.savedChats).filter(([key]) => key !== id) ) - void this.dbh.whenReady().then((db) => db?.delete('chats', id)) + void this.enqueueDbWrite(async (db) => { + await db.delete('chats', id) + const keys = await imageKeysForChat(db, id) + await Promise.all(keys.map((key) => db.delete('images', key))) + }).catch((err) => console.error('Could not delete chat', err)) } - loadPastChat(id: string) { + async loadPastChat(id: string) { const chat = this.savedChats[id] - if (chat) { - this.currentChatId = id - return chat - } + if (!chat) return + this.currentChatId = id + this.pruneImageIds(id) + // Hand back a hydrated clone: the stored record keeps its refs (matching + // what the DB holds) while the live chat gets real data URLs. Hydration + // runs even without a DB so refs degrade to placeholders instead of + // leaking into bubbles and requests. + const snapshot = $state.snapshot(chat) as typeof chat + const db = await this.dbh.whenReady() + await this.hydrateImages(db, id, snapshot.actualMessages, snapshot.displayMessages) + return snapshot } } diff --git a/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts b/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts index 540dfaf72e..ffcc065157 100644 --- a/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts @@ -90,6 +90,449 @@ describe('HistoryManager legacy chat-history migration', () => { expect(hm.getAllSavedChats()).toEqual([]) }) + it('persists image bytes out of the chat record and hydrates them back on load', async () => { + const png = 'data:image/png;base64,FULLBYTES' + const hm = new HistoryManager() + await hm.init() + const chatId = hm.getCurrentChatId() + await hm.saveChat( + [ + { role: 'user', content: 'look', images: [{ dataUrl: png, mediaType: 'image/png' }] }, + { role: 'tool', tool_call_id: 't1', content: 'shot', imageUrl: png } + ] as DisplayMessage[], + [ + { + role: 'user', + content: [ + { type: 'text', text: 'look' }, + { type: 'image_url', image_url: { url: png } } + ] + } + ] as ChatCompletionMessageParam[] + ) + + // The chat record holds refs, not bytes — and the shared data URL of the + // bubble, tool card, and API part dedups to a single blob record. + const db = await openDB('copilot-chat-history::admin@test') + const record = await db.get('chats' as never, chatId) + expect(JSON.stringify(record)).not.toContain('FULLBYTES') + expect((record as any).actualMessages[0].content[1].image_url.url).toMatch(/^wm-image:/) + expect(await db.count('images' as never)).toBe(1) + db.close() + + // A fresh instance (reload) hydrates the refs back to the original bytes. + const reloaded = new HistoryManager() + await reloaded.init() + const chat = await reloaded.loadPastChat(chatId) + expect((chat?.actualMessages[0].content as any[])[1].image_url.url).toBe(png) + expect((chat?.displayMessages[0] as any).images[0].dataUrl).toBe(png) + expect((chat?.displayMessages[1] as any).imageUrl).toBe(png) + }) + + it('re-saving the same conversation does not mint new blob records', async () => { + const png = 'data:image/png;base64,STABLE' + const display = [ + { role: 'user', content: 'x', images: [{ dataUrl: png, mediaType: 'image/png' }] } + ] as DisplayMessage[] + const hm = new HistoryManager() + await hm.init() + await hm.saveChat(display, [] as ChatCompletionMessageParam[]) + await hm.saveChat(display, [] as ChatCompletionMessageParam[]) + + const db = await openDB('copilot-chat-history::admin@test') + expect(await db.count('images' as never)).toBe(1) + db.close() + }) + + it('caps stored blobs per chat; an evicted ref hydrates to the omitted placeholder', async () => { + const hm = new HistoryManager() + await hm.init() + const chatId = hm.getCurrentChatId() + const urlFor = (i: number) => `data:image/png;base64,IMG${String(i).padStart(2, '0')}` + const messages = [] as ChatCompletionMessageParam[] + for (let i = 0; i <= 30; i++) { + messages.push({ + role: 'user', + content: [{ type: 'image_url', image_url: { url: urlFor(i) } }] + } as ChatCompletionMessageParam) + await hm.saveChat([{ role: 'user', content: 'x' }] as DisplayMessage[], messages) + } + // Re-saving the over-cap chat must not resurrect the evicted oldest blob + // (its live data URL is still in the arrays): a re-put would stamp it + // newest and push the eviction onto a newer image, and repeated saves + // would rotate the hole toward the latest attachment. + await hm.saveChat([{ role: 'user', content: 'x' }] as DisplayMessage[], messages) + await hm.saveChat([{ role: 'user', content: 'x' }] as DisplayMessage[], messages) + + const reloaded = new HistoryManager() + await reloaded.init() + const chat = await reloaded.loadPastChat(chatId) + expect((chat?.actualMessages[0].content as any[])[0]).toEqual({ + type: 'text', + text: '[image omitted]' + }) + expect((chat?.actualMessages[1].content as any[])[0].image_url.url).toBe(urlFor(1)) + expect((chat?.actualMessages[30].content as any[])[0].image_url.url).toBe(urlFor(30)) + }) + + it('keeps blob chronology when drop-oldest compaction removed old API counterparts', async () => { + const hm = new HistoryManager() + await hm.init() + const chatId = hm.getCurrentChatId() + const urlFor = (i: number) => `data:image/png;base64,IMG${String(i).padStart(2, '0')}` + // Transcript keeps all 31 bubbles; drop-oldest compaction pruned the API + // history down to the newest 4 image messages. + const display = Array.from({ length: 31 }, (_, i) => ({ + role: 'user', + content: 'x', + index: i - 27, + images: [{ dataUrl: urlFor(i), mediaType: 'image/png' }] + })) as DisplayMessage[] + const messages = Array.from({ length: 4 }, (_, i) => ({ + role: 'user', + content: [{ type: 'image_url', image_url: { url: urlFor(27 + i) } }] + })) as ChatCompletionMessageParam[] + await hm.saveChat(display, messages) + await hm.saveChat(display, messages) + + const reloaded = new HistoryManager() + await reloaded.init() + const chat = await reloaded.loadPastChat(chatId) + // The oldest transcript image is the one over the cap... + expect((chat?.displayMessages[0] as any).images).toBeUndefined() + // ...never a newer one that merely lost its API counterpart ordering. + expect((chat?.actualMessages[0].content as any[])[0].image_url.url).toBe(urlFor(27)) + expect((chat?.displayMessages[30] as any).images[0].dataUrl).toBe(urlFor(30)) + }) + + it('truncated turns release their blobs from the cap', async () => { + const hm = new HistoryManager() + await hm.init() + const chatId = hm.getCurrentChatId() + const urlFor = (i: number) => `data:image/png;base64,IMG${String(i).padStart(2, '0')}` + const imageMsg = (i: number) => + ({ + role: 'user', + content: [{ type: 'image_url', image_url: { url: urlFor(i) } }] + }) as ChatCompletionMessageParam + const display = [{ role: 'user', content: 'x' }] as DisplayMessage[] + // Fill the cap exactly, then retry/edit truncates the tail to 5 messages + // and adds one replacement image. The truncated turns' blobs must stop + // counting against the cap — image 0 is among the newest 6 *referenced* + // images and must survive. + await hm.saveChat( + display, + Array.from({ length: 30 }, (_, i) => imageMsg(i)) + ) + await hm.saveChat(display, [...Array.from({ length: 5 }, (_, i) => imageMsg(i)), imageMsg(99)]) + + const db = await openDB('copilot-chat-history::admin@test') + expect(await db.count('images' as never)).toBe(6) + db.close() + + const reloaded = new HistoryManager() + await reloaded.init() + const chat = await reloaded.loadPastChat(chatId) + expect((chat?.actualMessages[0].content as any[])[0].image_url.url).toBe(urlFor(0)) + expect((chat?.actualMessages[5].content as any[])[0].image_url.url).toBe(urlFor(99)) + }) + + it('re-attaching identical bytes ranks the image by its newest reference', async () => { + const hm = new HistoryManager() + await hm.init() + const chatId = hm.getCurrentChatId() + const urlFor = (i: number) => `data:image/png;base64,IMG${String(i).padStart(2, '0')}` + const imageMsg = (url: string) => + ({ + role: 'user', + content: [{ type: 'image_url', image_url: { url } }] + }) as ChatCompletionMessageParam + const display = [{ role: 'user', content: 'x' }] as DisplayMessage[] + const reused = 'data:image/png;base64,REUSED' + // The reused image appears first, 30 distinct images follow, then it is + // attached again. Its newest reference makes it one of the newest 30 + // distinct images, so the eviction must land on the oldest of the middle + // ones — not on the image the user just re-attached. + const messages = [ + imageMsg(reused), + ...Array.from({ length: 30 }, (_, i) => imageMsg(urlFor(i))), + imageMsg(reused) + ] + await hm.saveChat(display, [messages[0]]) + await hm.saveChat(display, messages) + + const reloaded = new HistoryManager() + await reloaded.init() + const chat = await reloaded.loadPastChat(chatId) + expect((chat?.actualMessages[0].content as any[])[0].image_url.url).toBe(reused) + expect((chat?.actualMessages[31].content as any[])[0].image_url.url).toBe(reused) + expect((chat?.actualMessages[1].content as any[])[0]).toEqual({ + type: 'text', + text: '[image omitted]' + }) + expect((chat?.actualMessages[2].content as any[])[0].image_url.url).toBe(urlFor(1)) + }) + + it('a save overlapping an older save keeps every blob its record references', async () => { + const hm = new HistoryManager() + await hm.init() + const chatId = hm.getCurrentChatId() + const X = 'data:image/png;base64,XBYTES' + const Y = 'data:image/png;base64,YBYTES' + const display = [{ role: 'user', content: 'x' }] as DisplayMessage[] + const msg = (url: string) => + ({ + role: 'user', + content: [{ type: 'image_url', image_url: { url } }] + }) as ChatCompletionMessageParam + await hm.saveChat(display, [msg(X)]) + + // An overlapping pair: the older snapshot no longer references X (retry + // truncation), the newer one re-references it (as its newest image) and + // adds Y. Un-serialized, the older save's delete pass removes X's blob + // after the newer save verified its existence and moved on, landing the + // winning record with a dangling ref. + const older = hm.saveChat(display, [ + { role: 'user', content: 'no images' } as ChatCompletionMessageParam + ]) + const newer = hm.saveChat(display, [msg(Y), msg(X)]) + await Promise.all([older, newer]) + + const reloaded = new HistoryManager() + await reloaded.init() + const chat = await reloaded.loadPastChat(chatId) + expect((chat?.actualMessages[0].content as any[])[0].image_url.url).toBe(Y) + expect((chat?.actualMessages[1].content as any[])[0].image_url.url).toBe(X) + }) + + it('reopening a rotated chat reuses its blob records on the next save', async () => { + const hm = new HistoryManager() + await hm.init() + const chatId = hm.getCurrentChatId() + const png = 'data:image/png;base64,STABLEBYTES' + await hm.save( + [ + { role: 'user', content: 'x', images: [{ dataUrl: png, mediaType: 'image/png' }] } + ] as DisplayMessage[], + [] as ChatCompletionMessageParam[] + ) // rotates to a fresh chat, pruning the id cache + + const before = await openDB('copilot-chat-history::admin@test') + const idsBefore = await before.getAllKeys('images' as never) + before.close() + + // Reopening must reseed the stable blob id — a re-save that minted a new + // id would rewrite every blob (and delete the old ones) on each reopen. + const chat = await hm.loadPastChat(chatId) + await hm.saveChat(chat!.displayMessages as DisplayMessage[], chat!.actualMessages) + + const after = await openDB('copilot-chat-history::admin@test') + const idsAfter = await after.getAllKeys('images' as never) + after.close() + expect(idsAfter).toEqual(idsBefore) + }) + + it('the same image in two chats gets two owned blobs; deleting one chat spares the other', async () => { + const png = 'data:image/png;base64,SHAREDBYTES' + const display = [ + { role: 'user', content: 'x', images: [{ dataUrl: png, mediaType: 'image/png' }] } + ] as DisplayMessage[] + const hm = new HistoryManager() + await hm.init() + const chatA = hm.getCurrentChatId() + await hm.save(display, [] as ChatCompletionMessageParam[]) // rotates to a new chat + const chatB = hm.getCurrentChatId() + await hm.saveChat(display, [] as ChatCompletionMessageParam[]) + + const db = await openDB('copilot-chat-history::admin@test') + expect(await db.count('images' as never)).toBe(2) + db.close() + + hm.deletePastChat(chatA) + await vi.waitFor(async () => { + const d = await openDB('copilot-chat-history::admin@test') + const count = await d.count('images' as never) + d.close() + expect(count).toBe(1) + }) + + const reloaded = new HistoryManager() + await reloaded.init() + const chat = await reloaded.loadPastChat(chatB) + expect((chat?.displayMessages[0] as any).images[0].dataUrl).toBe(png) + }) + + it("deletes a chat's image blobs along with the chat", async () => { + const hm = new HistoryManager() + await hm.init() + const chatId = hm.getCurrentChatId() + await hm.saveChat( + [ + { + role: 'user', + content: 'x', + images: [{ dataUrl: 'data:image/png;base64,GONE', mediaType: 'image/png' }] + } + ] as DisplayMessage[], + [] as ChatCompletionMessageParam[] + ) + + hm.deletePastChat(chatId) + + await vi.waitFor(async () => { + const db = await openDB('copilot-chat-history::admin@test') + const count = await db.count('images' as never) + db.close() + expect(count).toBe(0) + }) + }) + + it('loads pre-blob-store records with inline data URLs untouched', async () => { + const png = 'data:image/png;base64,LEGACYINLINE' + const hm = new HistoryManager() + await hm.init() + // Simulate a record persisted before the blob store existed. + const db = await openDB('copilot-chat-history::admin@test') + await db.put( + 'chats' as never, + { + id: 'legacy1', + title: 'legacy', + lastModified: 1, + actualMessages: [ + { role: 'user', content: [{ type: 'image_url', image_url: { url: png } }] } + ], + displayMessages: [ + { role: 'user', content: 'x', images: [{ dataUrl: png, mediaType: 'image/png' }] } + ] + } as never + ) + db.close() + + const reloaded = new HistoryManager() + await reloaded.init() + const chat = await reloaded.loadPastChat('legacy1') + expect((chat?.actualMessages[0].content as any[])[0].image_url.url).toBe(png) + expect((chat?.displayMessages[0] as any).images[0].dataUrl).toBe(png) + }) + + it("a failed record put cannot orphan the previous record's blobs", async () => { + const png = 'data:image/png;base64,SURVIVES' + const hm = new HistoryManager() + await hm.init() + const chatId = hm.getCurrentChatId() + await hm.saveChat( + [ + { role: 'user', content: 'x', images: [{ dataUrl: png, mediaType: 'image/png' }] } + ] as DisplayMessage[], + [] as ChatCompletionMessageParam[] + ) + + // Make the next `chats` put fail (quota/connection failure), on a save + // whose record drops the image — its blob is now stale, but deleting it + // before the record commit would corrupt the still-current OLD record. + const probe = await openDB('probe-proto', 1, { + upgrade: (d) => { + d.createObjectStore('s') + } + }) + const proto = Object.getPrototypeOf( + probe.transaction('s' as never, 'readwrite').objectStore('s' as never) + ) + probe.close() + const origPut = proto.put + let failNext = true + proto.put = function (this: { name: string }, ...args: unknown[]) { + if (this.name === 'chats' && failNext) { + failNext = false + throw new Error('simulated quota failure') + } + return origPut.apply(this, args) + } + try { + await expect( + hm.saveChat( + [{ role: 'user', content: 'no image' }] as DisplayMessage[], + [] as ChatCompletionMessageParam[] + ) + ).rejects.toThrow('simulated quota failure') + } finally { + proto.put = origPut + } + + const reloaded = new HistoryManager() + await reloaded.init() + const chat = await reloaded.loadPastChat(chatId) + expect((chat?.displayMessages[0] as any).images[0].dataUrl).toBe(png) + }) + + it('drops queued writes when the user switches before they execute (no cross-user leak)', async () => { + const hm = new HistoryManager() + await hm.init() + const display = [ + { + role: 'user', + content: 'private to A', + images: [{ dataUrl: 'data:image/png;base64,LEAKBYTES', mediaType: 'image/png' }] + } + ] as DisplayMessage[] + // Enqueue two writes under user A, then switch identity before either + // executes. Resolving the DB handle at execution time would write A's + // chat and image blob into B's database. + const first = hm.saveChat(display, [] as ChatCompletionMessageParam[]) + const second = hm.saveChat(display, [] as ChatCompletionMessageParam[]) + userStore.set(asUser('other@test')) + await Promise.all([first, second]) + + const db = await openDB('copilot-chat-history::other@test') + const chats = db.objectStoreNames.contains('chats') ? await db.count('chats' as never) : 0 + const images = db.objectStoreNames.contains('images') ? await db.count('images' as never) : 0 + db.close() + expect(chats).toBe(0) + expect(images).toBe(0) + }) + + it("a save finishing after another user's init cannot leak into their mirror", async () => { + const hm = new HistoryManager() + await hm.init() + + // The save passes enqueueDbWrite's identity checks under user A; the + // account switches (and re-inits, as the platform does on user change) + // while its transaction is still running. The committed save must still + // resolve (the switch closes A's handle, failing only the best-effort + // cleanup tail) and its convergence must not merge A's record into B's + // freshly adopted mirror. + const probe = await openDB('probe-proto2', 1, { + upgrade: (d) => { + d.createObjectStore('s') + } + }) + const proto = Object.getPrototypeOf( + probe.transaction('s' as never, 'readwrite').objectStore('s' as never) + ) + probe.close() + const origPut = proto.put + let initDone: Promise | undefined + proto.put = function (this: { name: string }, ...args: unknown[]) { + if (this.name === 'chats' && !initDone) { + userStore.set(asUser('other@test')) + initDone = hm.init() + } + return origPut.apply(this, args) + } + try { + await hm.saveChat( + [{ role: 'user', content: 'private to A' }] as DisplayMessage[], + [] as ChatCompletionMessageParam[] + ) + } finally { + proto.put = origPut + } + await initDone + + expect(hm.getAllSavedChats()).toEqual([]) + }) + it('writes land in the current user DB after an in-place user switch', async () => { const hm = new HistoryManager() await hm.init() @@ -115,6 +558,119 @@ describe('HistoryManager legacy chat-history migration', () => { }) }) +describe('HistoryManager image-only chats', () => { + it('titles an image-only chat from its attachment instead of leaving it blank', async () => { + const hm = new HistoryManager() + await hm.init() + const id = hm.getCurrentChatId() + await hm.saveChat( + [ + { + role: 'user', + content: '', + images: [ + { dataUrl: 'data:image/png;base64,A', mediaType: 'image/png', name: 'mockup.png' } + ] + } + ] as DisplayMessage[], + [] as ChatCompletionMessageParam[] + ) + expect(hm.getAllSavedChats().find((c) => c.id === id)?.title).toBe('mockup.png') + }) + + it('keeps the filename title when the evicted bubble re-saves as an omission marker', async () => { + const hm = new HistoryManager() + await hm.init() + const id = hm.getCurrentChatId() + await hm.saveChat( + [ + { + role: 'user', + content: '', + images: [ + { dataUrl: 'data:image/png;base64,A', mediaType: 'image/png', name: 'mockup.png' } + ] + } + ] as DisplayMessage[], + [] as ChatCompletionMessageParam[] + ) + // Post-reload shape of an evicted image-only first bubble: the omission + // marker as content, images gone. Re-saving must not adopt the marker as + // the chat's title. + await hm.saveChat( + [ + { role: 'user', content: '[image omitted]' }, + { role: 'user', content: 'follow-up' } + ] as DisplayMessage[], + [] as ChatCompletionMessageParam[] + ) + expect(hm.getAllSavedChats().find((c) => c.id === id)?.title).toBe('mockup.png') + }) + + it('shows an omission marker when an evicted image-only bubble reloads', async () => { + const hm = new HistoryManager() + await hm.init() + const chatId = hm.getCurrentChatId() + const urlFor = (i: number) => `data:image/png;base64,IMG${String(i).padStart(2, '0')}` + // 31 image-only turns: the oldest exceeds the blob cap, so its bubble + // reloads with no image AND no text — it must say what happened instead + // of rendering empty. + const display = Array.from({ length: 31 }, (_, i) => ({ + role: 'user', + content: '', + index: i, + images: [{ dataUrl: urlFor(i), mediaType: 'image/png' }] + })) as DisplayMessage[] + await hm.saveChat(display, [] as ChatCompletionMessageParam[]) + + const reloaded = new HistoryManager() + await reloaded.init() + const chat = await reloaded.loadPastChat(chatId) + expect((chat?.displayMessages[0] as any).images).toBeUndefined() + expect((chat?.displayMessages[0] as any).content).toBe('[image omitted]') + expect((chat?.displayMessages[1] as any).images[0].dataUrl).toBe(urlFor(1)) + expect((chat?.displayMessages[1] as any).content).toBe('') + }) +}) + +describe('HistoryManager without IndexedDB', () => { + it('degrades cleanly: refs never leak into bubbles or message content', async () => { + // whenReady() resolves undefined when opens fail (private browsing, + // blocked, corrupt). History then simply doesn't persist — like every + // other userScopedDb consumer — but a reloaded chat must degrade its + // unresolvable refs to omitted-image placeholders, never hand raw + // `wm-image:` URLs to an or an outgoing request. + ;(globalThis as any).indexedDB = { + open: () => { + throw new Error('blocked') + } + } + const hm = new HistoryManager() + await hm.init() + const chatId = hm.getCurrentChatId() + const png = 'data:image/png;base64,MEMORYONLY' + await hm.saveChat( + [ + { role: 'user', content: 'x', images: [{ dataUrl: png, mediaType: 'image/png' }] } + ] as DisplayMessage[], + [ + { + role: 'user', + content: [{ type: 'image_url', image_url: { url: png } }] + } + ] as ChatCompletionMessageParam[] + ) + + const chat = await hm.loadPastChat(chatId) + expect((chat?.displayMessages[0] as any).images).toBeUndefined() + expect((chat?.actualMessages[0].content as any[])[0]).toEqual({ + type: 'text', + text: '[image omitted]' + }) + expect(JSON.stringify(chat)).not.toContain('wm-image:') + }) +}) + describe('HistoryManager title across compaction', () => { it('keeps the original title once a summary boundary leads the transcript', async () => { const hm = new HistoryManager() @@ -142,6 +698,39 @@ describe('HistoryManager title across compaction', () => { }) }) +describe('HistoryManager mirror convergence under concurrent metadata saves', () => { + it('an older save completing mid-stream cannot erase newer metadata', async () => { + const hm = new HistoryManager() + await hm.init() + const id = hm.getCurrentChatId() + const display = [{ role: 'user', content: 'x', index: 0 }] as DisplayMessage[] + const job = { id: 'job1', status: 'running' } as unknown as ChatJob + + // s1 carries modifiedItems; s2 (overlapping) carries backgroundJobs and + // inherits s1's modifiedItems from the mirror. Awaiting s1 lets its + // convergence run while s2 is still queued — it must not rewind the + // mirror, or s3's backgroundJobs fallback below reads the stale record + // and permanently erases the job. + const p1 = hm.saveChat(display, [] as ChatCompletionMessageParam[], undefined, ['script:a']) + const p2 = hm.saveChat(display, [] as ChatCompletionMessageParam[], undefined, undefined, [job]) + await p1 + const p3 = hm.saveChat(display, [] as ChatCompletionMessageParam[], undefined, [ + 'script:a', + 'script:b' + ]) + await Promise.all([p2, p3]) + + expect(hm.getModifiedItems(id)).toEqual(['script:a', 'script:b']) + expect(hm.getBackgroundJobs(id)).toEqual([job]) + + const db = await openDB('copilot-chat-history::admin@test') + const record = (await db.get('chats' as never, id)) as any + db.close() + expect(record.modifiedItems).toEqual(['script:a', 'script:b']) + expect(record.backgroundJobs).toEqual([job]) + }) +}) + describe('HistoryManager modified-items mask persistence', () => { const msgs = [{ role: 'user', content: 'hello', index: 0 }] as DisplayMessage[] diff --git a/frontend/src/lib/components/copilot/chat/QueuedMessageChip.svelte b/frontend/src/lib/components/copilot/chat/QueuedMessageChip.svelte index 3d40225360..a6e6c248e6 100644 --- a/frontend/src/lib/components/copilot/chat/QueuedMessageChip.svelte +++ b/frontend/src/lib/components/copilot/chat/QueuedMessageChip.svelte @@ -11,15 +11,30 @@ const aiChatManager = getAiChatManager() -{#if aiChatManager.queuedMessage} + +{#if aiChatManager.queuedMessage || aiChatManager.queuedImages.length > 0}
-

- {aiChatManager.queuedMessage} -

+ {#if aiChatManager.queuedImages.length > 0} +
+ {#each aiChatManager.queuedImages as image, i (i)} + {image.name + {/each} +
+ {/if} + {#if aiChatManager.queuedMessage} +

+ {aiChatManager.queuedMessage} +

+ {/if}
+ + {#if message.imageUrl} +
+ +
+ {/if} + {#if isExpanded}
({ })) vi.mock('./shared', () => ({ - processToolCall: vi.fn() + processToolCall: vi.fn(), + appendPendingToolImages: vi.fn() })) describe('convertOpenAIToAnthropicMessages', () => { @@ -139,6 +140,42 @@ describe('convertOpenAIToAnthropicMessages', () => { expect(content[1]).toMatchObject({ type: 'tool_use', id: 'tool_old', name: 'list_resources' }) }) + it('converts a user message with an image_url part to an Anthropic base64 image block', () => { + const messages: ChatCompletionMessageParam[] = [ + { + role: 'user', + content: [ + { type: 'text', text: 'what is this?' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,AAAABBBB' } } + ] + } as any + ] + + const { messages: out } = convertOpenAIToAnthropicMessages(messages) + + expect(out).toHaveLength(1) + expect(out[0].role).toBe('user') + const content = out[0].content as any[] + expect(content[0]).toMatchObject({ type: 'text', text: 'what is this?' }) + expect(content[1]).toMatchObject({ + type: 'image', + source: { type: 'base64', media_type: 'image/png', data: 'AAAABBBB' } + }) + // The trailing block is the image — the ephemeral cache breakpoint may land on it + // (cache_control is valid on image blocks). + expect(content[1].cache_control).toEqual({ type: 'ephemeral' }) + }) + + it('keeps a plain string user message unchanged (no array wrapping)', () => { + // Non-trailing so the last-block cache_control wrapping doesn't obscure it. + const messages: ChatCompletionMessageParam[] = [ + { role: 'user', content: 'just text' }, + { role: 'assistant', content: 'ok' } + ] + const { messages: out } = convertOpenAIToAnthropicMessages(messages) + expect(out[0].content).toBe('just text') + }) + it('caches a trailing tool result even when the prior turn used no captured content', () => { const messages: ChatCompletionMessageParam[] = [ { role: 'user', content: 'q' }, diff --git a/frontend/src/lib/components/copilot/chat/anthropic.ts b/frontend/src/lib/components/copilot/chat/anthropic.ts index 2c19a611a0..209a3b7a83 100644 --- a/frontend/src/lib/components/copilot/chat/anthropic.ts +++ b/frontend/src/lib/components/copilot/chat/anthropic.ts @@ -16,8 +16,40 @@ import type { MessageStream } from '@anthropic-ai/sdk/lib/MessageStream' import type { AIProviderModel } from '$lib/gen' import { getProviderAndCompletionConfig, workspaceAIClients } from '../lib' import { applyReasoningToConfig } from '../reasoningRegistry' -import { processToolCall, type Tool, type ToolCallbacks } from './shared' +import { appendPendingToolImages, processToolCall, type Tool, type ToolCallbacks } from './shared' import { anthropicUsageToChatTokenUsage, type ChatTokenUsage } from './tokenUsage' +import { parseImageDataUrl } from './imageUtils' + +const ANTHROPIC_IMAGE_MEDIA_TYPES = new Set(['image/jpeg', 'image/png', 'image/gif', 'image/webp']) + +/** + * Convert an OpenAI user-message content array (text + image_url parts) to Anthropic + * content blocks. Returns a plain string when the content is a lone text part so + * simple messages stay unchanged. Non-image/text parts are dropped. + */ +function openAIUserContentToAnthropic(content: unknown): string | any[] { + if (typeof content === 'string') return content + if (!Array.isArray(content)) return JSON.stringify(content) + const blocks: any[] = [] + for (const part of content) { + if (part?.type === 'text' && typeof part.text === 'string') { + blocks.push({ type: 'text', text: part.text }) + } else if (part?.type === 'image_url' && part.image_url?.url) { + const { mediaType, base64 } = parseImageDataUrl(part.image_url.url) + if (!base64) continue + blocks.push({ + type: 'image', + source: { + type: 'base64', + media_type: ANTHROPIC_IMAGE_MEDIA_TYPES.has(mediaType) ? mediaType : 'image/png', + data: base64 + } + }) + } + } + if (blocks.length === 1 && blocks[0].type === 'text') return blocks[0].text + return blocks +} interface ParsedCompletionResult { shouldContinue: boolean @@ -310,6 +342,7 @@ export async function parseAnthropicCompletion( messages.push(messageToAdd) addedMessages.push(messageToAdd) } + appendPendingToolImages(messages, addedMessages, callbacks) return { shouldContinue: true, tokenUsage } } @@ -367,8 +400,7 @@ export function convertOpenAIToAnthropicMessages(messages: ChatCompletionMessage if (message.role === 'user') { anthropicMessages.push({ role: 'user', - content: - typeof message.content === 'string' ? message.content : JSON.stringify(message.content) + content: openAIUserContentToAnthropic(message.content) }) } else if (message.role === 'assistant') { // Replay a captured assistant turn verbatim so its thinking-block signatures @@ -443,8 +475,8 @@ export function convertOpenAIToAnthropicMessages(messages: ChatCompletionMessage // block of the last message. Each continuation only appends a tool result plus the // next turn, so everything up to here is read from cache — which is what keeps // replaying assistant turns verbatim (web-search results included) affordable. - // cache_control is valid on text/tool_use/tool_result blocks, but a thinking or - // redacted_thinking block must never be modified, so skip the breakpoint there. + // cache_control is valid on text/tool_use/tool_result/image blocks, but a thinking + // or redacted_thinking block must never be modified, so skip the breakpoint there. if (anthropicMessages.length > 0) { const lastMessage = anthropicMessages[anthropicMessages.length - 1] if (typeof lastMessage.content === 'string') { diff --git a/frontend/src/lib/components/copilot/chat/chatLoop.test.ts b/frontend/src/lib/components/copilot/chat/chatLoop.test.ts index b9581e3d74..b0bfa8b8ca 100644 --- a/frontend/src/lib/components/copilot/chat/chatLoop.test.ts +++ b/frontend/src/lib/components/copilot/chat/chatLoop.test.ts @@ -570,3 +570,108 @@ describe('runChatLoop history sanitization', () => { expect((poisoned as any).tool_calls[0].function.arguments).toContain('trunc') }) }) + +describe('runChatLoop per-iteration vision gating', () => { + beforeEach(() => { + vi.resetAllMocks() + mocks.providerSupportsWebSearch.mockReturnValue(false) + mocks.resolveRequestReasoning.mockReturnValue(undefined) + }) + + // The loop owns the vision strip entirely — the caller passes the full + // history even for a known text-only model (see AIChatManager.chatRequest). + it('strips image parts from the first iteration on a known text-only model', async () => { + const config = createConfig({ + workspace: `workspace-${randomUUID()}`, + modelProvider: { provider: 'groq', model: 'llama-3.3-70b-versatile' } + }) + config.messages.splice(0, config.messages.length, { + role: 'user', + content: [ + { type: 'text', text: 'earlier turn' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,IMG' } } + ] + } as any) + mocks.getCompletion.mockResolvedValue({}) + mocks.parseOpenAICompletion.mockResolvedValue({ shouldContinue: false, tokenUsage }) + + await runChatLoop(config) + + expect(mocks.getCompletion).toHaveBeenCalled() + expect(JSON.stringify(mocks.getCompletion.mock.calls[0][0])).not.toContain('image_url') + }) + + // The model selector stays enabled while the loop runs, and the loop re-reads + // the model each iteration. The vision gate has to be re-applied at the same + // cadence: filtering once at send start would ship the history's image parts + // to a text-only model the user switched to mid-turn. + it('strips image parts when the model switches to a text-only one mid-loop', async () => { + const config = createConfig({ workspace: `workspace-${randomUUID()}` }) + config.maxIterations = 2 + config.messages.splice(0, config.messages.length, { + role: 'user', + content: [ + { type: 'text', text: 'look at this' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,IMG' } } + ] + } as any) + // vision model on the first iteration, known text-only on the second + let iteration = 0 + Object.defineProperty(config, 'modelProvider', { + get: () => + iteration === 0 + ? { provider: 'openai', model: 'gpt-4.1' } + : { provider: 'groq', model: 'llama-3.3-70b-versatile' } + }) + mocks.getOpenAIResponsesCompletion.mockResolvedValue({}) + mocks.parseOpenAIResponsesCompletion.mockImplementation(async () => { + iteration++ + return { shouldContinue: true, tokenUsage } + }) + mocks.getCompletion.mockResolvedValue({}) + mocks.parseOpenAICompletion.mockResolvedValue({ shouldContinue: false, tokenUsage }) + + await runChatLoop(config) + + // the vision iteration carries the image... + const first = mocks.getOpenAIResponsesCompletion.mock.calls[0][0] + expect(JSON.stringify(first)).toContain('image_url') + // ...the text-only iteration must not + expect(mocks.getCompletion).toHaveBeenCalled() + const second = mocks.getCompletion.mock.calls[0][0] + expect(JSON.stringify(second)).not.toContain('image_url') + }) + + // A history whose images together exceed the provider request-size limit gets + // the whole request rejected with a 413 the vision-rejection fallback cannot + // classify — the loop must keep the outbound copy under the byte cap. + it('drops the oldest images when the history exceeds the total byte cap', async () => { + const config = createConfig({ workspace: `workspace-${randomUUID()}` }) + const bigImage = () => ({ + type: 'image_url', + // two of these exceed MAX_TOTAL_IMAGE_BYTES (12MB decoded) + image_url: { url: 'data:image/png;base64,' + 'A'.repeat(9_000_000) } + }) + config.messages.splice( + 0, + config.messages.length, + { role: 'user', content: [{ type: 'text', text: 'old' }, bigImage()] } as any, + { role: 'assistant', content: 'ok' }, + { role: 'user', content: [{ type: 'text', text: 'new' }, bigImage()] } as any + ) + mocks.getOpenAIResponsesCompletion.mockResolvedValue({}) + mocks.parseOpenAIResponsesCompletion.mockResolvedValue({ shouldContinue: false, tokenUsage }) + + await runChatLoop(config) + + const sent = mocks.getOpenAIResponsesCompletion.mock.calls[0][0] as any[] + const users = sent.filter((m) => m.role === 'user') + // the oldest message's image is stripped to a placeholder... + expect(users[0].content.some((p: any) => p.type === 'image_url')).toBe(false) + expect(JSON.stringify(users[0].content)).toContain('[image omitted]') + // ...the newest keeps its image part + expect(users[1].content.some((p: any) => p.type === 'image_url')).toBe(true) + // the stored history is untouched + expect((config.messages[0] as any).content.some((p: any) => p.type === 'image_url')).toBe(true) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/chatLoop.ts b/frontend/src/lib/components/copilot/chat/chatLoop.ts index cd1421dd0b..49afb03a55 100644 --- a/frontend/src/lib/components/copilot/chat/chatLoop.ts +++ b/frontend/src/lib/components/copilot/chat/chatLoop.ts @@ -12,7 +12,8 @@ import { type ReasoningProviderModel } from '../reasoningRegistry' import { getAnthropicCompletion, parseAnthropicCompletion } from './anthropic' -import { usesAnthropicMessagesApi } from '../modelConfig' +import { modelSupportsVision, usesAnthropicMessagesApi } from '../modelConfig' +import { boundImagePartBytes, stripImagePartsFromMessages } from './imageUtils' import { getOpenAIResponsesCompletion, parseOpenAIResponsesCompletion } from './openai-responses' import type { Tool, ToolCallbacks } from './shared' import { sanitizeToolCallArguments } from './toolCallArguments' @@ -65,8 +66,13 @@ export interface ChatLoopConfig { * lets the caller recover partial output if the loop throws or is aborted. */ addedMessages?: ChatCompletionMessageParam[] - /** Called before each iteration (e.g. to refresh tool schemas). */ - onBeforeIteration?: (tools: Tool[], helpers: any) => Promise + /** Called before each iteration (e.g. to refresh tool schemas, or to record + * which model the iteration is about to use). */ + onBeforeIteration?: ( + tools: Tool[], + helpers: any, + modelProvider: ReasoningProviderModel + ) => Promise } export interface ChatLoopResult { @@ -315,7 +321,7 @@ export async function runChatLoop(config: ChatLoopConfig): Promise t.def) diff --git a/frontend/src/lib/components/copilot/chat/global/core.test.ts b/frontend/src/lib/components/copilot/chat/global/core.test.ts index 407870433d..cf859efd34 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.test.ts @@ -3691,8 +3691,27 @@ describe('session-only preview tools gating', () => { expect(names).toContain('get_preview_status') expect(names).toContain('get_app_runtime_logs') expect(names).toContain('list_app_runs') - // session set is the full globalTools - expect(names.length).toBe(globalTools.length) + // The session set is the full globalTools minus capability-gated tools: + // this environment is not Chromium, so take_screenshot is withheld (DOM + // capture is only faithful on Blink). + expect(names).not.toContain('take_screenshot') + expect(names.length).toBe(globalTools.length - 1) + }) + + it('offers take_screenshot inside a session only on Chromium', () => { + vi.stubGlobal('navigator', { + userAgentData: { brands: [{ brand: 'Chromium', version: '138' }] }, + userAgent: 'stubbed' + }) + try { + const names = toolNames(true) + expect(names).toContain('take_screenshot') + expect(names.length).toBe(globalTools.length) + // still session-only, even on Chromium + expect(toolNames(false)).not.toContain('take_screenshot') + } finally { + vi.unstubAllGlobals() + } }) it('mentions open_preview / get_app_runtime_logs / list_app_runs in the system prompt only when preview tools are enabled', () => { diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index f9ba044f0a..b177233de4 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -46,6 +46,10 @@ import { } from '$lib/components/raw_apps/templates' import { DEFAULT_DATA as DEFAULT_RAW_APP_DATA } from '$lib/components/raw_apps/dataTableRefUtils' import { appSourceToDraftValue } from '$lib/components/raw_apps/rawAppDraftValue' +import { dataUrlToImagePart, normalizeImageDataUrl, type AttachedImage } from '../imageUtils' +import { modelSupportsVision } from '../../modelConfig' +import { tryGetCurrentModel } from '$lib/aiStore' +import { isChromiumBrowser } from '$lib/utils' import { applyEditableFlowJsonToFlow, buildEditableFlowJson, @@ -192,6 +196,8 @@ export type GlobalActiveEditorContext = { export type GlobalUserMessageOptions = { workspace?: string activeEditor?: GlobalActiveEditorContext + /** Images attached to this message; delivered as image_url content parts. */ + images?: AttachedImage[] } const itemTypeSchema = z.enum(ITEM_TYPES) @@ -805,6 +811,8 @@ const listAppRunsSchema = z.object({ .describe('How many of the most recent backend runs to return, newest first. Defaults to 20.') }) +const takeScreenshotSchema = z.object({}) + const FRAMEWORK_KEYS = [ 'react19', 'react18', @@ -930,6 +938,11 @@ Rules: - Building a data pipeline: call open_preview(kind="pipeline", path="") as the FIRST step, before creating any node — this opens the pipeline editor the user reviews in. path is the folder, not an item; an empty or not-yet-created folder is fine (create_folder first if needed, then open it). Opening it registers build_pipeline_node / edit_pipeline_node — use ONLY those to add or change pipeline nodes, never write_script for a pipeline node — they apply directly as unsaved drafts on the canvas (no separate accept/reject step) that the user reviews and deploys. Do not write pipeline scripts without first opening the editor. - When debugging a running raw app, call get_app_runtime_logs to read the live preview's browser console output. It needs the raw app preview open (open_preview kind="raw_app"). - get_app_runtime_logs only shows the app's browser console. For the server-side logs of a backend runnable the app invoked (a backend. call), call list_app_runs to get that run's job_id from the live preview, then get_job_logs with it. Use this when a backend call errors or returns something unexpected. +${ + isChromiumBrowser() + ? `- When the user raises how a raw app looks (something is off, or they want the design or layout improved), call take_screenshot to see what they are looking at before changing anything. Reach for it when the request is about appearance, not to review your own edits, which you can read back from the code. It needs the raw app preview open (open_preview kind="raw_app").` + : `- When the user raises how a raw app looks (something is off, or they want the design or layout improved) and their description alone isn't specific enough to pinpoint the problem, ask them to paste or drop a screenshot of it into the chat before changing anything.` +} - open_page opens its page as a tab in the side-panel preview next to the chat — the only way to show one of these pages there (open_preview only handles editable items). Changing filters on a page already open updates that same tab; only pass new_tab when the user explicitly asks for a separate tab. - create_artifact saves a persistent markdown document (a planning doc, design write-up, spec, or other longer structured output) shown in the session preview panel. Prefer it over a long inline reply for content the user will revisit; keep brief answers inline. To revise one, call list_artifacts then read_artifact for the current content, then update_artifact to overwrite it — never create a second artifact for the same document.` : '' @@ -2908,6 +2921,54 @@ export const globalTools: Tool<{}>[] = [ return result.aiResult } }, + { + def: createToolDef( + takeScreenshotSchema, + 'take_screenshot', + // Keep this short: every global session iteration re-sends it. How to read + // the result belongs on the result, where only a real capture pays for it. + 'Capture a screenshot of the raw app preview currently open in this AI session and attach it as an image so you can see the rendered UI. Use it when the user raises how the app looks, whether reporting a problem or asking for the design improved, rather than to check your own edits. The image is attached in the following message. Requires the raw app preview open (open_preview kind="raw_app").' + ), + showDetails: true, + fn: async (ctx) => { + // A known text-only model would reject the follow-up image message and fail + // the turn, so refuse before capturing rather than buffer an image it can + // never read. The model is re-read here because it can change between turns. + const model = tryGetCurrentModel() + if (model && !modelSupportsVision(model.provider, model.model)) { + const cannotSee = `${model.model} cannot read images, so a screenshot would be discarded. Ask the user to describe what looks wrong, or to switch to a model that supports images.` + ctx.toolCallbacks.setToolStatus(ctx.toolId, { + content: `${model.model} cannot read images`, + error: cannotSee + }) + return cannotSee + } + ctx.toolCallbacks.setToolStatus(ctx.toolId, { content: 'Capturing screenshot...' }) + const result = await getSessionScreenshot(sessionIdFromCtx(ctx)) + if (!result.dataUrl) { + ctx.toolCallbacks.setToolStatus(ctx.toolId, { + content: result.uiMessage ?? 'Screenshot unavailable', + error: result.error + }) + return result.error ?? 'Could not capture the app preview.' + } + // Normalize (downscale + png/jpeg) so history/context never carry a full-res blob; + // buffered here and flushed as a follow-up user image message once the tool batch + // completes (see appendPendingToolImages). + const image = await normalizeImageDataUrl(result.dataUrl) + ctx.toolCallbacks.attachToolImage?.(ctx.toolId, image) + // The card shows the same copy the model gets; sharing the exact data URL + // lets the history's blob store persist one copy for both. + ctx.toolCallbacks.setToolStatus(ctx.toolId, { + content: 'Screenshot captured', + imageUrl: image.dataUrl + }) + return ( + 'Screenshot captured; the image is attached in the following message.\n\n' + + 'It is rebuilt from the DOM rather than captured from the screen, so it can differ from what the user sees, and it differs by browser. Treat what you see as real and fix it. Before dismissing anything as a capture artifact, read the source for that element and name the specific cause; if you cannot, it is a real bug. If you are still unsure, say what looks wrong and ask the user to screenshot it themselves and drag the image into the chat rather than guessing.' + ) + } + }, // Workspace-scoped datatable tools (unrestricted: no whitelist, no creation policy) ...getDatatableTools(), // Read-only tools over files the user attached to the conversation @@ -2923,6 +2984,7 @@ export const SESSION_PREVIEW_TOOL_NAMES = new Set([ 'close_page', 'get_app_runtime_logs', 'list_app_runs', + 'take_screenshot', 'create_artifact', 'update_artifact', 'list_artifacts', @@ -2935,9 +2997,16 @@ export const SESSION_PREVIEW_TOOL_NAMES = new Set([ * global side-panel chat. */ export function globalToolsFor({ sessionPreview }: { sessionPreview: boolean }): Tool<{}>[] { - return sessionPreview + const tools = sessionPreview ? globalTools : globalTools.filter((t) => !SESSION_PREVIEW_TOOL_NAMES.has(t.def.function.name)) + // DOM capture re-renders the app through the engine's SVG-image path, which is + // only faithful on Blink — Gecko/WebKit shift text spacing and wrapping (font + // fallback, sub-pixel rounding). Elsewhere the tool is withheld entirely and + // the system prompt tells the agent to ask the user for a screenshot instead. + return isChromiumBrowser() + ? tools + : tools.filter((t) => t.def.function.name !== 'take_screenshot') } type WriteDraftCtx = { @@ -3149,6 +3218,28 @@ function getSessionAppRuns( return Promise.resolve(listAppRunsHandler({ sessionId, limit })) } +export type SessionScreenshotResult = { dataUrl?: string; error?: string; uiMessage?: string } +export type ScreenshotHandler = (req: { + sessionId: string | undefined +}) => Promise + +let screenshotHandler: ScreenshotHandler | undefined + +export function setScreenshotHandler(handler: ScreenshotHandler | undefined): void { + screenshotHandler = handler +} + +function getSessionScreenshot(sessionId: string | undefined): Promise { + if (!screenshotHandler) { + return Promise.resolve({ + error: + 'Error: take_screenshot is only available inside an AI session with a raw app preview open. Ask the user to open the raw app preview (open_preview kind="raw_app"), then try again.', + uiMessage: 'Screenshot unavailable' + }) + } + return screenshotHandler({ sessionId }) +} + // Registered by the session runtime to reload the open preview after a chat // deploy. Undefined outside a session. export type DeployedInSessionHandler = (req: { @@ -5264,6 +5355,20 @@ export function prepareGlobalUserMessage( content += `## INSTRUCTIONS:\n${instructions}` + const images = options.images ?? [] + if (images.length > 0) { + // Multimodal message: the text block plus one image_url part per attachment. + // The provider converters translate image_url for Anthropic/Responses; the + // OpenAI-compatible path sends it as-is. + return { + role: 'user', + content: [ + { type: 'text', text: content }, + ...images.map((img) => dataUrlToImagePart(img.dataUrl)) + ] + } + } + return { role: 'user', content diff --git a/frontend/src/lib/components/copilot/chat/imageUtils.test.ts b/frontend/src/lib/components/copilot/chat/imageUtils.test.ts new file mode 100644 index 0000000000..136c556258 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/imageUtils.test.ts @@ -0,0 +1,173 @@ +import { describe, expect, it } from 'vitest' +import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs' +import { + boundImagePartBytes, + captureScale, + fileToAttachedImage, + imagesFromContent, + MAX_IMAGE_BYTES, + messagesHaveImageParts, + parseImageDataUrl, + stripImagePartsFromMessages +} from './imageUtils' + +describe('fileToAttachedImage size bound', () => { + // Decoding allocates ~4 bytes per pixel before the downscale can run, so an + // oversized file must be refused up front. + it('rejects a file over the byte cap', async () => { + const blob = { size: MAX_IMAGE_BYTES + 1, type: 'image/png' } as unknown as Blob + await expect(fileToAttachedImage(blob)).rejects.toThrow(/too large/i) + }) +}) + +describe('parseImageDataUrl', () => { + it('splits media type and base64 payload', () => { + expect(parseImageDataUrl('data:image/png;base64,AAAA')).toEqual({ + mediaType: 'image/png', + base64: 'AAAA' + }) + expect(parseImageDataUrl('data:image/jpeg;base64,ZZ==')).toEqual({ + mediaType: 'image/jpeg', + base64: 'ZZ==' + }) + }) + + it('defaults to png and empty payload on a malformed url', () => { + expect(parseImageDataUrl('not-a-data-url')).toEqual({ mediaType: 'image/png', base64: '' }) + }) +}) + +describe('stripImagePartsFromMessages', () => { + it('replaces image parts with a placeholder and collapses to a string', () => { + const messages: ChatCompletionMessageParam[] = [ + { + role: 'user', + content: [ + { type: 'text', text: 'look at this' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,HUGEBLOB' } } + ] + } as any + ] + const out = stripImagePartsFromMessages(messages) + expect(out[0].content).toBe('look at this\n[image omitted]') + }) + + it('leaves image-free messages untouched (same reference)', () => { + const messages: ChatCompletionMessageParam[] = [{ role: 'user', content: 'plain' }] + const out = stripImagePartsFromMessages(messages) + expect(out[0]).toBe(messages[0]) + }) +}) + +describe('boundImagePartBytes', () => { + const imgMsg = (payloadChars: number, text: string): ChatCompletionMessageParam => + ({ + role: 'user', + content: [ + { type: 'text', text }, + { + type: 'image_url', + image_url: { url: 'data:image/png;base64,' + 'A'.repeat(payloadChars) } + } + ] + }) as any + + it('returns the same array when everything fits', () => { + const messages = [imgMsg(100, 'a')] + expect(boundImagePartBytes(messages, 1000)).toBe(messages) + }) + + const imageParts = (m: ChatCompletionMessageParam) => + (m.content as any[]).filter((p) => p?.type === 'image_url').length + + it('strips the oldest images first once the cap is exceeded', () => { + // 1000 base64 chars ≈ 750 bytes each: the newest fits alone, both together don't + const messages = [ + imgMsg(1000, 'old'), + { role: 'assistant', content: 'ok' } as ChatCompletionMessageParam, + imgMsg(1000, 'new') + ] + const out = boundImagePartBytes(messages, 1000) + expect(imageParts(out[0])).toBe(0) + expect(JSON.stringify(out[0].content)).toContain('[image omitted]') + expect(out[1]).toBe(messages[1]) + expect(imageParts(out[2])).toBe(1) + }) + + // An over-cap batch on the CURRENT turn must keep the subset that fits, not + // silently send a text-only message while the composer showed attached images. + // The newest parts win: for screenshot follow-ups the last image is the app's + // current state. + it('keeps the newest fitting subset when the newest message alone exceeds the cap', () => { + const url = (marker: string) => ({ + type: 'image_url', + image_url: { url: 'data:image/png;base64,' + marker.repeat(1000) } + }) + const messages = [ + { + role: 'user', + content: [{ type: 'text', text: 'batch' }, url('A'), url('B'), url('C')] + } as any + ] + const out = boundImagePartBytes(messages, 1600) + // 750 bytes each against a 1600-byte cap: the two NEWEST fit, the oldest drops + const content = out[0].content as any[] + expect(content[1]).toEqual({ type: 'text', text: '[image omitted]' }) + expect(content[2].image_url.url).toContain('B') + expect(content[3].image_url.url).toContain('C') + }) +}) + +describe('messagesHaveImageParts', () => { + it('detects an image part anywhere in the history', () => { + const messages: ChatCompletionMessageParam[] = [ + { role: 'user', content: 'plain' }, + { + role: 'user', + content: [{ type: 'image_url', image_url: { url: 'data:image/png;base64,A' } }] + } as any + ] + expect(messagesHaveImageParts(messages)).toBe(true) + }) + + it('is false for string content and image-free part arrays', () => { + const messages: ChatCompletionMessageParam[] = [ + { role: 'user', content: 'plain' }, + { role: 'user', content: [{ type: 'text', text: 'also plain' }] } as any + ] + expect(messagesHaveImageParts(messages)).toBe(false) + }) +}) + +describe('imagesFromContent', () => { + it('recovers image parts and skips text (including the omitted placeholder)', () => { + const content = [ + { type: 'text', text: 'look' }, + { type: 'image_url', image_url: { url: 'data:image/jpeg;base64,AAAA' } }, + { type: 'text', text: '[image omitted]' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,BBBB' } } + ] + expect(imagesFromContent(content)).toEqual([ + { dataUrl: 'data:image/jpeg;base64,AAAA', mediaType: 'image/jpeg' }, + { dataUrl: 'data:image/png;base64,BBBB', mediaType: 'image/png' } + ]) + }) + + it('is undefined for string content and image-free part arrays', () => { + expect(imagesFromContent('plain')).toBeUndefined() + expect(imagesFromContent([{ type: 'text', text: 'plain' }])).toBeUndefined() + }) +}) + +describe('captureScale', () => { + it('captures small targets above CSS resolution, capped at 2x', () => { + expect(captureScale(400)).toBe(2) + }) + + it('never yields a raster larger than MAX_IMAGE_EDGE, even below 1x', () => { + // A tall scrolling app body: rasterising at >=1x would allocate an + // unbounded canvas only for normalize to shrink or reject it. + expect(captureScale(10_000) * 10_000).toBe(1568) + expect(captureScale(10_000)).toBeLessThan(1) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/imageUtils.ts b/frontend/src/lib/components/copilot/chat/imageUtils.ts new file mode 100644 index 0000000000..eb8dd3b8ec --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/imageUtils.ts @@ -0,0 +1,273 @@ +/** + * Image handling shared by the two multimodal chat features: user-attached images + * (drag/drop/paste, GLOBAL mode) and the app agent's `take_screenshot` tool. + * + * Every image the model sees passes through here first so it is bounded in BOTH + * dimensions (≤ MAX_EDGE longest side — beyond this the provider downscales anyway + * and just bills more tokens) and bytes. Bounding bytes matters because the data URL + * rides every request (stateless APIs resend the whole history) and is persisted in + * the chat history's blob store. Everything is rasterised to PNG/JPEG so exotic + * inputs (SVG, WebP, HEIC where the browser can decode it) become a media type all + * providers accept. + */ +import type { + ChatCompletionContentPartImage, + ChatCompletionMessageParam +} from 'openai/resources/index.mjs' + +/** Longest-edge cap. Matches the point past which vision models downscale server-side. */ +export const MAX_IMAGE_EDGE = 1568 +/** Above this many bytes a PNG re-encodes to JPEG to keep history/storage bounded. */ +const PNG_SIZE_CAP = 700_000 +/** + * Refuse a file this large before reading it. Decoding allocates ~4 bytes per pixel + * — a 12MP photo is ~48MB of bitmap — and the downscale below can only run once that + * bitmap exists, so the cap has to bite before the read, not after. + */ +export const MAX_IMAGE_BYTES = 20_000_000 +/** Decoded-pixel ceiling, in case a small file expands to an absurd bitmap. */ +const MAX_IMAGE_PIXELS = 40_000_000 +/** + * Images one message may carry. Enforced wherever a message is assembled, not just + * at the composer: queuing clears the composer, so its own count would reset and let + * repeated sends stack an unbounded batch into a single message. + */ +export const MAX_ATTACHED_IMAGES = 8 + +export type ImageMediaType = 'image/png' | 'image/jpeg' + +/** A model-ready image: a normalised (bounded, png/jpeg) data URL plus its media type. */ +export type AttachedImage = { + dataUrl: string + mediaType: ImageMediaType + /** Original filename when it came from a user file; absent for screenshots. */ + name?: string +} + +/** Stands in for a stripped or evicted image part in message content. */ +export const IMAGE_OMITTED_PLACEHOLDER = '[image omitted]' + +/** + * Recover the model's own images from an API message's content parts. Anything + * resending a turn (retry, edit) must read images from here, never from the + * transcript bubble: a provider rejection strips them from history while the + * bubble keeps its copy so the user can still see what they sent — resending + * that copy would re-attach the image the provider just refused. + */ +export function imagesFromContent(content: unknown): AttachedImage[] | undefined { + if (!Array.isArray(content)) return undefined + const images = (content as any[]).flatMap((part): AttachedImage[] => { + if (part?.type !== 'image_url' || typeof part?.image_url?.url !== 'string') return [] + const dataUrl = part.image_url.url as string + return [ + { + dataUrl, + mediaType: + parseImageDataUrl(dataUrl).mediaType === 'image/jpeg' ? 'image/jpeg' : 'image/png' + } + ] + }) + return images.length > 0 ? images : undefined +} + +/** + * Raster scale for a DOM screenshot of a target whose longest CSS edge is + * `cssEdge`. Above CSS resolution (up to 2×) for small targets — the SVG + * re-render is vector, so the extra scale is real detail, not interpolation — + * but never a raster larger than MAX_IMAGE_EDGE: normalize would downscale the + * excess away, and rasterising an oversized body (a tall scrolling app) at ≥1× + * first can allocate a tab-freezing canvas. Sub-1× output is deliberate. + */ +export function captureScale(cssEdge: number): number { + return Math.min(2, MAX_IMAGE_EDGE / Math.max(1, cssEdge)) +} + +export function isImageFile(file: File | Blob): boolean { + return typeof file.type === 'string' && file.type.startsWith('image/') +} + +/** Byte size of a base64 data URL's payload (4 base64 chars → 3 bytes). */ +function base64Bytes(dataUrl: string): number { + const comma = dataUrl.indexOf(',') + const b64 = comma >= 0 ? dataUrl.slice(comma + 1) : dataUrl + const padding = b64.endsWith('==') ? 2 : b64.endsWith('=') ? 1 : 0 + return Math.max(0, Math.floor((b64.length * 3) / 4) - padding) +} + +function loadImage(src: string): Promise { + return new Promise((resolve, reject) => { + const img = new Image() + img.onload = () => resolve(img) + img.onerror = () => reject(new Error('Could not decode image')) + img.src = src + }) +} + +function blobToDataUrl(blob: Blob): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader() + reader.onload = () => resolve(reader.result as string) + reader.onerror = () => reject(reader.error ?? new Error('Could not read file')) + reader.readAsDataURL(blob) + }) +} + +/** PNG by default (lossless — crisp for the common UI-screenshot/diagram case); fall + * back to JPEG only when the PNG would blow the size cap (photographic content). */ +function encodeCanvas(canvas: HTMLCanvasElement): { dataUrl: string; mediaType: ImageMediaType } { + const png = canvas.toDataURL('image/png') + if (base64Bytes(png) <= PNG_SIZE_CAP) { + return { dataUrl: png, mediaType: 'image/png' } + } + // JPEG has no alpha channel and canvas encoders composite transparent pixels + // onto black, which hides dark strokes in a transparent diagram. Flatten onto + // white before encoding. + const flat = document.createElement('canvas') + flat.width = canvas.width + flat.height = canvas.height + const ctx = flat.getContext('2d') + if (ctx) { + ctx.fillStyle = '#ffffff' + ctx.fillRect(0, 0, flat.width, flat.height) + ctx.drawImage(canvas, 0, 0) + } + return { + dataUrl: (ctx ? flat : canvas).toDataURL('image/jpeg', 0.82), + mediaType: 'image/jpeg' + } +} + +/** + * Downscale a data URL to ≤ MAX_IMAGE_EDGE on its longest side and re-encode to + * png/jpeg. Used by both the file-attach path and the screenshot tool. + */ +export async function normalizeImageDataUrl( + dataUrl: string, + name?: string, + maxEdge: number = MAX_IMAGE_EDGE +): Promise { + const img = await loadImage(dataUrl) + const srcW = img.naturalWidth || img.width + const srcH = img.naturalHeight || img.height + if (!srcW || !srcH) throw new Error('Image has no dimensions') + if (srcW * srcH > MAX_IMAGE_PIXELS) throw new Error('Image resolution is too large') + const scale = Math.min(1, maxEdge / Math.max(srcW, srcH)) + const w = Math.max(1, Math.round(srcW * scale)) + const h = Math.max(1, Math.round(srcH * scale)) + const canvas = document.createElement('canvas') + canvas.width = w + canvas.height = h + const ctx = canvas.getContext('2d') + if (!ctx) throw new Error('Canvas 2D context unavailable') + ctx.drawImage(img, 0, 0, w, h) + return { ...encodeCanvas(canvas), name } +} + +/** Read a user-provided image file and produce a bounded, model-ready AttachedImage. */ +export async function fileToAttachedImage(file: File | Blob): Promise { + if (file.size > MAX_IMAGE_BYTES) throw new Error('Image file is too large') + const name = file instanceof File ? file.name : undefined + const dataUrl = await blobToDataUrl(file) + return await normalizeImageDataUrl(dataUrl, name) +} + +/** Split a data URL into its media type and base64 payload (for the Anthropic converter). */ +export function parseImageDataUrl(url: string): { mediaType: string; base64: string } { + const match = /^data:([^;,]+)?(;base64)?,(.*)$/s.exec(url) + if (!match) return { mediaType: 'image/png', base64: '' } + return { mediaType: match[1] || 'image/png', base64: match[2] ? match[3] : '' } +} + +/** Build the OpenAI-format image content part that all three provider paths convert from. */ +export function dataUrlToImagePart(dataUrl: string): ChatCompletionContentPartImage { + return { type: 'image_url', image_url: { url: dataUrl } } +} + +/** Whether any message still carries an image_url content part. */ +export function messagesHaveImageParts(messages: ChatCompletionMessageParam[]): boolean { + return messages.some( + (message) => + Array.isArray(message.content) && + (message.content as any[]).some((part) => part?.type === 'image_url') + ) +} + +/** + * Total decoded image bytes one request may carry. Providers reject the whole + * request body over a size limit (20MB on Bedrock, 32MB direct Anthropic), and + * that 413 never mentions images, so the vision-rejection fallback cannot + * recover it — each request must stay under the limit in the first place. + * Compaction cannot be relied on for this: it triggers on estimated tokens, + * and images are cheap in tokens relative to their bytes. 12MB decoded is + * ~16MB of base64 on the wire, safely under the tightest limit with text. + */ +export const MAX_TOTAL_IMAGE_BYTES = 12_000_000 + +/** + * Keep the request's cumulative image bytes under the cap by stripping the + * OLDEST image parts first (the newest images are the ones the conversation + * is about). Part-granular so a single over-cap batch keeps the subset that + * fits — the newest message never silently loses all its images (one bounded + * image alone cannot exceed the cap). Returns the input array unchanged when + * everything fits. + */ +export function boundImagePartBytes( + messages: ChatCompletionMessageParam[], + cap: number = MAX_TOTAL_IMAGE_BYTES +): ChatCompletionMessageParam[] { + let total = 0 + const drops = new Map>() + for (let i = messages.length - 1; i >= 0; i--) { + const content = messages[i].content + if (!Array.isArray(content)) continue + // Parts walk in reverse too: within a message they are in attachment order, + // and for screenshot follow-ups the last one is the app's current state. + for (let j = (content as any[]).length - 1; j >= 0; j--) { + const part = (content as any[])[j] + if (part?.type !== 'image_url' || typeof part?.image_url?.url !== 'string') continue + total += base64Bytes(part.image_url.url) + if (total > cap) { + if (!drops.has(i)) drops.set(i, new Set()) + drops.get(i)!.add(j) + } + } + } + if (drops.size === 0) return messages + return messages.map((message, i) => { + const drop = drops.get(i) + if (!drop) return message + return { + ...message, + content: (message.content as any[]).map((part, j) => + drop.has(j) ? { type: 'text', text: IMAGE_OMITTED_PLACEHOLDER } : part + ) + } as ChatCompletionMessageParam + }) +} + +/** + * Replace image_url content parts with a short text placeholder, collapsing the + * remaining parts back to a plain string. Used to keep base64 blobs out of the + * summarizer request during compaction (the summary text then stands in for them). + */ +export function stripImagePartsFromMessages( + messages: ChatCompletionMessageParam[] +): ChatCompletionMessageParam[] { + return messages.map((message) => { + if (!Array.isArray(message.content)) return message + let hadImage = false + const text = (message.content as any[]) + .map((part) => { + if (part?.type === 'text') return part.text ?? '' + if (part?.type === 'image_url') { + hadImage = true + return IMAGE_OMITTED_PLACEHOLDER + } + return '' + }) + .filter(Boolean) + .join('\n') + if (!hadImage) return message + return { ...message, content: text } as ChatCompletionMessageParam + }) +} diff --git a/frontend/src/lib/components/copilot/chat/openai-responses.test.ts b/frontend/src/lib/components/copilot/chat/openai-responses.test.ts new file mode 100644 index 0000000000..a91ddc5a17 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/openai-responses.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it, vi } from 'vitest' +import { toResponsesContent } from './openai-responses' + +// openai-responses.ts pulls in the chat client/registry layer at import time; the +// helper under test is pure, so stub those side-effecting modules away. +vi.mock('../lib', () => ({ + createOpenAIProxyClient: vi.fn(), + getAiProxyBaseURL: vi.fn(), + getProviderAndCompletionConfig: vi.fn(), + providerSupportsWebSearch: vi.fn(), + workspaceAIClients: {} +})) + +vi.mock('../reasoningRegistry', () => ({ + applyReasoningToConfig: vi.fn() +})) + +vi.mock('./shared', () => ({ + processToolCall: vi.fn(), + appendPendingToolImages: vi.fn() +})) + +describe('toResponsesContent', () => { + it('passes a plain string through unchanged', () => { + expect(toResponsesContent('hello')).toBe('hello') + }) + + it('maps text parts to input_text and image_url parts to input_image (string url)', () => { + const out = toResponsesContent([ + { type: 'text', text: 'describe this' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,ZZZZ' } } + ]) as any[] + + expect(out).toEqual([ + { type: 'input_text', text: 'describe this' }, + { type: 'input_image', image_url: 'data:image/png;base64,ZZZZ' } + ]) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/openai-responses.ts b/frontend/src/lib/components/copilot/chat/openai-responses.ts index 3a83595f8f..ec3f9dad38 100644 --- a/frontend/src/lib/components/copilot/chat/openai-responses.ts +++ b/frontend/src/lib/components/copilot/chat/openai-responses.ts @@ -13,7 +13,7 @@ import { workspaceAIClients } from '../lib' import { applyReasoningToConfig } from '../reasoningRegistry' -import { processToolCall, type Tool, type ToolCallbacks } from './shared' +import { appendPendingToolImages, processToolCall, type Tool, type ToolCallbacks } from './shared' import type { ResponseStream } from 'openai/lib/responses/ResponseStream.mjs' import type { AIProviderModel } from '$lib/gen' import { openAIResponsesUsageToChatTokenUsage, type ChatTokenUsage } from './tokenUsage' @@ -48,6 +48,23 @@ function setOpenAIWebSearchStatus( } // Conversion utilities for Responses API + +/** + * Translate Chat-Completions message content to Responses-native content. Strings + * pass through; a content-part array maps text→input_text and image_url→input_image + * (Responses takes image_url as a plain string, not the {url} object). + */ +export function toResponsesContent(content: unknown): unknown { + if (!Array.isArray(content)) return content + return content.map((part) => { + if (part?.type === 'text') return { type: 'input_text', text: part.text } + if (part?.type === 'image_url' && part.image_url?.url) { + return { type: 'input_image', image_url: part.image_url.url } + } + return part + }) +} + function convertMessagesToResponsesInput(messages: ChatCompletionMessageParam[]): { instructions?: string input: Array @@ -100,7 +117,7 @@ function convertMessagesToResponsesInput(messages: ChatCompletionMessageParam[]) input.push({ type: 'message' as const, role: m.role === 'developer' ? 'developer' : m.role === 'assistant' ? 'assistant' : 'user', - content: m.content + content: toResponsesContent(m.content) }) } } @@ -462,6 +479,7 @@ export async function parseOpenAIResponsesCompletion( messages.push(messageToAdd) addedMessages.push(messageToAdd) } + appendPendingToolImages(messages, addedMessages, callbacks) return { shouldContinue: true, tokenUsage } } diff --git a/frontend/src/lib/components/copilot/chat/shared.test.ts b/frontend/src/lib/components/copilot/chat/shared.test.ts index fcd848c60d..459ff6f5da 100644 --- a/frontend/src/lib/components/copilot/chat/shared.test.ts +++ b/frontend/src/lib/components/copilot/chat/shared.test.ts @@ -937,3 +937,39 @@ describe('trimJob', () => { expect(job.result).toBe(42) }) }) + +describe('appendPendingToolImages', () => { + // Tool results are string-only, so tool-produced images ride a follow-up + // user message appended after the whole tool batch. It must land in BOTH + // arrays (messages = sent next iteration, addedMessages = committed to + // history) and drain the buffer exactly once — a second flush appending the + // same screenshots again would duplicate them in history. + it('appends one user message to both arrays and drains the buffer once', async () => { + const { appendPendingToolImages } = await import('./shared') + let pending = [{ dataUrl: 'data:image/png;base64,SHOT', mediaType: 'image/png' as const }] + const toolCallbacks = { + setToolStatus: vi.fn(), + takePendingToolImages: () => { + const taken = pending + pending = [] + return taken + } + } + const messages: any[] = [] + const addedMessages: any[] = [] + + appendPendingToolImages(messages, addedMessages, toolCallbacks as any) + + expect(messages).toHaveLength(1) + expect(messages[0]).toBe(addedMessages[0]) + expect(messages[0].role).toBe('user') + expect(messages[0].content[1]).toEqual({ + type: 'image_url', + image_url: { url: 'data:image/png;base64,SHOT' } + }) + + appendPendingToolImages(messages, addedMessages, toolCallbacks as any) + expect(messages).toHaveLength(1) + expect(addedMessages).toHaveLength(1) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index 02f92519b7..1ffcf769e9 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -18,6 +18,7 @@ export const SPECIAL_MODULE_IDS = { } as const import { get } from 'svelte/store' import type { PasteAttachment } from './pasteTokens' +import { dataUrlToImagePart, type AttachedImage } from './imageUtils' import type { CodePieceElement, ContextElement, FlowModuleCodePieceElement } from './context' import { workspaceStore } from '$lib/stores' import type { ExtendedOpenFlow } from '$lib/components/flows/types' @@ -468,6 +469,9 @@ export type UserDisplayMessage = BaseDisplayMessage & { // Collapsed big-paste blobs referenced by tokens in `content`. Lets the // bubble render/expand chips; the LLM message stores the expanded text. pastes?: PasteAttachment[] + // Images the user attached to this message (drag/drop/paste), rendered as + // thumbnails in the bubble. The LLM message carries them as image_url parts. + images?: AttachedImage[] } export type CreatedResourceTriggerKind = @@ -539,6 +543,8 @@ export type ToolDisplayMessage = { showFade?: boolean actions?: ToolDisplayAction[] userQuestion?: UserQuestionDisplay + /** Data URL of an image the tool produced (e.g. take_screenshot), shown on the card. */ + imageUrl?: string } export type AssistantDisplayMessage = BaseDisplayMessage & { @@ -796,6 +802,32 @@ export async function processToolCall({ } } +/** + * Flush images buffered by tools during a batch (via toolCallbacks.attachToolImage) + * as ONE follow-up user message, appended to both `messages` (sent on later + * iterations) and `addedMessages` (committed to history). Call this once per + * completion, right after the whole tool loop — never mid-batch, so every tool_call + * id is already answered by its tool result before this non-tool message. The image + * parts ride the same `image_url` carrier that the provider converters translate. + */ +export function appendPendingToolImages( + messages: ChatCompletionMessageParam[], + addedMessages: ChatCompletionMessageParam[], + toolCallbacks: ToolCallbacks +): void { + const images = toolCallbacks.takePendingToolImages?.() ?? [] + if (images.length === 0) return + const message: ChatCompletionMessageParam = { + role: 'user', + content: [ + { type: 'text', text: 'Screenshot(s) of the app preview:' }, + ...images.map((img) => dataUrlToImagePart(img.dataUrl)) + ] + } + messages.push(message) + addedMessages.push(message) +} + export interface Tool { def: ChatCompletionFunctionTool fn: (p: { @@ -939,6 +971,16 @@ export interface ToolCallbacks { onItemDeployed?: (itemKind: UserDraftItemKind, storagePath: string, deployedPath: string) => void /** A tool discarded a draft: the chat's touch on the item is undone. */ onItemDiscarded?: (itemKind: UserDraftItemKind, storagePath: string) => void + /** + * Buffer an image a tool produced (e.g. take_screenshot). Tool results are + * string-only and OpenAI forbids images in tool messages, so buffered images are + * flushed as a follow-up user message once the whole tool batch is answered (see + * appendPendingToolImages) — appending mid-batch would leave sibling tool_call ids + * unanswered before a non-tool message. + */ + attachToolImage?: (toolId: string, image: AttachedImage) => void + /** Drain every image buffered this batch (insertion order), clearing the buffer. */ + takePendingToolImages?: () => AttachedImage[] } export function createToolDef( diff --git a/frontend/src/lib/components/copilot/lib.test.ts b/frontend/src/lib/components/copilot/lib.test.ts index ccc8b85a65..17a7ad6508 100644 --- a/frontend/src/lib/components/copilot/lib.test.ts +++ b/frontend/src/lib/components/copilot/lib.test.ts @@ -14,6 +14,7 @@ import { parseFimCompletionChoice } from './fim' import { getKnownModelContextWindow, getModelContextWindow, + modelSupportsVision, requiresMaxCompletionTokens } from './modelConfig' import { supportsAutocomplete } from './utils' @@ -255,3 +256,40 @@ describe('model context windows', () => { expect(getModelContextWindow('some-custom-model')).toBe(128000) }) }) + +describe('modelSupportsVision', () => { + // One listed pair pins the lookup mechanism (exact pair match, case-insensitive + // model ids); the set's contents are data, not behavior. + it('refuses images on a listed provider:model pair', () => { + expect(modelSupportsVision('groq' as any, 'llama-3.3-70b-versatile')).toBe(false) + expect(modelSupportsVision('azure_foundry' as any, 'DeepSeek-R1')).toBe(false) + }) + + // The reason this is an exact-match set. Each of these WOULD be wrongly blocked + // by a substring of an entry above, and each takes images via its API. + it.each([ + ['azure_foundry', 'Mistral-Large-3'], // substring of 'mistral-large-2411' + ['azure_foundry', 'Phi-4-multimodal-instruct'], // substring of 'phi-4' + ['openrouter', 'meta-llama/llama-3.2-90b-vision-instruct'], // 'llama-3.2-...' + ['groq', 'meta-llama/llama-4-scout-17b-16e-instruct'], + ['groq', 'qwen/qwen3.6-27b'] + ])('does not let a text-only id shadow the vision model %s/%s', (provider, model) => { + expect(modelSupportsVision(provider as any, model)).toBe(true) + }) + + // Permissive by design: a wrong "no" blocks a working model with no override, + // while a wrong "yes" costs one turn and the failure path recovers it. + it('allows unknown and custom models', () => { + expect(modelSupportsVision('customai' as any, 'some-internal-vlm')).toBe(true) + expect(modelSupportsVision('deepseek' as any, 'deepseek-v9-sees-everything')).toBe(true) + expect(modelSupportsVision(undefined, undefined)).toBe(true) + }) + + // The reason entries are keyed by provider, not id alone: an id proves nothing + // about a different endpoint. A Custom AI deployment serving a vision model + // under a colliding name must not inherit another provider's verdict. + it("does not apply one provider's text-only verdict to another provider's model", () => { + expect(modelSupportsVision('customai' as any, 'deepseek-chat')).toBe(true) + expect(modelSupportsVision('customai' as any, 'llama-3.3-70b-versatile')).toBe(true) + }) +}) diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index 6670789404..ed36c8e42d 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -17,7 +17,12 @@ import { EDIT_CONFIG, FIX_CONFIG, GEN_CONFIG } from './prompts' import { requiresMaxCompletionTokens, usesAnthropicMessagesApi } from './modelConfig' import { applyReasoningToConfig } from './reasoningRegistry' import { formatResourceTypes } from './utils' -import { processToolCall, type Tool, type ToolCallbacks } from './chat/shared' +import { + appendPendingToolImages, + processToolCall, + type Tool, + type ToolCallbacks +} from './chat/shared' import { hasValidToolCallArguments } from './chat/toolCallArguments' import { getNonStreamingOpenAIResponsesCompletion, @@ -1260,6 +1265,7 @@ export async function parseOpenAICompletion( messages.push(messageToAdd) addedMessages.push(messageToAdd) } + appendPendingToolImages(messages, addedMessages, callbacks) } else if (malformedFunctionCallError) { // Malformed function call with no tool calls - create artificial tool call to inform AI const fakeToolCallId = generateRandomString() diff --git a/frontend/src/lib/components/copilot/modelConfig.ts b/frontend/src/lib/components/copilot/modelConfig.ts index af0dd942b2..62e5a3542a 100644 --- a/frontend/src/lib/components/copilot/modelConfig.ts +++ b/frontend/src/lib/components/copilot/modelConfig.ts @@ -72,3 +72,69 @@ export function getModelContextWindow(model: string) { // Trim/compaction logic needs a number; assume a conservative window when unknown. return getKnownModelContextWindow(model) ?? 128000 } + +/** + * Best-effort check that a model can accept image input. There is no per-model vision + * metadata in the codebase, so this is deliberately permissive: it returns true unless + * the model is a known text-only one that would 400 on an image part. Used to gate the + * image-attach affordance and the screenshot follow-up; when unsure it allows the image + * (the user explicitly attached it — better to try than to silently drop it). + */ +export function modelSupportsVision( + provider: AIProvider | undefined, + model: string | undefined +): boolean { + if (!provider) return true + return !TEXT_ONLY_MODELS.has(`${provider}:${(model ?? '').toLowerCase()}`) +} + +/** + * Models whose provider API refuses image content, matched by exact + * `provider:model` pair — not by id alone, because an id proves nothing about a + * different endpoint (a Custom AI deployment may serve a vision model under a + * name that collides with someone's text-only id, and there is no override). + * + * The question is not whether a model can see, but whether its provider's API + * accepts image parts — the two diverge, and the divergence is invisible from a + * name: DeepSeek V4 ships vision in its chat UI while its API has no image + * content type, and o3-mini gained vision in ChatGPT that the API never exposed. + * So this is a cache of one provider's API surface at one moment, and it rots. + * Wrong entries are asymmetric: a missing one costs a single turn and + * self-corrects (the request fails, the image is dropped, the user is told), + * while a wrong one blocks a working model with no override. Hence exact pairs + * only, and only where a provider doc says so. + * + * Substrings are specifically avoided: `mistral-large` would also match + * Mistral Large 3, which does take images, and `phi-4` would match + * Phi-4-multimodal, which does too. + */ +const TEXT_ONLY_MODELS = new Set([ + 'openai:o1-mini', + 'openai:o3-mini', + 'azure_openai:o1-mini', + 'azure_openai:o3-mini', + 'mistral:codestral-latest', + // deepseek — vision exists in their chat product, not in the API + 'deepseek:deepseek-v4-pro', + 'deepseek:deepseek-v4-flash', + 'deepseek:deepseek-chat', + 'deepseek:deepseek-reasoner', + 'groq:llama-3.3-70b-versatile', + 'groq:llama-3.1-8b-instant', + // gpt-oss (text-only everywhere it is hosted) — on groq it succeeds the two + // llama defaults above, which retire 2026-08-16 + 'groq:openai/gpt-oss-120b', + 'groq:openai/gpt-oss-20b', + 'openrouter:openai/gpt-oss-120b', + 'openrouter:openai/gpt-oss-20b', + 'togetherai:openai/gpt-oss-120b', + 'togetherai:openai/gpt-oss-20b', + // azure_foundry serves DeepSeek-V4-Pro under the same id as deepseek's API + 'azure_foundry:deepseek-v4-pro', + 'azure_foundry:deepseek-r1', + 'azure_foundry:llama-3.3-70b-instruct', + 'azure_foundry:phi-4', + 'azure_foundry:mistral-large-2411', + 'openrouter:meta-llama/llama-3.2-3b-instruct:free', + 'togetherai:meta-llama/llama-3.3-70b-instruct-turbo' +]) diff --git a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte index 36988384f7..f97bef9706 100644 --- a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte @@ -23,7 +23,8 @@ type RawAppRuntimeLogEntry, type RawAppRuntimeLogRequester, type RawAppRunSummary, - type RawAppRunsProvider + type RawAppRunsProvider, + type RawAppScreenshotRequester } from './utils' import DarkModeObserver from '../DarkModeObserver.svelte' import RawAppSidebar from './RawAppSidebar.svelte' @@ -37,6 +38,7 @@ InspectorElementInfo } from '../copilot/chat/app/core' import { createAppSelectedContext, type AppCodeSelectionElement } from '../copilot/chat/context' + import { captureScale, MAX_IMAGE_EDGE } from '../copilot/chat/imageUtils' import { rawAppLintStore } from './lintStore' import { dbSchemas } from '$lib/stores' import { @@ -125,6 +127,7 @@ onOpenOthersDrafts?: () => void onRuntimeLogRequester?: (requester: RawAppRuntimeLogRequester | undefined) => void onRunsProvider?: (provider: RawAppRunsProvider | undefined) => void + onScreenshotRequester?: (requester: RawAppScreenshotRequester | undefined) => void // Restoring an older deployment from the history drawer. A callback prop // (not `on:restore` forwarding): forwarding a `createEventDispatcher` // event up through these runes-mode components silently drops it. @@ -166,6 +169,7 @@ onOpenOthersDrafts, onRuntimeLogRequester = undefined, onRunsProvider = undefined, + onScreenshotRequester = undefined, onRestore, onSavedNewAppPath, condensedHeader = false @@ -1352,12 +1356,121 @@ return out.reverse() } + // Only values whose non-wrapping counterpart collapses whitespace identically. + // `pre-line`/`break-spaces` have no such counterpart: forcing them to nowrap + // would eat their preserved newlines, so they are left to re-wrap. + const NON_WRAPPING_EQUIVALENT: Record = { + normal: 'nowrap', + 'pre-wrap': 'pre' + } + + // getClientRects yields a rect per contained node, not per line box, so the + // count alone says nothing: `a b` is two rects on one line. Rects + // sharing a line overlap vertically, and `top` alone would split a line that + // mixes font sizes — so count vertically disjoint runs. + function countLines(range: Range): number { + const rects = Array.from(range.getClientRects()).filter((r) => r.width > 0 || r.height > 0) + if (rects.length === 0) return 0 + rects.sort((a, b) => a.top - b.top) + let lines = 1 + let lineBottom = rects[0].bottom + for (const r of rects) { + if (r.top >= lineBottom) { + lines++ + lineBottom = r.bottom + } else { + lineBottom = Math.max(lineBottom, r.bottom) + } + } + return lines + } + + // A box that shrink-wraps its text can have zero sub-pixel slack (a 208.59px box + // holding a 208.59px text run). The capture re-runs layout in whole pixels, so + // the text no longer fits, wraps, and is then clipped out of the box entirely. + // Pinning runs that are already single-line is a no-op on the live DOM but stops + // the re-layout from re-deciding where they break. + function pinSingleLineText(root: HTMLElement): () => void { + const doc = root.ownerDocument + const view = doc.defaultView + if (!view) return () => {} + // Measure every candidate before mutating any of them: interleaving reads and + // writes forces a synchronous reflow per element. + const pending: Array<[HTMLElement, string]> = [] + const walker = doc.createTreeWalker(root, NodeFilter.SHOW_ELEMENT) + let node: Node | null + while ((node = walker.nextNode())) { + const el = node as HTMLElement + if (!(el instanceof view.HTMLElement)) continue + const hasOwnText = Array.from(el.childNodes).some( + (c) => c.nodeType === Node.TEXT_NODE && (c.textContent ?? '').trim() !== '' + ) + if (!hasOwnText) continue + const replacement = NON_WRAPPING_EQUIVALENT[view.getComputedStyle(el).whiteSpace] + if (!replacement) continue + const range = doc.createRange() + range.selectNodeContents(el) + if (countLines(range) !== 1) continue // already wraps — leave its breaks alone + pending.push([el, replacement]) + } + const restores = pending.map(([el, replacement]) => { + const prev = el.style.getPropertyValue('white-space') + const prio = el.style.getPropertyPriority('white-space') + el.style.setProperty('white-space', replacement, 'important') + return () => { + if (prev) el.style.setProperty('white-space', prev, prio) + else el.style.removeProperty('white-space') + } + }) + return () => restores.forEach((r) => r()) + } + + // Capture the live preview as a PNG data URL. The preview iframe + // (/ui_builder/app-preview.html) is same-origin with no sandbox, so its rendered + // document is reachable and can be serialized from here. There is no native + // element-screenshot API; modern-screenshot reconstructs the DOM into an SVG + // foreignObject, so a WebGL canvas is only captured when its context was created + // with preserveDrawingBuffer. Lazy-imported so the library only loads on demand. + const captureScreenshot: RawAppScreenshotRequester = async () => { + const target = previewIframe?.contentDocument?.body + if (!previewIframe || !previewIframeLoaded || !target) { + throw new Error('App preview is not ready') + } + // Collapsing the preview leaves the iframe mounted and populated at zero + // width, which passes every check above and then fails inside the rasteriser + // as an opaque decode error. Name the cause so the agent can act on it. + if (!target.clientWidth || !target.clientHeight) { + throw new Error( + 'The app preview is collapsed, so there is nothing to capture. Ask the user to expand the preview panel, then try again.' + ) + } + const { domToPng } = await import('modern-screenshot') + // Above CSS resolution for small previews (a 1× capture of a ~900px preview + // reads blurry next to the live render), sub-1× for oversized bodies — see + // captureScale. maximumCanvasSize is the belt over that math: the rasterised + // box can exceed the body's client size, and an unbounded canvas on a tall + // scrolling app can freeze the tab before normalize ever bounds the pixels. + const scale = captureScale(Math.max(target.clientWidth, target.clientHeight)) + const restore = pinSingleLineText(target) + try { + return await domToPng(target, { + backgroundColor: '#ffffff', + scale, + maximumCanvasSize: MAX_IMAGE_EDGE + }) + } finally { + restore() + } + } + onMount(() => { onRuntimeLogRequester?.(requestRuntimeLogs) onRunsProvider?.(getRuns) + onScreenshotRequester?.(captureScreenshot) return () => { onRuntimeLogRequester?.(undefined) onRunsProvider?.(undefined) + onScreenshotRequester?.(undefined) for (const requestId of Array.from(pendingRuntimeLogReqs.keys())) resolvePendingRuntimeLogRequest(requestId, undefined) } diff --git a/frontend/src/lib/components/raw_apps/utils.ts b/frontend/src/lib/components/raw_apps/utils.ts index c0fba48e18..12dc7d87d4 100644 --- a/frontend/src/lib/components/raw_apps/utils.ts +++ b/frontend/src/lib/components/raw_apps/utils.ts @@ -137,6 +137,11 @@ export type RawAppRunSummary = { } export type RawAppRunsProvider = () => RawAppRunSummary[] | undefined +/** Capture the live raw-app preview as a PNG data URL. Rejects when the preview + * isn't ready. Registered by the editor, dispatched by the session runtime for the + * global `take_screenshot` tool. */ +export type RawAppScreenshotRequester = () => Promise + export function formatAppRunsForChat(runs: RawAppRunSummary[]): string { return JSON.stringify(runs, null, 2) } diff --git a/frontend/src/lib/components/sessions/RawAppEditorView.svelte b/frontend/src/lib/components/sessions/RawAppEditorView.svelte index cda34b9dac..d276c6a93b 100644 --- a/frontend/src/lib/components/sessions/RawAppEditorView.svelte +++ b/frontend/src/lib/components/sessions/RawAppEditorView.svelte @@ -9,7 +9,8 @@ import { invalidateWorkspaceDrafts } from '$lib/workspaceDrafts.svelte' import type { RawAppRuntimeLogRequester, - RawAppRunsProvider + RawAppRunsProvider, + RawAppScreenshotRequester } from '$lib/components/raw_apps/utils' let { @@ -83,6 +84,21 @@ function registerRunsProvider(provider: RawAppRunsProvider | undefined) { runtime.setAppRunsProvider(provider) } + + // The preview host keeps every opened tab mounted, so registering on mount would + // leave a background tab owning the runtime's single screenshot slot and + // take_screenshot would capture an app the user isn't looking at. Ownership + // follows the visible tab instead, and only the owner may release it. + let screenshotRequester = $state(undefined) + function registerScreenshotRequester(requester: RawAppScreenshotRequester | undefined) { + screenshotRequester = requester + } + $effect(() => { + const requester = screenshotRequester + if (!active || !requester) return + runtime.setScreenshotRequester(requester) + return () => runtime.clearScreenshotRequester(requester) + }) {#if cell.saved.val} @@ -141,6 +157,7 @@ defaultSplitWithPreview={false} onRuntimeLogRequester={registerRuntimeLogRequester} onRunsProvider={registerRunsProvider} + onScreenshotRequester={registerScreenshotRequester} /> {/if} {/snippet} diff --git a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts index c8f523e799..81abca977b 100644 --- a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts @@ -60,6 +60,7 @@ import { setGetPreviewStatusHandler, setGetRuntimeLogsHandler, setListAppRunsHandler, + setScreenshotHandler, setOpenPagePreviewHandler, setOpenPreviewHandler } from '$lib/components/copilot/chat/global/core' @@ -69,7 +70,8 @@ import { type RawAppRuntimeLogEntry, type RawAppRuntimeLogRequester, type RawAppRunSummary, - type RawAppRunsProvider + type RawAppRunsProvider, + type RawAppScreenshotRequester } from '$lib/components/raw_apps/utils' import { getNonStreamingMetadataCompletion } from '$lib/components/copilot/lib' import type { DisplayMessage } from '$lib/components/copilot/chat/shared' @@ -174,6 +176,10 @@ export interface SessionRuntime { requestRuntimeLogs(limit: number): Promise setAppRunsProvider(provider: RawAppRunsProvider | undefined): void getAppRuns(): RawAppRunSummary[] | undefined + setScreenshotRequester(requester: RawAppScreenshotRequester | undefined): void + /** Release the slot only if `requester` still owns it. */ + clearScreenshotRequester(requester: RawAppScreenshotRequester): void + requestScreenshot(): Promise // Discard the local draft + force-reload the editor, so the preview matches // the deployed version. Used by editor onDeploy + the chat deploy handler. syncPreviewWithDeployed( @@ -446,6 +452,7 @@ function createRuntime(session: Session): SessionRuntime { let runtimeLogRequester: RawAppRuntimeLogRequester | undefined = undefined let appRunsProvider: RawAppRunsProvider | undefined = undefined + let screenshotRequester: RawAppScreenshotRequester | undefined = undefined return { sessionId: session.id, @@ -775,6 +782,19 @@ function createRuntime(session: Session): SessionRuntime { }, getAppRuns() { return appRunsProvider ? appRunsProvider() : undefined + }, + setScreenshotRequester(requester) { + screenshotRequester = requester + }, + clearScreenshotRequester(requester) { + // Tabs unmount in any order and several stay mounted at once, so a + // departing tab must not unregister whichever one now owns the slot. + if (screenshotRequester === requester) { + screenshotRequester = undefined + } + }, + async requestScreenshot() { + return screenshotRequester ? screenshotRequester() : undefined } } } @@ -1047,6 +1067,34 @@ setListAppRunsHandler(({ sessionId: callerSessionId, limit }) => { } }) +setScreenshotHandler(async ({ sessionId: callerSessionId }) => { + const sessionId = callerSessionId ?? sessionState.currentSessionId + const runtime = sessionId ? runtimes.get(sessionId) : undefined + if (!runtime) { + return { + error: + 'Error: take_screenshot is only available inside an AI session. Tell the user screenshots can only be captured from a session preview.', + uiMessage: 'Screenshot unavailable' + } + } + try { + const dataUrl = await runtime.requestScreenshot() + if (dataUrl === undefined) { + return { + error: + 'No raw app preview is open for this session, so there is nothing to screenshot. Next step: call open_preview with kind="raw_app" and the app path, wait for it to load, then call take_screenshot again.', + uiMessage: 'Screenshot unavailable' + } + } + return { dataUrl } + } catch (e) { + return { + error: `Could not capture the app preview: ${e instanceof Error ? e.message : String(e)}`, + uiMessage: 'Screenshot failed' + } + } +}) + export function getSessionChatStatus(runtime: SessionRuntime): SessionChatStatus { const m = runtime.manager if (m.loading) return 'streaming' diff --git a/frontend/src/lib/isChromiumBrowser.test.ts b/frontend/src/lib/isChromiumBrowser.test.ts new file mode 100644 index 0000000000..59e325e407 --- /dev/null +++ b/frontend/src/lib/isChromiumBrowser.test.ts @@ -0,0 +1,45 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { isChromiumBrowser } from './utils' + +// Gates Chromium-only capabilities (DOM screenshot capture). A wrong "true" on +// Gecko/WebKit re-enables the capture path whose spacing/wrapping artifacts the +// gate exists to avoid; a wrong "false" on Chromium silently drops the tool. +describe('isChromiumBrowser', () => { + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('detects Chromium via userAgentData brands (any Chromium-based browser)', () => { + vi.stubGlobal('navigator', { + userAgentData: { + brands: [ + { brand: 'Not-A.Brand', version: '99' }, + { brand: 'Chromium', version: '138' }, + { brand: 'Microsoft Edge', version: '138' } + ] + }, + userAgent: 'anything' + }) + expect(isChromiumBrowser()).toBe(true) + }) + + it('falls back to the UA string when userAgentData is absent (older Chromium)', () => { + vi.stubGlobal('navigator', { + userAgent: + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36' + }) + expect(isChromiumBrowser()).toBe(true) + }) + + it('is false on Firefox ("Chrome/" never appears in a Gecko UA)', () => { + vi.stubGlobal('navigator', { + userAgent: 'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0' + }) + expect(isChromiumBrowser()).toBe(false) + }) + + it('is false with no navigator at all (SSR)', () => { + vi.stubGlobal('navigator', undefined) + expect(isChromiumBrowser()).toBe(false) + }) +}) diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index fd57862f3b..3dd3c722bf 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -844,6 +844,21 @@ export function isMac(): boolean { return navigator.userAgent.indexOf('Mac OS X') !== -1 } +/** + * True on Chromium-based browsers (Chrome, Edge, Brave, Opera, ...). Gates + * capabilities that are only faithful on Blink, e.g. DOM screenshot capture. + * userAgentData is itself Chromium-only; the UA fallback covers Chromium + * versions predating it ("Chrome/" never appears in Gecko or WebKit UAs). + */ +export function isChromiumBrowser(): boolean { + if (typeof navigator === 'undefined') return false + const brands = (navigator as any).userAgentData?.brands + if (Array.isArray(brands)) { + return brands.some((entry) => typeof entry?.brand === 'string' && /chromium/i.test(entry.brand)) + } + return /chrome\//i.test(navigator.userAgent) +} + export function getModifierKey(): string { return isMac() ? '⌘' : 'Ctrl+' } From 8828341a2b0912cbf44d6c6d81b4f4e30c5fe631 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Fri, 17 Jul 2026 19:48:08 +0200 Subject: [PATCH 14/14] fix(frontend): scope session pipeline trigger editors to the session workspace (#10032) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(frontend): scope session pipeline trigger editors to the session workspace Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): hoist triggerWorkspace decl above GCP init-time getBaseUrl call Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): scope nested trigger pickers to the session workspace The triggerWorkspace resolver scoped direct trigger CRUD calls to the session's (forked) workspace, but nested pickers still defaulted to the nav `$workspaceStore`: in a fork session, resource lists/creation, variable creation, and path-existence checks ran against the parent workspace while save/delete targeted the fork — misleading options and false path-validation failures. Thread `wsId` into the nested controls of the 9 pipeline-canvas kinds: - `` (8 editors) — path + folder checks - `` (6 config sections; add the resolver to MqttEditorConfigSection, which lacked `wsId`) - SQS `` — variable creation Also drop the per-site `wsId` rationale comment repeated across ~20 files; the invariant is documented once in triggerWorkspace.ts. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): scope trigger runnable picker to the session workspace The runnable picker (`ScriptPicker`, reached via `TriggerRunnablePicker` and directly in the schedule editor) listed scripts/flows/apps from the nav `$workspaceStore` with no override, so a forked session offered the parent workspace's runnables when attaching a script/flow to a trigger. Add an optional `workspace` prop to `ScriptPicker` (defaults to `$workspaceStore` → no change for existing callers), pass it through `TriggerRunnablePicker`, and wire `wsId` from the 7 trigger editors that use it plus the schedule editor's 3 direct pickers. Completes the nested-picker workspace scoping; the excluded kinds (azure/http/websocket/native) keep their own ScriptPickers unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): scope trigger error-handler, folder default, and runnable actions Follow-up to the trigger-workspace scoping: three subtrees still read the nav `$workspaceStore` in a forked session. - ErrorOrRecoveryHandler (via TriggerRetriesAndErrorHandler in the 7 in-scope editors): add a `workspace` prop (defaults to `$workspaceStore`) and route handler lookup/schema, Slack/Teams settings, test jobs, and run links through it, so the error handler is resolved/tested/saved in the session workspace instead of A while the trigger lives in B. - useFolderDefaultPermissionedAs: accept an optional workspace getter so a `f/...` trigger's default permissioned-as is read from the session workspace, not the nav one (PermissionedAsLine passes `() => wsId`). - ScriptPicker actions: scope the View drawer (`getScriptByPath`) and `FlowPathViewer` to `effectiveWorkspace`, and carry `?workspace=` onto the Edit/View routes when an explicit override is set (the layout consumes the param, same mechanism as editInFork). The param is only appended when a workspace override is passed, so existing callers' links are unchanged. Also consolidate the repeated workspace-scoping comment in PipelineTriggerEditors (the invariant lives in triggerWorkspace.ts). Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): scope error-handler/schedule nested consumers to the session workspace Address CI Codex review on #10032 — the error-handler and schedule subtrees still had nested consumers reading the nav workspace: - ErrorOrRecoveryHandler: pass the resolved workspace to its own nested `ScriptPicker` (custom-handler list + View/Edit) and add a `workspace` prop to `ChannelSelector` (Teams channel listing); carry the acting workspace onto the "create from template" link. - ScheduleEditorInner: pass `workspace={wsId}` to the error/recovery/ success `ErrorOrRecoveryHandler` panels, `workspaceId={wsId}` to `WorkerTagPicker`, and the workspace query param onto the dynamic-skip template link. Template-link and picker overrides only diverge from `$workspaceStore` when a session override is set, so non-session callers are unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- .../src/lib/components/ChannelSelector.svelte | 20 +++---- .../components/ErrorOrRecoveryHandler.svelte | 43 +++++++++----- .../src/lib/components/ScriptPicker.svelte | 35 +++++++---- .../AssetGraph/PipelineTriggerEditors.svelte | 13 +++-- .../flows/content/FlowPathViewer.svelte | 7 ++- .../sessions/PipelineEditorView.svelte | 11 ++-- .../triggers/PermissionedAsLine.svelte | 14 +++-- .../triggers/TestTriggerConnection.svelte | 17 +++--- .../TriggerRetriesAndErrorHandler.svelte | 5 +- .../triggers/TriggerRunnablePicker.svelte | 3 + .../triggers/TriggerSuspendedJobsModal.svelte | 5 +- .../components/triggers/TriggerTokens.svelte | 9 ++- .../EmailTriggerEditorConfigSection.svelte | 7 ++- .../email/EmailTriggerEditorInner.svelte | 14 +++-- .../gcp/GcpTriggerEditorConfigSection.svelte | 15 +++-- .../triggers/gcp/GcpTriggerEditorInner.svelte | 14 +++-- .../kafka/KafkaTriggerEditorInner.svelte | 16 +++-- .../kafka/KafkaTriggersConfigSection.svelte | 6 +- .../mqtt/MqttEditorConfigSection.svelte | 12 +++- .../mqtt/MqttTriggerEditorInner.svelte | 14 +++-- .../nats/NatsTriggerEditorInner.svelte | 14 +++-- .../nats/NatsTriggersConfigSection.svelte | 8 ++- .../postgres/CheckPostgresRequirement.svelte | 5 +- .../PostgresTriggerEditorInner.svelte | 27 +++++---- .../postgres/PublicationPicker.svelte | 11 ++-- .../triggers/postgres/SlotPicker.svelte | 7 ++- .../schedules/ScheduleEditorInner.svelte | 58 ++++++++++++------- .../sqs/SqsTriggerEditorConfigSection.svelte | 15 +++-- .../triggers/sqs/SqsTriggerEditorInner.svelte | 14 +++-- .../components/triggers/triggerWorkspace.ts | 29 ++++++++++ .../webhook/WebhooksConfigSection.svelte | 11 ++-- .../useFolderDefaultPermissionedAs.svelte.ts | 13 +++-- 32 files changed, 346 insertions(+), 146 deletions(-) create mode 100644 frontend/src/lib/components/triggers/triggerWorkspace.ts diff --git a/frontend/src/lib/components/ChannelSelector.svelte b/frontend/src/lib/components/ChannelSelector.svelte index 3211fe914a..d7a720d877 100644 --- a/frontend/src/lib/components/ChannelSelector.svelte +++ b/frontend/src/lib/components/ChannelSelector.svelte @@ -21,6 +21,9 @@ showRefreshButton?: boolean onError?: (error: Error) => void onSelectedChannelChange?: (channel: ChannelItem | undefined) => void + /** Workspace to list Teams channels from; defaults to the nav + * `$workspaceStore`. A forked session passes its acting workspace. */ + workspace?: string } let { @@ -33,9 +36,12 @@ teamId, showRefreshButton = true, onError, - onSelectedChannelChange + onSelectedChannelChange, + workspace = undefined }: Props = $props() + let effectiveWorkspace = $derived(workspace ?? $workspaceStore) + let isFetching = $state(false) let loadedChannels = $state([]) let loadedForTeamId = $state(undefined) @@ -88,7 +94,7 @@ isFetching = true try { const response = await WorkspaceService.listAvailableTeamsChannels({ - workspace: $workspaceStore!, + workspace: effectiveWorkspace!, teamId: teamId }) @@ -130,10 +136,7 @@ clearable disabled={disabled || !teamId} loading={isFetching} - bind:value={ - () => selectedChannel?.channel_id, - (newId) => setSelectedChannelById(newId) - } + bind:value={() => selectedChannel?.channel_id, (newId) => setSelectedChannelById(newId)} /> {:else}