diff --git a/.release-please-manifest.json b/.release-please-manifest.json index ae93300697..e908485536 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.796.0" + ".": "1.800.0" } diff --git a/AGENTS.md b/AGENTS.md index 47919cfeda..71c59b479d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -165,6 +165,15 @@ $NAV --root backend callees "X" # what does X call? - Search for existing code to reuse before writing new code - Follow established patterns in the codebase - Keep changes focused — don't refactor beyond what's asked +- **A simpler design found late is still the design.** Work already spent is not an argument + for a shape, and neither is a clean review round, a passing suite, or a long PR thread. The + signal to stop and re-derive rather than patch again is a change that keeps growing to defend + its own structure: each review finding fixing an assumption the previous fix broke, the same + class of bug reappearing somewhere new, or most of the diff being consequences of one early + choice rather than the thing you set out to do. When that happens, say plainly what the + simpler design is and what switching costs — a migration, a review cycle restarted from zero, + work discarded — and let the user decide. Do not keep paying down the harder one because it + is nearly finished, and do not present the accumulated cost as a reason to continue. - **Ship only the tests the PR needs.** A committed test must pin behavior a future change could plausibly break, and be the smallest setup that exercises the new logic. While developing, write as many exhaustive tests and do as much manual testing as you need to convince yourself the change works — then remove that scaffolding before marking the PR ready, keeping only the essential regression guard(s). A test that merely re-exercises pre-existing behavior, or needs elaborate fixtures to assert something trivial, is scaffolding: delete it. If nothing meaningful is left to guard, ship no test rather than a ceremonial one. - **Comments record constraints, not narration.** Write a comment only for what the code can't show: why a non-obvious approach is required, what breaks if it's "simplified" away. State each invariant once, at the place where someone would break it, in ≤4 lines. Don't describe what the next line does, don't repeat the same rationale at multiple sites, and don't address the PR reviewer (justifying a change belongs in the PR description, not the code). Reference nothing ephemeral — no numbered steps from your dev flow, no "the poller / the test does X" scaffolding, no transient state that won't exist for the next reader; keep only the essential, durable rationale. Describe the code as it is, never its drafting history: "we no longer do X", "unchanged behavior", "instead of the previous approach" are meaningless to a reader who never saw the earlier iteration — before finishing, reread your comments as if the current state is the only state that ever existed. - **Never attribute work to a specific customer, account, or "requested by a customer" in repo-tracked content** (PR descriptions, commit messages, code comments, docs). Describe changes by their technical motivation instead. diff --git a/CHANGELOG.md b/CHANGELOG.md index 947e63f5c6..e27f2166a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,94 @@ # Changelog +## [1.800.0](https://github.com/windmill-labs/windmill/compare/v1.799.0...v1.800.0) (2026-08-31) + + +### Features + +* add --keep-deleted flag to wmill sync pull and push ([#10878](https://github.com/windmill-labs/windmill/issues/10878)) ([66123f3](https://github.com/windmill-labs/windmill/commit/66123f3a9b8978c0084b02f50b44cffba125a13a)) +* day-partition the service log index and expire whole chunks ([#10893](https://github.com/windmill-labs/windmill/issues/10893)) ([d91ee46](https://github.com/windmill-labs/windmill/commit/d91ee4614a70f20a194f47e190327129f499ec63)) +* free AI tokens + home search/filter revamp ([#10020](https://github.com/windmill-labs/windmill/issues/10020)) ([716ce2e](https://github.com/windmill-labs/windmill/commit/716ce2ece00cd5cfb8641afc6432636bc4aa46e9)) +* make the service log retention period an instance setting ([#10889](https://github.com/windmill-labs/windmill/issues/10889)) ([815de49](https://github.com/windmill-labs/windmill/commit/815de49e2322f85ca92b1e41a2bcd22591ebe93f)) +* rework the evals dataset drawer and run navigation ([#10884](https://github.com/windmill-labs/windmill/issues/10884)) ([1462f17](https://github.com/windmill-labs/windmill/commit/1462f17643302127b4bd76bacfde80cc03f9d606)) +* serve service log context from parquet and retire the raw log files ([#10892](https://github.com/windmill-labs/windmill/issues/10892)) ([338d75c](https://github.com/windmill-labs/windmill/commit/338d75cc5227e352cb84828c99bfd3b984cf0fa5)) +* serve service log retrieval from a columnar parquet store ([#10886](https://github.com/windmill-labs/windmill/issues/10886)) ([7c1a785](https://github.com/windmill-labs/windmill/commit/7c1a785f756ed27e4425f6534709b19971a73a97)) + + +### Bug Fixes + +* correct the service log ingest flush boundary ([#10898](https://github.com/windmill-labs/windmill/issues/10898)) ([ac56586](https://github.com/windmill-labs/windmill/commit/ac56586c0e56d4022761d3c80306a03d57f8bfcb)) +* harden the service log indexer's recovery and read paths ([#10904](https://github.com/windmill-labs/windmill/issues/10904)) ([831370c](https://github.com/windmill-labs/windmill/commit/831370cdde8e06f4298b17baa1a0041bacdd98c7)) +* keep raw-app editor selection consistent across sidebar and tabs ([#10885](https://github.com/windmill-labs/windmill/issues/10885)) ([b57e231](https://github.com/windmill-labs/windmill/commit/b57e231c2bf5e5fe007f0aa7b958a51e32b47141)) +* register every rotated service log file exactly once ([#10891](https://github.com/windmill-labs/windmill/issues/10891)) ([c817248](https://github.com/windmill-labs/windmill/commit/c8172480b0b1be6c57210212afc71d6ec8711235)) +* show a loading indicator while the initial data table migration is generated ([#10900](https://github.com/windmill-labs/windmill/issues/10900)) ([b998267](https://github.com/windmill-labs/windmill/commit/b998267c91b9dcf02787768f6205cc5aeda494fb)) +* track outstanding service log files on the rows themselves ([#10894](https://github.com/windmill-labs/windmill/issues/10894)) ([aa4a6ff](https://github.com/windmill-labs/windmill/commit/aa4a6ffd66813010a79c07741b01a984ed4e7df6)) + + +### Performance Improvements + +* add service log documents to the index one batch at a time ([#10906](https://github.com/windmill-labs/windmill/issues/10906)) ([0c2eb0a](https://github.com/windmill-labs/windmill/commit/0c2eb0ae3d18f49c21370131d15011e8dd103746)) + +## [1.799.0](https://github.com/windmill-labs/windmill/compare/v1.798.1...v1.799.0) (2026-08-28) + + +### Features + +* enable Anthropic prompt caching on Vertex AI agent steps ([#10876](https://github.com/windmill-labs/windmill/issues/10876)) ([320f400](https://github.com/windmill-labs/windmill/commit/320f4005124202852e6e9c70b394e7f87231d278)) +* instrument AI fill/fix, evals, agents and the debugger ([#10853](https://github.com/windmill-labs/windmill/issues/10853)) ([0bbd559](https://github.com/windmill-labs/windmill/commit/0bbd559ac8a35dba04ba5e8d6f2fd8d1d1124891)) + + +### Bug Fixes + +* **datatables:** stop a fork's pg_dump restore from failing silently ([#10830](https://github.com/windmill-labs/windmill/issues/10830)) ([3ce9bbc](https://github.com/windmill-labs/windmill/commit/3ce9bbc7168b837cb2111aabd533bb67803502b8)) +* key build artifact caches on a runnable's inline modules ([#10819](https://github.com/windmill-labs/windmill/issues/10819)) ([b72ccc3](https://github.com/windmill-labs/windmill/commit/b72ccc35934165b4bad112b947ca5af064aab26f)) +* nested template literals in step inputs, and unresolvable $args tags ([#10856](https://github.com/windmill-labs/windmill/issues/10856)) ([8f349c0](https://github.com/windmill-labs/windmill/commit/8f349c032a0d75fc3350292075e5050a030f6166)) +* pre-fill the test panel JSON args editor and align its placeholder ([#10871](https://github.com/windmill-labs/windmill/issues/10871)) ([fb82f36](https://github.com/windmill-labs/windmill/commit/fb82f36e6d6492dd0740984d8d78ea4eaa30361e)) +* reject a prefixed error_handler_path on triggers ([#10847](https://github.com/windmill-labs/windmill/issues/10847)) ([d334831](https://github.com/windmill-labs/windmill/commit/d33483173526a3b352d2829ac8a2e1e229cc1127)) +* unify billable seat counting and prevent fork subscriptions ([#10818](https://github.com/windmill-labs/windmill/issues/10818)) ([7dd88c4](https://github.com/windmill-labs/windmill/commit/7dd88c470caee5f095dc240667aa7550c55696bc)) + +## [1.798.1](https://github.com/windmill-labs/windmill/compare/v1.798.0...v1.798.1) (2026-08-27) + + +### Bug Fixes + +* allow job tokens to read the automate_username_creation setting ([#10869](https://github.com/windmill-labs/windmill/issues/10869)) ([c2279db](https://github.com/windmill-labs/windmill/commit/c2279db8a96ac76382eafe254627dafd24d173fd)) + +## [1.798.0](https://github.com/windmill-labs/windmill/compare/v1.797.0...v1.798.0) (2026-08-27) + + +### Features + +* a wizard for importing a hub project, and finishing what the import cannot ([#10729](https://github.com/windmill-labs/windmill/issues/10729)) ([2913339](https://github.com/windmill-labs/windmill/commit/29133398f99cd2dd5b33057ee9df4492d82e067a)) + +## [1.797.0](https://github.com/windmill-labs/windmill/compare/v1.796.0...v1.797.0) (2026-08-26) + + +### Features + +* configurable expiry for presigned s3 public url signatures ([#10835](https://github.com/windmill-labs/windmill/issues/10835)) ([8a6dc27](https://github.com/windmill-labs/windmill/commit/8a6dc27236aca67f0efe941d9606b787c2305ea8)) +* **frontend:** flag the fork-compare datatable schema diff as legacy ([#10829](https://github.com/windmill-labs/windmill/issues/10829)) ([07c77ea](https://github.com/windmill-labs/windmill/commit/07c77ead7425f1877372d358d867445a4c525c96)) +* keep a Hub project live while an update is under review ([#10814](https://github.com/windmill-labs/windmill/issues/10814)) ([c04b570](https://github.com/windmill-labs/windmill/commit/c04b5705745c36ecbb3a551ac59459218d2e3807)) + + +### Bug Fixes + +* **cli:** keep svelte component styles in the raw-app bundle ([#10838](https://github.com/windmill-labs/windmill/issues/10838)) ([b8bf539](https://github.com/windmill-labs/windmill/commit/b8bf539c3fe2b4db9c74dd73f04b3029287acdc6)) +* **debugger:** parse bun 1.4's UUID inspector token ([#10828](https://github.com/windmill-labs/windmill/issues/10828)) ([4658224](https://github.com/windmill-labs/windmill/commit/46582245926a7f8ea961bcd125a58fbfba3530cf)) +* force HTTP router rebuild on trigger-change notification ([#10849](https://github.com/windmill-labs/windmill/issues/10849)) ([ffdf17e](https://github.com/windmill-labs/windmill/commit/ffdf17ef8dc5575dd92d62d0d0ba887c1e378576)) +* **frontend:** follow the operating workspace in step input forms ([#10834](https://github.com/windmill-labs/windmill/issues/10834)) ([6b73145](https://github.com/windmill-labs/windmill/commit/6b73145e7220232601538b801ebc9dc73fe79bbb)) +* **frontend:** key the GitHub App installation selector on installation_id ([#10831](https://github.com/windmill-labs/windmill/issues/10831)) ([78331fd](https://github.com/windmill-labs/windmill/commit/78331fda8b290a2d9a5dd92b8362ff32c8b39432)) +* **frontend:** operator menu opens on hover, pins on click ([#10824](https://github.com/windmill-labs/windmill/issues/10824)) ([665f83e](https://github.com/windmill-labs/windmill/commit/665f83e1f438e34d006429889d51a5fb6a6b6176)) +* keep connection string query parameters under token auth ([#10859](https://github.com/windmill-labs/windmill/issues/10859)) ([f131c39](https://github.com/windmill-labs/windmill/commit/f131c3920f50f9fa18cd637eac39609495999aef)) +* migrate slack resource-connect oauth to v2 ([#10836](https://github.com/windmill-labs/windmill/issues/10836)) ([9fa8159](https://github.com/windmill-labs/windmill/commit/9fa8159ad16204cab52fd18a34a48ebf13f800f6)) +* recover from unresolvable AI session links instead of a dead end ([#10854](https://github.com/windmill-labs/windmill/issues/10854)) ([e38c449](https://github.com/windmill-labs/windmill/commit/e38c449007f27b952808cba5aa812441f2ce5946)) +* require admin on workspace tarball settings export ([#10817](https://github.com/windmill-labs/windmill/issues/10817)) ([46c363f](https://github.com/windmill-labs/windmill/commit/46c363ffa4bc72bef6b367ece4bdbeef5e0eadc9)) +* restrict filesystem workspace storage to debug builds ([#10864](https://github.com/windmill-labs/windmill/issues/10864)) ([8b80b09](https://github.com/windmill-labs/windmill/commit/8b80b09f33d311f0881678577ca6004c12d97c22)) + + +### Performance Improvements + +* index the suspended-job resume test instead of filtering it ([#10863](https://github.com/windmill-labs/windmill/issues/10863)) ([69320b2](https://github.com/windmill-labs/windmill/commit/69320b28f615b897a92f580bd5961c41e5c29951)) + ## [1.796.0](https://github.com/windmill-labs/windmill/compare/v1.795.0...v1.796.0) (2026-08-24) diff --git a/backend/.sqlx/query-94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033.json b/backend/.sqlx/query-0e03dc960c0a22e042e54af719ac90c4b8506acefc85ef5b2f92a7bc451b1c5e.json similarity index 51% rename from backend/.sqlx/query-94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033.json rename to backend/.sqlx/query-0e03dc960c0a22e042e54af719ac90c4b8506acefc85ef5b2f92a7bc451b1c5e.json index 24a5ee62dd..42bac71afd 100644 --- a/backend/.sqlx/query-94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033.json +++ b/backend/.sqlx/query-0e03dc960c0a22e042e54af719ac90c4b8506acefc85ef5b2f92a7bc451b1c5e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval RETURNING file_path, hostname", + "query": "DELETE FROM log_file WHERE (hostname, log_ts) IN (\n SELECT hostname, log_ts FROM log_file\n WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval\n LIMIT $2\n ) RETURNING file_path, hostname", "describe": { "columns": [ { @@ -16,6 +16,7 @@ ], "parameters": { "Left": [ + "Int8", "Int8" ] }, @@ -24,5 +25,5 @@ false ] }, - "hash": "94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033" + "hash": "0e03dc960c0a22e042e54af719ac90c4b8506acefc85ef5b2f92a7bc451b1c5e" } diff --git a/backend/.sqlx/query-247486558e023ec3adf0c1e8f5664fc5e65995ce81e622c174a89befc1a527e5.json b/backend/.sqlx/query-247486558e023ec3adf0c1e8f5664fc5e65995ce81e622c174a89befc1a527e5.json new file mode 100644 index 0000000000..4ae74ff53c --- /dev/null +++ b/backend/.sqlx/query-247486558e023ec3adf0c1e8f5664fc5e65995ce81e622c174a89befc1a527e5.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT cost_nanos FROM ai_free_token_usage WHERE email = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "cost_nanos", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "247486558e023ec3adf0c1e8f5664fc5e65995ce81e622c174a89befc1a527e5" +} diff --git a/backend/.sqlx/query-44b9400fed2082c0df9d57f940923f37b8d452ea8e199ccc207da238368f1996.json b/backend/.sqlx/query-44b9400fed2082c0df9d57f940923f37b8d452ea8e199ccc207da238368f1996.json new file mode 100644 index 0000000000..2253944269 --- /dev/null +++ b/backend/.sqlx/query-44b9400fed2082c0df9d57f940923f37b8d452ea8e199ccc207da238368f1996.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_free_token_daily_usage (day, cost_nanos, updated_at)\n VALUES ($2::date, $1::bigint, now())\n ON CONFLICT (day) DO UPDATE\n SET cost_nanos = ai_free_token_daily_usage.cost_nanos + $1::bigint,\n updated_at = now()\n RETURNING cost_nanos", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "cost_nanos", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8", + "Date" + ] + }, + "nullable": [ + false + ] + }, + "hash": "44b9400fed2082c0df9d57f940923f37b8d452ea8e199ccc207da238368f1996" +} diff --git a/backend/.sqlx/query-624a7dbc6cc951a199b0e70d86c463a0e7b5248c226ee92d95df94c3099cc400.json b/backend/.sqlx/query-624a7dbc6cc951a199b0e70d86c463a0e7b5248c226ee92d95df94c3099cc400.json new file mode 100644 index 0000000000..3713af1486 --- /dev/null +++ b/backend/.sqlx/query-624a7dbc6cc951a199b0e70d86c463a0e7b5248c226ee92d95df94c3099cc400.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE log_file SET indexed_at = now()\n FROM unnest($1::text[], $2::text[]) AS c(hostname, file_path)\n WHERE log_file.indexed_at IS NULL\n AND log_file.hostname = c.hostname\n AND log_file.file_path = c.file_path", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "624a7dbc6cc951a199b0e70d86c463a0e7b5248c226ee92d95df94c3099cc400" +} diff --git a/backend/.sqlx/query-67a83afb708c90b2132cba81a0701dfc8b5e7aedf2f26fc1fe5be8f685cf709c.json b/backend/.sqlx/query-67a83afb708c90b2132cba81a0701dfc8b5e7aedf2f26fc1fe5be8f685cf709c.json new file mode 100644 index 0000000000..4481fa3dea --- /dev/null +++ b/backend/.sqlx/query-67a83afb708c90b2132cba81a0701dfc8b5e7aedf2f26fc1fe5be8f685cf709c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT max(log_ts) FROM log_file\n WHERE hostname = $1 AND log_ts < (SELECT max(log_ts) FROM log_file WHERE hostname = $1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "max", + "type_info": "Timestamp" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "67a83afb708c90b2132cba81a0701dfc8b5e7aedf2f26fc1fe5be8f685cf709c" +} diff --git a/backend/.sqlx/query-b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a.json b/backend/.sqlx/query-6bbcb27a3bb70302076c559c8394b14b842f595f68dd885248abaaeabd2d0bf1.json similarity index 86% rename from backend/.sqlx/query-b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a.json rename to backend/.sqlx/query-6bbcb27a3bb70302076c559c8394b14b842f595f68dd885248abaaeabd2d0bf1.json index ba9f3814e8..13d1c9b1e8 100644 --- a/backend/.sqlx/query-b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a.json +++ b/backend/.sqlx/query-6bbcb27a3bb70302076c559c8394b14b842f595f68dd885248abaaeabd2d0bf1.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n hostname,\n mode::text,\n worker_group,\n log_ts,\n file_path,\n ok_lines,\n err_lines,\n json_fmt\n FROM log_file\n WHERE log_ts > $1\n ORDER BY log_ts ASC LIMIT $2", + "query": "SELECT\n hostname,\n mode::text,\n worker_group,\n log_ts,\n file_path,\n ok_lines,\n err_lines,\n json_fmt\n FROM log_file\n WHERE indexed_at IS NULL\n ORDER BY log_ts ASC, hostname ASC LIMIT $1", "describe": { "columns": [ { @@ -46,7 +46,6 @@ ], "parameters": { "Left": [ - "Timestamp", "Int8" ] }, @@ -61,5 +60,5 @@ true ] }, - "hash": "b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a" + "hash": "6bbcb27a3bb70302076c559c8394b14b842f595f68dd885248abaaeabd2d0bf1" } diff --git a/backend/.sqlx/query-872be44352d4a27a2005f6bdc38f927ac4886755fd17ec755037763aa92f6c0e.json b/backend/.sqlx/query-872be44352d4a27a2005f6bdc38f927ac4886755fd17ec755037763aa92f6c0e.json new file mode 100644 index 0000000000..98e08b8167 --- /dev/null +++ b/backend/.sqlx/query-872be44352d4a27a2005f6bdc38f927ac4886755fd17ec755037763aa92f6c0e.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH moved AS (\n UPDATE log_file SET indexed_at = CASE\n WHEN log_ts > NOW() - make_interval(secs => $1) THEN NULL\n ELSE now() END\n WHERE indexed_at = 'epoch' RETURNING 1)\n SELECT count(*) AS \"n!\" FROM moved", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "n!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Float8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "872be44352d4a27a2005f6bdc38f927ac4886755fd17ec755037763aa92f6c0e" +} diff --git a/backend/.sqlx/query-87e8e034b1cf5ea7ce43870d77d33fa0fd05b067454f0d4686d8b155fe562ffe.json b/backend/.sqlx/query-87e8e034b1cf5ea7ce43870d77d33fa0fd05b067454f0d4686d8b155fe562ffe.json new file mode 100644 index 0000000000..76456ea8fb --- /dev/null +++ b/backend/.sqlx/query-87e8e034b1cf5ea7ce43870d77d33fa0fd05b067454f0d4686d8b155fe562ffe.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE log_file SET indexed_at = now()\n FROM unnest($1::text[], $2::timestamp[]) AS c(hostname, log_ts)\n WHERE log_file.hostname = c.hostname AND log_file.log_ts = c.log_ts", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "TimestampArray" + ] + }, + "nullable": [] + }, + "hash": "87e8e034b1cf5ea7ce43870d77d33fa0fd05b067454f0d4686d8b155fe562ffe" +} diff --git a/backend/.sqlx/query-8d207cc9ed101ff116b617d25a94633c1531170ded1fdf09114718b941f5e1db.json b/backend/.sqlx/query-8d207cc9ed101ff116b617d25a94633c1531170ded1fdf09114718b941f5e1db.json deleted file mode 100644 index d0b96444ee..0000000000 --- a/backend/.sqlx/query-8d207cc9ed101ff116b617d25a94633c1531170ded1fdf09114718b941f5e1db.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n hostname,\n mode::text,\n worker_group,\n log_ts,\n file_path,\n ok_lines,\n err_lines,\n json_fmt\n FROM log_file\n WHERE log_ts > NOW() - make_interval(secs => $1)\n ORDER BY log_ts ASC LIMIT $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "hostname", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "mode", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "worker_group", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "log_ts", - "type_info": "Timestamp" - }, - { - "ordinal": 4, - "name": "file_path", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "ok_lines", - "type_info": "Int8" - }, - { - "ordinal": 6, - "name": "err_lines", - "type_info": "Int8" - }, - { - "ordinal": 7, - "name": "json_fmt", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Float8", - "Int8" - ] - }, - "nullable": [ - false, - null, - true, - false, - false, - true, - true, - true - ] - }, - "hash": "8d207cc9ed101ff116b617d25a94633c1531170ded1fdf09114718b941f5e1db" -} diff --git a/backend/.sqlx/query-a7d450e34084d561f69e588bd76fd56e616ee79d895b7dcc37ad9442789e1574.json b/backend/.sqlx/query-a7d450e34084d561f69e588bd76fd56e616ee79d895b7dcc37ad9442789e1574.json new file mode 100644 index 0000000000..a83c34e0db --- /dev/null +++ b/backend/.sqlx/query-a7d450e34084d561f69e588bd76fd56e616ee79d895b7dcc37ad9442789e1574.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH retired AS (\n UPDATE log_file SET indexed_at = now()\n WHERE indexed_at IS NULL\n AND log_ts <= NOW() - make_interval(secs => $1) RETURNING 1)\n SELECT count(*) AS \"n!\" FROM retired", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "n!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Float8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a7d450e34084d561f69e588bd76fd56e616ee79d895b7dcc37ad9442789e1574" +} diff --git a/backend/.sqlx/query-acf4a5f4b67ebd06157819677fee6be4cc3adc6aa859c3dbaa57fa50b63741cf.json b/backend/.sqlx/query-acf4a5f4b67ebd06157819677fee6be4cc3adc6aa859c3dbaa57fa50b63741cf.json new file mode 100644 index 0000000000..a741ee1c0d --- /dev/null +++ b/backend/.sqlx/query-acf4a5f4b67ebd06157819677fee6be4cc3adc6aa859c3dbaa57fa50b63741cf.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_free_token_daily_usage (day, cost_nanos, updated_at)\n VALUES ($2::date, GREATEST(0, $1::bigint), now())\n ON CONFLICT (day) DO UPDATE\n SET cost_nanos = GREATEST(0, ai_free_token_daily_usage.cost_nanos + $1::bigint),\n updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Date" + ] + }, + "nullable": [] + }, + "hash": "acf4a5f4b67ebd06157819677fee6be4cc3adc6aa859c3dbaa57fa50b63741cf" +} diff --git a/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json b/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json index b336210daf..b892061f56 100644 --- a/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json +++ b/backend/.sqlx/query-b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384.json @@ -98,12 +98,12 @@ null, null, null, - true, + false, null, null, null, - true, - true + false, + false ] }, "hash": "b8e732a03969666444f73397ac153d23ec2af084b2f93da24c920532c1916384" diff --git a/backend/.sqlx/query-c29d815cc38493f17950f12e9e5fe34b6c4ed1b2a49ed2b61dc9fb494026e956.json b/backend/.sqlx/query-c29d815cc38493f17950f12e9e5fe34b6c4ed1b2a49ed2b61dc9fb494026e956.json new file mode 100644 index 0000000000..7078c32f1a --- /dev/null +++ b/backend/.sqlx/query-c29d815cc38493f17950f12e9e5fe34b6c4ed1b2a49ed2b61dc9fb494026e956.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_free_token_usage (email, cost_nanos, updated_at)\n VALUES ($1, $2::bigint, now())\n ON CONFLICT (email) DO UPDATE\n SET cost_nanos = ai_free_token_usage.cost_nanos + $2::bigint,\n updated_at = now()\n RETURNING cost_nanos", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "cost_nanos", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "c29d815cc38493f17950f12e9e5fe34b6c4ed1b2a49ed2b61dc9fb494026e956" +} diff --git a/backend/.sqlx/query-daa5b57290cd1f821a53eebe96434f1befe6b16eee363c843faa1f836d53ca8d.json b/backend/.sqlx/query-daa5b57290cd1f821a53eebe96434f1befe6b16eee363c843faa1f836d53ca8d.json new file mode 100644 index 0000000000..d64f007d7b --- /dev/null +++ b/backend/.sqlx/query-daa5b57290cd1f821a53eebe96434f1befe6b16eee363c843faa1f836d53ca8d.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT mode::text AS \"mode!\", log_ts FROM log_file WHERE hostname = $1 AND file_path = $2 ORDER BY log_ts DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "mode!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "log_ts", + "type_info": "Timestamp" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null, + false + ] + }, + "hash": "daa5b57290cd1f821a53eebe96434f1befe6b16eee363c843faa1f836d53ca8d" +} diff --git a/backend/.sqlx/query-f08ef43b4523c74fcbfc7879c147e1bf2c9a0d11e9841caafca62d85c9fb4c8f.json b/backend/.sqlx/query-f08ef43b4523c74fcbfc7879c147e1bf2c9a0d11e9841caafca62d85c9fb4c8f.json new file mode 100644 index 0000000000..32f7e8cf1f --- /dev/null +++ b/backend/.sqlx/query-f08ef43b4523c74fcbfc7879c147e1bf2c9a0d11e9841caafca62d85c9fb4c8f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_free_token_usage (email, cost_nanos, updated_at)\n VALUES ($1, GREATEST(0, $2::bigint), now())\n ON CONFLICT (email) DO UPDATE\n SET cost_nanos = GREATEST(0, ai_free_token_usage.cost_nanos + $2::bigint),\n updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "f08ef43b4523c74fcbfc7879c147e1bf2c9a0d11e9841caafca62d85c9fb4c8f" +} diff --git a/backend/.sqlx/query-92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b.json b/backend/.sqlx/query-f277db0459ff311d8a396aa4e03876dac75ba6df5f8eaa9f185300483e3ee36f.json similarity index 51% rename from backend/.sqlx/query-92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b.json rename to backend/.sqlx/query-f277db0459ff311d8a396aa4e03876dac75ba6df5f8eaa9f185300483e3ee36f.json index 7df22ca7b7..976cdfec06 100644 --- a/backend/.sqlx/query-92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b.json +++ b/backend/.sqlx/query-f277db0459ff311d8a396aa4e03876dac75ba6df5f8eaa9f185300483e3ee36f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt)\n VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8)\n ON CONFLICT (hostname, log_ts) DO UPDATE SET ok_lines = log_file.ok_lines + $6, err_lines = log_file.err_lines + $7", + "query": "INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt)\n VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8)\n ON CONFLICT (hostname, log_ts) DO UPDATE SET ok_lines = log_file.ok_lines + $6, err_lines = log_file.err_lines + $7", "describe": { "columns": [], "parameters": { @@ -17,5 +17,5 @@ }, "nullable": [] }, - "hash": "92faee8714a45a403b623e04d789f7f99067a05e9dfe270223164db8a1df2e4b" + "hash": "f277db0459ff311d8a396aa4e03876dac75ba6df5f8eaa9f185300483e3ee36f" } diff --git a/backend/.sqlx/query-f47cafb6e9a6ec58ccafb06cf5e806e3fe749119214863b9111b58fff0bb9583.json b/backend/.sqlx/query-f47cafb6e9a6ec58ccafb06cf5e806e3fe749119214863b9111b58fff0bb9583.json new file mode 100644 index 0000000000..cce644b300 --- /dev/null +++ b/backend/.sqlx/query-f47cafb6e9a6ec58ccafb06cf5e806e3fe749119214863b9111b58fff0bb9583.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT plan FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "plan", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "f47cafb6e9a6ec58ccafb06cf5e806e3fe749119214863b9111b58fff0bb9583" +} diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 0d9639c6c0..7bd84dfca3 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -131,9 +131,11 @@ minimal explicit set for dev. ## Workspace object storage in dev — use the local filesystem For a dev workspace you don't need MinIO/S3: use the built-in **`FilesystemStorage`** large-file -storage (a root path on local disk). It is intentionally hidden from the settings-UI storage -dropdown (dev-only), so set it via the API. Requires the backend built with `parquet` (+ `private` -for the real S3 helpers, + `enterprise` if you want advanced permission rules enforced): +storage (a root path on local disk). It is a **debug-build affordance only** — every site that +builds a filesystem object store calls `ensure_filesystem_storage_allowed`, so release builds +refuse it, and the settings UI never offers it — so set it via the API on a `cargo run`/`cargo +test` binary. Requires the backend built with `parquet` (+ `private` for the real S3 helpers, ++ `enterprise` if you want advanced permission rules enforced): ```bash curl -X POST "$BASE/api/w//workspaces/edit_large_file_storage_config" \ diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 2721fc7283..2067bfd789 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -23,7 +23,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "generic-array", ] @@ -261,13 +261,13 @@ dependencies = [ [[package]] name = "argon2" -version = "0.5.3" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" +checksum = "134c52ddac6d63c576bef8168db10c83c49c26444ecbc68060fef078925a901c" dependencies = [ "base64ct", - "blake2", - "cpufeatures 0.2.17", + "blake2 0.11.0", + "cpufeatures 0.3.1", "password-hash", ] @@ -430,7 +430,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 2.14.0", + "indexmap 2.14.1", "lexical-core", "memchr", "num", @@ -1338,7 +1338,7 @@ dependencies = [ "http 1.5.0", "http-body 0.4.6", "hyper 0.14.32", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-rustls 0.24.2", "hyper-rustls 0.27.9", "hyper-util", @@ -1532,7 +1532,7 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "itoa", "matchit 0.8.4", @@ -1629,7 +1629,7 @@ dependencies = [ "addr2line", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.8.9", "object 0.37.3", "rustc-demangle", "windows-link 0.2.1", @@ -1729,7 +1729,7 @@ dependencies = [ "clang-sys", "itertools 0.13.0", "log", - "prettyplease", + "prettyplease 0.2.37", "proc-macro2", "quote", "regex", @@ -1749,7 +1749,7 @@ dependencies = [ "clang-sys", "itertools 0.13.0", "log", - "prettyplease", + "prettyplease 0.2.37", "proc-macro2", "quote", "regex", @@ -1824,6 +1824,15 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "blake2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b5d4d889834ee8ecfc0f8426ad30faf7cdcb10f741a8e6d7224d95325479f6f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "blake3" version = "1.8.7" @@ -1834,7 +1843,7 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq 0.4.2", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", ] [[package]] @@ -1856,6 +1865,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "block-modes" version = "0.8.1" @@ -1908,7 +1926,7 @@ dependencies = [ "hex", "http 1.5.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-named-pipe", "hyper-util", "hyperlocal", @@ -1940,34 +1958,32 @@ dependencies = [ [[package]] name = "bon" -version = "3.9.3" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a602c73c7b0148ec6d12af6fd5cc7a46e2eacc8878271a999abac56eed12f561" +checksum = "9e3fac94a66da67200398458a25412bcc3f9b6443b5119a6cad9cf3ccfcd8cc6" dependencies = [ "bon-macros", - "rustversion", ] [[package]] name = "bon-macros" -version = "3.9.3" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" +checksum = "d4654961ad0494e4774c5c60b4cb4cd0ae9b9d92d039d901638b1dba97ebebf5" dependencies = [ - "darling 0.23.0", + "darling 0.24.1", "ident_case", - "prettyplease", + "prettyplease 0.3.0", "proc-macro2", "quote", - "rustversion", - "syn 2.0.119", + "syn 3.0.4", ] [[package]] name = "borsh" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88b7ea17d208c4193f2c1e6de3c35fe71f98c96982d5ced308bdcc749ff6e1f" +checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a" dependencies = [ "borsh-derive", "bytes", @@ -1976,15 +1992,15 @@ dependencies = [ [[package]] name = "borsh-derive" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8f347189c62a579b8cd5f80714efa178f52e461dc2e6d701d264f5ff22e566c" +checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56" dependencies = [ "once_cell", "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.4", ] [[package]] @@ -2334,12 +2350,12 @@ checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] name = "chacha20" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", "rand_core 0.10.1", ] @@ -2392,7 +2408,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "inout", ] @@ -2456,6 +2472,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "cms" version = "0.2.3" @@ -2476,9 +2498,9 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "combine" -version = "4.6.7" +version = "4.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" dependencies = [ "bytes", "memchr", @@ -2678,9 +2700,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" dependencies = [ "libc", ] @@ -2820,6 +2842,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "csv" version = "1.4.0" @@ -2850,6 +2881,15 @@ dependencies = [ "cipher 0.4.4", ] +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "curl-sys" version = "0.4.90+curl-8.21.0" @@ -3207,7 +3247,7 @@ dependencies = [ "base64 0.22.1", "half", "hashbrown 0.14.5", - "indexmap 2.14.0", + "indexmap 2.14.1", "libc", "log", "object_store", @@ -3386,7 +3426,7 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", - "indexmap 2.14.0", + "indexmap 2.14.1", "paste", "recursive", "serde_json", @@ -3401,7 +3441,7 @@ checksum = "422ac9cf3b22bbbae8cdf8ceb33039107fde1b5492693168f13bd566b1bcc839" dependencies = [ "arrow", "datafusion-common", - "indexmap 2.14.0", + "indexmap 2.14.1", "itertools 0.14.0", "paste", ] @@ -3415,7 +3455,7 @@ dependencies = [ "arrow", "arrow-buffer", "base64 0.22.1", - "blake2", + "blake2 0.10.6", "blake3", "chrono", "datafusion-common", @@ -3555,7 +3595,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-physical-expr", - "indexmap 2.14.0", + "indexmap 2.14.1", "itertools 0.14.0", "log", "recursive", @@ -3578,7 +3618,7 @@ dependencies = [ "datafusion-physical-expr-common", "half", "hashbrown 0.14.5", - "indexmap 2.14.0", + "indexmap 2.14.1", "itertools 0.14.0", "log", "paste", @@ -3640,7 +3680,7 @@ dependencies = [ "futures", "half", "hashbrown 0.14.5", - "indexmap 2.14.0", + "indexmap 2.14.1", "itertools 0.14.0", "log", "parking_lot", @@ -3682,7 +3722,7 @@ dependencies = [ "bigdecimal", "datafusion-common", "datafusion-expr", - "indexmap 2.14.0", + "indexmap 2.14.1", "log", "recursive", "regex", @@ -3691,9 +3731,9 @@ dependencies = [ [[package]] name = "datasketches" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745" +checksum = "46c4cf71a36b46dcfc00e5014c0c20ccad2b1b6a008304d7d57d2749b2d41b3d" [[package]] name = "debug-helper" @@ -3813,7 +3853,7 @@ dependencies = [ "deno_path_util", "deno_unsync", "futures", - "indexmap 2.14.0", + "indexmap 2.14.1", "libc", "parking_lot", "percent-encoding", @@ -3955,7 +3995,7 @@ dependencies = [ "hickory-resolver", "http 1.5.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-rustls 0.27.9", "hyper-util", "ipnet", @@ -4082,7 +4122,7 @@ version = "0.228.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bf8dbe5abf37d270bb853c5dfe45fbe3b1b6c453877cc11d7fe84e9862a6dbc" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.1", "proc-macro-rules", "proc-macro2", "quote", @@ -4158,7 +4198,7 @@ dependencies = [ "deno_error 0.6.1", "deno_tls", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-rustls 0.27.9", "hyper-util", "log", @@ -4457,10 +4497,21 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "const-oid", - "crypto-common", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "dirs" version = "4.0.0" @@ -5032,13 +5083,13 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" dependencies = [ "crc32fast", "libz-sys", - "miniz_oxide", + "miniz_oxide 0.9.1", "zlib-rs", ] @@ -5124,6 +5175,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "frostem" +version = "1.20260821.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2e81eab151ba68484704bb3d21b4b4d2747314d3081fa86fa6c7300a4c42e2" + [[package]] name = "fs3" version = "0.5.0" @@ -5698,9 +5755,9 @@ dependencies = [ [[package]] name = "gosyn" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9758a950dc61a15bc65162f72f5bec7e8efde91f102dc7dce7317f67019a6ba9" +checksum = "28f4ed3cbcb66ddad22553bb3b94087573cf492a34501bc025a23167ed57e856" dependencies = [ "anyhow", "strum", @@ -5740,7 +5797,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.14.0", + "indexmap 2.14.1", "slab", "tokio", "tokio-util", @@ -5759,7 +5816,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.5.0", - "indexmap 2.14.0", + "indexmap 2.14.1", "slab", "tokio", "tokio-util", @@ -5841,7 +5898,7 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd1246c0e5493286aeb2dde35b1f4eb9c4ce00e628641210a5e553fc001a1f26" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.1", "proc-macro2", "quote", "syn 2.0.119", @@ -5894,9 +5951,9 @@ checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "hermit-abi" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" [[package]] name = "hex" @@ -6111,7 +6168,7 @@ dependencies = [ "futures", "http 1.5.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-rustls 0.26.0", "hyper-tls", "hyper-tungstenite", @@ -6135,6 +6192,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "0.14.32" @@ -6161,9 +6227,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" dependencies = [ "atomic-waker", "bytes", @@ -6191,7 +6257,7 @@ dependencies = [ "futures-util", "headers", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-rustls 0.27.9", "hyper-util", "pin-project-lite", @@ -6207,7 +6273,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fab3637d6b04a8037af8a266fdf6cf92ea957e8c53981a2bf6136572531025bf" dependencies = [ "hex", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "pin-project-lite", "tokio", @@ -6238,7 +6304,7 @@ checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" dependencies = [ "futures-util", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "log", "rustls 0.22.4", @@ -6256,7 +6322,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "log", "rustls 0.23.35", @@ -6273,7 +6339,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "pin-project-lite", "tokio", @@ -6288,7 +6354,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "native-tls", "tokio", @@ -6303,7 +6369,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a343d17fe7885302ed7252767dc7bb83609a874b6ff581142241ec4b73957ad" dependencies = [ "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "pin-project-lite", "tokio", @@ -6323,7 +6389,7 @@ dependencies = [ "futures-util", "http 1.5.0", "http-body 1.1.0", - "hyper 1.11.0", + "hyper 1.11.1", "ipnet", "libc", "percent-encoding", @@ -6344,7 +6410,7 @@ checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" dependencies = [ "hex", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-util", "pin-project-lite", "tokio", @@ -6504,9 +6570,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.14.0" +version = "2.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "07aa2048142242915a31d35844fb311e0e53fcca590c3a0a40dcf1b841fa09eb" dependencies = [ "equivalent", "hashbrown 0.17.1", @@ -6892,7 +6958,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.1", ] [[package]] @@ -6948,7 +7014,7 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-http-proxy", "hyper-rustls 0.27.9", "hyper-timeout", @@ -7196,9 +7262,9 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d0a00925a9f930d679b6789b721e3a7f9ed110f41b86d2497caa780c3a070a" +checksum = "d7955dfc218a8afb29dfeffd540e3a6e96baeb94fe7138228dd7cc6937fbbf96" dependencies = [ "bitflags 2.13.1", "libc", @@ -7321,18 +7387,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.16.4" +version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" -dependencies = [ - "hashbrown 0.16.1", -] - -[[package]] -name = "lru" -version = "0.18.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d2f2f9b4ba7e6b24d95e7e899329d35be83bcded72c8540cdd5368932d1d90a" +checksum = "0d317b4b9eb398e6acce275758ec6125535505e7a146fb1a9b8bda2451b0ff4c" dependencies = [ "hashbrown 0.17.1", ] @@ -7363,9 +7420,9 @@ dependencies = [ [[package]] name = "lz4_flex" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" [[package]] name = "lzma-sys" @@ -7703,6 +7760,15 @@ name = "miniz_oxide" version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" dependencies = [ "adler2", "simd-adler32", @@ -7780,9 +7846,9 @@ dependencies = [ [[package]] name = "murmurhash32" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" +checksum = "a8afc6df942f4c022d70c5725e18df1a705773870e5b7f96fde94ca0334ce77a" [[package]] name = "mysql-common-derive" @@ -7816,7 +7882,7 @@ dependencies = [ "futures-sink", "futures-util", "keyed_priority_queue", - "lru 0.18.2", + "lru 0.18.3", "mysql_common", "native-tls", "pem 3.0.6", @@ -8058,7 +8124,7 @@ dependencies = [ "dirs-sys 0.4.1", "fancy-regex 0.14.0", "heck", - "indexmap 2.14.0", + "indexmap 2.14.1", "log", "lru 0.12.5", "miette", @@ -8236,7 +8302,7 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" dependencies = [ - "hermit-abi 0.5.2", + "hermit-abi 0.5.3", "libc", ] @@ -8321,7 +8387,7 @@ dependencies = [ "http-body-util", "httparse", "humantime", - "hyper 1.11.0", + "hyper 1.11.1", "itertools 0.14.0", "md-5 0.10.6", "parking_lot", @@ -8773,16 +8839,16 @@ checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "ownedbytes" version = "0.9.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "stable_deref_trait", ] [[package]] name = "owo-colors" -version = "4.3.0" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" +checksum = "13c45bb4a6ae1280ec0803b1ef9d3455eb50f01efbbe1447ab020f1d54fba9d8" [[package]] name = "p12-keystore" @@ -8920,13 +8986,12 @@ dependencies = [ [[package]] name = "password-hash" -version = "0.5.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" +checksum = "aab41826031698d6ffcd9cff78ef56ef998e39dc7e5067cdfebe373842d4723b" dependencies = [ - "base64ct", - "rand_core 0.6.4", - "subtle", + "getrandom 0.4.3", + "phc", ] [[package]] @@ -9058,7 +9123,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset", - "indexmap 2.14.0", + "indexmap 2.14.1", ] [[package]] @@ -9070,6 +9135,17 @@ dependencies = [ "phf 0.11.3", ] +[[package]] +name = "phc" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44dc769b75f93afdddd8c7fa12d685292ddeff1e66f7f0f3a234cf1818afe892" +dependencies = [ + "base64ct", + "ctutils", + "getrandom 0.4.3", +] + [[package]] name = "phf" version = "0.11.3" @@ -9313,7 +9389,7 @@ checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" dependencies = [ "cfg-if", "concurrent-queue", - "hermit-abi 0.5.2", + "hermit-abi 0.5.3", "pin-project-lite", "rustix 1.1.4", "windows-sys 0.61.2", @@ -9442,6 +9518,16 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "prettyplease" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" +dependencies = [ + "proc-macro2", + "syn 3.0.4", +] + [[package]] name = "primeorder" version = "0.13.6" @@ -9533,7 +9619,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3ef4f2f0422f23a82ec9f628ea2acd12871c81a9362b02c43c1aa86acfc3ba1" dependencies = [ "futures", - "indexmap 2.14.0", + "indexmap 2.14.1", "nix 0.30.1", "tokio", "tracing", @@ -10204,7 +10290,7 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-rustls 0.27.9", "hyper-tls", "hyper-util", @@ -10252,7 +10338,7 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-rustls 0.27.9", "hyper-util", "js-sys", @@ -10308,7 +10394,7 @@ dependencies = [ "futures", "getrandom 0.2.17", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "reqwest 0.13.4", "reqwest-middleware", "retry-policies", @@ -10479,7 +10565,7 @@ dependencies = [ "convert_case 0.10.0", "fnv", "ident_case", - "indexmap 2.14.0", + "indexmap 2.14.1", "proc-macro-crate", "proc-macro2", "quote", @@ -10571,16 +10657,6 @@ dependencies = [ "walkdir", ] -[[package]] -name = "rust-stemmers" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" -dependencies = [ - "serde", - "serde_derive", -] - [[package]] name = "rust_decimal" version = "1.42.1" @@ -11322,7 +11398,7 @@ version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.1", "itoa", "memchr", "serde", @@ -11416,7 +11492,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.14.0", + "indexmap 2.14.1", "jiff", "schemars 0.9.0", "schemars 1.2.2", @@ -11444,7 +11520,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.1", "itoa", "ryu", "serde", @@ -11457,7 +11533,7 @@ version = "0.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59e2dd588bf1597a252c3b920e0143eb99b0f76e4e082f4c92ce34fbc9e71ddd" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.1", "itoa", "libyml", "memchr", @@ -11877,7 +11953,7 @@ dependencies = [ "futures-util", "hashbrown 0.15.5", "hashlink", - "indexmap 2.14.0", + "indexmap 2.14.1", "log", "memchr", "once_cell", @@ -12234,7 +12310,7 @@ checksum = "72e90b52ee734ded867104612218101722ad87ff4cf74fe30383bd244a533f97" dependencies = [ "anyhow", "bytes-str", - "indexmap 2.14.0", + "indexmap 2.14.1", "serde", "serde_json", "swc_config_macro", @@ -12368,7 +12444,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c6f1b8f4232e7a7f614ff7c0f6ccb89c2d028cdf7629f79ad710cff5b28b62c" dependencies = [ "better_scoped_tls", - "indexmap 2.14.0", + "indexmap 2.14.1", "once_cell", "par-core", "phf 0.11.3", @@ -12434,7 +12510,7 @@ checksum = "69ea0052ac23b5b9fbc85bbdb1791b36b918f9d55f594b0ed8e25babb4c32d16" dependencies = [ "base64 0.22.1", "bytes-str", - "indexmap 2.14.0", + "indexmap 2.14.1", "once_cell", "rustc-hash 2.1.3", "serde", @@ -12474,7 +12550,7 @@ version = "21.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83259addd99ed4022aa9fc4d39428c008d3d42533769e1a005529da18cde4568" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.1", "num_cpus", "once_cell", "par-core", @@ -12714,12 +12790,12 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" [[package]] name = "tantivy" -version = "0.26.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.27.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "aho-corasick", "arc-swap", - "base64 0.22.1", + "base64 0.23.1", "bitpacking", "bon", "byteorder", @@ -12730,20 +12806,20 @@ dependencies = [ "downcast-rs", "fastdivide", "fnv", + "frostem", "fs4", "htmlescape", "itertools 0.14.0", "levenshtein_automata", "log", - "lru 0.16.4", - "lz4_flex 0.13.1", + "lru 0.18.3", + "lz4_flex 0.14.0", "measure_time", "memmap2", "once_cell", "oneshot", "rayon", "regex", - "rust-stemmers", "rustc-hash 2.1.3", "serde", "serde_json", @@ -12760,22 +12836,23 @@ dependencies = [ "thiserror 2.0.20", "time", "typetag", + "unwrap-infallible", "uuid", "winapi", ] [[package]] name = "tantivy-bitpacker" -version = "0.9.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.10.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "bitpacking", ] [[package]] name = "tantivy-columnar" -version = "0.6.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.7.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "downcast-rs", "fastdivide", @@ -12789,8 +12866,8 @@ dependencies = [ [[package]] name = "tantivy-common" -version = "0.10.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.11.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "async-trait", "byteorder", @@ -12812,8 +12889,8 @@ dependencies = [ [[package]] name = "tantivy-query-grammar" -version = "0.25.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.26.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "fnv", "nom", @@ -12824,8 +12901,8 @@ dependencies = [ [[package]] name = "tantivy-sstable" -version = "0.6.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.7.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "futures-util", "itertools 0.14.0", @@ -12837,8 +12914,8 @@ dependencies = [ [[package]] name = "tantivy-stacker" -version = "0.6.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.7.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "murmurhash32", "tantivy-common", @@ -12846,8 +12923,8 @@ dependencies = [ [[package]] name = "tantivy-tokenizer-api" -version = "0.6.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" +version = "0.7.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=ea3b818c7b93db0c2b5db4f5e7ffef38f339060a#ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" dependencies = [ "serde", ] @@ -13472,7 +13549,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.1", "serde", "serde_spanned", "toml_datetime 0.6.11", @@ -13485,7 +13562,7 @@ version = "0.25.13+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.1", "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", "winnow 1.0.4", @@ -13516,7 +13593,7 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-timeout", "hyper-util", "percent-encoding", @@ -13548,7 +13625,7 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.0", + "hyper 1.11.1", "hyper-timeout", "hyper-util", "percent-encoding", @@ -13593,7 +13670,7 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", - "indexmap 2.14.0", + "indexmap 2.14.1", "pin-project-lite", "slab", "sync_wrapper", @@ -13808,9 +13885,9 @@ dependencies = [ [[package]] name = "tree-sitter-language" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782" +checksum = "ca0d1bf6fdd806e43ae5198f82f527056d359def39e54e67a0f478ac09dac081" [[package]] name = "tree-sitter-r" @@ -13901,9 +13978,9 @@ dependencies = [ [[package]] name = "twox-hash" -version = "2.1.3" +version = "2.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" +checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a" [[package]] name = "typed-path" @@ -14140,7 +14217,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "subtle", ] @@ -14168,6 +14245,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "unwrap-infallible" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e497bb1f828cc9fb236722c2eaa100dcf201563f38f4da6252357a59037adf31" + [[package]] name = "ureq" version = "2.12.1" @@ -14251,9 +14334,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.25.0" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f053576934f05a761a402421fbbe3d425d9366f75f978806a037b3ca481abecc" +checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -14272,7 +14355,7 @@ dependencies = [ "fslock", "gzip-header", "home", - "miniz_oxide", + "miniz_oxide 0.8.9", "paste", "which 6.0.3", ] @@ -14663,7 +14746,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "async-nats", @@ -14748,7 +14831,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.796.0" +version = "1.800.0" dependencies = [ "async-stream", "async-trait", @@ -14781,7 +14864,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.796.0" +version = "1.800.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14794,7 +14877,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "argon2", @@ -14824,8 +14907,8 @@ dependencies = [ "hex", "hmac", "http 1.5.0", - "hyper 1.11.0", - "indexmap 2.14.0", + "hyper 1.11.1", + "indexmap 2.14.1", "itertools 0.14.0", "jsonwebtoken 8.3.0", "lazy_static", @@ -14934,12 +15017,12 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.796.0" +version = "1.800.0" dependencies = [ "axum 0.8.9", "chrono", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "lazy_static", "quick_cache", "serde", @@ -14957,7 +15040,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.796.0" +version = "1.800.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14974,7 +15057,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15000,7 +15083,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.796.0" +version = "1.800.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15010,7 +15093,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.796.0" +version = "1.800.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15027,7 +15110,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.796.0" +version = "1.800.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -15049,7 +15132,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15072,7 +15155,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.796.0" +version = "1.800.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15088,11 +15171,11 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.796.0" +version = "1.800.0" dependencies = [ "axum 0.8.9", "chrono", - "hyper 1.11.0", + "hyper 1.11.1", "serde", "serde_json", "sql-builder", @@ -15110,7 +15193,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.796.0" +version = "1.800.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15131,7 +15214,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.796.0" +version = "1.800.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15145,7 +15228,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "async-nats", @@ -15180,14 +15263,14 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "axum 0.8.9", "base64 0.22.1", "chrono", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "lazy_static", "serde", "serde_json", @@ -15205,7 +15288,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.796.0" +version = "1.800.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15233,12 +15316,12 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "axum 0.8.9", "http 1.5.0", - "indexmap 2.14.0", + "indexmap 2.14.1", "itertools 0.14.0", "lazy_static", "serde", @@ -15255,7 +15338,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.796.0" +version = "1.800.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15275,13 +15358,13 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.796.0" +version = "1.800.0" dependencies = [ "axum 0.8.9", "chrono", "futures", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "itertools 0.14.0", "lazy_static", "prometheus", @@ -15313,7 +15396,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15341,7 +15424,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.796.0" +version = "1.800.0" dependencies = [ "lazy_static", "serde", @@ -15353,13 +15436,13 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.796.0" +version = "1.800.0" dependencies = [ "argon2", "axum 0.8.9", "chrono", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "lazy_static", "serde", "serde_json", @@ -15377,7 +15460,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.796.0" +version = "1.800.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15391,14 +15474,14 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.796.0" +version = "1.800.0" dependencies = [ "axum 0.8.9", "chrono", "futures", "hex", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "lazy_static", "magic-crypt", "regex", @@ -15426,7 +15509,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.796.0" +version = "1.800.0" dependencies = [ "chrono", "lazy_static", @@ -15440,7 +15523,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15459,7 +15542,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.796.0" +version = "1.800.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15496,8 +15579,8 @@ dependencies = [ "globset", "hex", "hmac", - "hyper 1.11.0", - "indexmap 2.14.0", + "hyper 1.11.1", + "indexmap 2.14.1", "itertools 0.14.0", "jsonwebtoken 8.3.0", "lazy_static", @@ -15563,7 +15646,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.796.0" +version = "1.800.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -15582,7 +15665,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.796.0" +version = "1.800.0" dependencies = [ "regex", "serde", @@ -15597,16 +15680,18 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "astral-tokio-tar", "bytes", "chrono", "const_format", + "datafusion", "flume", "futures", "lazy_static", + "object_store", "serde", "serde_json", "sqlx", @@ -15614,6 +15699,7 @@ dependencies = [ "tempfile", "tokio", "tracing", + "url", "uuid", "windmill-common", "windmill-object-store", @@ -15621,7 +15707,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "futures", @@ -15638,7 +15724,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.796.0" +version = "1.800.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15654,7 +15740,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "async-trait", @@ -15675,7 +15761,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "async-trait", @@ -15706,7 +15792,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "arc-swap", @@ -15731,7 +15817,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "async-stream", @@ -15765,7 +15851,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "futures", @@ -15783,7 +15869,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.796.0" +version = "1.800.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15792,7 +15878,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "lazy_static", @@ -15804,7 +15890,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "serde_json", @@ -15816,7 +15902,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "gosyn", @@ -15828,7 +15914,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "lazy_static", @@ -15840,7 +15926,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "serde_json", @@ -15852,7 +15938,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "nu-parser", @@ -15863,7 +15949,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15874,7 +15960,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15886,7 +15972,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15897,7 +15983,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "async-recursion", @@ -15919,7 +16005,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "serde_json", @@ -15931,7 +16017,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "lazy_static", @@ -15945,7 +16031,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15962,7 +16048,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "lazy_static", @@ -15975,7 +16061,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "serde", @@ -15987,7 +16073,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "lazy_static", @@ -16005,7 +16091,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16021,7 +16107,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16037,7 +16123,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "lazy_static", @@ -16051,7 +16137,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "async-recursion", @@ -16090,7 +16176,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "const_format", @@ -16130,7 +16216,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.796.0" +version = "1.800.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16141,7 +16227,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "async-recursion", @@ -16152,7 +16238,7 @@ dependencies = [ "futures", "hex", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "lazy_static", "magic-crypt", "quick_cache", @@ -16176,7 +16262,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "async-trait", @@ -16200,14 +16286,14 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "async-trait", "axum 0.8.9", "chrono", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "itertools 0.14.0", "lazy_static", "rand 0.9.0", @@ -16233,7 +16319,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "async-trait", @@ -16260,7 +16346,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "async-trait", @@ -16293,7 +16379,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "async-trait", @@ -16313,7 +16399,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "async-trait", @@ -16347,7 +16433,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "async-trait", @@ -16359,7 +16445,7 @@ dependencies = [ "hex", "hmac", "http 1.5.0", - "hyper 1.11.0", + "hyper 1.11.1", "itertools 0.14.0", "lazy_static", "matchit 0.7.3", @@ -16383,7 +16469,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "async-trait", @@ -16406,7 +16492,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "async-trait", @@ -16430,7 +16516,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "async-nats", @@ -16454,7 +16540,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "async-trait", @@ -16489,7 +16575,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "async-trait", @@ -16517,7 +16603,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "async-trait", @@ -16542,7 +16628,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "bitflags 2.13.1", @@ -16561,7 +16647,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "async-once-cell", @@ -16678,7 +16764,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.796.0" +version = "1.800.0" dependencies = [ "bytes", "futures", @@ -17477,7 +17563,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c42e33efc22a0650c311c2ef19115ce232583abbe80850bc8b66509ebef02de0" dependencies = [ "crc32fast", - "indexmap 2.14.0", + "indexmap 2.14.1", "memchr", "typed-path", ] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 6dd9e452b8..966f820d57 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.796.0" +version = "1.800.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.796.0" +version = "1.800.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -477,7 +477,10 @@ rust-embed = { version = "^6", features = ["interpolate-folder-path"] } mime_guess = "^2" hex = "^0" sql-builder = "^3" -argon2 = "^0" +# Minor-pinned rather than the `^0` used elsewhere in this file: argon2's 0.x +# minors are API-breaking (0.6 moved `SaltString` into `phc`, put `rand_core` +# behind a feature and changed `hash_password`), so a float breaks the build. +argon2 = "0.6" quick_cache = "^0" rand = "=0.9.0" rand_core = { version = "^0", features = ["std"] } @@ -697,7 +700,7 @@ tikv-jemalloc-ctl = { version = "^0.5" } triomphe = "^0" pin-project-lite = "^0" -tantivy = { git="https://github.com/windmill-labs/tantivy", rev="6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" } +tantivy = { git="https://github.com/windmill-labs/tantivy", rev="ea3b818c7b93db0c2b5db4f5e7ffef38f339060a" } backon = "1.3.0" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 1b781f8bd1..3a820786e3 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -d63dfdf143d743a57295a63284b61001a0a491f3 +56f5e6a82056fa63323d6d4e1ec44832e64512fc diff --git a/backend/migrations/20260622072905_ai_free_token_usage.down.sql b/backend/migrations/20260622072905_ai_free_token_usage.down.sql new file mode 100644 index 0000000000..95879034e4 --- /dev/null +++ b/backend/migrations/20260622072905_ai_free_token_usage.down.sql @@ -0,0 +1,2 @@ +DROP TABLE ai_free_token_daily_usage; +DROP TABLE ai_free_token_usage; diff --git a/backend/migrations/20260622072905_ai_free_token_usage.up.sql b/backend/migrations/20260622072905_ai_free_token_usage.up.sql new file mode 100644 index 0000000000..ae0cf47e45 --- /dev/null +++ b/backend/migrations/20260622072905_ai_free_token_usage.up.sql @@ -0,0 +1,19 @@ +-- One-time grant of the Windmill-provided free AI tier, measured as cost in nano-dollars +-- (1e-9 USD) rather than raw tokens — a prompt-cache hit costs a fraction of a fresh input +-- token, so a token count wildly overstates the real bill. The grant never resets: once +-- spent, the user must bring their own API key. Keyed by normalized email so the allowance +-- is shared across a user's workspaces (and is resistant to +tag / gmail-dot aliasing). +CREATE TABLE ai_free_token_usage ( + email VARCHAR(255) PRIMARY KEY, + cost_nanos BIGINT NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- Instance-wide daily cost ceiling (nano-dollars) for the free tier — a kill-switch +-- independent of the per-user grant, bounding the blast radius of a bad day. One row per +-- UTC day. +CREATE TABLE ai_free_token_daily_usage ( + day DATE PRIMARY KEY, + cost_nanos BIGINT NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); diff --git a/backend/migrations/20260826202939_queue_suspended_resume_at_index.down.sql b/backend/migrations/20260826202939_queue_suspended_resume_at_index.down.sql new file mode 100644 index 0000000000..fe727f6761 --- /dev/null +++ b/backend/migrations/20260826202939_queue_suspended_resume_at_index.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS queue_suspended_v2; diff --git a/backend/migrations/20260826202939_queue_suspended_resume_at_index.up.sql b/backend/migrations/20260826202939_queue_suspended_resume_at_index.up.sql new file mode 100644 index 0000000000..6d91a64d5f --- /dev/null +++ b/backend/migrations/20260826202939_queue_suspended_resume_at_index.up.sql @@ -0,0 +1,20 @@ +-- Serves the suspended-job pull in windmill-common/src/worker.rs, whose resume test is the +-- indexed CASE expression. Two things about the shape are load-bearing: +-- * (priority DESC NULLS LAST, created_at) leads, so the scan yields that query's ORDER BY +-- and stops at the first match rather than sorting. +-- * the index is dropped before it is built rather than relying on IF NOT EXISTS. The +-- OVERRIDDEN_MIGRATIONS rewrite in windmill-api/src/db.rs runs these CONCURRENTLY, and an +-- interrupted concurrent build leaves the index present but invalid, which IF NOT EXISTS +-- would then skip rebuilding. Retiring the index this replaces is left to the migration +-- that follows, so this one can only ever be replayed while that index is still there to +-- cover the rebuild. +DROP INDEX IF EXISTS queue_suspended_v2; + +CREATE INDEX IF NOT EXISTS queue_suspended_v2 + ON v2_job_queue ( + priority DESC NULLS LAST, + created_at, + (CASE WHEN suspend <= 0 THEN '-infinity'::timestamptz ELSE suspend_until END), + tag + ) + WHERE suspend_until IS NOT NULL; diff --git a/backend/migrations/20260826214706_queue_suspended_drop_legacy_index.down.sql b/backend/migrations/20260826214706_queue_suspended_drop_legacy_index.down.sql new file mode 100644 index 0000000000..011105927a --- /dev/null +++ b/backend/migrations/20260826214706_queue_suspended_drop_legacy_index.down.sql @@ -0,0 +1,3 @@ +CREATE INDEX IF NOT EXISTS queue_suspended + ON v2_job_queue (priority DESC NULLS LAST, created_at, suspend_until, suspend, tag) + WHERE suspend_until IS NOT NULL; diff --git a/backend/migrations/20260826214706_queue_suspended_drop_legacy_index.up.sql b/backend/migrations/20260826214706_queue_suspended_drop_legacy_index.up.sql new file mode 100644 index 0000000000..b8851b99e0 --- /dev/null +++ b/backend/migrations/20260826214706_queue_suspended_drop_legacy_index.up.sql @@ -0,0 +1,6 @@ +-- Retires the index queue_suspended_v2 replaces. Separate from the migration that builds it +-- so that one is only ever replayed while this index still exists: sqlx records a migration +-- only after all its statements run, so a process that dies before the record is written +-- replays the build, and its leading DROP would otherwise be destroying the sole usable +-- index rather than an interrupted build. +DROP INDEX IF EXISTS queue_suspended; diff --git a/backend/migrations/20260830085453_log_file_indexed_at.down.sql b/backend/migrations/20260830085453_log_file_indexed_at.down.sql new file mode 100644 index 0000000000..ac3c92f3eb --- /dev/null +++ b/backend/migrations/20260830085453_log_file_indexed_at.down.sql @@ -0,0 +1,4 @@ +DROP INDEX IF EXISTS index_log_file_premigration; +DROP INDEX IF EXISTS index_log_file_pending_path; +DROP INDEX IF EXISTS index_log_file_pending; +ALTER TABLE log_file DROP COLUMN IF EXISTS indexed_at; diff --git a/backend/migrations/20260830085453_log_file_indexed_at.up.sql b/backend/migrations/20260830085453_log_file_indexed_at.up.sql new file mode 100644 index 0000000000..3421a5082a --- /dev/null +++ b/backend/migrations/20260830085453_log_file_indexed_at.up.sql @@ -0,0 +1,35 @@ +-- The service log ingest walked `log_file` with a cursor over `log_ts`, which is when a line +-- was written rather than when its row appeared. Rows do not arrive in that order — an upload +-- retried after a failure, a host that has just started, a batch the row limit cut mid-minute — +-- and a row that becomes visible behind the cursor is never read: it stays in `log_file` and its +-- lines stay out of search until retention drops them. +-- +-- No ordering fixes this. A cursor over arrival order fails the same way, because `nextval` is +-- allocated before its INSERT commits: a row can be assigned a lower value and commit after a +-- higher one has already moved the cursor past it. Which rows are outstanding is a property of +-- the rows, so it is recorded on them. +ALTER TABLE log_file ADD COLUMN indexed_at TIMESTAMPTZ; + +-- Rows that already existed are marked, not queued: on a 14-day window most were ingested long +-- ago and their raw files are gone. A sentinel rather than a timestamp, because the indexer has to +-- tell them apart from rows registered since — those start NULL — and it puts the window's worth of +-- them back on the queue on its first pass, keeping only what the columnar store can vouch for. +-- +-- Not split here on the cursor the old ingest had reached. Below that cursor sits every row it +-- skipped, which is the loss this migration exists to stop; recording those as done would carry the +-- bug into its own fix. +UPDATE log_file SET indexed_at = 'epoch' WHERE indexed_at IS NULL; + +-- The work queue, and the only index the ingest query needs: outstanding rows are a small +-- fraction of the table, so this stays proportional to what is left to do rather than to the +-- retention window. +CREATE INDEX index_log_file_pending ON log_file (log_ts) WHERE indexed_at IS NULL; + +-- A rebuild takes rows out of the queue by the file it just read out of the store, which is +-- the one lookup that arrives without a `log_ts`: the primary key is `(hostname, log_ts)`, so +-- nothing else covers it and each batch would scan every outstanding row instead. +CREATE INDEX index_log_file_pending_path ON log_file (hostname, file_path) WHERE indexed_at IS NULL; + +-- Reached once per pass while pre-migration rows survive, and never again after the first +-- conversion clears them. +CREATE INDEX index_log_file_premigration ON log_file (log_ts) WHERE indexed_at = 'epoch'; diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 2f2fafedf4..b5ed9dff9e 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.796.0" +version = "1.800.0" dependencies = [ "aho-corasick", "anyhow", @@ -6274,7 +6274,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.796.0" +version = "1.800.0" dependencies = [ "proc-macro2", "quote", @@ -6286,7 +6286,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.796.0" +version = "1.800.0" dependencies = [ "convert_case", "serde", @@ -6295,7 +6295,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "lazy_static", @@ -6307,7 +6307,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "serde_json", @@ -6319,7 +6319,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "gosyn", @@ -6331,7 +6331,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "lazy_static", @@ -6343,7 +6343,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "serde_json", @@ -6355,7 +6355,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "nu-parser", @@ -6366,7 +6366,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6377,7 +6377,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6389,7 +6389,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6400,7 +6400,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "async-recursion", @@ -6422,7 +6422,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "serde_json", @@ -6434,7 +6434,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "lazy_static", @@ -6448,7 +6448,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "convert_case", @@ -6465,7 +6465,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "lazy_static", @@ -6478,7 +6478,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "serde", @@ -6490,7 +6490,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "lazy_static", @@ -6508,7 +6508,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6524,7 +6524,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6540,7 +6540,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6572,7 +6572,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "lazy_static", @@ -6586,7 +6586,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.796.0" +version = "1.800.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 826e279c39..ea7160821b 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.796.0" +version = "1.800.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/src/main.rs b/backend/src/main.rs index f8b150383a..fa88cf6cdc 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -14,7 +14,7 @@ use monitor::{ reload_nuget_config_setting, reload_powershell_repo_pat_setting, reload_powershell_repo_url_setting, reload_ruby_repos_setting, reload_timeout_wait_result_setting, reload_workspace_registries_setting, - send_current_log_file_to_object_store, send_logs_to_object_store, WORKERS_NAMES, + flush_pending_log_files_to_object_store, send_logs_to_object_store, WORKERS_NAMES, }; use rand::Rng; use sqlx::{Pool, Postgres}; @@ -61,8 +61,9 @@ use windmill_common::{ SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, SANDBOX_IMAGE_PULL_POLICY_SETTING, SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, - SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, - UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, + SERVICE_LOG_RETENTION_SECS_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, + TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, + UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, WORKSPACE_MAX_QUEUED_JOBS_SETTING, WORKSPACE_REGISTRIES_SETTING, @@ -143,7 +144,8 @@ use crate::monitor::{ reload_pip_index_url_setting, reload_retention_period_setting, reload_sandbox_image_cache_max_setting, reload_sandbox_image_default_registry_setting, reload_sandbox_image_max_size_setting, reload_sandbox_image_pull_policy_setting, - reload_sandbox_registry_auth_setting, reload_scim_token_setting, reload_smtp_config, + reload_sandbox_registry_auth_setting, reload_scim_token_setting, + reload_service_log_retention_secs_setting, reload_smtp_config, reload_store_audit_logs_s3_setting, reload_uv_exclude_newer_setting, reload_uv_index_strategy_setting, reload_uv_python_install_mirror_setting, reload_worker_config, MonitorIteration, @@ -380,6 +382,7 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { "", &mut None, &None, + None, ) .await { @@ -1261,10 +1264,12 @@ Windmill Community Edition {GIT_VERSION} #[cfg(all(feature = "tantivy", feature = "parquet"))] let log_indexer_f = { let log_indexer_rx = killpill_rx.resubscribe(); - let log_index_writer2 = log_index_writer.clone(); + // Moved, not cloned: sealing a chunk takes sole ownership of its + // tantivy writer, which a second live handle would silently prevent. + let moved_log_index_writer = log_index_writer; async { if let Some(db) = conn.as_sql() { - if let Some(log_index_writer) = log_index_writer2 { + if let Some(log_index_writer) = moved_log_index_writer { windmill_indexer::service_logs_oss::run_indexer( db.clone(), log_index_writer, @@ -1661,7 +1666,7 @@ Windmill Community Edition {GIT_VERSION} } else { tracing::info!("Nothing to do, exiting."); } - send_current_log_file_to_object_store(&conn, &hostname, &mode).await; + flush_pending_log_files_to_object_store(&conn, &hostname, &mode).await; if let Some(db) = conn.as_sql() { tracing::info!("Exiting connection pool"); @@ -1952,6 +1957,9 @@ async fn process_notify_event( } TIMEOUT_WAIT_RESULT_SETTING => reload_timeout_wait_result_setting(conn).await, RETENTION_PERIOD_SECS_SETTING => reload_retention_period_setting(conn).await, + SERVICE_LOG_RETENTION_SECS_SETTING => { + reload_service_log_retention_secs_setting(conn).await + } RETENTION_PERIOD_SECS_OVERRIDES_SETTING => { if let Err(e) = load_retention_period_overrides(db).await { tracing::error!("Error loading per-workspace retention overrides: {e:#}"); diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 430f3659c3..ce09cebe71 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -70,18 +70,17 @@ use windmill_common::{ RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, SANDBOX_IMAGE_PULL_POLICY_SETTING, SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, - SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TIMEOUT_WAIT_RESULT_SETTING, - UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, - WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING, - WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, - WORKSPACE_MAX_QUEUED_JOBS_SETTING, + SERVICE_LOG_RETENTION_SECS_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, + TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, + UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, + WORKSPACE_FAIRNESS_ENABLED_SETTING, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, + WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, WORKSPACE_MAX_QUEUED_JOBS_SETTING, }, indexer::load_indexer_config, jobs::delete_jobs, jwt::JWT_SECRET, oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH, server::load_smtp_config, - tracing_init::JSON_FMT, users::truncate_token, utils::{empty_as_none, now_from_db, report_critical_error, Mode, HUB_API_SECRET}, worker::{ @@ -98,10 +97,10 @@ use windmill_common::{ KillpillSender, AUDIT_LOG_RETENTION_DAYS, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERTS_ON_TOKEN_EXPIRY, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, - HUB_BASE_URL, JOB_RETENTION_SECS, JOB_RETENTION_SECS_OVERRIDES, - JOB_RETENTION_SECS_OVERRIDES_LOADED, METRICS_DEBUG_ENABLED, METRICS_ENABLED, - MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, - SERVICE_LOG_RETENTION_SECS, STORE_AUDIT_LOGS_S3, + DEFAULT_SERVICE_LOG_RETENTION_SECS, HUB_BASE_URL, JOB_RETENTION_SECS, + JOB_RETENTION_SECS_OVERRIDES, JOB_RETENTION_SECS_OVERRIDES_LOADED, METRICS_DEBUG_ENABLED, + METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, + OTEL_TRACING_ENABLED, STORE_AUDIT_LOGS_S3, }; use windmill_common::{ client::AuthedClient, @@ -476,6 +475,19 @@ pub async fn initial_load( |v: Option| async move { HUB_API_SECRET.store(std::sync::Arc::new(v)) }, ); + // Outside the `server_mode` guard below: every mode reads this. A worker registers its + // rotated log files against the cutoff, and a dedicated indexer trims the search index to a + // window derived from it — neither is a server. + pass.setting(SERVICE_LOG_RETENTION_SECS_SETTING, true, |v| async move { + windmill_common::set_service_log_retention_secs(parse_setting_value::( + v, + SERVICE_LOG_RETENTION_SECS_SETTING, + "SERVICE_LOG_RETENTION_SECS", + DEFAULT_SERVICE_LOG_RETENTION_SECS, + |x| x, + )) + }); + if server_mode { pass.setting(RETENTION_PERIOD_SECS_SETTING, true, |v| async move { JOB_RETENTION_SECS.store( @@ -1221,32 +1233,60 @@ async fn sleep_until_next_minute_start_plus_one_s() { } use windmill_common::tracing_init::TMP_WINDMILL_LOGS_SERVICE; -async fn find_two_highest_files(hostname: &str) -> (Option, Option) { + +/// The minutely rolling appender names each file `.log.<%Y-%m-%d-%H-%M>`; +/// anything else in the directory is not a rotated log file. +fn parse_log_file_ts(file_name: &str) -> Option { + NaiveDateTime::parse_from_str( + file_name.rsplit('.').next()?, + windmill_common::tracing_init::LOG_TIMESTAMP_FMT, + ) + .ok() +} + +/// Oldest first. Readdir order is filesystem-dependent — tmpfs hands back the +/// newest entry first, ext4 hashes the names — so the listing has to be sorted +/// before anything picks a file out of it. +fn sorted_log_files(file_names: impl Iterator) -> Vec<(NaiveDateTime, String)> { + let mut files = file_names + .filter_map(|name| parse_log_file_ts(&name).map(|ts| (ts, name))) + .collect::>(); + files.sort(); + files +} + +/// Every log file but the newest one: that one is still being appended to, every +/// older one is final. +fn rotated_log_files(file_names: impl Iterator) -> Vec<(NaiveDateTime, String)> { + let mut files = sorted_log_files(file_names); + files.pop(); + files +} + +async fn read_log_file_names(hostname: &str) -> Vec { let log_dir = format!("{}/{}/", *TMP_WINDMILL_LOGS_SERVICE, hostname); - let rd_dir = tokio::fs::read_dir(log_dir).await; - if let Ok(mut log_files) = rd_dir { - let mut highest_file: Option = None; - let mut second_highest_file: Option = None; - while let Ok(Some(file)) = log_files.next_entry().await { - let file_name = file - .file_name() - .to_str() - .map(|x| x.to_string()) - .unwrap_or_default(); - if file_name > highest_file.clone().unwrap_or_default() { - second_highest_file = highest_file; - highest_file = Some(file_name); - } + let mut rd_dir = match tokio::fs::read_dir(&log_dir).await { + Ok(rd_dir) => rd_dir, + Err(e) => { + tracing::error!("Error reading log files: {}, {:#?}", log_dir, e); + return vec![]; + } + }; + let mut file_names = vec![]; + while let Ok(Some(file)) = rd_dir.next_entry().await { + if let Some(file_name) = file.file_name().to_str() { + file_names.push(file_name.to_string()); } - (highest_file, second_highest_file) - } else { - tracing::error!( - "Error reading log files: {}, {:#?}", - *TMP_WINDMILL_LOGS_SERVICE, - rd_dir.unwrap_err() - ); - (None, None) } + file_names +} + +async fn list_log_files(hostname: &str) -> Vec<(NaiveDateTime, String)> { + sorted_log_files(read_log_file_names(hostname).await.into_iter()) +} + +async fn list_rotated_log_files(hostname: &str) -> Vec<(NaiveDateTime, String)> { + rotated_log_files(read_log_file_names(hostname).await.into_iter()) } fn get_worker_group(mode: &Mode) -> Option { @@ -1266,133 +1306,188 @@ pub fn send_logs_to_object_store(conn: &Connection, hostname: &str, mode: &Mode) tokio::spawn(async move { let mut interval = tokio::time::interval(Duration::from_secs(10)); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + init_last_log_file_sent(&conn, &hostname).await; sleep_until_next_minute_start_plus_one_s().await; loop { interval.tick().await; - let (_, snd_highest_file) = find_two_highest_files(&hostname).await; - send_log_file_to_object_store( - &hostname, - &mode, - &worker_group, - &conn, - snd_highest_file, - false, - ) - .await; + let files = list_rotated_log_files(&hostname).await; + send_log_files_to_object_store(&hostname, &mode, &worker_group, &conn, files).await; } }); } -pub async fn send_current_log_file_to_object_store(conn: &Connection, hostname: &str, mode: &Mode) { - tracing::info!("Sending current log file to object store"); - let (highest_file, _) = find_two_highest_files(hostname).await; +pub async fn flush_pending_log_files_to_object_store( + conn: &Connection, + hostname: &str, + mode: &Mode, +) { + tracing::info!("Sending pending log files to object store"); let worker_group = get_worker_group(&mode); - send_log_file_to_object_store(hostname, mode, &worker_group, conn, highest_file, true).await; -} - -fn get_now_and_str() -> (NaiveDateTime, String) { - let ts = Utc::now().naive_utc(); - ( - ts, - ts.format(windmill_common::tracing_init::LOG_TIMESTAMP_FMT) - .to_string(), - ) + // Nothing rotates after this, so the file still being appended to is registered + // here, along with any rotated one the loop had not reached yet. Bounded like the + // pool close that follows: a backlog against a slow object store would otherwise + // hold the process past its termination grace period. Whatever is left over is + // registered by the next run's catch-up. + let flush = async { + let files = list_log_files(hostname).await; + send_log_files_to_object_store(hostname, mode, &worker_group, conn, files).await; + }; + if timeout(Duration::from_secs(15), flush).await.is_err() { + tracing::warn!("Could not send all pending log files in time (15s). Exiting anyway."); + } } lazy_static::lazy_static! { static ref LAST_LOG_FILE_SENT: Arc>> = Arc::new(Mutex::new(None)); + /// Serializes the periodic uploader against the shutdown flush. The uploader is a + /// detached task that keeps ticking while the flush runs and both walk the same + /// files, so without this both can clear the watermark for one file and count its + /// lines twice through the additive upsert. + static ref SENDING_LOG_FILES: tokio::sync::Mutex<()> = tokio::sync::Mutex::new(()); } +fn last_log_file_sent() -> Option { + LAST_LOG_FILE_SENT.lock().ok().and_then(|ts| *ts) +} + +/// Resume from what this host already registered, so a previous run's leftovers reach +/// the object store rather than being dropped. Their line counts come out zero, this +/// run having counted none of them, which only flattens their bars in the UI. +/// +/// The newest registered minute is left out on purpose: the shutdown flush registers +/// the file that was still open and the appender reopens that minute in append mode, +/// so a restart inside it would otherwise strand everything written afterwards. +/// +/// A row rewritten this way restores the object and sums the counters, but it keeps the +/// `indexed_at` it already had, so one the indexers have taken is not offered again and +/// the lines added by the rewrite stay out of search. +async fn init_last_log_file_sent(conn: &Connection, hostname: &str) { + let Some(db) = conn.as_sql() else { + return; + }; + match sqlx::query_scalar!( + "SELECT max(log_ts) FROM log_file + WHERE hostname = $1 AND log_ts < (SELECT max(log_ts) FROM log_file WHERE hostname = $1)", + hostname + ) + .fetch_one(db) + .await + { + Ok(Some(ts)) => { + if let Err(e) = LAST_LOG_FILE_SENT.lock().map(|mut last_log_file_sent| { + last_log_file_sent.replace(ts); + }) { + tracing::error!("Error initializing last log file sent: {:?}", e); + } + } + Ok(None) => {} + Err(e) => tracing::error!("Error loading last log file sent: {:?}", e), + } +} + +async fn send_log_files_to_object_store( + hostname: &str, + mode: &Mode, + worker_group: &Option, + conn: &Connection, + files: Vec<(NaiveDateTime, String)>, +) { + let _guard = SENDING_LOG_FILES.lock().await; + let retention_cutoff = Utc::now().naive_utc() + - chrono::Duration::seconds(windmill_common::service_log_retention_secs()); + for (ts, file_name) in files { + if last_log_file_sent().is_some_and(|last| last >= ts) { + continue; + } + // A run coming back from a long outage still finds its predecessor's files on + // disk. Registering one past the retention cutoff inserts a row + // `delete_expired_items` drops on its next pass, once the indexers have already + // paid to parse it. + if ts < retention_cutoff { + continue; + } + // Stop at the first failure rather than moving on, so a file is never + // registered before an older one that has not made it to the store yet. + // The indexers do not depend on that ordering — every row is offered until + // it is marked — but a gap here would still be visible while it lasts. + if !send_log_file_to_object_store(hostname, mode, worker_group, conn, &file_name, ts).await + { + break; + } + } +} + +/// Returns whether the file ended up registered in `log_file`. async fn send_log_file_to_object_store( hostname: &str, mode: &Mode, worker_group: &Option, conn: &Connection, - snd_highest_file: Option, - use_now: bool, -) { - if let Some(highest_file) = snd_highest_file { - //parse datetime frome file xxxx.yyyy-MM-dd-HH-mm - let (ts, ts_str) = if use_now { - get_now_and_str() - } else { - highest_file - .split(".") - .last() - .and_then(|x| { - NaiveDateTime::parse_from_str( - x, - windmill_common::tracing_init::LOG_TIMESTAMP_FMT, - ) - .ok() - .map(|y| (y, x.to_string())) - }) - .unwrap_or_else(get_now_and_str) - }; + file_name: &str, + ts: NaiveDateTime, +) -> bool { + #[cfg(feature = "parquet")] + if let Some(s3_client) = windmill_object_store::get_object_store().await { + let path = std::path::Path::new(&*TMP_WINDMILL_LOGS_SERVICE) + .join(hostname) + .join(file_name); - let exists = LAST_LOG_FILE_SENT.lock().map(|last_log_file_sent| { - last_log_file_sent - .map(|last_log_file_sent| last_log_file_sent >= ts) - .unwrap_or(false) - }); - - if exists.unwrap_or(false) { - return; - } - - #[cfg(feature = "parquet")] - let s3_client = windmill_object_store::get_object_store().await; - #[cfg(feature = "parquet")] - if let Some(s3_client) = s3_client { - let path = std::path::Path::new(&*TMP_WINDMILL_LOGS_SERVICE) - .join(hostname) - .join(&highest_file); - - //read file as byte stream - let bytes = tokio::fs::read(&path).await; - if let Err(e) = bytes { + //read file as byte stream + let bytes = match tokio::fs::read(&path).await { + Ok(bytes) => bytes, + Err(e) => { tracing::error!("Error reading log file: {:?}", e); - return; + return false; } - let path = windmill_object_store::object_store_reexports::Path::from_url_path(format!( - "{}{hostname}/{highest_file}", - windmill_common::tracing_init::LOGS_SERVICE - )); - if let Err(e) = path { + }; + let path = windmill_object_store::object_store_reexports::Path::from_url_path(format!( + "{}{hostname}/{file_name}", + windmill_common::tracing_init::LOGS_SERVICE + )); + let path = match path { + Ok(path) => path, + Err(e) => { tracing::error!("Error creating log file path: {:?}", e); - return; - } - if let Err(e) = s3_client.put(&path.unwrap(), bytes.unwrap().into()).await { - tracing::error!("Error sending logs to object store: {:?}", e); + return false; } + }; + if let Err(e) = s3_client.put(&path, bytes.into()).await { + tracing::error!("Error sending logs to object store: {:?}", e); + return false; } + } - let (ok_lines, err_lines) = read_log_counters(ts_str); + let ts_str = ts + .format(windmill_common::tracing_init::LOG_TIMESTAMP_FMT) + .to_string(); + let (ok_lines, err_lines) = read_log_counters(ts_str); - if let Some(db) = conn.as_sql() { - match timeout(Duration::from_secs(10), sqlx::query!("INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt) - VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8) - ON CONFLICT (hostname, log_ts) DO UPDATE SET ok_lines = log_file.ok_lines + $6, err_lines = log_file.err_lines + $7", - hostname, mode.to_string(), worker_group.clone(), ts, highest_file, ok_lines as i64, err_lines as i64, *JSON_FMT) - .execute(db)).await { - Ok(Ok(_)) => { - if let Err(e) = LAST_LOG_FILE_SENT.lock().map(|mut last_log_file_sent| { - last_log_file_sent.replace(ts); - }) { - tracing::error!("Error updating last log file sent: {:?}", e); - } - tracing::info!("Log file sent: {}", highest_file); - } - Ok(Err(e)) => { - tracing::error!("Error inserting log file: {:?}", e); - } - Err(e) => { - tracing::error!("Error inserting log file, timeout elapsed: {:?}", e); - } + let Some(db) = conn.as_sql() else { + // not sending log file to object store in agent mode + return false; + }; + + match timeout(Duration::from_secs(10), sqlx::query!("INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt) + VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8) + ON CONFLICT (hostname, log_ts) DO UPDATE SET ok_lines = log_file.ok_lines + $6, err_lines = log_file.err_lines + $7", + hostname, mode.to_string(), worker_group.clone(), ts, file_name, ok_lines as i64, err_lines as i64, true) + .execute(db)).await { + Ok(Ok(_)) => { + if let Err(e) = LAST_LOG_FILE_SENT.lock().map(|mut last_log_file_sent| { + last_log_file_sent.replace(ts); + }) { + tracing::error!("Error updating last log file sent: {:?}", e); } - } else { - // tracing::warn!("Not sending log file to object store in agent mode"); - () + tracing::info!("Log file sent: {}", file_name); + true + } + Ok(Err(e)) => { + tracing::error!("Error inserting log file: {:?}", e); + false + } + Err(e) => { + tracing::error!("Error inserting log file, timeout elapsed: {:?}", e); + false } } } @@ -1580,6 +1675,13 @@ pub async fn trim_resource_versions(db: &DB) -> () { } } +/// Matches the batch the settings-page cleanup uses for the same table. +const SERVICE_LOG_DELETE_BATCH: i64 = 2_000; +/// Batches per pass. `monitor_db` runs under a 600s timeout that cancels every maintenance +/// future in the same `join!` and reports a critical error, so a large backlog has to drain +/// across ticks rather than inside one, the way the neighbouring sweeps already do. +const SERVICE_LOG_DELETE_MAX_BATCHES: usize = 10; + pub async fn delete_expired_items(db: &DB) -> () { let expired_tokens_r = sqlx::query_as!( TokenRow, @@ -1662,23 +1764,48 @@ pub async fn delete_expired_items(db: &DB) -> () { Err(e) => tracing::error!("Error deleting cache resource {}", e.to_string()), } - match sqlx::query_as!( - LogFile, - "DELETE FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval RETURNING file_path, hostname", - SERVICE_LOG_RETENTION_SECS, - ) - .fetch_all(db) - .await - { - Ok(log_files_to_delete) => { + // Batched: every process rotates a log file a minute, so lowering the retention makes one + // ordinary setting change expire millions of rows at once. An unbounded `DELETE ... + // RETURNING` would materialize all of them, and their deletion futures, in this one tick. + for _ in 0..SERVICE_LOG_DELETE_MAX_BATCHES { + let batch = sqlx::query_as!( + LogFile, + "DELETE FROM log_file WHERE (hostname, log_ts) IN ( + SELECT hostname, log_ts FROM log_file + WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval + LIMIT $2 + ) RETURNING file_path, hostname", + windmill_common::service_log_retention_secs(), + SERVICE_LOG_DELETE_BATCH, + ) + .fetch_all(db) + .await; + + match batch { + Ok(log_files_to_delete) => { + if log_files_to_delete.is_empty() { + break; + } + let n = log_files_to_delete.len(); let paths = log_files_to_delete .iter() .map(|f| format!("{}/{}", f.hostname, f.file_path)) .collect(); - delete_log_files_from_disk_and_store(paths, &*TMP_WINDMILL_LOGS_SERVICE, windmill_common::tracing_init::LOGS_SERVICE).await; - + delete_log_files_from_disk_and_store( + paths, + &*TMP_WINDMILL_LOGS_SERVICE, + windmill_common::tracing_init::LOGS_SERVICE, + ) + .await; + if (n as i64) < SERVICE_LOG_DELETE_BATCH { + break; + } + } + Err(e) => { + tracing::error!("Error deleting log file: {:?}", e); + break; + } } - Err(e) => tracing::error!("Error deleting log file: {:?}", e), } let audit_retention_days = audit_log_retention_days().await; @@ -2785,6 +2912,21 @@ pub async fn reload_retention_period_setting(conn: &Connection) { } } +pub async fn reload_service_log_retention_secs_setting(conn: &Connection) { + match load_setting_value::( + conn, + SERVICE_LOG_RETENTION_SECS_SETTING, + "SERVICE_LOG_RETENTION_SECS", + DEFAULT_SERVICE_LOG_RETENTION_SECS, + |x| x, + ) + .await + { + Ok(v) => windmill_common::set_service_log_retention_secs(v), + Err(e) => tracing::error!("Error reloading service log retention period: {:?}", e), + } +} + pub async fn reload_audit_log_retention_days_setting(conn: &Connection) { match load_setting_value::( conn, @@ -6832,3 +6974,54 @@ mod zombie_worker_memory_pct_tests { ); } } + +#[cfg(test)] +mod log_file_listing_tests { + use super::{rotated_log_files, sorted_log_files}; + + fn names(files: Vec<(chrono::NaiveDateTime, String)>) -> Vec { + files.into_iter().map(|(_, n)| n).collect() + } + + /// A directory read newest-entry-first is what tmpfs actually hands back. + #[test] + fn orders_by_minute_whatever_order_readdir_used() { + let newest_first = [ + "h.log.2026-08-29-06-49", + "h.log.2026-08-29-06-46", + "h.log.2026-08-29-06-48", + "h.log.2026-08-29-06-47", + ]; + assert_eq!( + names(sorted_log_files(newest_first.iter().map(|x| x.to_string()))), + vec![ + "h.log.2026-08-29-06-46", + "h.log.2026-08-29-06-47", + "h.log.2026-08-29-06-48", + "h.log.2026-08-29-06-49", + ] + ); + assert_eq!( + names(rotated_log_files( + newest_first.iter().map(|x| x.to_string()) + )), + vec![ + "h.log.2026-08-29-06-46", + "h.log.2026-08-29-06-47", + "h.log.2026-08-29-06-48", + ] + ); + } + + #[test] + fn drops_names_that_are_not_rotated_log_files() { + let files = sorted_log_files( + ["h.log", "not-a-log-file", "h.log.2026-08-29-06-46"] + .iter() + .map(|x| x.to_string()), + ); + assert_eq!(files.len(), 1); + assert_eq!(files[0].1, "h.log.2026-08-29-06-46"); + assert_eq!(files[0].0.to_string(), "2026-08-29 06:46:00"); + } +} diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index a9865cebd3..dfd0305e57 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -38,6 +38,8 @@ account: workspace_id(char), id(int), expires_at(ts), refresh_token(char), clien FK: (workspace_id) -> workspace(id) agent_token_blacklist: token(char), expires_at(ts), blacklisted_at(ts), blacklisted_by(char) ai_agent_memory: workspace_id(char), conversation_id(uuid), step_id(char), messages(jsonb), created_at(ts), updated_at(ts) +ai_free_token_daily_usage: day(date), cost_nanos(bigint), updated_at(ts) +ai_free_token_usage: email(char), cost_nanos(bigint), updated_at(ts) ai_token_usage: workspace_id(char), day(date), email(char), provider(char), model(char), session_id(char), input_tokens(bigint), cache_read_tokens(bigint), cache_write_tokens(bigint), output_tokens(bigint), reported_cost_nano_usd(bigint), requests(bigint), updated_at(ts) FK: (workspace_id) -> workspace(id) alerts: id(int), alert_type(char), message(text), created_at(ts), acknowledged(bool), workspace_id(text), acknowledged_workspace(bool), resource(text) @@ -128,7 +130,7 @@ job_stats: workspace_id(char), job_id(uuid), metric_id(char), metric_name(char), kafka_pending_commits: id(bigint), workspace_id(char), kafka_trigger_path(char), topic(char), partition(int), offset(bigint), created_at(ts) FK: (workspace_id, kafka_trigger_path) -> kafka_trigger(workspace_id, path) kafka_trigger: path(char), kafka_resource_path(char), topics(char), group_id(char), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), server_id(char), last_server_ping(ts), error(text), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), mode(trigger_mode), filters(jsonb[]), auto_commit(bool), labels(text[]) -log_file: hostname(char), log_ts(ts), ok_lines(bigint), err_lines(bigint), mode(log_mode), worker_group(char), file_path(char), json_fmt(bool) +log_file: hostname(char), log_ts(ts), ok_lines(bigint), err_lines(bigint), mode(log_mode), worker_group(char), file_path(char), json_fmt(bool), indexed_at(ts) macro_definition: workspace_id(char), name(char), provider_path(char), params(text), body(text), is_table_macro(bool), created_at(ts) FK: (workspace_id) -> workspace(id) macro_usage: workspace_id(char), consumer_path(char), macro_name(char) diff --git a/backend/tests/suspended_pull_index.rs b/backend/tests/suspended_pull_index.rs new file mode 100644 index 0000000000..4b31f4d45e --- /dev/null +++ b/backend/tests/suspended_pull_index.rs @@ -0,0 +1,77 @@ +//! Pins the plan of the suspended-job pull. Its resume test degrades silently: once the +//! query expression and `queue_suspended_v2` stop matching, Postgres still returns the right +//! job, just by falling back to a heap filter and fetching one tuple per suspended row on +//! every worker poll. No functional test can see that, so assert on the plan instead. + +use serde_json::Value; +use sqlx::{Pool, Postgres}; +use windmill_common::worker::make_suspended_pull_query; + +/// Depth-first walk of an `EXPLAIN (FORMAT JSON)` plan tree. +fn nodes(plan: &Value, out: &mut Vec) { + out.push(plan.clone()); + for child in plan["Plans"].as_array().unwrap_or(&vec![]) { + nodes(child, out); + } +} + +#[sqlx::test(fixtures("base"))] +async fn suspended_pull_tests_resume_time_inside_the_index( + db: Pool, +) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, created_at, scheduled_for, running, suspend, suspend_until, tag) + SELECT gen_random_uuid(), 'test-workspace', now() - make_interval(secs => i), + now(), true, 1 + (i % 3), now() + interval '7 day', 'flow' + FROM generate_series(1, 2000) i", + ) + .execute(&db) + .await?; + sqlx::query("ANALYZE v2_job_queue").execute(&db).await?; + + // Both plans are cheap on a 2000-row table, and which one wins there says nothing + // about a queue with a large suspended backlog. Force the index path, which is the + // one production takes, and assert on how it evaluates the resume test. + let mut conn = db.acquire().await?; + sqlx::query("SET enable_seqscan = off") + .execute(&mut *conn) + .await?; + let version: String = sqlx::query_scalar("SELECT version()") + .fetch_one(&mut *conn) + .await?; + // FORMAT JSON rather than the default: `Index Cond` and `Filter` are separate keys on the + // node, so this does not ride on EXPLAIN's line layout staying put across a major bump. + let explained: Value = sqlx::query_scalar(&format!( + "EXPLAIN (FORMAT JSON) {}", + make_suspended_pull_query(&["flow".to_string()]) + )) + .bind("test-worker") + .fetch_one(&mut *conn) + .await?; + + let mut all = vec![]; + nodes(&explained[0]["Plan"], &mut all); + let pretty = serde_json::to_string_pretty(&explained)?; + let scan = all + .iter() + .find(|n| n["Index Name"] == "queue_suspended_v2") + .unwrap_or_else(|| { + panic!("suspended pull did not scan queue_suspended_v2 on {version}:\n{pretty}") + }); + // Only `Index Cond` is checked against the index tuple, so that is where the resume test + // has to land — as a `Filter` it would cost a heap fetch per suspended row. The residual + // `suspend_until IS NOT NULL` filter is not that: it is always true for rows the partial + // index holds, and only ever runs on the row LIMIT 1 already fetched. + let cond = scan["Index Cond"].as_str().unwrap_or_else(|| { + panic!("no Index Cond on the suspended pull scan on {version}:\n{pretty}") + }); + assert!( + cond.contains("CASE WHEN"), + "resume test is not an index condition on {version}:\n{pretty}" + ); + assert!( + !scan["Filter"].as_str().unwrap_or("").contains("CASE WHEN"), + "resume test fell back to a heap filter on {version}:\n{pretty}" + ); + Ok(()) +} diff --git a/backend/tests/wm_token_confinement.rs b/backend/tests/wm_token_confinement.rs index 5eb885fba9..2d3f1373fe 100644 --- a/backend/tests/wm_token_confinement.rs +++ b/backend/tests/wm_token_confinement.rs @@ -318,6 +318,33 @@ async fn test_wm_token_is_confined_to_its_workspace(db: Pool) -> anyho resp.text().await? ); } + // ...and the one `settings/global` key on the allowlist, which the CLI reads before + // creating a user on a git-sync push. `ws_base_url` is the control: the handler leaves + // it as ungated as `automate_username_creation`, so only the allowlist stops it. + let resp = authed( + client().get(format!("{api}/settings/global/automate_username_creation")), + &user_wm, + ) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "WM_TOKEN must still read automate_username_creation: {}", + resp.text().await? + ); + let resp = authed( + client().get(format!("{api}/settings/global/ws_base_url")), + &user_wm, + ) + .send() + .await?; + assert_eq!( + resp.status(), + 403, + "WM_TOKEN must not read any other global setting: {}", + resp.text().await? + ); let resp = authed(client().post(format!("{api}/schedules/preview")), &user_wm) .json(&json!({ "schedule": "0 0 12 * * *", "timezone": "UTC" })) .send() diff --git a/backend/windmill-ai/src/ai_providers.rs b/backend/windmill-ai/src/ai_providers.rs index bf16877f7b..a4dcbc4f35 100644 --- a/backend/windmill-ai/src/ai_providers.rs +++ b/backend/windmill-ai/src/ai_providers.rs @@ -24,6 +24,16 @@ lazy_static::lazy_static! { .ok() .map(|v| v == "true" || v == "1") .unwrap_or(false); + /// Drops the cache breakpoints from agent-step requests on every Anthropic platform, + /// not just the one that motivates it: a Google Cloud project can have explicit prompt + /// caching turned off (by request to Cloud support), and Vertex then rejects any request + /// carrying breakpoints. An instance that sets this to unblock such a project also gives + /// up caching on its direct-Anthropic and Foundry resources. + pub static ref DISABLE_ANTHROPIC_PROMPT_CACHING: bool = + std::env::var("DISABLE_ANTHROPIC_PROMPT_CACHING") + .ok() + .map(|v| v == "true" || v == "1") + .unwrap_or(false); } pub const OPENAI_BASE_URL: &str = "https://api.openai.com/v1"; diff --git a/backend/windmill-ai/src/providers/anthropic.rs b/backend/windmill-ai/src/providers/anthropic.rs index f22f28de14..486b08b739 100644 --- a/backend/windmill-ai/src/providers/anthropic.rs +++ b/backend/windmill-ai/src/providers/anthropic.rs @@ -1,7 +1,7 @@ use super::{anthropic_model_rejects_sampling_params, REASONING_OFF_SENTINEL}; use crate::{ ai_google::parse_data_url, - ai_providers::{AIPlatform, AIProvider}, + ai_providers::{AIPlatform, AIProvider, DISABLE_ANTHROPIC_PROMPT_CACHING}, image_handler::prepare_messages_for_api, proxy::{ add_user_to_body, common_outbound_headers, credential_header, ProxyBuildArgs, ProxyRequest, @@ -632,15 +632,13 @@ impl AnthropicQueryBuilder { } } + let caching = !*DISABLE_ANTHROPIC_PROMPT_CACHING; + let system = collect_system_prompt(&prepared_messages, args.system_prompt).map(|text| { vec![AnthropicSystemContent { r#type: "text".to_string(), text, - cache_control: if self.is_vertex() { - None - } else { - Some(CacheControl::ephemeral()) - }, + cache_control: caching.then(CacheControl::ephemeral), }] }); @@ -665,7 +663,7 @@ impl AnthropicQueryBuilder { let max_tokens = Some(args.max_tokens.unwrap_or(64000)); // Apply cache_control on the last custom tool - if !self.is_vertex() { + if caching { if let Some(ref mut tools_vec) = tools_option { if let Some(AnthropicTool::Custom(ref mut custom)) = tools_vec.last_mut() { custom.cache_control = Some(CacheControl::ephemeral()); @@ -674,7 +672,7 @@ impl AnthropicQueryBuilder { } // Apply cache_control on the last content block of the last message - if !self.is_vertex() { + if caching { if let Some(last_msg) = anthropic_messages.last_mut() { if let Some(last_block) = last_msg.content.last_mut() { match last_block { @@ -882,10 +880,15 @@ mod tests { } } - async fn build_text_body(messages: &[OpenAIMessage], system_prompt: Option<&str>) -> String { + async fn build_text_body_on( + platform: AIPlatform, + messages: &[OpenAIMessage], + system_prompt: Option<&str>, + tools: Option<&[ToolDef]>, + ) -> String { let args = BuildRequestArgs { messages, - tools: None, + tools, model: "claude-sonnet-4", temperature: None, reasoning_effort: None, @@ -899,12 +902,16 @@ mod tests { prompt_cache_key: None, }; - AnthropicQueryBuilder::new(AIProvider::Anthropic, AIPlatform::Standard) + AnthropicQueryBuilder::new(AIProvider::Anthropic, platform) .build_request(&args, &authed_client(), "test-workspace") .await .unwrap() } + async fn build_text_body(messages: &[OpenAIMessage], system_prompt: Option<&str>) -> String { + build_text_body_on(AIPlatform::Standard, messages, system_prompt, None).await + } + /// The worker prepends the system prompt as a system message *and* passes it as /// `system_prompt`; the request must still carry it exactly once. #[tokio::test] @@ -946,6 +953,35 @@ mod tests { assert!(request.get("system").is_none()); } + /// Vertex serves the same Messages API and honours `cache_control` breakpoints, so + /// its requests must carry the same three the standard platform gets. + #[tokio::test] + async fn sets_cache_breakpoints_on_every_platform() { + let messages = vec![message("system", SYSTEM_PROMPT), message("user", "hi")]; + let tools = vec![ToolDef { + r#type: "function".to_string(), + function: ToolDefFunction { + name: "get_weather".to_string(), + description: None, + parameters: RawValue::from_string("{}".to_string()).unwrap(), + }, + }]; + let ephemeral = serde_json::json!({ "type": "ephemeral" }); + + for platform in [AIPlatform::Standard, AIPlatform::GoogleVertexAi] { + let body = + build_text_body_on(platform, &messages, Some(SYSTEM_PROMPT), Some(&tools)).await; + let request: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert_eq!(request["system"][0]["cache_control"], ephemeral); + let sent_tools = request["tools"].as_array().unwrap(); + assert_eq!(sent_tools.last().unwrap()["cache_control"], ephemeral); + let sent = request["messages"].as_array().unwrap(); + let content = sent.last().unwrap()["content"].as_array().unwrap(); + assert_eq!(content.last().unwrap()["cache_control"], ephemeral); + } + } + fn has_header(headers: &[(String, String)], name: &str, value: &str) -> bool { headers .iter() diff --git a/backend/windmill-api-auth/src/scopes.rs b/backend/windmill-api-auth/src/scopes.rs index 7109a04425..d99d955de9 100644 --- a/backend/windmill-api-auth/src/scopes.rs +++ b/backend/windmill-api-auth/src/scopes.rs @@ -996,6 +996,13 @@ fn scope_grants_access( /// the caller's own row; `email` and `allowed_domain_auto_invite` are derived from the /// token itself and touch no table. /// +/// `settings/global/automate_username_creation` is the one instance setting on the list. +/// `get_global_setting` exempts a handful of keys from its own super-admin gate, that one +/// among them, so the boolean is already readable by every authenticated user; it is here +/// because the CLI reads it before creating a user during a git-sync push, which runs as a +/// job. The other ungated keys have no such caller, so they stay confined — being ungated +/// earns a key nothing on its own. +/// /// Deliberately absent, as each crosses that line: `users/list_invites` (returns the /// workspace ids the identity was invited to), `users/tokens/list` (credential metadata /// of the borrowed identity), `users/exists/{email}` (an oracle over arbitrary @@ -1011,6 +1018,7 @@ fn is_global_read_open_to_job_token(route_path: &str) -> bool { | "/api/users/usage" | "/api/users/tutorial_progress" | "/api/workspaces/allowed_domain_auto_invite" + | "/api/settings/global/automate_username_creation" | "/api/docs/search" | "/api/docs/page" | "/api/integrations/hub/list" diff --git a/backend/windmill-api-debug/src/lib.rs b/backend/windmill-api-debug/src/lib.rs index 7b2f569821..ab89df988d 100644 --- a/backend/windmill-api-debug/src/lib.rs +++ b/backend/windmill-api-debug/src/lib.rs @@ -468,6 +468,9 @@ async fn sign_debug_request( // Parse the language let script_lang: ScriptLang = request.language.parse().unwrap_or(ScriptLang::Bun); + // Taken from the parsed language, not the request's string: the telemetry key vocabulary has + // to stay the closed set of languages rather than whatever a caller sent. + let lang_key = script_lang.as_str(); // Hash the code (we don't include full code in JWT to keep it small) let mut hasher = Sha256::new(); @@ -578,6 +581,8 @@ async fn sign_debug_request( tx.commit().await?; + windmill_common::feature_usage::log_feature_usage("debugger", "session", lang_key); + Ok(Json(SignedDebugPayload { token, code: request.code, diff --git a/backend/windmill-api-integration-tests/tests/users.rs b/backend/windmill-api-integration-tests/tests/users.rs index fb64d9eb77..bec59ac45a 100644 --- a/backend/windmill-api-integration-tests/tests/users.rs +++ b/backend/windmill-api-integration-tests/tests/users.rs @@ -308,14 +308,17 @@ async fn test_user_endpoints(db: Pool) -> anyhow::Result<()> { let auth_base = format!("http://localhost:{port}/api/auth"); // --- login (will fail: password hash in fixture is fake) --- + // An unparseable stored hash must read as a failed login, not as a server error + // relaying the hash parser's message to an unauthenticated caller. let resp = client() .post(format!("{auth_base}/login")) .json(&json!({"email": "test@windmill.dev", "password": "wrong-password"})) .send() .await .unwrap(); - assert!( - resp.status() == 400 || resp.status() == 401 || resp.status() == 500, + assert_eq!( + resp.status(), + 400, "login: unexpected status {}", resp.status() ); @@ -804,12 +807,16 @@ async fn test_change_user_email_leaves_group_identities(db: Pool) -> a let server = ApiServer::start(db.clone()).await?; let global_base = format!("http://localhost:{}/api/users", server.addr.port()); - sqlx::query!("UPDATE password SET email = 'group-ops@windmill.dev' WHERE email = 'test2@windmill.dev'") - .execute(&db) - .await?; - sqlx::query!("UPDATE usr SET email = 'group-ops@windmill.dev' WHERE email = 'test2@windmill.dev'") - .execute(&db) - .await?; + sqlx::query!( + "UPDATE password SET email = 'group-ops@windmill.dev' WHERE email = 'test2@windmill.dev'" + ) + .execute(&db) + .await?; + sqlx::query!( + "UPDATE usr SET email = 'group-ops@windmill.dev' WHERE email = 'test2@windmill.dev'" + ) + .execute(&db) + .await?; sqlx::query!( "INSERT INTO group_(workspace_id, name, summary, extra_perms) VALUES ('test-workspace', 'ops', '', '{}')" ) diff --git a/backend/windmill-api-settings/src/log_cleanup.rs b/backend/windmill-api-settings/src/log_cleanup.rs index ecbc38fcb7..82080cb61c 100644 --- a/backend/windmill-api-settings/src/log_cleanup.rs +++ b/backend/windmill-api-settings/src/log_cleanup.rs @@ -32,7 +32,7 @@ use windmill_common::tracing_init::{LOGS_SERVICE, TMP_WINDMILL_LOGS_SERVICE}; use windmill_common::worker::WINDMILL_DIR; use windmill_common::{ DB, INSTANCE_NAME, JOB_RETENTION_SECS, JOB_RETENTION_SECS_OVERRIDES, - JOB_RETENTION_SECS_OVERRIDES_LOADED, SERVICE_LOG_RETENTION_SECS, + JOB_RETENTION_SECS_OVERRIDES_LOADED, }; use windmill_object_store::object_store_reexports::{ @@ -249,7 +249,7 @@ async fn cleanup_service_logs( // Count candidates upfront for progress reporting. let total: i64 = sqlx::query_scalar!( "SELECT COUNT(*) FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval", - SERVICE_LOG_RETENTION_SECS, + windmill_common::service_log_retention_secs(), ) .fetch_one(db) .await? @@ -274,7 +274,7 @@ async fn cleanup_service_logs( WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval LIMIT $2 ) RETURNING file_path, hostname", - SERVICE_LOG_RETENTION_SECS, + windmill_common::service_log_retention_secs(), SERVICE_LOG_BATCH, ) .fetch_all(db) @@ -680,9 +680,10 @@ async fn cleanup_s3_orphans( ) -> error::Result<()> { let job_retention_secs = JOB_RETENTION_SECS.load(std::sync::atomic::Ordering::Relaxed); let now = Utc::now(); - // Service logs always have a retention (hardcoded SERVICE_LOG_RETENTION_SECS), - // so we scan for service-log orphans regardless of JOB_RETENTION_SECS. - let service_cutoff = now - chrono::Duration::seconds(SERVICE_LOG_RETENTION_SECS); + // Service logs always have a retention, so we scan for service-log orphans regardless of + // JOB_RETENTION_SECS. + let service_cutoff = + now - chrono::Duration::seconds(windmill_common::service_log_retention_secs()); // Job-log orphans are only considered once past a job's effective retention window. That window // is the instance one OR, for an override workspace (EE), its own — and jobs orphan their logs as diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index 8ea3bd0995..8daa414b46 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -19,7 +19,7 @@ use windmill_api_auth::ApiAuthed; pub use windmill_api_auth::Tokened; -use argon2::{Argon2, PasswordHash, PasswordVerifier}; +use argon2::{Argon2, PasswordVerifier}; use axum::{ extract::{Extension, Path, Query}, response::{IntoResponse, Response}, @@ -2680,10 +2680,8 @@ async fn login( .await?; if let Some((email, hash, super_admin)) = email_w_h { - let parsed_hash = - PasswordHash::new(&hash).map_err(|e| Error::internal_err(e.to_string()))?; if argon2 - .verify_password(password.as_bytes(), &parsed_hash) + .verify_password(password.as_bytes(), hash.as_str()) .is_err() { audit_log( @@ -3710,3 +3708,23 @@ async fn request_password_reset( } // NOTE: reset_password is in windmill-api (depends on users_oss::hash_password EE dispatch) + +#[cfg(test)] +mod tests { + use super::*; + + /// Stored hashes outlive the hashing crate: every instance still holds hashes minted by + /// older argon2 releases, and an upgrade that stopped reading them locks their users out. + #[test] + fn verifies_a_hash_minted_by_an_older_argon2() { + // The seeded admin hash from migration 20220508150023, m=4096,t=3,p=1. + let seeded = "$argon2id$v=19$m=4096,t=3,p=1$oLJo/lPn/gezXCuFOEyaNw$i0T2tCkw3xUFsrBIKZwr8jVNHlIfoxQe+HfDnLtd12I"; + + assert!(Argon2::default() + .verify_password(b"changeme", seeded) + .is_ok()); + assert!(Argon2::default() + .verify_password(b"not-the-password", seeded) + .is_err()); + } +} diff --git a/backend/windmill-api-workspaces/src/datatable_migrations.rs b/backend/windmill-api-workspaces/src/datatable_migrations.rs index 23026c10b8..b2d1f28453 100644 --- a/backend/windmill-api-workspaces/src/datatable_migrations.rs +++ b/backend/windmill-api-workspaces/src/datatable_migrations.rs @@ -10,7 +10,10 @@ //! management, and the workspace-merge diff helper. Split out of `workspaces.rs` //! to keep that file focused on core workspace configuration. -use crate::workspaces::{pg_dump_database, ItemComparison}; +use crate::workspaces::{ + is_instance_datatable, pg_dump_database, strip_unreplayable_dump_lines, ItemComparison, + PgDumpOptions, +}; use axum::{ extract::{Extension, Path, Query}, @@ -1448,18 +1451,25 @@ async fn generate_initial_datatable_migration( let pg_db: PgDatabase = serde_json::from_value(db_resource) .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?; - // Snapshot the schema, excluding Windmill's own migration bookkeeping table. - let dump_file = pg_dump_database(&pg_db, true, &["_wm_migrations"]).await?; + // Snapshot the schema without `_wm_migrations`, Windmill's own bookkeeping table, and + // without what a replay elsewhere cannot run: the replaying user owns none of this + // database's objects, and the grants Windmill plants in an instance database (`ALTER + // DEFAULT PRIVILEGES FOR ROLE ...`) fail even replaying onto the same server. + let no_acl = is_instance_datatable(&db, &w_id, &datatable_name).await?; + let dump_file = pg_dump_database( + &pg_db, + PgDumpOptions { + schema_only: true, + exclude_tables: &["_wm_migrations"], + no_owner: true, + no_acl, + }, + ) + .await?; let raw_dump = tokio::fs::read_to_string(&dump_file.path) .await .map_err(|e| Error::internal_err(format!("Failed to read schema dump: {}", e)))?; - // pg_dump emits psql meta-commands (\restrict / \unrestrict) that aren't - // valid SQL; drop them so the migration body can run via a plain query. - let code_up: String = raw_dump - .lines() - .filter(|line| !line.trim_start().starts_with('\\')) - .collect::>() - .join("\n"); + let code_up = strip_unreplayable_dump_lines(&raw_dump); // Record the definition first, then mark it installed. If marking fails we // delete the definition, so a failure leaves no phantom "initial" (rather diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 3d23b747d0..4c461915ff 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -119,6 +119,7 @@ pub fn workspaced_service() -> Router { get(get_secondary_storage_names), ) .route("/is_premium", get(is_premium)) + .route("/billable_seats", get(get_billable_seats)) .route("/edit_error_handler", post(edit_error_handler)) .route("/edit_success_handler", post(edit_success_handler)) .route( @@ -690,6 +691,48 @@ async fn is_premium( Ok(Json(premium)) } +#[derive(Serialize)] +struct BillableSeatsResponse { + /// Both omitted when the seats counted are another workspace's: a fork member need not be a + /// member of the billing root, so the root's headcount is not theirs to read. The total is, + /// since it is the divisor of the quota their own executions draw on. + #[serde(skip_serializing_if = "Option::is_none")] + developers: Option, + #[serde(skip_serializing_if = "Option::is_none")] + operators: Option, + seats: i64, +} + +async fn get_billable_seats( + _authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult { + // Readable by any workspace member, like `is_premium`: this is what the sidebar usage meter + // divides by, and that meter is shown to non-admin developers too. + // + // On cloud a fork draws its plan, quota and bill from the root, so the seats its usage is + // measured against are the root's. Resolved here rather than by the caller: a fork member need + // not be a member of that root, and so cannot count its seats from the member list. Off cloud + // a fork is not billed through a root at all, so the workspace answers for itself. + #[cfg(feature = "cloud")] + let billing_w_id = if *CLOUD_HOSTED { + windmill_common::workspaces::get_billing_workspace_id(&db, &w_id).await? + } else { + w_id.clone() + }; + #[cfg(not(feature = "cloud"))] + let billing_w_id = w_id.clone(); + + let counted = windmill_common::workspaces::billable_seats(&db, &billing_w_id).await?; + let own = billing_w_id == w_id; + Ok(Json(BillableSeatsResponse { + developers: own.then_some(counted.developers), + operators: own.then_some(counted.operators), + seats: counted.seats, + })) +} + async fn exists_workspace( authed: ApiAuthed, Extension(user_db): Extension, @@ -1975,6 +2018,23 @@ async fn edit_large_file_storage_config( ))); } + if !windmill_common::workspaces::filesystem_storage_allowed() { + let named = std::iter::once(("primary storage", &lfs_config.large_file_storage)).chain( + lfs_config + .secondary_storage + .iter() + .map(|(name, storage)| (name.as_str(), storage)), + ); + for (name, storage) in named { + if matches!(storage, LargeFileStorage::FilesystemStorage(_)) { + return Err(Error::BadRequest(format!( + "{name}: {}", + windmill_common::workspaces::FILESYSTEM_STORAGE_DEV_ONLY_MSG + ))); + } + } + } + let serialized_lfs_config = serde_json::to_value::(lfs_config) .map_err(|err| Error::internal_err(err.to_string()))?; @@ -2757,6 +2817,66 @@ fn truncate_column_default(default: String) -> String { mod tests { use super::*; + /// The header of a pg_dump, followed by an object whose body also holds a `SET`. + const DUMP: &str = "--\n\ + -- PostgreSQL database dump\n\ + --\n\ + \n\ + \\restrict aBcD\n\ + \n\ + SET statement_timeout = 0;\n\ + SET transaction_timeout = 0;\n\ + SET client_encoding = 'UTF8';\n\ + SELECT pg_catalog.set_config('search_path', '', false);\n\ + \n\ + SET default_table_access_method = heap;\n\ + \n\ + CREATE FUNCTION public.f() RETURNS void LANGUAGE plpgsql AS $$\n\ + BEGIN\n\ + SET transaction_timeout = 0;\n\ + END;\n\ + $$;\n"; + + #[test] + fn replayable_dump_keeps_everything_but_meta_commands_and_session_timeouts() { + let replayable = strip_unreplayable_dump_lines(DUMP); + + assert!(!replayable.contains("\\restrict")); + assert!(!replayable.contains("SET statement_timeout")); + assert!(!replayable.contains("SET transaction_timeout = 0;\nSET client_encoding")); + assert!(replayable.contains("SET client_encoding = 'UTF8';")); + assert!(replayable.contains("SET default_table_access_method = heap;")); + // Past the preamble the dump is an object's own text: left exactly as it is. + assert!(replayable.contains("BEGIN\nSET transaction_timeout = 0;\nEND;")); + } + + #[tokio::test] + async fn dump_preamble_only_drops_settings_the_server_lacks() { + let dump_file = DumpFile::new().unwrap(); + tokio::fs::write(&dump_file.path, DUMP).await.unwrap(); + let supported = [ + "statement_timeout", + "client_encoding", + "default_table_access_method", + ] + .map(String::from) + .into_iter() + .collect(); + + comment_out_unsupported_settings(&dump_file, &supported) + .await + .unwrap(); + + let patched = tokio::fs::read_to_string(&dump_file.path).await.unwrap(); + // Rewriting the header must not shift the rest of the dump. + assert_eq!(patched.len(), DUMP.len()); + assert!(patched.contains("-- transaction_timeout = 0;")); + assert!(patched.contains("SET statement_timeout = 0;")); + assert!(patched.contains("SET default_table_access_method = heap;")); + // The `SET` inside the function body is past the preamble: never touched. + assert!(patched.contains("BEGIN\nSET transaction_timeout = 0;\nEND;")); + } + #[test] fn compact_column_type_truncates_multibyte_defaults_safely() { let default = "é".repeat(31); @@ -2859,6 +2979,35 @@ pub(crate) async fn resolve_pg_source_checked( .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e))) } +/// Whether the data table `name` is backed by the Windmill instance's own PostgreSQL +/// rather than a user resource. +pub(crate) async fn is_instance_datatable(db: &DB, w_id: &str, name: &str) -> Result { + let config = sqlx::query_scalar!( + "SELECT datatable->'datatables'->$2 FROM workspace_settings WHERE workspace_id = $1", + w_id, + name + ) + .fetch_optional(db) + .await? + .flatten(); + Ok(config + .and_then(|v| { + v.get("database") + .and_then(|d| d.get("resource_type")) + .and_then(|r| r.as_str()) + .map(|s| s == "instance") + }) + .unwrap_or(false)) +} + +/// Same, for the `datatable://` / `$res:` form the import endpoints take. +async fn is_instance_datatable_source(db: &DB, w_id: &str, source: &str) -> Result { + match source.strip_prefix("datatable://") { + Some(name) => is_instance_datatable(db, w_id, name).await, + None => Ok(false), + } +} + /// A temporary file for pg_dump output that is automatically deleted when dropped. pub(crate) struct DumpFile { pub(crate) path: std::path::PathBuf, @@ -2906,12 +3055,21 @@ impl Drop for DumpFile { } } +#[derive(Default)] +pub(crate) struct PgDumpOptions<'a> { + pub(crate) schema_only: bool, + pub(crate) exclude_tables: &'a [&'a str], + /// Leave out `ALTER ... OWNER TO`. + pub(crate) no_owner: bool, + /// Leave out `GRANT`, `REVOKE` and `ALTER DEFAULT PRIVILEGES`. + pub(crate) no_acl: bool, +} + /// Run pg_dump against a PgDatabase, writing output to a temp file on disk. /// Returns a DumpFile handle; the file is deleted when the handle is dropped. pub(crate) async fn pg_dump_database( pg_db: &PgDatabase, - schema_only: bool, - exclude_tables: &[&str], + opts: PgDumpOptions<'_>, ) -> Result { let dump_file = DumpFile::new()?; @@ -2922,10 +3080,16 @@ pub(crate) async fn pg_dump_database( let mut cmd = tokio::process::Command::new("pg_dump"); cmd.arg("--format=plain").arg("--file").arg(&dump_file.path); - if schema_only { + if opts.schema_only { cmd.arg("--schema-only"); } - for table in exclude_tables { + if opts.no_owner { + cmd.arg("--no-owner"); + } + if opts.no_acl { + cmd.arg("--no-privileges"); + } + for table in opts.exclude_tables { cmd.arg(format!("--exclude-table={table}")); } cmd.arg("--host") @@ -2957,37 +3121,179 @@ pub(crate) async fn pg_dump_database( Ok(dump_file) } -/// Import a pg_dump file into a target database using psql. -async fn pg_import_dump(target_db: &PgDatabase, dump_file: &DumpFile) -> Result<()> { - let host = &target_db.host; - let port = target_db.port.unwrap_or(5432).to_string(); - let user = target_db.login_name(); - let dbname = &target_db.dbname; +/// Whether `line` still belongs to the preamble pg_dump emits before the first +/// dumped object: comments, blank lines, psql meta-commands and the session `SET`s. +fn is_dump_preamble_line(line: &[u8]) -> bool { + let line = line.trim_ascii_start(); + line.is_empty() + || line.starts_with(b"--") + || line.starts_with(b"\\") + || line.starts_with(b"SET ") + || line.starts_with(b"SELECT pg_catalog.set_config(") +} +/// The GUCs pg_dump's preamble sets only to keep the dumping session out of the way. +/// They are also the ones that come and go across versions (`transaction_timeout` is +/// PG 17+), so they are what a dump replayed on an older server trips over first. +const DUMP_SESSION_TIMEOUTS: [&str; 4] = [ + "statement_timeout", + "lock_timeout", + "idle_in_transaction_session_timeout", + "transaction_timeout", +]; + +/// Turn a dump into SQL that can be replayed on another database: drop pg_dump's psql +/// meta-commands (`\restrict` / `\unrestrict`, not valid SQL) and the session timeouts +/// its preamble sets, which the replaying server may not have as GUCs at all. Only the +/// preamble is filtered, so an object's body keeps whatever it holds. +pub(crate) fn strip_unreplayable_dump_lines(dump: &str) -> String { + let mut in_preamble = true; + dump.lines() + .filter(|line| { + in_preamble = in_preamble && is_dump_preamble_line(line.as_bytes()); + if line.trim_start().starts_with('\\') { + return false; + } + !(in_preamble + && preamble_setting_name(line.as_bytes()) + .is_some_and(|name| DUMP_SESSION_TIMEOUTS.contains(&name))) + }) + .collect::>() + .join("\n") +} + +/// The GUC a preamble `SET = ...;` line assigns, if the line is one. +fn preamble_setting_name(line: &[u8]) -> Option<&str> { + let name = line.strip_prefix(b"SET ")?.split(|c| *c == b' ').next()?; + std::str::from_utf8(name).ok() +} + +/// The preamble Windmill's postgres client writes can set GUCs an older server does not +/// have — harmless session tuning, but one failing statement aborts a restore that stops +/// on the first error. Comment those out in place, three bytes each, so the data +/// section's offsets stay put. +async fn comment_out_unsupported_settings( + dump_file: &DumpFile, + supported_settings: &HashSet, +) -> Result<()> { + use tokio::io::{AsyncBufReadExt, AsyncWriteExt}; + + let file = tokio::fs::File::open(&dump_file.path) + .await + .map_err(|e| Error::internal_err(format!("Failed to open dump file: {}", e)))?; + let mut reader = tokio::io::BufReader::new(file); + + let mut preamble: Vec = Vec::new(); + let mut patched = false; + loop { + let start = preamble.len(); + let read = reader + .read_until(b'\n', &mut preamble) + .await + .map_err(|e| Error::internal_err(format!("Failed to read dump file: {}", e)))?; + if read == 0 { + break; + } + let line = &preamble[start..]; + if !is_dump_preamble_line(line) { + preamble.truncate(start); + break; + } + if preamble_setting_name(line).is_some_and(|name| !supported_settings.contains(name)) { + preamble[start..start + 3].copy_from_slice(b"-- "); + patched = true; + } + } + if !patched { + return Ok(()); + } + + let mut file = tokio::fs::OpenOptions::new() + .write(true) + .open(&dump_file.path) + .await + .map_err(|e| Error::internal_err(format!("Failed to open dump file: {}", e)))?; + file.write_all(&preamble) + .await + .map_err(|e| Error::internal_err(format!("Failed to rewrite dump preamble: {}", e)))?; + file.flush() + .await + .map_err(|e| Error::internal_err(format!("Failed to rewrite dump preamble: {}", e)))?; + Ok(()) +} + +/// A psql invocation against `pg_db`, carrying the connection settings the CLI reads +/// from the environment. +fn psql_command(pg_db: &PgDatabase) -> tokio::process::Command { let mut cmd = tokio::process::Command::new("psql"); cmd.arg("--host") - .arg(host) + .arg(&pg_db.host) .arg("--port") - .arg(&port) + .arg(pg_db.port.unwrap_or(5432).to_string()) .arg("--username") - .arg(user) + .arg(pg_db.login_name()) .arg("--dbname") - .arg(dbname) + .arg(&pg_db.dbname) .arg("--no-psqlrc") - .arg("--file") - .arg(&dump_file.path) .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()); - if let Some(ref password) = target_db.password { + if let Some(ref password) = pg_db.password { cmd.env("PGPASSWORD", password); } - - if let Some(ref sslmode) = target_db.sslmode { + if let Some(ref sslmode) = pg_db.sslmode { cmd.env("PGSSLMODE", sslmode); } + cmd +} - let output = cmd +/// GUC names the server backing `pg_db` knows about. +/// +/// Asked through psql rather than a tokio-postgres connection so the lookup reaches +/// exactly the servers the restore itself can: libpq negotiates TLS for `sslmode=prefer` +/// and an unset mode, where `PgDatabase::connect` would hand a TLS-only server a +/// plaintext socket and fail before the import ever starts. +async fn server_setting_names(pg_db: &PgDatabase) -> Result> { + let output = psql_command(pg_db) + .arg("--tuples-only") + .arg("--no-align") + .arg("--command") + .arg("SELECT name FROM pg_settings") + .output() + .await + .map_err(|e| Error::internal_err(format!("Failed to execute psql: {}", e)))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(Error::internal_err(format!( + "Failed to list the settings of the target server: {}", + stderr + ))); + } + + Ok(String::from_utf8_lossy(&output.stdout) + .lines() + .map(|name| name.trim().to_string()) + .filter(|name| !name.is_empty()) + .collect()) +} + +/// Import a pg_dump file into a target database using psql. +/// +/// Left to its defaults psql reports a failed statement, carries on and still exits 0, +/// so a dump that breaks partway through imports partially and reads as a success. +/// ON_ERROR_STOP surfaces the failure and --single-transaction makes the restore +/// all-or-nothing, leaving the target as it was and the import retryable. +async fn pg_import_dump(target_db: &PgDatabase, dump_file: &DumpFile) -> Result<()> { + let supported_settings = server_setting_names(target_db).await?; + comment_out_unsupported_settings(dump_file, &supported_settings).await?; + + let output = psql_command(target_db) + .arg("--set") + .arg("ON_ERROR_STOP=1") + .arg("--single-transaction") + .arg("--file") + .arg(&dump_file.path) .output() .await .map_err(|e| Error::internal_err(format!("Failed to execute psql: {}", e)))?; @@ -3032,29 +3338,7 @@ async fn create_pg_database( } } - // Determine if this is an instance or resource-backed datatable - let is_instance_datatable = if let Some(dt_name) = req.source.strip_prefix("datatable://") { - let config = sqlx::query_scalar!( - "SELECT datatable->'datatables'->$2 FROM workspace_settings WHERE workspace_id = $1", - &w_id, - dt_name - ) - .fetch_optional(&db) - .await? - .flatten(); - config - .and_then(|v| { - v.get("database") - .and_then(|d| d.get("resource_type")) - .and_then(|r| r.as_str()) - .map(|s| s == "instance") - }) - .unwrap_or(false) - } else { - false - }; - - if is_instance_datatable { + if is_instance_datatable_source(&db, &w_id, &req.source).await? { windmill_common::create_custom_instance_database(&db, &req.target_dbname, "datatable") .await?; } else { @@ -3152,7 +3436,18 @@ async fn import_pg_database( } windmill_common::validate_dbname(&target_pg.dbname)?; - let dump_file = pg_dump_database(&source_pg, schema_only, &[]).await?; + // Ownership never replays: the restore runs as the target's own connection user, and + // what it creates it owns. Grants do, except around an instance data table — Windmill + // plants `custom_instance_user` grants in one, which nothing else can replay. Elsewhere + // the ACLs are user intent (`REVOKE ... FROM PUBLIC`) and dropping them widens access. + let no_acl = is_instance_datatable_source(&db, &w_id, &req.target).await? + || is_instance_datatable_source(&db, &w_id, &req.source).await?; + + let dump_file = pg_dump_database( + &source_pg, + PgDumpOptions { schema_only, no_owner: true, no_acl, ..Default::default() }, + ) + .await?; pg_import_dump(&target_pg, &dump_file).await?; Ok(format!( @@ -3174,7 +3469,11 @@ async fn export_pg_schema( Json(req): Json, ) -> Result { let pg = resolve_pg_source_checked(&db, &user_db, &authed, &w_id, &req.source).await?; - let dump_file = pg_dump_database(&pg, true, &[]).await?; + let dump_file = pg_dump_database( + &pg, + PgDumpOptions { schema_only: true, ..Default::default() }, + ) + .await?; tokio::fs::read_to_string(&dump_file.path) .await .map_err(|e| Error::internal_err(format!("Failed to read dump file: {}", e))) @@ -7296,6 +7595,76 @@ async fn enforce_cloud_fork_cap(db: &DB, parent_workspace_id: &str) -> Result<() enforce_cloud_fork_count(db, &root, 1).await } +/// Cloud: refuse to attach a workspace that already has a paid plan of its own. +/// +/// Once attached it draws the root's plan and meters its usage there, so a subscription of its own +/// bills a second time for one plan. Only an attach can reach this state: a fork is created as a +/// fresh workspace and never had a plan to keep. +/// +/// Asked only of a candidate joining this family, never of one already under the same root: that +/// one is already in the double-billed state, where the settings page surfaces the leftover +/// subscription and the portal that cancels it, and refusing there would block re-designating a +/// renamed dev workspace over a billing problem the attach did not cause. +#[cfg(feature = "cloud")] +async fn reject_attach_of_subscribed_workspace(db: &DB, dev_w_id: &str) -> Result<()> { + let plan = sqlx::query_scalar!( + "SELECT plan FROM workspace_settings WHERE workspace_id = $1", + dev_w_id + ) + .fetch_optional(db) + .await? + .flatten(); + // Any plan, not just `'team'`: the column is written by the subscription webhook, and a plan + // value it does not write yet would otherwise walk straight past this. An enterprise + // arrangement is deliberately not covered — it sets `premium` without a plan and has no + // self-serve portal, so refusing there would be a dead end rather than something to act on. + if plan.is_some() { + return Err(Error::BadRequest(format!( + "Workspace {dev_w_id} is on a paid plan of its own. A dev or fork workspace runs on its parent's plan and is never invoiced separately, so cancel that subscription from its own billing settings before attaching it." + ))); + } + Ok(()) +} + +#[cfg(all(test, feature = "cloud"))] +mod attach_billing_guard_tests { + use super::reject_attach_of_subscribed_workspace; + use sqlx::{Pool, Postgres}; + + async fn workspace_on_plan(db: &Pool, id: &str, plan: Option<&str>) { + sqlx::query("INSERT INTO workspace (id, name, owner) VALUES ($1, $1, 'test-user')") + .bind(id) + .execute(db) + .await + .expect("insert workspace"); + sqlx::query("INSERT INTO workspace_settings (workspace_id, plan) VALUES ($1, $2)") + .bind(id) + .bind(plan) + .execute(db) + .await + .expect("insert workspace_settings"); + } + + #[sqlx::test(migrations = "../migrations")] + async fn refuses_a_candidate_that_still_pays_for_itself(db: Pool) { + workspace_on_plan(&db, "subscribed", Some("team")).await; + workspace_on_plan(&db, "cancelled", None).await; + + let err = reject_attach_of_subscribed_workspace(&db, "subscribed") + .await + .expect_err("a workspace on a paid plan of its own must not be attachable"); + assert!(err.to_string().contains("paid plan of its own"), "{err}"); + + // Cancelling clears `plan` but keeps `customer_id`, so the plan column is what decides. + reject_attach_of_subscribed_workspace(&db, "cancelled") + .await + .expect("a workspace with no plan is attachable"); + reject_attach_of_subscribed_workspace(&db, "no-settings-row") + .await + .expect("a workspace with no settings row is attachable"); + } +} + /// General guardrail (all builds): reject creating a fork/dev under `parent` when it would nest deeper /// than `MAX_FORK_DEPTH`. `added_subtree_height` is the height of the subtree grafted below the new /// node — 0 for a plain fork, or the candidate's own subtree height for an attach. @@ -7864,6 +8233,17 @@ async fn attach_dev_workspace( ))); } + // Deliberately below the admin-of-candidate check, unlike the cap enforcement above: the + // refusal names the candidate's plan, so running it earlier would tell any admin of any + // premium workspace whether an arbitrary workspace id is on a team plan. + #[cfg(feature = "cloud")] + if *CLOUD_HOSTED { + let root = windmill_common::workspaces::get_billing_workspace_id(&db, &prod_w_id).await?; + if windmill_common::workspaces::get_billing_workspace_id(&db, &dev_w_id).await? != root { + reject_attach_of_subscribed_workspace(&db, &dev_w_id).await?; + } + } + let mut tx = db.begin().await?; // Everything above ran outside a transaction, so prod's eligibility and the chain's labels could // have changed under us: re-decide both here, under the pairing lock. diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 99a6cd90f7..ad7f12453c 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -10,7 +10,7 @@ path = "src/lib.rs" [features] default = [] -private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-assets/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-amqp?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-azure?/private", "windmill-trigger-email?/private", "windmill-git-sync/private", "windmill-autoscaling?/private", "windmill-object-store/private", "windmill-api-npm-proxy/private"] +private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-assets/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-amqp?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-azure?/private", "windmill-trigger-email?/private", "windmill-git-sync/private", "windmill-autoscaling?/private", "windmill-object-store/private", "windmill-api-npm-proxy/private", "windmill-indexer?/private"] enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker?/enterprise", "windmill-api-auth/enterprise", "windmill-store/enterprise", "windmill-api-jobs/enterprise", "windmill-api-scripts/enterprise", "windmill-api-flows/enterprise", "windmill-api-users/enterprise", "windmill-api-workspaces/enterprise", "windmill-api-groups/enterprise", "windmill-api-configs/enterprise", "windmill-api-settings/enterprise", "windmill-api-schedule/enterprise", "windmill-api-debug/enterprise", "windmill-api-agent-workers?/enterprise", "windmill-trigger/enterprise", "windmill-trigger-kafka?/enterprise", "windmill-trigger-postgres?/enterprise", "windmill-trigger-mqtt?/enterprise", "windmill-trigger-amqp?/enterprise", "windmill-trigger-websocket?/enterprise", "windmill-trigger-email?/enterprise", "windmill-trigger-nats?/enterprise", "windmill-trigger-sqs?/enterprise", "windmill-trigger-gcp?/enterprise", "windmill-trigger-azure?/enterprise", "windmill-trigger-http?/enterprise", "windmill-native-triggers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-api-npm-proxy/enterprise", "license"] stripe = [] run_inline = ["dep:windmill-worker", "windmill-api-configs/run_inline"] @@ -18,10 +18,12 @@ agent_worker_server = ["dep:windmill-worker", "dep:windmill-api-agent-workers"] enterprise_saml = ["dep:samael", "dep:libxml"] benchmark = [] embedding = ["windmill-api-embeddings/embedding"] -parquet = ["dep:datafusion", "windmill-common/parquet", "windmill-object-store/parquet", "windmill-worker?/parquet", "windmill-api-users/parquet", "windmill-api-settings/parquet", "windmill-api-workspaces/parquet", "windmill-api-npm-proxy/parquet", "dep:aws-sigv4", "dep:aws-sdk-config", "dep:quick-xml"] +parquet = ["dep:datafusion", "windmill-common/parquet", "windmill-object-store/parquet", "windmill-worker?/parquet", "windmill-indexer?/parquet", "windmill-api-users/parquet", "windmill-api-settings/parquet", "windmill-api-workspaces/parquet", "windmill-api-npm-proxy/parquet", "dep:aws-sigv4", "dep:aws-sdk-config", "dep:quick-xml"] prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker?/prometheus", "windmill-api-scripts/prometheus"] openidconnect = ["dep:openidconnect", "windmill-common/openidconnect", "windmill-store/openidconnect"] -tantivy = ["dep:windmill-indexer"] +# The service log search handler reads the columnar store, so tantivy alone is +# not enough for this crate to build on its own. +tantivy = ["dep:windmill-indexer", "parquet"] kafka = ["dep:windmill-trigger-kafka", "windmill-store/kafka"] kafka-gssapi = ["kafka", "windmill-trigger-kafka/kafka-gssapi"] nats = ["dep:windmill-trigger-nats", "windmill-store/nats"] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 0830b6ffbc..469a2cd3ab 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.796.0 + version: 1.800.0 title: Windmill API contact: @@ -3788,6 +3788,38 @@ paths: schema: type: boolean + /w/{workspace}/workspaces/billable_seats: + get: + summary: get the billable seats of the workspace the plan is billed on + operationId: getBillableSeats + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + + responses: + "200": + description: billable seats + content: + application/json: + schema: + type: object + properties: + developers: + type: integer + description: >- + Omitted when the seats counted are another workspace's, as they are for a + fork resolving to its billing root. + operators: + type: integer + description: >- + Omitted when the seats counted are another workspace's, as they are for a + fork resolving to its billing root. + seats: + type: integer + required: + - seats + /w/{workspace}/workspaces/premium_info: get: summary: get premium info @@ -24085,7 +24117,7 @@ paths: items: type: string hits: - description: log files that matched the query + description: the log lines that matched the query, newest first type: array items: $ref: "#/components/schemas/LogSearchHit" @@ -25514,6 +25546,68 @@ paths: schema: type: string + /w/{workspace}/hub/projects/{slug}/withdraw: + post: + summary: take a hub project submission back out of review + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. Everything pushed for the + submission is kept, so it can be fixed and submitted again. + operationId: withdrawHubProject + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: slug + in: path + required: true + description: hub project slug (3-50 chars, lowercase alphanumeric and hyphens, no leading/trailing hyphen) + schema: + type: string + minLength: 3 + maxLength: 50 + pattern: "^[a-z0-9][a-z0-9-]*[a-z0-9]$" + - $ref: "#/components/parameters/HubPublishFolder" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/projects/{slug}/discard_update: + post: + summary: discard the pending update to a published hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. The published project is + left untouched. + operationId: discardHubProjectUpdate + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: slug + in: path + required: true + description: hub project slug (3-50 chars, lowercase alphanumeric and hyphens, no leading/trailing hyphen) + schema: + type: string + minLength: 3 + maxLength: 50 + pattern: "^[a-z0-9][a-z0-9-]*[a-z0-9]$" + - $ref: "#/components/parameters/HubPublishFolder" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + /w/{workspace}/hub/project: get: summary: get the hub project linked to a workspace folder @@ -27389,11 +27483,29 @@ components: type: integer minimum: 1 maximum: 2000000 + free_tier: + $ref: "#/components/schemas/FreeTierInfo" model_pricing: type: object additionalProperties: $ref: "#/components/schemas/ModelPriceOverride" + FreeTierInfo: + type: object + description: >- + Read-only. Present when the workspace has no AI provider of its own and is running + on Windmill's free tier. Ignored on write. + properties: + exhausted: + type: boolean + description: The one-time grant is spent; no provider is served and the user must add their own API key. + used_ratio: + type: number + description: Fraction of the grant consumed, 0 to 1. + required: + - exhausted + - used_ratio + ModelPriceOverride: type: object description: negotiated rates in USD per million tokens, keyed `provider:model` @@ -28155,7 +28267,10 @@ components: type: string args: $ref: "#/components/schemas/ScriptArgs" - result: {} + result: + description: | + For large results, this may be the placeholder string 'WINDMILL_TOO_BIG'. + Use the completed job result endpoint to retrieve the full result. logs: type: string deleted: @@ -30429,7 +30544,9 @@ components: description: If true, passes the request body as a raw string instead of parsing as JSON error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -30518,7 +30635,9 @@ components: description: If true, passes the request body as a raw string instead of parsing as JSON error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -30614,7 +30733,9 @@ components: description: If true, passes the request body as a raw string instead of parsing as JSON error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -30806,7 +30927,9 @@ components: description: Optional periodic heartbeat message configuration error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: description: Arguments to pass to the error handler $ref: "#/components/schemas/ScriptArgs" @@ -30871,7 +30994,9 @@ components: description: Optional periodic heartbeat message configuration error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: description: Arguments to pass to the error handler $ref: "#/components/schemas/ScriptArgs" @@ -30947,7 +31072,9 @@ components: description: Optional periodic heartbeat message configuration error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: description: Arguments to pass to the error handler $ref: "#/components/schemas/ScriptArgs" @@ -31075,7 +31202,9 @@ components: description: Last error message if the trigger failed error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -31126,7 +31255,9 @@ components: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -31190,7 +31321,9 @@ components: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -31273,7 +31406,9 @@ components: description: Last error message if the trigger failed error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -31314,7 +31449,9 @@ components: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -31368,7 +31505,9 @@ components: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -31766,7 +31905,9 @@ components: description: Last error message if the trigger failed error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -31868,7 +32009,9 @@ components: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -31924,7 +32067,9 @@ components: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -32052,7 +32197,9 @@ components: description: Timestamp of last server heartbeat (internal) error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -32092,7 +32239,9 @@ components: description: Configuration for creating/managing the publication (tables, operations) error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -32144,7 +32293,9 @@ components: description: Configuration for creating/managing the publication (tables, operations) error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -32221,7 +32372,9 @@ components: description: Last error message if the trigger failed error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -32285,7 +32438,9 @@ components: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -32360,7 +32515,9 @@ components: description: True if script_path points to a flow, false if it points to a script error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -32423,7 +32580,9 @@ components: description: Last error message if the trigger failed error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -32471,7 +32630,9 @@ components: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -32530,7 +32691,9 @@ components: description: True if script_path points to a flow, false if it points to a script error_handler_path: type: string - description: Path to a script or flow to run when the triggered job fails + description: Path to a script to run when the triggered job fails. A bare + path, without the script/ or flow/ prefix a schedule error handler takes; + it cannot be a flow. error_handler_args: $ref: "#/components/schemas/ScriptArgs" description: Arguments to pass to the error handler @@ -34580,8 +34743,34 @@ components: LogSearchHit: type: object properties: - dancer: + ts: + description: timestamp of the log line itself, not of the file containing it type: string + format: date-time + host: + type: string + level: + type: string + enum: [TRACE, DEBUG, INFO, WARN, ERROR] + target: + description: the tracing target that emitted the line + type: string + nullable: true + message: + type: string + file_path: + description: the log file the line came from + type: string + line_no: + description: offset of the line within its file + type: integer + required: + - ts + - host + - level + - message + - file_path + - line_no AutoscalingEvent: type: object diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 1a5f9dba87..c684830b27 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -409,6 +409,19 @@ impl ExpiringProviderCredentials { } } +/// Set on the copilot config when the workspace has no AI provider of its own and is +/// running on Windmill's free tier, so the client can label the lent model as free, warn +/// before the grant runs out, and tell the user to add their own key once it has — rather +/// than showing the same "no provider configured" state a never-configured workspace gets. +#[derive(Serialize, Deserialize, Debug, Default, Clone)] +pub struct FreeTierInfo { + /// The grant is spent: no provider is served and the user must bring their own key. + pub exhausted: bool, + /// Fraction of the grant consumed, 0.0..=1.0. A ratio, not a dollar amount — the + /// pricing model stays server-side. + pub used_ratio: f64, +} + #[derive(Serialize, Deserialize, Debug, Default)] pub struct AIConfig { #[serde(skip_serializing_if = "Option::is_none")] @@ -423,6 +436,11 @@ pub struct AIConfig { pub custom_prompts: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub max_tokens_per_model: Option>, + /// Response-only: this same struct is the request body for saving a workspace's AI + /// config, and `skip_deserializing` is what stops a client from storing a forged + /// free-tier marker. Only the server sets it, per-request. + #[serde(skip_serializing_if = "Option::is_none", skip_deserializing)] + pub free_tier: Option, /// Per-model price overrides, keyed `provider:model` like `max_tokens_per_model`. /// Only models whose rates differ from the built-in table are stored. #[serde(skip_serializing_if = "Option::is_none")] @@ -1013,83 +1031,119 @@ async fn proxy( check_scopes(&authed, || format!("resources:read:{}", resource_path))?; } - let mut credentials = match workspace_cache { - Some(request_cache) if !request_cache.is_expired() && forced_resource_path.is_none() => { - request_cache.credentials - } - _ => { - let (resource_path, save_to_cache, resource_workspace, instance_ai_config_revision) = - if let Some(resource_path) = forced_resource_path { - // forced resource path - (resource_path, false, w_id.clone(), None) - } else { - let workspace_ai_config = sqlx::query_scalar!( - "SELECT ai_config FROM workspace_settings WHERE workspace_id = $1", - &w_id - ) - .fetch_one(&db) - .await?; + // Set when serving the request through Windmill's free AI tier (the lent key). Holds + // the per-user concurrency lock and drives response metering. + let mut free_lease: Option = None; + let mut credentials = 'cred: { + match workspace_cache { + Some(request_cache) + if !request_cache.is_expired() && forced_resource_path.is_none() => + { + request_cache.credentials + } + _ => { + let (resource_path, save_to_cache, resource_workspace, instance_ai_config_revision) = + if let Some(resource_path) = forced_resource_path { + // forced resource path + (resource_path, false, w_id.clone(), None) + } else { + let workspace_ai_config = sqlx::query_scalar!( + "SELECT ai_config FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_one(&db) + .await?; - let (ai_config_value, resource_workspace, instance_ai_config_revision) = { - let ws_has_config = workspace_ai_config - .as_ref() - .and_then(|v| serde_json::from_value::(v.clone()).ok()) - .is_some_and(|config| config.has_providers()); + let (ai_config_value, resource_workspace, instance_ai_config_revision) = { + let ws_has_config = workspace_ai_config + .as_ref() + .and_then(|v| serde_json::from_value::(v.clone()).ok()) + .is_some_and(|config| config.has_providers()); - if ws_has_config { - (workspace_ai_config.unwrap(), w_id.clone(), None) - } else { - let instance_config = sqlx::query_scalar!( - "SELECT value FROM global_settings WHERE name = 'ai_config'" - ) - .fetch_optional(&db) - .await?; + if ws_has_config { + (workspace_ai_config.unwrap(), w_id.clone(), None) + } else { + let instance_config = sqlx::query_scalar!( + "SELECT value FROM global_settings WHERE name = 'ai_config'" + ) + .fetch_optional(&db) + .await?; - match instance_config { - Some(config) => ( - config, - "admins".to_string(), - Some(current_instance_ai_config_revision()), - ), - None => { - return Err(Error::internal_err( - "AI resource not configured".to_string(), - )); + let instance_has_config = + instance_config.as_ref().is_some_and(|v| { + serde_json::from_value::(v.clone()) + .ok() + .is_some_and(|c| c.has_providers()) + }); + match instance_config { + // An instance `ai_config` row with no usable provider (e.g. `{}` + // or `{"providers":{}}`) is treated as unconfigured, exactly as + // build_copilot_settings_state does — otherwise its mere presence + // would suppress the free-tier fallback below. + Some(config) if instance_has_config => ( + config, + "admins".to_string(), + Some(current_instance_ai_config_revision()), + ), + _ => { + // Nothing configured: fall back to Windmill's free AI tier + // (EE-only) if a lent key is set and both the user's + // one-time grant and the instance's daily cap have room. + // Errors once the grant is spent, the day is capped, or the + // user already has a request in flight; None otherwise. + // Ineligible identities (e.g. service accounts) are refused + // inside the helper, so every path treats them alike. + let free = + crate::ai_free_tier_oss::resolve_free_tier_credentials( + &provider, + &db, + &ai_path, + &authed.email, + &body, + ) + .await?; + if let Some((free_credentials, lease)) = free { + free_lease = Some(lease); + break 'cred free_credentials; + } + return Err(Error::internal_err( + "AI resource not configured".to_string(), + )); + } } } + }; + + let mut ai_config = serde_json::from_value::(ai_config_value) + .map_err(|e| Error::BadRequest(e.to_string()))?; + + let provider_config = ai_config + .providers + .as_mut() + .and_then(|providers| providers.remove(&provider)) + .ok_or_else(|| { + Error::BadRequest(format!("Provider {:?} not configured", provider)) + })?; + + if provider_config.resource_path.is_empty() { + return Err(Error::BadRequest("Resource path is empty".to_string())); } + + ( + provider_config.resource_path, + true, + resource_workspace, + instance_ai_config_revision, + ) }; - let mut ai_config = serde_json::from_value::(ai_config_value) - .map_err(|e| Error::BadRequest(e.to_string()))?; - - let provider_config = ai_config - .providers - .as_mut() - .and_then(|providers| providers.remove(&provider)) - .ok_or_else(|| { - Error::BadRequest(format!("Provider {:?} not configured", provider)) - })?; - - if provider_config.resource_path.is_empty() { - return Err(Error::BadRequest("Resource path is empty".to_string())); - } - - ( - provider_config.resource_path, - true, - resource_workspace, - instance_ai_config_revision, - ) - }; - - // For user-specified resources, fetch through an RLS-scoped - // connection so PostgreSQL row-level security enforces the same - // folder/group boundaries as the regular resource API. For the - // workspace/instance ai_config path, the resource_path was already - // validated by an admin/devops user when configuring the workspace, - // so the raw pool is used. - let resource = if is_user_specified_resource { + // For user-specified resources, fetch through an RLS-scoped + // connection so PostgreSQL row-level security enforces the same + // folder/group boundaries as the regular resource API. For the + // workspace/instance ai_config path, the resource_path was already + // validated by an admin/devops user when configuring the workspace, + // so the raw pool is used. + let resource = if is_user_specified_resource { let mut tx = user_db.clone().begin(&authed).await?; let res = sqlx::query_scalar::<_, Option>>>( "SELECT value FROM resource WHERE path = $1 AND workspace_id = $2", @@ -1112,38 +1166,45 @@ async fn proxy( .ok_or_else(|| Error::NotFound(format!("Could not find the resource {}, update the resource path in the workspace settings", resource_path)))? .ok_or_else(|| Error::BadRequest(format!("Empty resource value for {}", resource_path)))?; - let resource = serde_json::from_str::(resource.0.get()) - .map_err(|e| Error::BadRequest(e.to_string()))?; + let resource = serde_json::from_str::(resource.0.get()) + .map_err(|e| Error::BadRequest(e.to_string()))?; - // Enforce RLS on $var: resolution when the resource path was - // user-specified (X-Resource-Path header) so users can only read - // variables they have permission to access. - let enforce_authed = if is_user_specified_resource { - Some(&authed) - } else { - None - }; - let credentials = resolve_provider_credentials( - &provider, - &db, - &resource_workspace, - resource, - enforce_authed, - ) - .await?; - if save_to_cache { - AI_REQUEST_CACHE.insert( - (w_id.clone(), provider.clone()), - ExpiringProviderCredentials::new( - credentials.clone(), - instance_ai_config_revision, - ), - ); + // Enforce RLS on $var: resolution when the resource path was + // user-specified (X-Resource-Path header) so users can only read + // variables they have permission to access. + let enforce_authed = if is_user_specified_resource { + Some(&authed) + } else { + None + }; + let credentials = resolve_provider_credentials( + &provider, + &db, + &resource_workspace, + resource, + enforce_authed, + ) + .await?; + if save_to_cache { + AI_REQUEST_CACHE.insert( + (w_id.clone(), provider.clone()), + ExpiringProviderCredentials::new( + credentials.clone(), + instance_ai_config_revision, + ), + ); + } + credentials } - credentials } }; + // Free tier: pin the model and clamp max_tokens server-side before forwarding, + // since the request body is otherwise client-controlled. + if free_lease.is_some() { + body = crate::ai_free_tier_oss::enforce_free_tier_body(&body)?; + } + if let Some(fim_transform) = maybe_transform_fim_request(&provider, &ai_path, &credentials.base_url, &body)? { @@ -1291,8 +1352,32 @@ async fn proxy( let status_code = response.status(); let headers = response.headers().clone(); + let is_sse = is_sse_response(&headers); + + // Free tier: reconcile the cost reserved up-front against what the response actually + // used, holding the per-user lock (via the lease) until it is recorded. The chat + // streams (SSE), where the usage report only arrives in the final chunk; the + // non-streaming JSON path is handled for completeness. + if let Some(lease) = free_lease { + let body = if is_sse { + axum::body::Body::from_stream(inject_keepalives( + Box::pin(crate::ai_free_tier_oss::meter_usage( + response.bytes_stream(), + db.clone(), + lease, + )), + Duration::from_secs(KEEPALIVE_INTERVAL_SECS), + )) + } else { + let bytes = response.bytes().await.map_err(to_anyhow)?; + crate::ai_free_tier_oss::record_json_usage(db.clone(), lease, &bytes); + axum::body::Body::from(bytes) + }; + return Ok((status_code, headers, body)); + } + let stream = response.bytes_stream(); - let body = if is_sse_response(&headers) { + let body = if is_sse { axum::body::Body::from_stream(inject_keepalives( stream, Duration::from_secs(KEEPALIVE_INTERVAL_SECS), diff --git a/backend/windmill-api/src/ai_evals/datasets.rs b/backend/windmill-api/src/ai_evals/datasets.rs index da901276b0..8e1acb3d2a 100644 --- a/backend/windmill-api/src/ai_evals/datasets.rs +++ b/backend/windmill-api/src/ai_evals/datasets.rs @@ -190,6 +190,8 @@ pub async fn create_dataset( } tx.commit().await?; + windmill_common::feature_usage::log_feature_usage("ai_agent_eval", "dataset_created", ""); + Ok(format!("Created eval dataset {}", payload.path)) } diff --git a/backend/windmill-api/src/ai_evals/run.rs b/backend/windmill-api/src/ai_evals/run.rs index e102abaf4c..5d282c7e5b 100644 --- a/backend/windmill-api/src/ai_evals/run.rs +++ b/backend/windmill-api/src/ai_evals/run.rs @@ -707,6 +707,17 @@ pub async fn run_experiment( .await?; return Err(e); } + // Which state of the agent was measured is the whole key vocabulary: it is what separates + // running what is deployed from measuring edits or an older version. + windmill_common::feature_usage::log_feature_usage( + "ai_agent_eval", + "run", + match subject.kind { + EvalSubjectKind::Agent => "agent", + EvalSubjectKind::AgentDraft => "agent_draft", + EvalSubjectKind::AgentVersion => "agent_version", + }, + ); Ok(experiment_id.to_string()) } diff --git a/backend/windmill-api/src/ai_free_tier_oss.rs b/backend/windmill-api/src/ai_free_tier_oss.rs new file mode 100644 index 0000000000..7a183ff8ed --- /dev/null +++ b/backend/windmill-api/src/ai_free_tier_oss.rs @@ -0,0 +1,65 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::ai_free_tier_ee::*; + +// Open-source build: Windmill's free AI tier does not exist. These stubs make the +// callers in `ai.rs` / `workspaces.rs` compile while disabling the feature entirely — +// `resolve_free_tier_credentials` never opts in, so the proxy falls through to its +// normal "AI resource not configured" path and the copilot stays hidden. +// +// Caller contract (enforced by the private impl, restated here for parity): the `email` +// passed to `resolve_free_tier_credentials` / `free_tier_copilot_config` MUST be the +// authenticated caller's own identity (an `ApiAuthed` email), never a client-supplied one — +// it selects whose lent-key grant is spent and whose usage is read. + +#[cfg(not(feature = "private"))] +use crate::ai::AIConfig; +#[cfg(not(feature = "private"))] +use crate::db::DB; +#[cfg(not(feature = "private"))] +use axum::body::Bytes; +#[cfg(not(feature = "private"))] +use windmill_ai::ai_providers::AIProvider; +#[cfg(not(feature = "private"))] +use windmill_ai::credentials::ProviderCredentials; +#[cfg(not(feature = "private"))] +use windmill_common::error::Result; + +#[cfg(not(feature = "private"))] +pub struct FreeTierLease; + +#[cfg(not(feature = "private"))] +pub async fn resolve_free_tier_credentials( + _provider: &AIProvider, + _db: &DB, + _ai_path: &str, + _email: &str, + _body: &Bytes, +) -> Result> { + Ok(None) +} + +#[cfg(not(feature = "private"))] +pub fn enforce_free_tier_body(body: &Bytes) -> Result { + Ok(body.clone()) +} + +#[cfg(not(feature = "private"))] +pub async fn free_tier_copilot_config(_db: &DB, _email: &str) -> Result> { + Ok(None) +} + +#[cfg(not(feature = "private"))] +pub fn record_json_usage(_db: DB, _lease: FreeTierLease, _bytes: &[u8]) {} + +#[cfg(not(feature = "private"))] +pub fn meter_usage( + upstream: S, + _db: DB, + _lease: FreeTierLease, +) -> impl futures::Stream> +where + S: futures::Stream> + Unpin, +{ + upstream +} diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index 04d21c7b53..d781e229d2 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -102,6 +102,12 @@ lazy_static::lazy_static! { (20260727151319, include_str!( "../../migrations/20260727151319_draft_only_listing_indexes.up.sql" ).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY")), + (20260826202939, include_str!( + "../../migrations/20260826202939_queue_suspended_resume_at_index.up.sql" + ).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY").replace("DROP INDEX", "DROP INDEX CONCURRENTLY")), + (20260826214706, include_str!( + "../../migrations/20260826214706_queue_suspended_drop_legacy_index.up.sql" + ).replace("DROP INDEX", "DROP INDEX CONCURRENTLY")), ].into_iter().collect(); } @@ -228,6 +234,8 @@ impl Migrate for CustomMigrator { // CONCURRENTLY operations cannot run inside a transaction block // or a multi-statement query (PostgreSQL requires top-level execution). // Split into individual statements and execute each separately. + // The split is naive, so a `;` anywhere in an overridden migration — + // inside a comment or a string literal included — splits mid-statement. for stmt in migration_sql.split(';') { let stmt = stmt.trim(); if !stmt.is_empty() diff --git a/backend/windmill-api/src/hub_publish.rs b/backend/windmill-api/src/hub_publish.rs index 033c6ebce5..ccb530ce20 100644 --- a/backend/windmill-api/src/hub_publish.rs +++ b/backend/windmill-api/src/hub_publish.rs @@ -42,6 +42,11 @@ pub fn workspaced_service() -> Router { .route("/migrations", post(publish_migrations)) .route("/projects/{slug}/export", get(get_project_export)) .route("/projects/{slug}/submit", post(submit_project)) + .route("/projects/{slug}/withdraw", post(withdraw_project)) + .route( + "/projects/{slug}/discard_update", + post(discard_project_update), + ) .route("/project", get(get_project_by_source)) } @@ -554,9 +559,35 @@ async fn submit_project( .await } -// The Hub has no auth of its own: it validates bearer tokens by calling this -// instance's /api/users/whoami. Forwarding the caller's own token logs them in -// on the Hub as themselves (account auto-created on first use). +// Take a submission back out of review, keeping what was pushed for it. +async fn withdraw_project( + ctx: HubPublishCtx, + Path((_workspace, slug)): Path<(String, ProjectSlug)>, +) -> Result { + ctx.post( + &format!("/projects/{}/withdraw", slug), + &serde_json::json!({}), + ) + .await +} + +// Throw away the pending update to an already-published project. The published +// version is untouched — it never saw the update. +async fn discard_project_update( + ctx: HubPublishCtx, + Path((_workspace, slug)): Path<(String, ProjectSlug)>, +) -> Result { + ctx.post( + &format!("/projects/{}/discard_update", slug), + &serde_json::json!({}), + ) + .await +} + +// The Hub has no auth of its own: it validates bearer tokens by calling +// /api/users/whoami — on app.windmill.dev for the public Hub, on the paired +// instance for a private one. Forwarding the caller's own token logs them in on +// the Hub as themselves (account auto-created on first use). async fn get_from_hub( path: &str, source_id: &str, diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index de49949043..fc98020d1b 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -66,6 +66,9 @@ use crate::scim_oss::has_scim_token; use windmill_common::error::AppError; mod ai; +#[cfg(feature = "private")] +mod ai_free_tier_ee; +mod ai_free_tier_oss; mod ai_skills; mod apps; mod apps_raw_bundle; diff --git a/backend/windmill-api/src/service_logs.rs b/backend/windmill-api/src/service_logs.rs index 99dc7c41a8..a54f275f46 100644 --- a/backend/windmill-api/src/service_logs.rs +++ b/backend/windmill-api/src/service_logs.rs @@ -88,6 +88,66 @@ async fn list_files( Ok(Json(rows)) } +/// Rebuild one source log file from the columnar store. +/// +/// Not the original bytes: the store holds a line's fields rather than its text, +/// so the JSON is re-serialized here and key order and whitespace are this +/// writer's. Everything a reader can see survives — the drawer this feeds +/// renders a prettified view of each line either way, and a line that was never +/// JSON comes back exactly as it was written. +#[cfg(all(feature = "tantivy", feature = "private"))] +async fn get_log_file_from_store( + db: &DB, + store: &windmill_indexer::service_logs_store_ee::Store, + path: &str, +) -> windmill_common::error::Result { + let (hostname, file_name) = path + .split_once('/') + .ok_or_else(|| Error::BadRequest("Invalid path".to_string()))?; + + // The store is partitioned by day and mode, neither of which the path + // carries. `log_file` names both, and its primary key starts with hostname. + let file = sqlx::query!( + // `mode!` because the column is NOT NULL and only the cast makes sqlx + // think otherwise; a silent default would look up a `mode=` partition + // that matches nothing and read as a missing file. + "SELECT mode::text AS \"mode!\", log_ts FROM log_file WHERE hostname = $1 AND file_path = $2 ORDER BY log_ts DESC LIMIT 1", + hostname, + file_name + ) + .fetch_optional(db) + .await? + .ok_or_else(|| Error::NotFound(format!("File {path} not found")))?; + + // A row registered by this version carries the minute in the file's own name, + // so the two agree and the second is redundant. One written before the + // uploader derived `log_ts` from the name carries a wall clock instead, and + // those outlive an upgrade by the retention period — which is also what makes + // the `ORDER BY` above worth having. The name is authoritative, so both go. + let mut known_ts = vec![chrono::DateTime::from_naive_utc_and_offset( + file.log_ts, + chrono::Utc, + )]; + if let Some(named) = file_name.rsplit('.').next().and_then(|s| { + chrono::NaiveDateTime::parse_from_str(s, windmill_common::tracing_init::LOG_TIMESTAMP_FMT) + .ok() + }) { + known_ts.push(chrono::DateTime::from_naive_utc_and_offset( + named, + chrono::Utc, + )); + } + + let text = windmill_indexer::service_logs_store_ee::read_log_file( + store, &file.mode, hostname, file_name, &known_ts, + ) + .await + .map_err(|e| Error::internal_err(format!("Error reading the service log store: {e}")))? + .ok_or_else(|| Error::NotFound(format!("File {path} not found")))?; + + Ok(content_plain(Body::from(text))) +} + async fn get_log_file( authed: ApiAuthed, Extension(db): Extension, @@ -104,27 +164,30 @@ async fn get_log_file( let s3_client = windmill_object_store::get_object_store().await; #[cfg(feature = "parquet")] if let Some(s3_client) = s3_client { - let path = format!("{}{}", windmill_common::tracing_init::LOGS_SERVICE, path); - let file = s3_client + use windmill_object_store::object_store_reexports::ObjectStoreError; + + // The raw file, for as long as it is there. It outlives its ingestion by + // one indexer pass at most, so this covers the most recent minutes of a + // host's logs byte for byte; everything older is rebuilt from the store. + let object_path = format!("{}{}", windmill_common::tracing_init::LOGS_SERVICE, path); + match s3_client .get(&windmill_object_store::object_store_reexports::Path::from( - path, + object_path, )) - .await; - match file { - Ok(file) => { - let bytes = file.bytes().await; - match bytes { - Ok(bytes) => { - return Ok(content_plain(Body::from(bytes::Bytes::from(bytes)))); - } - Err(e) => { - return Err(Error::internal_err(format!( - "Error pulling the bytes: {}", - e - ))); - } + .await + { + Ok(file) => match file.bytes().await { + Ok(bytes) => { + return Ok(content_plain(Body::from(bytes::Bytes::from(bytes)))); } - } + Err(e) => { + return Err(Error::internal_err(format!( + "Error pulling the bytes: {}", + e + ))); + } + }, + Err(ObjectStoreError::NotFound { .. }) => {} Err(e) => { return Err(Error::internal_err(format!( "Error fetching the file: {}", @@ -132,6 +195,11 @@ async fn get_log_file( ))); } } + + #[cfg(all(feature = "tantivy", feature = "private"))] + return get_log_file_from_store(&db, &s3_client, &path).await; + #[cfg(not(all(feature = "tantivy", feature = "private")))] + return Err(Error::NotFound(format!("File {path} not found"))); } let full_path = format!("{}{}", *TMP_WINDMILL_LOGS_SERVICE, path); // SECURITY (defense in depth): refuse to read through a symlink so a planted diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 01eb32b7f6..76378e335a 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -18,7 +18,6 @@ use crate::teams_oss::{ connect_teams, edit_teams_command, run_teams_message_test_job, workspaces_list_available_teams_channels, workspaces_list_available_teams_ids, }; - use axum::{ extract::{Extension, Path}, routing::{get, post}, @@ -153,10 +152,23 @@ async fn edit_copilot_config( .await?; let settings_state = build_copilot_settings_state(workspace_has_config, instance_ai_config.as_ref()); + // A provider-less instance config (e.g. `{}`) is unconfigured, same as build_copilot_settings_state + // treats it — so it must not shadow the free-tier fallback here either. + let instance_config_with_providers = instance_ai_config + .as_ref() + .and_then(|v| serde_json::from_value::(v.clone()).ok()) + .filter(|c| c.has_providers()); let effective_ai_config = if workspace_has_config { ai_config - } else if let Some(instance_ai_config) = instance_ai_config { - serde_json::from_value::(instance_ai_config).unwrap_or_default() + } else if let Some(instance_config) = instance_config_with_providers { + instance_config + } else if let Some(free_config) = + crate::ai_free_tier_oss::free_tier_copilot_config(&db, &authed.email).await? + { + // Same fallback as get_copilot_info: with nothing configured, surface Windmill's free + // tier (EE-only) so clearing a workspace provider activates it immediately, instead of + // returning an empty config that disables AI until the next page reload re-fetches it. + free_config } else { AIConfig::default() }; @@ -179,6 +191,7 @@ struct EditCopilotConfigResponse { } async fn get_copilot_info( + authed: ApiAuthed, Extension(db): Extension, Path(w_id): Path, ) -> JsonResult { @@ -194,16 +207,25 @@ async fn get_copilot_info( )) })?; - if let Some(workspace_ai_config) = workspace_ai_config.filter(|c| c.0.has_providers()) { - Ok(Json(workspace_ai_config.0)) - } else if let Some(instance_config) = + let instance_config = sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'ai_config'") .fetch_optional(&db) .await? + .and_then(|v| serde_json::from_value::(v).ok()) + // A provider-less instance config (e.g. `{}`) is unconfigured; don't let it shadow the + // free-tier fallback, matching the proxy and edit_copilot_config paths. + .filter(|c| c.has_providers()); + if let Some(workspace_ai_config) = workspace_ai_config.filter(|c| c.0.has_providers()) { + Ok(Json(workspace_ai_config.0)) + } else if let Some(instance_config) = instance_config { + Ok(Json(instance_config)) + } else if let Some(free_config) = + crate::ai_free_tier_oss::free_tier_copilot_config(&db, &authed.email).await? { - Ok(Json( - serde_json::from_value::(instance_config).unwrap_or_default(), - )) + // Nothing configured: fall back to Windmill's free tier (EE-only). The config + // carries a `free_tier` marker even once the user's grant is spent — with no + // providers, but telling the client *why* AI is off. + Ok(Json(free_config)) } else { Ok(Json(AIConfig::default())) } @@ -216,7 +238,14 @@ pub async fn get_critical_alerts( authed: ApiAuthed, Query(params): Query, ) -> JsonResult { - require_admin_or_devops(authed.is_admin, &authed.username, &authed.email, authed.job_id.is_some(), &db).await?; + require_admin_or_devops( + authed.is_admin, + &authed.username, + &authed.email, + authed.job_id.is_some(), + &db, + ) + .await?; crate::utils::get_critical_alerts(db, params, Some(w_id)).await } @@ -232,7 +261,14 @@ pub async fn acknowledge_critical_alert( Path((w_id, id)): Path<(String, i32)>, authed: ApiAuthed, ) -> Result { - require_admin_or_devops(authed.is_admin, &authed.username, &authed.email, authed.job_id.is_some(), &db).await?; + require_admin_or_devops( + authed.is_admin, + &authed.username, + &authed.email, + authed.job_id.is_some(), + &db, + ) + .await?; crate::utils::acknowledge_critical_alert(db, Some(w_id), id).await } diff --git a/backend/windmill-common/src/db_params.rs b/backend/windmill-common/src/db_params.rs deleted file mode 100644 index d52700376d..0000000000 --- a/backend/windmill-common/src/db_params.rs +++ /dev/null @@ -1,45 +0,0 @@ -use anyhow::Result; - -/// Parsed database connection parameters, shared across DB auth providers (IAM RDS, Entra ID, etc.) -#[derive(Debug, Clone)] -pub struct DatabaseParams { - pub hostname: String, - pub port: u64, - pub username: String, - pub database: String, -} - -/// Extract database connection parameters from a PostgreSQL URL -pub fn extract_database_params(database_url: &str) -> Result { - let url = url::Url::parse(database_url) - .map_err(|e| anyhow::anyhow!("Failed to parse database URL: {}", e))?; - - let hostname = url - .host_str() - .ok_or_else(|| anyhow::anyhow!("Database URL missing hostname"))? - .to_string(); - - let port = url.port().unwrap_or(5432) as u64; - - let username = if url.username().is_empty() { - return Err(anyhow::anyhow!("Database URL missing username")); - } else { - urlencoding::decode(url.username())?.to_string() - }; - - let database = url - .path() - .trim_start_matches('/') - .split('/') - .next() - .filter(|s| !s.is_empty()) - .ok_or_else(|| anyhow::anyhow!("Database URL missing database name"))? - .to_string(); - - Ok(DatabaseParams { - hostname, - port, - username, - database: urlencoding::decode(&database)?.to_string(), - }) -} diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index b539c1c6b3..b1ad2edd01 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -15,6 +15,7 @@ pub const OAUTH_SETTING: &str = "oauths"; pub const AI_CONFIG_SETTING: &str = "ai_config"; pub const RETENTION_PERIOD_SECS_SETTING: &str = "retention_period_secs"; pub const RETENTION_PERIOD_SECS_OVERRIDES_SETTING: &str = "retention_period_secs_overrides"; +pub const SERVICE_LOG_RETENTION_SECS_SETTING: &str = "service_log_retention_secs"; /// Upper bound on how many per-workspace retention overrides may be configured. The periodic monitor /// sweeps each override workspace in its own transaction every pass, so this keeps a pass bounded /// (and the feature is a targeted escape hatch for a handful of special workspaces, not a bulk knob). diff --git a/backend/windmill-common/src/indexer.rs b/backend/windmill-common/src/indexer.rs index 7b1ed407ab..559eba3bf9 100644 --- a/backend/windmill-common/src/indexer.rs +++ b/backend/windmill-common/src/indexer.rs @@ -94,6 +94,21 @@ pub async fn load_indexer_config(db: &DB) -> error::Result i64 { + let retention = crate::service_log_retention_secs(); + if max_index_time_window_secs > 0 { + std::cmp::min(max_index_time_window_secs, retention) + } else { + retention + } +} + pub fn get_env_var(env_var: &str) -> Option { match std::env::var(env_var).map(|x| x.parse()) { Ok(Ok(i)) => Some(i), @@ -136,3 +151,60 @@ pub fn get_indexer_rates_from_env() -> TantivyIndexerSettings { settings } + +#[cfg(test)] +mod tests { + use super::*; + + // One test rather than several: both halves share the process-wide retention, and the + // setter half writes it, which parallel tests would race. + #[test] + fn retention_rejects_unusable_values_and_the_index_window_clamps_to_it() { + use crate::{ + service_log_retention_secs, set_service_log_retention_secs, + DEFAULT_SERVICE_LOG_RETENTION_SECS, + }; + + // See `set_service_log_retention_secs` for why the two unusable directions land apart: + // too large keeps the intent by capping, non-positive cannot and falls back. + let rejected: Vec = [0, -1, i64::MIN] + .iter() + .map(|v| { + set_service_log_retention_secs(*v); + service_log_retention_secs() + }) + .collect(); + let capped: Vec = [i64::MAX, 60 * 60 * 24 * 365 * 101] + .iter() + .map(|v| { + set_service_log_retention_secs(*v); + service_log_retention_secs() + }) + .collect(); + + set_service_log_retention_secs(60 * 60 * 24 * 3); + let retention = service_log_retention_secs(); + let windows = [ + // `0` disables the extra shrinking rather than lifting the ceiling — the trap that + // makes an unset setting look unbounded. + service_log_index_window_secs(0), + // Retention is the ceiling: the index cannot reach lines whose `log_file` row is gone. + service_log_index_window_secs(retention * 2), + service_log_index_window_secs(60), + ]; + set_service_log_retention_secs(DEFAULT_SERVICE_LOG_RETENTION_SECS); + + assert_eq!( + rejected, + vec![DEFAULT_SERVICE_LOG_RETENTION_SECS; 3], + "a value that would expire everything must fall back to the default" + ); + assert_eq!( + capped, + vec![60 * 60 * 24 * 365 * 100; 2], + "an oversized value must cap, not shorten retention to the default" + ); + assert_eq!(retention, 60 * 60 * 24 * 3); + assert_eq!(windows, [retention, retention, 60]); + } +} diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index ea4b8e5045..cb5adce068 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -42,7 +42,6 @@ pub mod db; mod db_entra_ee; #[cfg(all(feature = "enterprise", feature = "private"))] mod db_iam_ee; -pub mod db_params; pub mod dbt_manifest; pub mod deploy_origin; #[cfg(feature = "private")] @@ -148,9 +147,53 @@ pub const DEFAULT_MAX_CONNECTIONS_INDEXER: u32 = 5; pub const DEFAULT_HUB_BASE_URL: &str = "https://hub.windmill.dev"; pub const PRIVATE_HUB_MIN_VERSION: i32 = 10_000_000; -pub const SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 14; // 2 weeks retention period for logs +pub const DEFAULT_SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 14; // 2 weeks retention period for logs pub const WM_DEPLOYERS_GROUP: &str = "wm_deployers"; +/// A century. Every consumer has to survive `now - retention`, and the ceilings are much lower +/// than an `i64`: `DateTime` subtraction panics past year 262143, and the `( s)::interval` +/// the cleanup queries build overflows Postgres' microsecond field. +const MAX_SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 365 * 100; + +/// Apply a configured service log retention, in seconds. +/// +/// The only way into [`SERVICE_LOG_RETENTION_SECS`], so an unusable value can never reach a +/// cutoff. The two unusable directions are not the same mistake and must not share a landing +/// point: too large still says "keep these for a very long time", so it is capped and the +/// intent survives, whereas falling back would delete logs the operator meant to keep. A +/// non-positive value has no such reading — every cutoff is `now - retention`, so it lands at +/// or after `now` and the next sweep expires the entire history, rows and object-storage files +/// alike. Unlike job retention there is no "keep forever" spelling here, so `0` — what an +/// operator types by analogy with it, and what the settings UI writes into a field that was +/// merely focused — falls back to the default. +pub fn set_service_log_retention_secs(configured: i64) { + let effective = if configured > MAX_SERVICE_LOG_RETENTION_SECS { + tracing::warn!( + "service log retention of {configured}s exceeds the maximum of \ + {MAX_SERVICE_LOG_RETENTION_SECS}s, capping it there" + ); + MAX_SERVICE_LOG_RETENTION_SECS + } else if configured >= 1 { + configured + } else { + tracing::warn!( + "service log retention of {configured}s would expire every service log, \ + falling back to the default of {DEFAULT_SERVICE_LOG_RETENTION_SECS}s" + ); + DEFAULT_SERVICE_LOG_RETENTION_SECS + }; + SERVICE_LOG_RETENTION_SECS.store(effective, std::sync::atomic::Ordering::Relaxed); +} + +/// How long a service log line stays retrievable, in seconds. +/// +/// The outer bound on everything service-log: the `log_file` rows, the raw files in object +/// storage, the columnar store queried by retrieval, and — through +/// [`indexer::service_log_index_window_secs`] — the search index. +pub fn service_log_retention_secs() -> i64 { + SERVICE_LOG_RETENTION_SECS.load(std::sync::atomic::Ordering::Relaxed) +} + /// Canonical form of a base URL, used as one of the inputs to the offline-license /// instance hash (`compute_instance_hash`). /// @@ -376,6 +419,10 @@ lazy_static::lazy_static! { /// workspace configured before its override could be read. pub static ref JOB_RETENTION_SECS_OVERRIDES_LOADED: AtomicBool = AtomicBool::new(false); pub static ref AUDIT_LOG_RETENTION_DAYS: AtomicI64 = AtomicI64::new(0); + /// Private on purpose: [`set_service_log_retention_secs`] is the only writer, so a value that + /// would expire every service log cannot reach a cutoff. Read it with + /// [`service_log_retention_secs`]. + static ref SERVICE_LOG_RETENTION_SECS: AtomicI64 = AtomicI64::new(DEFAULT_SERVICE_LOG_RETENTION_SECS); pub static ref MONITOR_LOGS_ON_OBJECT_STORE: AtomicBool = AtomicBool::new(false); @@ -1519,6 +1566,17 @@ pub async fn create_custom_instance_database( Ok(()) } +/// Connection options parsed from a database URL. +/// +/// The only place a database URL becomes `PgConnectOptions`. Providers that mint the password +/// themselves override it on these and keep the rest: options assembled field by field instead +/// would drop every query parameter, `sslmode` and `sslrootcert` above all, leaving the +/// connection on sqlx's default TLS policy rather than the operator's. +pub fn base_connect_options(database_url: &str) -> Result { + sqlx::postgres::PgConnectOptions::from_str(database_url) + .map_err(|e| Error::InternalErr(format!("Failed to parse database URL: {}", e))) +} + #[derive(Clone)] pub enum DatabaseUrl { #[cfg(all(feature = "enterprise", feature = "private"))] @@ -1549,8 +1607,8 @@ impl DatabaseUrl { } /// Get PgConnectOptions for this database URL. - /// For token-based auth (IAM RDS, Entra ID), this returns options built directly from the - /// token to avoid double-encoding issues with temporary credentials. + /// For token-based auth (IAM RDS, Entra ID), this returns options carrying the current + /// token, set on the builder to avoid double-encoding temporary credentials. /// For static URLs, this parses the URL string. pub async fn connect_options(&self) -> Result { match self { @@ -1564,8 +1622,7 @@ impl DatabaseUrl { let guard = entra_url.read().await; Ok(guard.connect_options()) } - DatabaseUrl::Static(url) => sqlx::postgres::PgConnectOptions::from_str(url) - .map_err(|e| Error::InternalErr(format!("Failed to parse database URL: {}", e))), + DatabaseUrl::Static(url) => base_connect_options(url), } } diff --git a/backend/windmill-common/src/tracing_init.rs b/backend/windmill-common/src/tracing_init.rs index 5091c7e16d..b456e370fc 100644 --- a/backend/windmill-common/src/tracing_init.rs +++ b/backend/windmill-common/src/tracing_init.rs @@ -178,56 +178,40 @@ pub fn initialize_tracing( .with(logs_bridge.with_filter(otel_logs_filter)) .with(opentelemetry_filtered); - match *JSON_FMT { - true => { - // Stdout layer with its own filter - let stdout_layer = json_layer() - .with_writer(std::io::stdout) - .flatten_event(true) - .with_filter(stdout_env_filter) - .with_filter(create_targets_filter(default_env_filter)); + // The service log files are written to be indexed, not tailed, so they always carry the + // JSON format: it is what preserves level, target and the current span as fields rather + // than as text the index would have to recover by regex. JSON_FMT governs stdout only. + let file_layer = json_layer() + .with_writer(log_file_writer) + .flatten_event(true) + .with_filter(file_env_filter) + .with_filter(create_targets_filter(default_env_filter)); - // File layer with its own filter - let file_layer = json_layer() - .with_writer(log_file_writer) - .flatten_event(true) - .with_filter(file_env_filter) - .with_filter(create_targets_filter(default_env_filter)); + // Boxed so both arms have one type: the file layer is a single value and could not + // otherwise be typed against two different subscriber stacks. + let stdout_layer = match *JSON_FMT { + true => json_layer() + .with_writer(std::io::stdout) + .flatten_event(true) + .with_filter(stdout_env_filter) + .with_filter(create_targets_filter(default_env_filter)) + .boxed(), + false => compact_layer() + .with_writer(std::io::stdout) + .with_ansi(style.to_lowercase() != "never") + .with_file(true) + .with_line_number(true) + .with_target(false) + .with_filter(stdout_env_filter) + .with_filter(create_targets_filter(default_env_filter)) + .boxed(), + }; - base_layer - .with(stdout_layer) - .with(file_layer) - .with(CountingLayer::new()) - .init() - } - false => { - // Stdout layer with its own filter - let stdout_layer = compact_layer() - .with_writer(std::io::stdout) - .with_ansi(style.to_lowercase() != "never") - .with_file(true) - .with_line_number(true) - .with_target(false) - .with_filter(stdout_env_filter) - .with_filter(create_targets_filter(default_env_filter)); - - // File layer with its own filter - let file_layer = compact_layer() - .with_writer(log_file_writer) - .with_ansi(false) // No ANSI codes in log files - .with_file(true) - .with_line_number(true) - .with_target(false) - .with_filter(file_env_filter) - .with_filter(create_targets_filter(default_env_filter)); - - base_layer - .with(stdout_layer) - .with(file_layer) - .with(CountingLayer::new()) - .init() - } - } + base_layer + .with(stdout_layer) + .with(file_layer) + .with(CountingLayer::new()) + .init(); (_guard, meter_provider) } diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 631eb2fd3e..fa38d0c86e 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -734,11 +734,17 @@ fn format_pull_query(peek: String) -> String { r } +// The `CASE` is `suspend <= 0 OR suspend_until <= now()` written as one indexable +// expression, equivalent only under the `suspend_until IS NOT NULL` guard. It must stay in +// sync with `queue_suspended_v2` (migration 20260826202939): if it no longer matches, the +// test silently reverts to a heap filter over every suspended row on every worker poll. pub fn make_suspended_pull_query(tags: &[String]) -> String { format_pull_query(format!( "SELECT id FROM v2_job_queue - WHERE suspend_until IS NOT NULL AND (suspend <= 0 OR suspend_until <= now()) AND tag IN ({}) + WHERE suspend_until IS NOT NULL + AND (CASE WHEN suspend <= 0 THEN '-infinity'::timestamptz ELSE suspend_until END) <= now() + AND tag IN ({}) ORDER BY priority DESC NULLS LAST, created_at FOR UPDATE SKIP LOCKED LIMIT 1", diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 0063eef381..398b13ee93 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -759,15 +759,24 @@ pub async fn count_workspace_forks(db: &crate::DB, root: &str) -> Result { Ok(count) } -/// Approximate paid seats of a workspace as `ceil(developers + operators/2)`, excluding disabled and -/// service-account members. Reuses billing's author/operator weighting, but counts provisioned -/// members rather than the active-user population billing meters, so it only ever loosens the fork -/// cap (never blocks a paid seat) — good enough for a soft guardrail. +/// The billable members of a workspace and the seats they add up to. +#[derive(Clone, Debug, Serialize)] +pub struct BillableSeats { + pub developers: i64, + pub operators: i64, + pub seats: i64, +} + +/// Billable members of `w_id` and the seats they cost, as `ceil(developers + operators/2)`. Service +/// accounts cannot log in and do not take a seat; a disabled member is not billed either. +/// +/// The workspace is invoiced by a job outside this codebase that counts the same rows with its own +/// SQL. The two must be changed together: this rule disagreeing with that one is what bills a +/// workspace for seats the product never credits it for. /// /// Unauthenticated metering helper: reads member counts for any `w_id`, so callers must already be /// authorized for that workspace (or run in trusted server-side code). -#[cfg(feature = "cloud")] -pub async fn count_paid_seats(db: &crate::DB, w_id: &str) -> Result { +pub async fn billable_seats(db: &crate::DB, w_id: &str) -> Result { let row = sqlx::query!( r#"SELECT COUNT(*) FILTER (WHERE NOT operator AND NOT disabled AND NOT is_service_account) AS "developers!", @@ -777,8 +786,18 @@ pub async fn count_paid_seats(db: &crate::DB, w_id: &str) -> Result { ) .fetch_one(db) .await - .map_err(|e| Error::internal_err(format!("counting paid seats of {w_id}: {e:#}")))?; - Ok(((row.developers as f64) + 0.5 * (row.operators as f64)).ceil() as i64) + .map_err(|e| Error::internal_err(format!("counting billable seats of {w_id}: {e:#}")))?; + Ok(BillableSeats { + developers: row.developers, + operators: row.operators, + seats: ((row.developers as f64) + 0.5 * (row.operators as f64)).ceil() as i64, + }) +} + +/// Seats only, for the fork cap. See [`billable_seats`]. +#[cfg(feature = "cloud")] +pub async fn count_paid_seats(db: &crate::DB, w_id: &str) -> Result { + Ok(billable_seats(db, w_id).await?.seats) } #[cfg(feature = "cloud")] @@ -2531,6 +2550,32 @@ pub fn lfs_entry_storage_ref(entry: &serde_json::Value) -> Option { Some(format!("{typ}:{path}")) } +pub const FILESYSTEM_STORAGE_DEV_ONLY_MSG: &str = + "Filesystem storage is only available in development builds of Windmill: it points the \ + workspace at a directory on the server's own disk rather than at a resource. Use an S3, \ + Azure Blob or Google Cloud Storage backend instead."; + +/// A filesystem workspace storage names a directory on the server's own disk, so it hands whoever +/// configures it — a workspace admin, or any member who can write a `filesystem` resource — +/// whatever the server process can reach, and it only resolves when server and workers share that +/// disk. It is there so local development can skip MinIO, hence debug builds only. Instance object +/// storage on local disk is a separate, superadmin-only setting and stays allowed everywhere. +pub fn filesystem_storage_allowed() -> bool { + cfg!(debug_assertions) +} + +/// Guards every site that builds an `ObjectStoreResource::Filesystem`, so nothing downstream can +/// reach a local-disk store: a stored config outlives the build that accepted it, and the resource +/// route never passes through the workspace-storage settings at all. +pub fn ensure_filesystem_storage_allowed() -> Result<()> { + if !filesystem_storage_allowed() { + return Err(Error::BadRequest( + FILESYSTEM_STORAGE_DEV_ONLY_MSG.to_string(), + )); + } + Ok(()) +} + /// Resolve a `$res:`/`$var:` reference tree to its concrete value (recursively, secrets /// decrypted). No permission checks — trusted server-side callers only; never echo the result /// to a user. diff --git a/backend/windmill-common/tests/billing_workspace.rs b/backend/windmill-common/tests/billing_workspace.rs index 1b0186a5cf..3025061249 100644 --- a/backend/windmill-common/tests/billing_workspace.rs +++ b/backend/windmill-common/tests/billing_workspace.rs @@ -3,7 +3,7 @@ use sqlx::{Pool, Postgres}; use windmill_common::workspaces::{ - count_paid_seats, count_workspace_forks, fork_chain_depth, fork_subtree_height, + billable_seats, count_paid_seats, count_workspace_forks, fork_chain_depth, fork_subtree_height, get_billing_workspace_id, invalidate_billing_workspace_cache, list_fork_descendants, }; @@ -106,11 +106,17 @@ async fn paid_seats_and_fork_count(db: Pool) { insert_member(&db, "seat-root", "dev2@w.dev", false, false, false).await; insert_member(&db, "seat-root", "op1@w.dev", true, false, false).await; insert_member(&db, "seat-root", "op2@w.dev", true, false, false).await; - // These must NOT count towards seats. + // These must NOT count towards seats. The service account is a non-operator, so counting it + // would inflate the developer tally the invoice line is written from, not the operator one. insert_member(&db, "seat-root", "disabled@w.dev", false, true, false).await; insert_member(&db, "seat-root", "svc@w.dev", false, false, true).await; assert_eq!(count_paid_seats(&db, "seat-root").await.unwrap(), 3); + let breakdown = billable_seats(&db, "seat-root").await.unwrap(); + assert_eq!( + (breakdown.developers, breakdown.operators, breakdown.seats), + (2, 2, 3) + ); insert_ws(&db, "seat-fork1", Some("seat-root"), false).await; insert_ws(&db, "seat-fork2", Some("seat-root"), false).await; diff --git a/backend/windmill-indexer/Cargo.toml b/backend/windmill-indexer/Cargo.toml index acf90ab57a..09b2bbe5b1 100644 --- a/backend/windmill-indexer/Cargo.toml +++ b/backend/windmill-indexer/Cargo.toml @@ -10,7 +10,13 @@ path = "src/lib.rs" [features] default = [] -parquet = ["windmill-common/parquet", "windmill-object-store/parquet"] +parquet = [ + "windmill-common/parquet", + "windmill-object-store/parquet", + "dep:datafusion", + "dep:object_store", + "dep:url", +] private = ["windmill-common/private"] enterprise = ["windmill-common/enterprise", "windmill-object-store/enterprise"] @@ -33,3 +39,6 @@ astral-tokio-tar.workspace = true lazy_static.workspace = true const_format.workspace = true flume.workspace = true +datafusion = { workspace = true, optional = true } +object_store = { workspace = true, optional = true } +url = { workspace = true, optional = true } diff --git a/backend/windmill-indexer/src/lib.rs b/backend/windmill-indexer/src/lib.rs index 6c13b551d1..f107ea911d 100644 --- a/backend/windmill-indexer/src/lib.rs +++ b/backend/windmill-indexer/src/lib.rs @@ -7,3 +7,5 @@ pub mod indexer_oss; #[cfg(feature = "private")] pub mod service_logs_ee; pub mod service_logs_oss; +#[cfg(all(feature = "private", feature = "parquet"))] +pub mod service_logs_store_ee; diff --git a/backend/windmill-object-store/src/lib.rs b/backend/windmill-object-store/src/lib.rs index 0f919f1048..cc588d485f 100644 --- a/backend/windmill-object-store/src/lib.rs +++ b/backend/windmill-object-store/src/lib.rs @@ -1171,6 +1171,7 @@ pub fn lfs_to_object_store_resource( Ok(ObjectStoreResource::Gcs(gcs_resource)) } LargeFileStorage::FilesystemStorage(fs) => { + windmill_common::workspaces::ensure_filesystem_storage_allowed()?; Ok(ObjectStoreResource::Filesystem(FilesystemSettings { root_path: fs.root_path.clone(), })) diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index a1bd8034f0..c7c119a5a7 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -4671,6 +4671,12 @@ pub async fn check_debouncing_within_limits( } } +/// Whether the tag's queue name is computed from the job's arguments, so that a caller holding +/// arguments it could not build knows the tag cannot be built either. +pub fn tag_reads_args(tag: &str) -> bool { + RE_ARG_TAG.is_match(tag) +} + pub fn interpolate_args(x: String, args: &PushArgs, workspace_id: &str) -> String { // Save this value to avoid parsing twice let workspaced = x.as_str().replace("$workspace", workspace_id).to_string(); diff --git a/backend/windmill-trigger-http/src/handler.rs b/backend/windmill-trigger-http/src/handler.rs index 14e1b9a4d2..68986bb6f9 100644 --- a/backend/windmill-trigger-http/src/handler.rs +++ b/backend/windmill-trigger-http/src/handler.rs @@ -268,6 +268,15 @@ pub async fn create_many_http_triggers( format!("http_triggers:write:{}", &new_http_trigger.base.path) })?; + // This route inserts directly, bypassing the shared create handler. + // `error_wrapper` would turn the rejection into a 500. + new_http_trigger.error_handling.validate().map_err(|err| { + Error::BadRequest(format!( + "Error occurred for HTTP route at route path: {}, error: {}", + new_http_trigger.config.route_path, err + )) + })?; + handler .validate_new(&db, &w_id, &new_http_trigger.config) .await @@ -570,8 +579,8 @@ impl TriggerCrud for HttpTrigger { route_path, &route_path_key, Some(effective_workspaced), - trigger.config.wrap_body, - trigger.config.raw_string, + trigger.config.wrap_body.unwrap_or(false), + trigger.config.raw_string.unwrap_or(false), trigger.config.authentication_resource_path, trigger.base.script_path, trigger.base.path, @@ -626,8 +635,8 @@ impl TriggerCrud for HttpTrigger { workspace_id = $20 AND path = $21 "#, - trigger.config.wrap_body, - trigger.config.raw_string, + trigger.config.wrap_body.unwrap_or(false), + trigger.config.raw_string.unwrap_or(false), trigger.config.authentication_resource_path, trigger.base.script_path, trigger.base.path, diff --git a/backend/windmill-trigger/src/handler.rs b/backend/windmill-trigger/src/handler.rs index 28898489f4..8ef7c1ee39 100644 --- a/backend/windmill-trigger/src/handler.rs +++ b/backend/windmill-trigger/src/handler.rs @@ -554,6 +554,8 @@ async fn create_trigger( ))); } + new_trigger.error_handling.validate()?; + handler .validate_new(&db, &workspace_id, &new_trigger.config) .await?; @@ -815,6 +817,8 @@ async fn update_trigger( ) })?; + edit_trigger.error_handling.validate()?; + handler .validate_edit(&db, &workspace_id, &edit_trigger.config, path) .await?; diff --git a/backend/windmill-trigger/src/types.rs b/backend/windmill-trigger/src/types.rs index dcf9a136be..2c1b854396 100644 --- a/backend/windmill-trigger/src/types.rs +++ b/backend/windmill-trigger/src/types.rs @@ -84,6 +84,31 @@ pub struct TriggerErrorHandling { pub retry: Option>, } +impl TriggerErrorHandling { + /// Schedule and workspace error handlers encode script-vs-flow as a + /// `script/`/`flow/` prefix; a trigger's handler is always a script, so a + /// prefixed path here would be looked up verbatim as a script name and fail + /// only once the trigger errors, which is when the handler is needed. + pub fn validate(&self) -> windmill_common::error::Result<()> { + let Some(path) = self.error_handler_path.as_deref() else { + return Ok(()); + }; + if let Some(bare) = path.strip_prefix("script/") { + return Err(windmill_common::error::Error::BadRequest(format!( + "error_handler_path is a plain script path, not the prefixed form a schedule \ + error handler takes: got '{path}', use '{bare}'" + ))); + } + if path.starts_with("flow/") { + return Err(windmill_common::error::Error::BadRequest(format!( + "error_handler_path must be a script: a trigger error handler cannot be a flow \ + (got '{path}')" + ))); + } + Ok(()) + } +} + #[derive(Serialize, Deserialize, Clone)] pub struct Trigger where @@ -253,6 +278,43 @@ mod tests { use super::*; use serde_json::json; + // --- TriggerErrorHandling::validate --- + + fn error_handling(path: Option<&str>) -> TriggerErrorHandling { + TriggerErrorHandling { + error_handler_path: path.map(str::to_string), + error_handler_args: None, + retry: None, + } + } + + #[test] + fn test_error_handler_path_accepts_bare_and_hub_paths() { + for path in [None, Some("f/team/handler"), Some("u/admin/handler")] { + assert!(error_handling(path).validate().is_ok(), "{path:?}"); + } + assert!(error_handling(Some("hub/13953/windmill/handler")) + .validate() + .is_ok()); + } + + #[test] + fn test_error_handler_path_rejects_prefixed_paths() { + // The rejection names the bare path to use, so the caller can fix it + // without knowing which of the two conventions a trigger follows. + let err = error_handling(Some("script/f/team/handler")) + .validate() + .unwrap_err() + .to_string(); + assert!(err.contains("f/team/handler"), "{err}"); + + let err = error_handling(Some("flow/f/team/handler")) + .validate() + .unwrap_err() + .to_string(); + assert!(err.contains("cannot be a flow"), "{err}"); + } + // --- TriggerMode serde --- #[test] diff --git a/backend/windmill-types/src/jobs.rs b/backend/windmill-types/src/jobs.rs index db20bc1251..5190efe380 100644 --- a/backend/windmill-types/src/jobs.rs +++ b/backend/windmill-types/src/jobs.rs @@ -632,6 +632,28 @@ pub enum JobPayload { }, } +impl JobPayload { + /// Whether the payload itself declares a dedicated worker, in which case `push` replaces + /// whatever tag it is handed and the caller's tag never reaches the queue. + /// + /// This reads what the payload carries, not what `push` will conclude: a `SingleStepFlow` + /// loads the flag from the script row at push time and reports `false` here. That only + /// matters to a caller reasoning about the tag, and for those the answer is the same either + /// way, since `push` replaces the tag in exactly the case this misses. + pub fn is_dedicated_worker(&self) -> bool { + let dedicated_worker = match self { + JobPayload::ScriptHash { dedicated_worker, .. } + | JobPayload::FlowScript { dedicated_worker, .. } + | JobPayload::Dependencies { dedicated_worker, .. } + | JobPayload::FlowDependencies { dedicated_worker, .. } + | JobPayload::Flow { dedicated_worker, .. } => dedicated_worker, + JobPayload::Code(raw) => &raw.dedicated_worker, + _ => &None, + }; + dedicated_worker.is_some_and(|x| x) + } +} + #[derive(Clone, Serialize, Deserialize, Debug)] pub struct SkipHandler { pub path: String, diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 1fb566e57f..77221edf22 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -1256,6 +1256,7 @@ pub async fn prebundle_bun_script( token: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, temp_script_refs: &Option>, + modules: Option<&HashMap>, ) -> Result<()> { let (local_path, remote_path) = compute_bundle_local_and_remote_path( inner_content, @@ -1264,6 +1265,7 @@ pub async fn prebundle_bun_script( db, w_id, temp_script_refs, + modules, ) .await; if exists_in_cache(&local_path, &remote_path).await { @@ -1442,6 +1444,7 @@ pub async fn compute_bundle_local_and_remote_path( db: Option<&DB>, w_id: &str, temp_script_refs: &Option>, + modules: Option<&HashMap>, ) -> (String, String) { let mut input_src = format!("{inner_content}{lock}",); @@ -1470,7 +1473,10 @@ pub async fn compute_bundle_local_and_remote_path( let ws_suffix = crate::workspace_registry_cache_suffix(w_id).await; input_src.push_str(&ws_suffix); - let hash = windmill_common::utils::calculate_hash(&input_src); + + // The loader resolves relative imports against the module files in the job dir, so + // their content is inlined into the bundle this name covers. + let hash = crate::worker::artifact_cache_name(input_src, modules); let local_path = format!("{}/{hash}", *BUN_BUNDLE_CACHE_DIR); #[cfg(windows)] @@ -1569,6 +1575,7 @@ pub async fn handle_bun_job( Some(db), &job.workspace_id, &temp_script_refs, + modules.as_ref(), ) .await } @@ -2962,9 +2969,8 @@ pub async fn handle_wac_v2_output( version: flow_info.version, labels: flow_info.labels.clone(), }; - let on_behalf_of = flow_info - .on_behalf_of(&job.workspace_id, db) - .await?; + let on_behalf_of = + flow_info.on_behalf_of(&job.workspace_id, db).await?; let step_args: HashMap> = step .args .iter() @@ -4382,4 +4388,49 @@ export function main(x: number) { return x; }"#; assert!(wrapper.contains(r#"line.startsWith("exec_preprocess:")"#)); assert!(wrapper.contains(r#"line.startsWith("exec:")"#)); } + + /// The bundle cache is global and content-keyed, so a key that ignores the inline + /// modules hands one workspace's bundle — attacker helper code and all — to the next + /// job whose main content and lockfile happen to match. + #[tokio::test] + async fn bundle_cache_key_separates_inline_module_content() { + use windmill_common::scripts::ScriptModule; + + async fn key_for(modules: Option<&HashMap>) -> String { + compute_bundle_local_and_remote_path( + "import { h } from './helper.ts';\nexport async function main() { return h(); }", + "{}\n//bun.lock\n", + "u/alice/script", + None, + "w1", + &None, + modules, + ) + .await + .1 + } + fn modules(content: &str) -> HashMap { + HashMap::from([( + "helper.ts".to_string(), + ScriptModule { + content: content.to_string(), + language: ScriptLang::Bun, + lock: None, + }, + )]) + } + + let attacker = key_for(Some(&modules("export const h = () => 'attacker'"))).await; + let victim = key_for(Some(&modules("export const h = () => 'victim'"))).await; + assert_ne!(attacker, victim); + assert_eq!( + attacker, + key_for(Some(&modules("export const h = () => 'attacker'"))).await, + "same modules must still share a cache slot" + ); + + // An absent map and an empty one are the same script, so they share a slot. + assert_eq!(key_for(None).await, key_for(Some(&HashMap::new())).await); + assert_ne!(key_for(None).await, attacker); + } } diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 6ee22fd882..f6391eab07 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -1640,6 +1640,7 @@ pub(crate) async fn get_workspace_s3_resource_path( ) } Some(LargeFileStorage::FilesystemStorage(fs)) => { + windmill_common::workspaces::ensure_filesystem_storage_allowed()?; return Ok(Some( windmill_object_store::ObjectStoreResource::Filesystem( windmill_object_store::FilesystemSettings { root_path: fs.root_path.clone() }, diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs index c95107d504..d393171f7f 100644 --- a/backend/windmill-worker/src/csharp_executor.rs +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -13,7 +13,7 @@ use itertools::Itertools; #[cfg(feature = "csharp")] use tokio::{fs::File, io::AsyncReadExt, process::Command}; #[cfg(feature = "csharp")] -use windmill_common::{utils::calculate_hash, worker::write_file}; +use windmill_common::worker::write_file; #[cfg(feature = "csharp")] use crate::global_cache::save_cache; @@ -72,13 +72,21 @@ const CSHARP_OBJECT_STORE_PREFIX: &str = /// Cache key of a C# build. The run path and the deploy-time prebuild must derive it the /// same way or the prebuilt binary is never found and gets rebuilt on first run. #[cfg(feature = "csharp")] -async fn csharp_cache_key(code: &str, requirements_o: Option<&str>, w_id: &str) -> String { - let mut hash = calculate_hash(&format!( +async fn csharp_cache_key( + code: &str, + requirements_o: Option<&str>, + w_id: &str, + modules: Option<&HashMap>, +) -> String { + // The SDK project globs every `.cs` under the job dir, so companion modules are + // compiled into the binary this key names and have to be part of it. + let base = format!( "{}{}{}", code, requirements_o.unwrap_or(""), DOTNET_TARGET_FRAMEWORK.as_str() - )); + ); + let mut hash = crate::worker::artifact_cache_name(base, modules); hash.push_str(&crate::workspace_registry_cache_suffix(w_id).await); hash } @@ -487,10 +495,11 @@ pub async fn prebuild_csharp_binary( worker_name: &str, base_internal_url: &str, occupancy_metrics: &mut OccupancyMetrics, + modules: Option<&HashMap>, ) -> error::Result> { check_executor_binary_exists("dotnet", DOTNET_PATH.as_str(), "C#")?; - let hash = csharp_cache_key(code, Some(lock), &job.workspace_id).await; + let hash = csharp_cache_key(code, Some(lock), &job.workspace_id, modules).await; let remote_path = format!("{CSHARP_OBJECT_STORE_PREFIX}{hash}"); if crate::global_cache::exists_in_object_store(&remote_path).await { return Ok(None); @@ -549,6 +558,7 @@ pub async fn handle_csharp_job( _worker_name: &str, _envs: HashMap, _occupancy_metrics: &mut OccupancyMetrics, + _modules: Option<&HashMap>, ) -> Result, Error> { Err(anyhow!("C# is not available because the feature is not enabled").into()) } @@ -568,6 +578,7 @@ pub async fn handle_csharp_job( worker_name: &str, envs: HashMap, occupancy_metrics: &mut OccupancyMetrics, + modules: Option<&HashMap>, ) -> Result, Error> { check_executor_binary_exists("dotnet", DOTNET_PATH.as_str(), "C#")?; @@ -575,6 +586,7 @@ pub async fn handle_csharp_job( inner_content, requirements_o.map(|x| x.as_str()), &job.workspace_id, + modules, ) .await; let bin_path = format!("{}/{hash}", *CSHARP_CACHE_DIR); diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index 1d2009c5c0..9fe6d4a616 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -230,8 +230,15 @@ fn go_runtime_int32(v: &str) -> Option { /// Cache key of a Go build. The run path and the deploy-time prebuild must derive it the /// same way or the prebuilt binary is never found and gets rebuilt on first run. -fn go_cache_key(code: &str, maybe_lock: &MaybeLock) -> String { - calculate_hash(&format!("{}{:?}v2", code, maybe_lock)) +fn go_cache_key( + code: &str, + maybe_lock: &MaybeLock, + modules: Option<&HashMap>, +) -> String { + // A module whose path starts with `go/` lands inside the module dir this builds, so + // its content ends up in the binary the key names. + let base = format!("{}{:?}v2", code, maybe_lock); + crate::worker::artifact_cache_name(base, modules) } /// Install the deps, generate the entrypoint wrapper, `go build`, and push the binary to @@ -459,9 +466,10 @@ pub async fn prebuild_go_binary( worker_name: &str, base_internal_url: &str, occupancy_metrics: &mut OccupancyMetrics, + modules: Option<&HashMap>, ) -> Result, Error> { let maybe_lock = MaybeLock::Resolved { lock: lock.to_string() }; - let hash = go_cache_key(code, &maybe_lock); + let hash = go_cache_key(code, &maybe_lock, modules); let remote_path = format!("{GO_OBJECT_STORE_PREFIX}{hash}"); if crate::global_cache::exists_in_object_store(&remote_path).await { return Ok(None); @@ -512,6 +520,7 @@ pub async fn handle_go_job( envs: HashMap, occupation_metrics: &mut OccupancyMetrics, maybe_lock: MaybeLock, + modules: Option<&HashMap>, ) -> Result, Error> { //go does not like executing modules at temp root let job_dir = &format!("{job_dir}/go"); @@ -520,7 +529,7 @@ pub async fn handle_go_job( .create(&job_dir) .expect("could not create go job dir"); - let hash = go_cache_key(inner_content, &maybe_lock); + let hash = go_cache_key(inner_content, &maybe_lock, modules); let bin_path = format!("{}/{hash}", *GO_BIN_CACHE_DIR); let remote_path = format!("{GO_OBJECT_STORE_PREFIX}{hash}"); let (cache, cache_logs) = crate::global_cache::load_cache(&bin_path, &remote_path, false).await; diff --git a/backend/windmill-worker/src/java_executor.rs b/backend/windmill-worker/src/java_executor.rs index bb0518f22d..2e47fced18 100644 --- a/backend/windmill-worker/src/java_executor.rs +++ b/backend/windmill-worker/src/java_executor.rs @@ -66,6 +66,7 @@ pub(crate) struct JobHandlerInput<'a> { pub requirements_o: Option<&'a String>, pub shared_mount: &'a str, pub worker_name: &'a str, + pub modules: Option<&'a HashMap>, } pub async fn handle_java_job<'a>(mut args: JobHandlerInput<'a>) -> Result, Error> { @@ -612,25 +613,33 @@ async fn compile<'a>( inner_content, requirements_o, parent_runnable_path, + modules, .. }: &mut JobHandlerInput<'a>, classpath: &'a str, // plugins: Vec<&'a str>, ) -> Result<(), Error> { - fn compute_hash(code: &str, requirements_o: Option<&String>) -> String { - calculate_hash(&format!( + // The cached artifact is the whole `target/` dir, and companion modules are written + // into the job dir before this runs, so their content can land in it. + fn compute_hash( + code: &str, + requirements_o: Option<&String>, + modules: Option<&HashMap>, + ) -> String { + let base = format!( "{}{}", code, requirements_o .as_ref() .map(|x| x.to_string()) .unwrap_or_default() - )) + ); + crate::worker::artifact_cache_name(base, modules) } let reserved_variables = get_reserved_variables(job, &client.token, conn, parent_runnable_path.clone()).await?; let ws_suffix = crate::workspace_registry_cache_suffix(&job.workspace_id).await; - let mut hash = compute_hash(inner_content, *requirements_o); + let mut hash = compute_hash(inner_content, *requirements_o, *modules); hash.push_str(&ws_suffix); let bin_path = format!("{}/{hash}", *JAVA_CACHE_DIR); let remote_path = format!("java_jar/{hash}"); diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index b9d53a801b..dff7d58c2d 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -14,7 +14,6 @@ use tokio::{ }; use windmill_common::{ error::{self, Error}, - utils::calculate_hash, worker::{write_file, Connection}, }; use windmill_queue::MiniPulledJob; @@ -602,8 +601,13 @@ pub async fn build_rust_crate( /// Cache key of a Rust build. The run path and the deploy-time prebuild must derive it /// the same way or the prebuilt binary is never found and gets rebuilt on first run. -async fn rust_cache_key(code: &str, requirements_o: Option<&String>, w_id: &str) -> String { - let mut hash = compute_rust_hash(code, requirements_o); +async fn rust_cache_key( + code: &str, + requirements_o: Option<&String>, + w_id: &str, + modules: Option<&HashMap>, +) -> String { + let mut hash = compute_rust_hash(code, requirements_o, modules); hash.push_str(&crate::workspace_registry_cache_suffix(w_id).await); hash } @@ -621,11 +625,12 @@ pub async fn prebuild_rust_binary( worker_name: &str, base_internal_url: &str, occupancy_metrics: &mut OccupancyMetrics, + modules: Option<&HashMap>, ) -> error::Result> { ensure_rust_runtime_dirs(); check_executor_binary_exists("cargo", CARGO_PATH.as_str(), "rust")?; - let hash = rust_cache_key(code, Some(&lock.to_string()), &job.workspace_id).await; + let hash = rust_cache_key(code, Some(&lock.to_string()), &job.workspace_id, modules).await; let remote_path = format!("{RUST_OBJECT_STORE_PREFIX}{hash}"); if crate::global_cache::exists_in_object_store(&remote_path).await { return Ok(None); @@ -652,15 +657,22 @@ pub async fn prebuild_rust_binary( Ok(Some(logs)) } -pub fn compute_rust_hash(code: &str, requirements_o: Option<&String>) -> String { - calculate_hash(&format!( +pub fn compute_rust_hash( + code: &str, + requirements_o: Option<&String>, + // Companion modules are written into the crate dir and compiled into the binary this + // key names, so leaving them out shares one script's binary with another. + modules: Option<&HashMap>, +) -> String { + let base = format!( "{}{}", code, requirements_o .as_ref() .map(|x| x.to_string()) .unwrap_or_default() - )) + ); + crate::worker::artifact_cache_name(base, modules) } #[tracing::instrument(level = "trace", skip_all)] @@ -679,11 +691,12 @@ pub async fn handle_rust_job( worker_name: &str, envs: HashMap, occupancy_metrics: &mut OccupancyMetrics, + modules: Option<&HashMap>, ) -> Result, Error> { ensure_rust_runtime_dirs(); check_executor_binary_exists("cargo", CARGO_PATH.as_str(), "rust")?; - let hash = rust_cache_key(inner_content, requirements_o, &job.workspace_id).await; + let hash = rust_cache_key(inner_content, requirements_o, &job.workspace_id, modules).await; let bin_path = format!("{}/{hash}", *RUST_CACHE_DIR); let remote_path = format!("{RUST_OBJECT_STORE_PREFIX}{hash}"); diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 350f0c5a61..7e620439b0 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -950,6 +950,51 @@ pub async fn workspace_registry_cache_suffix(w_id: &str) -> String { } } +/// The name a build artifact is cached under, derived from `base` — the runnable's own +/// cache-key input — and its inline modules. +/// +/// `write_module_files` puts module content in the job dir where the build inlines it into +/// the artifact, so a name without it serves one runnable's modules to another whose main +/// content and lockfile match — across workspaces, the cache being global. +/// +/// Only the path and content may name the artifact, because they are all the build reads. +/// `ScriptModule::lock` especially must stay out: deploy regenerates it *after* the parent +/// has prebuilt, so naming it would strand every prebuilt artifact. +pub(crate) fn artifact_cache_name( + base: String, + modules: Option<&std::collections::HashMap>, +) -> String { + let Some(modules) = modules.filter(|m| !m.is_empty()) else { + // Byte-identical to the name a module-free runnable had before modules entered this, + // so its cached artifacts stay reachable. A pre-fix multi-file runnable also stored + // here, so one of those stays reachable too — accepted over invalidating every cache, + // and once this ships nothing can be stored here with module content in it again. + return windmill_common::utils::calculate_hash(&base); + }; + let mut entries: Vec<(&String, &ScriptModule)> = modules.iter().collect(); + entries.sort_by(|a, b| a.0.cmp(b.0)); + // `base` ends in caller-supplied bytes (a preview brings its own lockfile), so it is + // sealed to a fixed width before the module block is appended — raw, a crafted lockfile + // could spell out another runnable's block and reach its slot. + let mut keyed = format!( + "{}:modules:{}", + windmill_common::utils::calculate_hash(&base), + entries.len() + ); + for (path, module) in entries { + // Both length-prefixed, else `{"a": "bc"}` and `{"ab": "c"}` encode alike. + keyed.push_str(&format!( + ":{}:{path}:{}:{}", + path.len(), + module.content.len(), + module.content, + )); + } + // Its own namespace: `calculate_hash` emits hex, so however a module-free runnable + // crafts its content and lockfile it can never land on a module-bearing name. + format!("mod-{}", windmill_common::utils::calculate_hash(&keyed)) +} + pub fn is_sandboxing_enabled() -> bool { if !*DISABLE_NSJAIL { return true; @@ -3532,7 +3577,13 @@ pub async fn run_worker( job.kind, JobKind::Script | JobKind::Preview | JobKind::FlowScript ) { - if !dedicated_workers.is_empty() { + // A job carrying a pre-run error never runs its code: it only has to be + // pulled so `handle_queued_job` can fail it. Both hand-off paths below + // dispatch by path and return before that check, so a job sent down them + // would run with whatever arguments survived the failure. + let fails_before_running = job.pre_run_error.is_some(); + + if !dedicated_workers.is_empty() && !fails_before_running { let dedicated_worker_tx = job.runnable_path.as_ref().and_then(|path| { // For flow steps inside branches/loops, runnable_path includes // nesting segments (e.g. f/flow/branchone-0/a) but the dedicated @@ -3577,7 +3628,7 @@ pub async fn run_worker( NextJob::Http(_) => None, }; - if let Some(flow_runners) = flow_runners { + if let Some(flow_runners) = flow_runners.filter(|_| !fails_before_running) { let key_o = job.flow_step_id.as_ref().map(|x| x.to_string()); if let Some(key) = key_o { if let Some(flow_runner_tx) = flow_runners.runners.get(&key) { @@ -5453,7 +5504,9 @@ async fn handle_code_execution_job( None => job, }; - // For preview jobs, extract modules from args._MODULES if not already set + // Any job kind, not just previews: whatever is here is what gets written to the job dir + // and built in, so the agent-worker server precomputing a cache name has to resolve + // modules the same way (`windmill-api-agent-workers`, `get_code_and_lock`). let modules = modules_from_data.clone().or_else(|| { job.args.as_ref().and_then(|args| { args.get("_MODULES").and_then(|raw| { @@ -5646,11 +5699,106 @@ mod write_module_files_tests { use super::*; use std::collections::HashMap; use windmill_common::scripts::ScriptLang; + use windmill_common::utils::calculate_hash; fn module(content: &str) -> ScriptModule { ScriptModule { content: content.to_string(), language: ScriptLang::Python3, lock: None } } + /// Every language's artifact cache name funnels module content through this, so an + /// ambiguous encoding puts two different runnables back on one name. + #[test] + fn artifact_name_cannot_be_re_cut_into_another_module_map() { + fn name(entries: &[(&str, &str)]) -> String { + let map: HashMap = entries + .iter() + .map(|(p, c)| (p.to_string(), module(c))) + .collect(); + artifact_cache_name("base".to_string(), Some(&map)) + } + + // Naive `path + content` concatenation renders both of these as "abc". + assert_ne!(name(&[("a", "bc")]), name(&[("ab", "c")])); + // Splitting one module into two must not read back as the joined one. + assert_ne!(name(&[("a", "b"), ("c", "d")]), name(&[("ac", "bd")])); + // Iteration order of the map must not move the name. + assert_eq!( + name(&[("a", "1"), ("b", "2")]), + name(&[("b", "2"), ("a", "1")]) + ); + } + + /// A module-free runnable must keep the exact name it had before modules entered the + /// derivation, or upgrading strands every artifact already in the cache. + #[test] + fn artifact_name_is_unchanged_without_modules() { + assert_eq!( + artifact_cache_name("code+lock".to_string(), None), + calculate_hash("code+lock") + ); + assert_eq!( + artifact_cache_name("code+lock".to_string(), Some(&HashMap::new())), + calculate_hash("code+lock") + ); + } + + /// A preview brings its own source and lockfile, so a module-free runnable picks its + /// whole `base`. Module-bearing names live in their own namespace precisely so that no + /// crafted `base` can be made to land on one. + #[test] + fn a_module_free_runnable_cannot_forge_a_module_bearing_name() { + let modules = HashMap::from([("h.ts".to_string(), module("evil"))]); + let victim = artifact_cache_name("code+lock".to_string(), Some(&modules)); + + // `calculate_hash` emits hex, so the namespace is unreachable however `base` is + // chosen — including by feeding it the victim's own name. + assert!(victim.starts_with("mod-")); + assert_ne!(artifact_cache_name(victim.clone(), None), victim); + assert!(!artifact_cache_name("anything".to_string(), None).starts_with("mod-")); + } + + /// The `mod-` namespace separates module-free from module-bearing, and nothing separates + /// two module-bearing runnables — only the seal does. Unsealed, `base` is variable-width, + /// so the split between it and the module block is ambiguous and a preview (which brings + /// its own source *and* lockfile) can absorb part of another runnable's block. + #[test] + fn a_module_bearing_runnable_cannot_absorb_another_ones_block() { + let victim = artifact_cache_name( + "V".to_string(), + Some(&HashMap::from([( + "h.ts".to_string(), + module(":modules:1:1:a:1:b"), + )])), + ); + // Byte-identical to the victim's without the seal: the forger's `base` spells out the + // victim's leading block, leaving its own single module to supply the tail. + let forged = artifact_cache_name( + "V:modules:1:4:h.ts:18:".to_string(), + Some(&HashMap::from([("a".to_string(), module("b"))])), + ); + + assert_ne!(victim, forged); + } + + /// Deploy fills a module's lock in after the parent has prebuilt, so a name that moved + /// with it would leave every prebuilt artifact unreachable by the runs it was built for. + #[test] + fn artifact_name_ignores_the_lock_deploy_fills_in_later() { + let prebuild = artifact_cache_name( + "base".to_string(), + Some(&HashMap::from([("h.ts".to_string(), module("x"))])), + ); + + let mut locked = module("x"); + locked.lock = Some("{}\n//bun.lock\n".to_string()); + let after_deploy = artifact_cache_name( + "base".to_string(), + Some(&HashMap::from([("h.ts".to_string(), locked)])), + ); + + assert_eq!(prebuild, after_deploy); + } + #[test] fn contained_relative_path_rejects_traversal_and_absolute() { assert!(is_contained_relative_path("u/admin/pkg")); @@ -6380,6 +6528,7 @@ mount {{ envs, occupancy_metrics, maybe_lock, + modules.as_ref(), )) .await } @@ -6509,6 +6658,7 @@ mount {{ worker_name, envs, occupancy_metrics, + modules.as_ref(), )) .await } @@ -6567,6 +6717,7 @@ mount {{ worker_name, envs, occupancy_metrics, + modules.as_ref(), )) .await } @@ -6631,6 +6782,7 @@ mount {{ worker_name, envs, occupancy_metrics, + modules: modules.as_ref(), })) .await } diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index bd39afa8c7..34f46307cf 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -70,9 +70,9 @@ use windmill_queue::schedule::get_schedule_opt; use windmill_queue::{ add_completed_job, add_completed_job_error, append_logs, get_mini_pulled_job, insert_concurrency_key_capped, interpolate_args, - report_error_to_workspace_handler_or_critical_side_channel, try_schedule_next_job, CanceledBy, - FlowRunners, MiniCompletedJob, MiniPulledJob, PushArgs, PushIsolationLevel, SameWorkerPayload, - WrappedError, + report_error_to_workspace_handler_or_critical_side_channel, tag_reads_args, + try_schedule_next_job, CanceledBy, FlowRunners, MiniCompletedJob, MiniPulledJob, PushArgs, + PushIsolationLevel, SameWorkerPayload, WrappedError, }; use windmill_audit::audit_oss::audit_log; @@ -4403,6 +4403,23 @@ async fn push_next_flow_job( payload_tag.tag.as_deref(), ); + // `push_args` is empty once the input transforms failed, so a tag reading `$args[...]` + // interpolates to a queue nobody serves and the step sits there instead of reporting + // the error. Send it to the flow's tag, which a worker is provably serving right now. + // + // A step handed over by id, or one whose tag `push` replaces, never reaches a worker + // through its tag, so rewriting theirs would be noise. + let step_is_pulled_by_tag = !continue_on_same_worker + && !continue_with_runners + && !payload_tag.payload.is_dedicated_worker(); + let reroute_to_flow_tag = + err.is_some() && step_is_pulled_by_tag && tag.as_deref().is_some_and(tag_reads_args); + let tag = if reroute_to_flow_tag { + Some(flow_job.tag.clone()) + } else { + tag + }; + let (email, permissioned_as) = if let Some(on_behalf_of) = payload_tag.on_behalf_of.as_ref() { (&on_behalf_of.email, on_behalf_of.permissioned_as.clone()) @@ -4421,8 +4438,7 @@ async fn push_next_flow_job( .as_deref() .filter(|t| !t.is_empty() && *t != flow_job.tag.as_str()) { - let is_super_admin = - windmill_common::auth::is_super_admin_email(db, email).await?; + let is_super_admin = windmill_common::auth::is_super_admin_email(db, email).await?; check_tag_available_for_workspace_internal( db, &flow_job.workspace_id, @@ -6155,9 +6171,7 @@ pub async fn script_to_payload( .await? .prefetch_cached(&db) .await?; - let on_behalf_of = script_info - .on_behalf_of(&flow_job.workspace_id, db) - .await?; + let on_behalf_of = script_info.on_behalf_of(&flow_job.workspace_id, db).await?; let ScriptHashInfo { tag, cache_ttl, diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 5cfb9f08ce..57b859075a 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -199,6 +199,7 @@ async fn handle_build_binary_job( worker_name, base_internal_url, occupancy_metrics, + script_data.modules.as_ref(), ) .await? } @@ -214,6 +215,7 @@ async fn handle_build_binary_job( worker_name, base_internal_url, occupancy_metrics, + script_data.modules.as_ref(), ) .await? } @@ -235,6 +237,7 @@ async fn handle_build_binary_job( worker_name, base_internal_url, occupancy_metrics, + script_data.modules.as_ref(), ) .await? } @@ -3155,7 +3158,11 @@ async fn capture_dependency_job( ) .await? { - if !wd_exist { + // Nothing here writes the module files, so a bundle built now resolves a + // multi-file script's relative imports remotely rather than from its + // modules; caching that under a key naming them would serve the wrong + // code to every run. Leave it to the first run instead. + if !wd_exist && modules.map_or(true, |m| m.is_empty()) { crate::bun_executor::prebundle_bun_script( job_raw_code, &lock, @@ -3169,6 +3176,7 @@ async fn capture_dependency_job( &token, &mut Some(occupancy_metrics), temp_script_refs, + modules, ) .await?; } diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 55734e52e8..f9584c3091 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.796.0"; +export const VERSION = "v1.800.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/commands/app/bundle.ts b/cli/src/commands/app/bundle.ts index 0ebbdaa076..286e878cfb 100644 --- a/cli/src/commands/app/bundle.ts +++ b/cli/src/commands/app/bundle.ts @@ -191,7 +191,14 @@ function createSveltePlugin(appDir: string): any { // Convert Svelte syntax to JavaScript try { - const { js, warnings } = svelte.compile(source, { filename }); + // The raw-app editor's in-browser bundler compiles with + // `css: "injected"`, so this must too, or the same app renders + // styled there and unstyled once the CLI builds it: Svelte's default + // ("external") hands the +`, + "styles_entry.ts": `import Styled from './Styled.svelte'; +export default Styled; +`, + }); + + const js = await bundle("styles_entry.ts"); + + const scopeClass = js.match(/

= {}; +let remoteUnreadable = false; +let pushedFiles: Record | undefined; mock.module("../gen/services.gen.ts", () => ({ - getSharedUi: async (_args: { workspace: string }) => ({ files: remoteFiles }), + getSharedUi: async (_args: { workspace: string }) => { + if (remoteUnreadable) throw new Error("shared UI store unreadable"); + return { files: remoteFiles }; + }, + updateSharedUi: async (args: { + workspace: string; + requestBody: { files: Record }; + }) => { + pushedFiles = args.requestBody.files; + }, })); -const { diffSharedUi } = await import("../src/commands/shared_ui.ts"); +const { diffSharedUi, pushSharedUi } = await import( + "../src/commands/shared_ui.ts" +); describe("diffSharedUi", () => { const ws = "test-workspace"; @@ -25,6 +38,8 @@ describe("diffSharedUi", () => { beforeEach(() => { remoteFiles = {}; + remoteUnreadable = false; + pushedFiles = undefined; prevCwd = process.cwd(); tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "wm-shared-ui-")); process.chdir(tmpDir); @@ -89,4 +104,46 @@ describe("diffSharedUi", () => { const changes = await diffSharedUi(ws); expect(changes).toEqual([]); }); + + test("emits nothing under keepDeleted when the remote store is unreadable", async () => { + // pushSharedUi skips the push rather than clearing a store it can't read, + // so the preview must show that same nothing. + remoteUnreadable = true; + writeUi("theme.json", "{}"); + expect(await diffSharedUi(ws, true)).toEqual([]); + }); +}); + +describe("pushSharedUi with keepDeleted", () => { + const ws = "test-workspace"; + let tmpDir: string; + let prevCwd: string; + + beforeEach(() => { + remoteFiles = {}; + remoteUnreadable = false; + pushedFiles = undefined; + prevCwd = process.cwd(); + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "wm-shared-ui-push-")); + process.chdir(tmpDir); + }); + + afterEach(() => { + process.chdir(prevCwd); + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + test("carries remote-only files, including ui/__proto__, into the pushed map", async () => { + // JSON.parse, not a literal: `{__proto__: …}` sets the prototype instead of + // creating the own property the API response really has. + remoteFiles = JSON.parse('{"__proto__":"keep me","extra.json":"1"}'); + fs.mkdirSync(path.join(tmpDir, "ui"), { recursive: true }); + fs.writeFileSync(path.join(tmpDir, "ui", "theme.json"), "{}", "utf-8"); + + expect(await pushSharedUi(ws, true)).toBe(true); + // The store is written whole, so anything missing here is deleted. + expect(pushedFiles!["extra.json"]).toEqual("1"); + expect(Object.getOwnPropertyDescriptor(pushedFiles!, "__proto__")?.value) + .toEqual("keep me"); + }); }); diff --git a/cli/test/sync_pull_push.test.ts b/cli/test/sync_pull_push.test.ts index 343ed49ef7..24c4a282a3 100644 --- a/cli/test/sync_pull_push.test.ts +++ b/cli/test/sync_pull_push.test.ts @@ -2774,3 +2774,102 @@ kind: script }); }); }); + +describe("keep deleted", () => { + test("Integration: --keep-deleted keeps items absent from the other side", async () => { + await withTestBackend(async (backend, tempDir) => { + const uniqueId = Date.now(); + const scriptPath = `f/test/keep_deleted_${uniqueId}`; + + const resp = await backend.apiRequest!( + `/api/w/${backend.workspace}/scripts/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: scriptPath, + content: 'export async function main() { return "keep me"; }', + language: "bun", + summary: "Kept by --keep-deleted", + schema: { + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + properties: {}, + required: [], + }, + }), + } + ); + expect(resp.status).toBeLessThan(300); + await resp.text(); + + await writeWmillYaml(tempDir); + expect( + (await backend.runCLICommand(["sync", "pull", "--yes"], tempDir)).code + ).toEqual(0); + + const contentFile = `${scriptPath}.ts`; + const metadataFile = `${scriptPath}.script.yaml`; + expect(await listFilesRecursive(tempDir)).toContain(contentFile); + + // Push direction: the remote script survives losing its local files. + await rm(join(tempDir, contentFile)); + await rm(join(tempDir, metadataFile)); + expect( + ( + await backend.runCLICommand( + ["sync", "push", "--yes", "--keep-deleted"], + tempDir + ) + ).code + ).toEqual(0); + // A push deletion archives the script rather than removing the row, so + // `archived` — not the status code — is what says it survived. + const remote = await backend.apiRequest!( + `/api/w/${backend.workspace}/scripts/get/p/${scriptPath}` + ); + expect(remote.status).toEqual(200); + expect((await remote.json()).archived).not.toEqual(true); + + // Pull direction: a file with no remote counterpart survives the pull. + const localOnly = `f/test/local_only_${uniqueId}.ts`; + await writeFile( + join(tempDir, localOnly), + 'export async function main() { return "local only"; }', + "utf-8" + ); + expect( + ( + await backend.runCLICommand( + ["sync", "pull", "--yes", "--keep-deleted"], + tempDir + ) + ).code + ).toEqual(0); + const afterPull = await listFilesRecursive(tempDir); + expect(afterPull).toContain(localOnly); + // Adds still apply: the script deleted above is written back. + expect(afterPull).toContain(contentFile); + + // An empty changeset falls past the dry-run return, on to the shared-UI + // step — which writes to disk, so a dry run must skip it. + // Including the metadata and lock the pull's auto-fill generated for it. + for (const ext of [".ts", ".script.yaml", ".script.lock"]) { + await rm(join(tempDir, `f/test/local_only_${uniqueId}${ext}`), { + force: true, + }); + } + await mkdir(join(tempDir, "ui"), { recursive: true }); + await writeFile(join(tempDir, "ui", "custom.css"), "body{}", "utf-8"); + const dryRun = await backend.runCLICommand( + ["sync", "pull", "--dry-run"], + tempDir + ); + expect(dryRun.code).toEqual(0); + // Guards against a vacuous pass: a non-empty changeset would return at + // the dry-run check above and never reach the shared-UI step. + expect(dryRun.stdout + dryRun.stderr).toContain("0 changes to apply"); + expect(await listFilesRecursive(tempDir)).toContain("ui/custom.css"); + }); + }); +}); diff --git a/docker-compose.yml b/docker-compose.yml index a801a0ce7c..bd1e060f22 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,15 +8,28 @@ x-logging: &default-logging compress: "true" services: + ## UPGRADING FROM POSTGRES 16: db_data holds a cluster 18 cannot read, so the + ## container exits with an explanatory error rather than coming up blank. Migrating + ## means dumping the WHOLE cluster (pg_dumpall), never just the windmill database: + ## Windmill keeps datatable, DuckLake and wm_fork_* databases beside it and grants + ## its RLS policies to cluster-level roles, and a single-database dump loses both + ## silently. Full procedure, and why 16 is still a valid choice until Nov 2028: + ## https://www.windmill.dev/docs/advanced/self_host#upgrade-postgresql-to-18 db: deploy: # To use an external database, set replicas to 0 and set DATABASE_URL to the external database url in the .env file replicas: 1 - image: postgres:16 + image: postgres:18 shm_size: 1g restart: unless-stopped volumes: - - db_data:/var/lib/postgresql/data + # From 18 on the official image keeps the cluster in a major-version + # subdirectory (/var/lib/postgresql/18/docker), so the mount has to be the + # parent directory: that is what lets pg_upgrade see an old and a new + # cluster inside a single mount point. Mounting the pre-18 .../data path + # instead makes the image exit rather than start, which is what turns a + # stale 16 cluster into a loud failure instead of an empty instance. + - db_data:/var/lib/postgresql expose: - 5432 environment: diff --git a/docs/feature-telemetry.md b/docs/feature-telemetry.md index 57467e25ec..5d7f0b2c29 100644 --- a/docs/feature-telemetry.md +++ b/docs/feature-telemetry.md @@ -4,10 +4,10 @@ anonymous usage-stats payload. It answers "does anyone use this, and which variant do they pick" without any identifying data leaving the instance. -It currently carries 21 registered actions across nine features (`ai_session`, `ai_chat`, -`flow_editor`, `flow_run`, `flow_step`, `trigger`, `command_script`, `hub_script`, -`usage_meter`). Nearly all of the product is uninstrumented, so new user-facing work is the -opportunity to change that. +It currently carries 28 registered actions across fourteen features (`ai_session`, `ai_chat`, +`ai_fix`, `ai_agent`, `ai_agent_eval`, `flow_editor`, `flow_run`, `flow_step`, `run_form`, +`debugger`, `trigger`, `command_script`, `hub_script`, `usage_meter`). Nearly all of the +product is uninstrumented, so new user-facing work is the opportunity to change that. ## When to instrument diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 6d72964274..20592d0371 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.796.0", + "version": "1.800.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.796.0", + "version": "1.800.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { @@ -29,6 +29,7 @@ "@windmill-labs/svelte-dnd-action": "^0.9.44", "@xterm/addon-fit": "^0.10.0", "@xyflow/svelte": "^1.0.0", + "acorn": "^8.15.0", "ag-charts-community": "^9.0.1", "ag-charts-enterprise": "^9.0.1", "ag-grid-community": "^31.3.4", @@ -13195,9 +13196,9 @@ } }, "node_modules/svelte-check/node_modules/picomatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", - "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", "dev": true, "license": "MIT", "optional": true, diff --git a/frontend/package.json b/frontend/package.json index 1d984c8587..08d8c427ff 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.796.0", + "version": "1.800.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", @@ -105,6 +105,7 @@ "@windmill-labs/svelte-dnd-action": "^0.9.44", "@xterm/addon-fit": "^0.10.0", "@xyflow/svelte": "^1.0.0", + "acorn": "^8.15.0", "ag-charts-community": "^9.0.1", "ag-charts-enterprise": "^9.0.1", "ag-grid-community": "^31.3.4", diff --git a/frontend/src/lib/aiStore.ts b/frontend/src/lib/aiStore.ts index 0955f68ee5..2d6a98c625 100644 --- a/frontend/src/lib/aiStore.ts +++ b/frontend/src/lib/aiStore.ts @@ -7,6 +7,7 @@ import { type AIProviderModel, type AIProvider, type AIConfig, + type FreeTierInfo, type ModelPriceOverride } from './gen' import { @@ -49,6 +50,10 @@ export const copilotInfo = writable<{ /** Negotiated rates per `provider:model`, overriding the built-in price table. */ modelPricing?: Record webSearchEnabledProviders?: Partial> + // Set only when the workspace has no AI provider of its own and is running on + // Windmill's free tier. `exhausted` means the grant is spent: there is no model, but + // that is a different state from "never configured" and the UI must say so. + freeTier?: FreeTierInfo }>({ enabled: false, codeCompletionModel: undefined, @@ -132,8 +137,9 @@ export function setCopilotInfo(aiConfig: AIConfig) { aiModels: aiModels, customPrompts: aiConfig.custom_prompts ?? {}, maxTokensPerModel: aiConfig.max_tokens_per_model ?? {}, + webSearchEnabledProviders, modelPricing: aiConfig.model_pricing ?? {}, - webSearchEnabledProviders + freeTier: aiConfig.free_tier }) } else { copilotSessionModel.set(undefined) @@ -146,8 +152,11 @@ export function setCopilotInfo(aiConfig: AIConfig) { aiModels: [], customPrompts: {}, maxTokensPerModel: {}, + webSearchEnabledProviders: {}, modelPricing: {}, - webSearchEnabledProviders: {} + // An exhausted free grant lands here — no providers, but the reason AI is off + // is "you used it up", not "you never set it up". + freeTier: aiConfig.free_tier }) } } diff --git a/frontend/src/lib/components/AppConnectDrawer.svelte b/frontend/src/lib/components/AppConnectDrawer.svelte index 67c19b0fe7..8de484929e 100644 --- a/frontend/src/lib/components/AppConnectDrawer.svelte +++ b/frontend/src/lib/components/AppConnectDrawer.svelte @@ -6,7 +6,7 @@ import DrawerContent from './common/drawer/DrawerContent.svelte' import AppConnectInner from './AppConnectInner.svelte' - import DarkModeObserver from './DarkModeObserver.svelte' + import GoogleSigninButton from './GoogleSigninButton.svelte' import IconedResourceType from './IconedResourceType.svelte' import { addResourceTitle } from './resourceTypeDisplay' @@ -22,6 +22,10 @@ disableChatOffset = false }: Props = $props() + /** Set by `open(rt, fillPath)`, not by the parent: which resource this run fills is a + * property of the click, and a prop would go stale between two different rows. */ + let fillPath: string | undefined = $state(undefined) + let drawer: Drawer | undefined = $state() let resourceType = $state('') let step = $state(1) @@ -32,29 +36,46 @@ let appConnectInner: AppConnectInner | undefined = $state(undefined) let rtToLoad: string | undefined = $state('') - export async function open(rt?: string) { + /** `fill` connects into a resource that already exists, instead of creating one. */ + export async function open(rt?: string, fill?: string) { + fillPath = fill + handedOff = false rtToLoad = rt drawer?.openDrawer?.() } + /** + * Hand off to the inner component exactly once per opening. The reactive statement below + * re-runs both when `rtToLoad` changes and when `appConnectInner` binds — and it binds + * afresh on every opening, since the drawer destroys its content on close. A second + * `open()` runs `next()` a second time, which walks a drawer opened on a resource type + * straight past the Connect button and into `window.open`; a popup opened from a reactive + * effect rather than from the click is blocked, leaving "Finish connection in popup + * window" with no popup behind it. + * + * A flag rather than the last resource type: `open()` with no argument leaves `rtToLoad` + * undefined, which compares equal to the initial state and would skip the hand-off + * entirely — the resources page opens it that way. + */ + let handedOff = false function onRtToLoadChange(rtToLoad: string | undefined) { + if (handedOff) return + handedOff = true appConnectInner?.open(rtToLoad) } const dispatch = createEventDispatcher() - let darkMode: boolean = $state(false) run(() => { appConnectInner && onRtToLoadChange(rtToLoad) }) - - { step = 1 + handedOff = false dispatch('close') }} size="700px" @@ -83,22 +104,26 @@ on:refresh express={expressOAuthSetup} {workspace} + {fillPath} /> {#snippet actions()}
- {#if step > 1} - + + {#if step > 1 && !rtToLoad} + {/if} {#if isGoogleSignin} - + appConnectInner?.next()} /> {:else} -
{:else} -
+ +
{#if step > 2} - + {/if} - + {#if isGoogleSignin} + appConnect?.next()} /> + {:else} + + {/if}
{/if} @@ -64,6 +72,7 @@ bind:resourceType bind:disabled bind:manual + bind:isGoogleSignin on:error on:refresh /> diff --git a/frontend/src/lib/components/CenteredModal.svelte b/frontend/src/lib/components/CenteredModal.svelte index e3f8ea1604..3250b3a16a 100644 --- a/frontend/src/lib/components/CenteredModal.svelte +++ b/frontend/src/lib/components/CenteredModal.svelte @@ -6,6 +6,9 @@ interface Props { subtitle?: string | undefined + /** Rendered under the title, for a subtitle that needs markup (a link, say). + * Sits below `subtitle` when both are given. */ + subtitleSnippet?: import('svelte').Snippet title?: string large?: boolean centerVertically?: boolean @@ -16,6 +19,7 @@ let { subtitle = undefined, + subtitleSnippet = undefined, title = 'Windmill', large = false, centerVertically = true, @@ -60,6 +64,9 @@ {subtitle}

{/if} + {#if subtitleSnippet} +
{@render subtitleSnippet()}
+ {/if}
{#if children} diff --git a/frontend/src/lib/components/Dev.svelte b/frontend/src/lib/components/Dev.svelte index 54a68dd5a3..c7e83b8a80 100644 --- a/frontend/src/lib/components/Dev.svelte +++ b/frontend/src/lib/components/Dev.svelte @@ -1,6 +1,7 @@ @@ -223,7 +209,7 @@ Files
diff --git a/frontend/src/lib/components/FilterSearchbar.svelte b/frontend/src/lib/components/FilterSearchbar.svelte index 5078e0a83b..51026f10fd 100644 --- a/frontend/src/lib/components/FilterSearchbar.svelte +++ b/frontend/src/lib/components/FilterSearchbar.svelte @@ -9,6 +9,11 @@ type: 'string' | 'number' | 'boolean' allowMultiple?: boolean format?: 'json' + /** Boolean only: the value the filter holds while unset (defaults to false). + * Selecting a filter whose default is false sets it to true immediately rather + * than opening a true/false picker whose only useful choice is true. A default-true + * boolean still opens the picker, since choosing false is the meaningful action. */ + default?: boolean } | { type: 'date' @@ -121,16 +126,34 @@ // Create the filter instance object const filterInstance: { val: Partial> } = $state({ val: {} }) - // Sync URL params to filter instance on initialization and when URL changes + // Sync URL params to filter instance, reactively. Reading urlFilter[key] tracked + // means browser Back/Forward — which mutates useSearchParams' cells on popstate — + // flows into the instance (chips, kind toggle, results), not just the first render. + // The write happens untracked so it can't self-trigger, and the equality check plus + // the reverse effect's own guard keep the two directions from ping-ponging. for (const key of Object.keys(schemaRec)) { - let urlValue = urlFilter[key] - if (schemaRec[key].type === 'date' && typeof urlValue === 'string') { - const d = new Date(urlValue) - urlValue = isNaN(d.getTime()) ? null : d - } - if (urlValue !== undefined && urlValue !== null) { - ;(filterInstance.val as any)[key] = urlValue - } + $effect(() => { + let urlValue = urlFilter[key] + if (schemaRec[key].type === 'date' && typeof urlValue === 'string') { + const d = new Date(urlValue) + urlValue = isNaN(d.getTime()) ? null : d + } + untrack(() => { + const current = (filterInstance.val as any)[key] + const same = + urlValue instanceof Date && current instanceof Date + ? urlValue.getTime() === current.getTime() + : current === (urlValue ?? undefined) + if (same) return + if (urlValue !== undefined && urlValue !== null) { + ;(filterInstance.val as any)[key] = urlValue + } else if (current !== undefined) { + // Key dropped from the URL (Back to a state without it): clear it so a + // stale chip / filter doesn't linger against the navigated-to URL. + delete (filterInstance.val as any)[key] + } + }) + }) } // Sync filter instance changes back to URL params @@ -275,6 +298,17 @@ class?: string placeholder?: string autofocus?: boolean + // Applied as the id of the underlying editable, so a parent can focus it or recognise its + // key events by id (the searchbar is a contenteditable, not an ). + inputId?: string + // Free-text mode: while the input holds only free text (no specific filter tag is + // being edited and no non-default filter is set), suppress the suggestions dropdown + // so it behaves like a plain search box. This frees the arrow keys for the + // surrounding UI (e.g. a results list). The dropdown returns the moment a specific + // filter is present (e.g. `path: u/me/abc`). + hideDropdownOnFreeText?: boolean + // Notified whenever the dropdown's effective visibility changes + onDropdownVisibleChange?: (visible: boolean) => void } type SchemaT = FilterSchemaRec // TODO: Generic @@ -284,7 +318,10 @@ presets: _presets = [], class: className, placeholder = 'Filter...', - autofocus + autofocus, + hideDropdownOnFreeText = false, + onDropdownVisibleChange, + inputId }: Props = $props() let _value = new DebouncedTempValue( @@ -298,6 +335,24 @@ let currentTag: keyof SchemaT | undefined = $state() let currentTextSegment = $state({ text: '', start: 0, end: 0 }) let open = $state(false) + + // A specific filter is in play when a tag is being edited or any non-free-text filter + // is set. + let hasSpecificFilter = $derived( + !!currentTag || Object.keys(value).some((k) => k !== '_default_') + ) + // A plain search term is being typed (free text, no specific filter). + let hasFreeText = $derived(!!String(value['_default_'] ?? '').trim()) + // Effective dropdown visibility. Free-text mode suppresses the dropdown ONLY while the + // user is typing a plain search term: it still opens when the input is empty (so the + // available filters stay discoverable) and whenever a specific filter is set or being + // edited. That leaves the arrow keys for the surrounding list only during free-text search. + let dropdownVisible = $derived( + open && (!hideDropdownOnFreeText || hasSpecificFilter || !hasFreeText) + ) + $effect(() => { + onDropdownVisibleChange?.(dropdownVisible) + }) let inputElement: HTMLDivElement | undefined = $state() let highlightedIndex = $state(0) let taggedTextInput: TaggedTextInput | undefined = $state() @@ -347,9 +402,17 @@ key, filterSchema, onClick: () => { - // Replace the text segment with the new filter tag const before = asText.val.slice(0, currentTextSegment.start) const after = asText.val.slice(currentTextSegment.end) + if (schema[key].type === 'boolean' && schema[key].default !== true) { + // Set the only useful value and reparse to canonical text. The space is + // required: dropping the segment must not fuse the tags that flanked it. + asText.val = `${before} ${after}` + value[key] = true as any + asText.reparse() + return + } + // Replace the text segment with the new (empty) filter tag; the value picker opens. asText.val = `${before}${before && !before.endsWith(' ') ? ' ' : ''}${key}:\\\u00A0${after}`.trim() + '\u00A0' @@ -406,6 +469,28 @@ onClick: () => setValueForCurrentTag(false) } ] + } else if (filter.type === 'string' && filter.format !== 'json') { + // A plain string filter has no fixed options, but any presets targeting this tag + // (`:`) are exactly its useful values — surface them as suggestions so + // picking one is a click, matching the top-level preset row. Unescape the tagged + // syntax's `\ ` back to a real space for the stored value. + const prefix = `${String(currentTag)}:` + const suffix = String(value[currentTag!] ?? '') + .trim() + .toLowerCase() + return _presets + .filter((p) => p.value.startsWith(prefix) && !asText.val.includes(p.value)) + .map((p) => { + const raw = p.value.slice(prefix.length).replace(/^\\ /, '').replace(/\\ /g, ' ') + return { name: p.name, raw } + }) + .filter((p) => !suffix || p.raw.toLowerCase().includes(suffix)) + .map((p) => ({ + type: 'option' as const, + option: { value: p.raw, label: p.name }, + onClick: () => appendOrSetValueForCurrentTag(p.raw), + onNegativeClick: undefined + })) } } return [] @@ -514,7 +599,9 @@ } function handleKeyDown(e: KeyboardEvent) { - if (!open) return + // In free-text mode the dropdown is hidden; let arrow/enter keys pass through to + // the surrounding UI (e.g. list navigation) rather than steering a hidden menu. + if (!dropdownVisible) return if (e.key === 'Escape') { open = false return @@ -601,6 +688,7 @@ > (open = true)} + onKeyDown={(e) => { + // In free-text mode the searchbar coexists with a list that owns Arrow/Enter, so opening + // the dropdown on a bare navigation key would steal them from an empty box. Typing, click, + // or an already-open dropdown still open/keep it. Other searchbars keep opening on any key. + if ( + !hideDropdownOnFreeText || + !['ArrowDown', 'ArrowUp', 'ArrowLeft', 'ArrowRight', 'Enter', 'Escape', 'Tab'].includes( + e.key + ) + ) { + open = true + } + }} {autofocus} /> {#if asText.val} @@ -630,9 +730,10 @@
inputElement?.getBoundingClientRect() ?? new DOMRect()} - innerClass="!max-h-[30rem]" + innerClass="!max-h-[25rem]" strictWidth > @@ -747,6 +848,20 @@ class="border border-border-light rounded min-h-[4rem]" /> + {:else if filter.type === 'string'} + {#if menuItems.length} +
+ {#each menuItems as item, index} + {#if item.type === 'option' && item.option} + {@render menuItem({ + onClick: item.onClick, + label: item.option.label || item.option.value, + highlighted: index === highlightedIndex + })} + {/if} + {/each} +
+ {/if} {/if} {/snippet} diff --git a/frontend/src/lib/components/FlowPreviewContent.svelte b/frontend/src/lib/components/FlowPreviewContent.svelte index dc42f9828c..d895cb4e2e 100644 --- a/frontend/src/lib/components/FlowPreviewContent.svelte +++ b/frontend/src/lib/components/FlowPreviewContent.svelte @@ -34,6 +34,8 @@ import InputSelectedBadge from './schema/InputSelectedBadge.svelte' import Toggle from './Toggle.svelte' import JsonInputs from './JsonInputs.svelte' + import { argsToJsonPayload } from '$lib/schema' + import type { Schema } from '$lib/common' import FlowHistoryJobPicker from './FlowHistoryJobPicker.svelte' import type { DurationStatus, GraphModuleState } from './graph' import { getStepHistoryLoaderContext } from './stepHistoryLoader.svelte' @@ -264,8 +266,12 @@ previewArgs.val = input inputSelected = type preventEscape = true - jsonEditor?.setCode(JSON.stringify(previewArgs.val ?? {}, null, '\t')) } + // Deselecting restores the args the same way selecting replaced them, so both branches + // owe the editor an overwrite — it holds a payload for the input being left behind. + jsonEditor?.setCode( + argsToJsonPayload(flowStore.val.schema as Schema | undefined, previewArgs.val) + ) } export function refresh() { @@ -510,8 +516,7 @@ rightTooltip: 'Fill args from JSON' }} lightMode - on:change={(e) => { - jsonEditor?.setCode(JSON.stringify(previewArgs.val ?? {}, null, '\t')) + on:change={() => { refresh() }} /> @@ -526,6 +531,10 @@ previewArgs.val = e.detail } }} + initialCode={argsToJsonPayload( + flowStore.val.schema as Schema | undefined, + previewArgs.val + )} updateOnBlur={false} placeholder={`Write args as JSON.

Example:

{
  "foo": "12"
}`} /> diff --git a/frontend/src/lib/components/FolderPicker.svelte b/frontend/src/lib/components/FolderPicker.svelte index 69c219d0a1..325d36346d 100644 --- a/frontend/src/lib/components/FolderPicker.svelte +++ b/frontend/src/lib/components/FolderPicker.svelte @@ -1,5 +1,5 @@ diff --git a/frontend/src/lib/components/GoogleSigninButton.svelte b/frontend/src/lib/components/GoogleSigninButton.svelte new file mode 100644 index 0000000000..d7d2ab102c --- /dev/null +++ b/frontend/src/lib/components/GoogleSigninButton.svelte @@ -0,0 +1,21 @@ + + + + diff --git a/frontend/src/lib/components/ImportProjectCard.svelte b/frontend/src/lib/components/ImportProjectCard.svelte new file mode 100644 index 0000000000..26bb3f7b18 --- /dev/null +++ b/frontend/src/lib/components/ImportProjectCard.svelte @@ -0,0 +1,115 @@ + + + + +
+ +
+
+ +
+ {#if project.logoUrl} + + {:else if icons[0]} + {@const Icon = icons[0]} + + + + {:else} + + {/if} +
+ +
+ + + {project.name} + + +

{project.summary}

+

+ by {project.author} + · {project.slug} +

+ + +
+ +
+
+ + + {#if restIcons.length > 0} +
+ {#each restIcons as Icon, i (i)} + + + + {/each} +
+ {/if} +
+
+ + +
diff --git a/frontend/src/lib/components/ImportProjectStep.svelte b/frontend/src/lib/components/ImportProjectStep.svelte new file mode 100644 index 0000000000..04c169d41c --- /dev/null +++ b/frontend/src/lib/components/ImportProjectStep.svelte @@ -0,0 +1,524 @@ + + +
+ + {#if existingWorkspace} +
+ + + Folder inside {existingWorkspace} + + +

+ Everything the project ships is imported into this folder. +

+ + + {#if folder.trim() && !folderValid} +

Letters, digits, dashes and underscores only.

+ {/if} +
+ {/if} + + +

What this will do

+ + + + + {#if execution?.error} + + {execution.error} + + {/if} + + + + Resources are imported as empty stubs — set their values after import; one whose path is + already in the workspace is left exactly as it is and reported as already there, so a value + you have since filled in is never overwritten. Trigger kinds are + recreated disabled, except GCP and Azure triggers, which manage cloud subscriptions at creation + and must be re-created manually after filling their resource. Kafka, NATS, SQS, GCP and Azure + triggers all require Enterprise. Triggers that reference a resource depend on stubs imported + empty, so fill in the resource value before re-enabling the trigger. + + +
+ + {#if !execution?.done} + + {:else} + + {/if} + +
+ + {#if execution?.createdWorkspace && !execution.done} + + {/if} + + {#if execution?.done} + + {#if execution.error} + + {/if} + + + {:else} + + {/if} +
+
+
+ + + + + + + closeMigrationReview(false)}> + closeMigrationReview(false)}> +
+ + + {reviewList.length === 1 ? 'This data table is' : 'These data tables are'} already set up{existingWorkspace + ? ` in ${existingWorkspace}` + : ''} and may already hold data. These migrations were written to create the project's tables, + so running them here can alter or drop what is in them. Read the SQL before you run it, and skip + anything you are unsure of. + +

+ Review and edit the SQL, then choose which to run. A migration runs against the data table + of the same name in the destination workspace; if that data table has migrations enabled it + is recorded, otherwise it runs once as a preview job. +

+ {#each reviewList as m (m.datatable_name)} +
+
+ {m.datatable_name} + +
+ {#if m.run} + + {/if} +
+ {/each} +
+ {#snippet actions()} + + + {/snippet} + + diff --git a/frontend/src/lib/components/ImportSetupRow.svelte b/frontend/src/lib/components/ImportSetupRow.svelte new file mode 100644 index 0000000000..11f95b333c --- /dev/null +++ b/frontend/src/lib/components/ImportSetupRow.svelte @@ -0,0 +1,48 @@ + + + +
  • +
    +
    {@render icon()}
    + +
    + {@render title()} + {@render detail?.()} +
    + +
    + {@render action()} + {#if flash} +
    + +
    + {/if} +
    +
    + {@render extra?.()} +
    • diff --git a/frontend/src/lib/components/ImportSetupStep.svelte b/frontend/src/lib/components/ImportSetupStep.svelte new file mode 100644 index 0000000000..86670b0817 --- /dev/null +++ b/frontend/src/lib/components/ImportSetupStep.svelte @@ -0,0 +1,861 @@ + + +
    +
    +

    Finish setting up

    + +

    + Your project is imported. For its apps and flows to actually run, they need a place to store + data and credentials for the services they use — the import can't supply those for you. +

    +
    + + {#if loading} +
    + Checking what this project needs… +
    + {:else if loadError} + + {loadError}. You can finish and configure them later in Workspace settings → Data tables. + + {:else} + {#if rows.length > 0} + +
    + + Data table{rows.length === 1 ? '' : 's'} to set up ({rows.length}) + +

    + Where apps and flows keep the data they read and write. +

    +
    + {/if} +
      + {#each rows as row (row.name)} + {@const sql = row.migrations + .map((m) => m.sql) + .filter(Boolean) + .join('\n\n')} + {@const hasTable = configuredNames.some((c) => c.name === row.name)} + + {#snippet icon()} + {#if row.status === 'done'} + + {:else if row.status === 'running'} + + {:else if row.status === 'failed'} + + {:else if row.status === 'unknown'} + + {:else} + + {/if} + {/snippet} + {#snippet title()} + {row.name} + {/snippet} + {#snippet detail()} + + {#if row.status === 'done'} + {row.migrations.length} migration{row.migrations.length === 1 ? '' : 's'} run + {:else if row.status === 'running'} + running migrations… + {:else if row.status === 'failed'} + {row.error} + {:else if row.status === 'unknown'} + set up, but its tables could not be read — the database may be unreachable + {:else} + not configured yet + {/if} + + {/snippet} + {#snippet extra()} + + {#if sql && row.status !== 'done'} +
      + + {row.status === 'unknown' + ? 'Show the SQL this project ships' + : 'Show the SQL this will run'} + + 
      {sql}
      +
      + {/if} + {/snippet} + {#snippet action()} + + {#if row.status === 'unknown'} + + + {:else if hasTable && row.status !== 'done' && row.status !== 'running'} + + + {:else} + + + {/if} + {/snippet} +       + {/each} +
    + + {#if blanks.length > 0} +
    + Credentials to fill ({blanks.length}) +
      + {#each blanks as b (b.path)} + {@const blocked = !!b.occupiedBy || !!b.unreadable} + {@const canConnect = !b.done && !blocked && canConnectType(b.resourceType)} + + + {#snippet icon()} + {#if b.done} + + {:else if blocked} + + {:else} + + {/if} + {/snippet} + {#snippet title()} +
      + + {resourceTypeDisplayName(b.resourceType)} + + + {b.path} + +
      + {/snippet} + {#snippet detail()} + {#if b.occupiedBy} + + a {resourceTypeDisplayName(b.occupiedBy)} resource already holds this path — the + project did not get this one + + {:else if b.unreadable} + + could not be read, so whether it needs filling is unknown + + {:else if !b.done && b.missing.length > 0} + + Missing {b.missing.join(', ')} + + {/if} + {/snippet} + {#snippet action()} + + {#if blocked} + + + {b.occupiedBy ? 'Resolve in the workspace' : 'Check the workspace'} + + {:else} + + {/if} + {/snippet} +       + {/each} +
    +
    + {/if} + + + {#if outstanding === 0} + + Everything this project needs is configured. Finish, and it is ready to run. + + {:else if pendingTables.length > 0} + 0 + ? 'The project will not run without this' + : 'This could not be checked'} + size="xs" + > + {#if missingTables.length > 0} + The tables {missingTables.length === 1 ? 'this data table holds' : 'these data tables hold'} + do not exist, and the project's apps and flows read them. Every one of those fails as soon + as it opens. + {/if} + {#if uncheckedTables.length > 0} + {#if missingTables.length > 0}

    {/if} + {uncheckedTables.length === 1 ? 'One data table is' : 'Some data tables are'} set up, but + {uncheckedTables.length === 1 ? 'its' : 'their'} schema could not be read, so whether the + project's tables are there is unknown. Check again once the database is reachable. + {/if} +     + {:else} + + The project's apps and flows will fail wherever they read a credential that is still + missing. Everything else it imported works either way, and you can fill these in from the + workspace at any time. + + {/if} + {/if} + +
    + {#if onBack} + + {:else} + + {/if} +
    + + {#if outstanding > 0 && !loading && !loadError} + + {/if} + +
    +
    +
    + +{#if wizardOpen || wizardFor} + runMigrationsFor(retryTarget ?? '')} + existingNames={configuredNames.map((c) => c.name)} + existingDataTables={configuredNames} + onDone={() => void afterWizard()} + {customInstanceDbs} + {confirmationModal} + {defaultInstanceDbName} + /> +{/if} + + + + + + + void refreshBlanks()} + onRestored={() => void refreshBlanks()} +/> + + + void refreshBlanks()} /> diff --git a/frontend/src/lib/components/ImportWizardSteps.svelte b/frontend/src/lib/components/ImportWizardSteps.svelte new file mode 100644 index 0000000000..5712ede180 --- /dev/null +++ b/frontend/src/lib/components/ImportWizardSteps.svelte @@ -0,0 +1,71 @@ + + + + +
    + + onStepClick(e.detail.index)} + /> +
    diff --git a/frontend/src/lib/components/InputTransformForm.svelte b/frontend/src/lib/components/InputTransformForm.svelte index 630d052ce4..d766c06603 100644 --- a/frontend/src/lib/components/InputTransformForm.svelte +++ b/frontend/src/lib/components/InputTransformForm.svelte @@ -30,6 +30,7 @@ import type { InputTransform } from '$lib/gen' import TemplateEditor from './TemplateEditor.svelte' import { setInputCat as computeInputCat, isCodeInjection } from '$lib/utils' + import { escapeTemplateBackticks } from '$lib/utils/templateLiteral' import { FunctionSquare, InfoIcon } from 'lucide-svelte' import { getResourceTypes } from './resourceTypesStore' import type { FlowCopilotContext } from './copilot/flow' @@ -253,7 +254,7 @@ arg.expr = getDefaultExpr( argName, previousModuleId, - `\`${rawValue.toString().replaceAll('`', '\\`')}\`` + `\`${escapeTemplateBackticks(rawValue.toString())}\`` ) arg.type = 'javascript' propertyType = 'static' @@ -687,7 +688,7 @@ argName, previousModuleId, staticTemplate - ? `\`${arg?.value?.toString().replaceAll('`', '\\`') ?? ''}\`` + ? `\`${escapeTemplateBackticks(arg?.value?.toString() ?? '')}\`` : arg.value ? '(' + JSON.stringify(arg?.value, null, 4) + ')' : '' diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index 62bbe1ef93..4a0bd58715 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -1069,8 +1069,9 @@
  • git sync repo count (sync vs promotion mode)
  • feature usage (counts of which product features are used, including AI provider and - model identifiers, the names of public hub scripts used, and the plan tier and quota - shown when the execution meter is opened, last 30 days)
  • feature adoption (counts of which flow, script, trigger and worker features your @@ -1121,8 +1122,9 @@
  • development instance status
  • feature usage (counts of which product features are used, including AI provider and - model identifiers, the names of public hub scripts used, and the plan tier and quota - shown when the execution meter is opened, last 30 days)
  • feature adoption (counts of which flow, script, trigger and worker features your @@ -1141,6 +1143,32 @@ description="Configure default timeouts and retention policies for job execution." link="https://www.windmill.dev/docs/advanced/instance_settings#jobs" /> + {:else if category == 'Service logs'} + + {#if !$values['object_store_cache_config']} +
    + + Instance object storage is not configured, so every server and worker keeps its log + files on its own disk. This page lists what each host wrote, but can only open the + files belonging to the replica serving the request — another host's are listed and + not readable — and a host's files go with it when it is replaced. Retention below + still governs the entries in the database and the files on disk. + +
    + {:else if !$enterpriseLicense} +
    + + Log files are uploaded to instance object storage, and the indexer that would ingest + them into the columnar store and delete each one afterwards is an enterprise + feature. Retention below expires the database entries and the local files; the + uploaded copies are only removed when Delete logs from s3 periodically is on + under Object Storage. + +
    + {/if} {:else if category == 'Object Storage'} import SimpleEditor from '$lib/components/SimpleEditor.svelte' - import { createEventDispatcher } from 'svelte' + import { createEventDispatcher, untrack } from 'svelte' const dispatch = createEventDispatcher() @@ -8,18 +8,48 @@ updateOnBlur?: boolean placeholder?: string selected?: boolean + /** Content the editor opens with, and keeps following while the buffer is untouched — so a + * payload nobody has typed into tracks the schema instead of going stale. The first edit + * hands the buffer to the user and later changes stop overwriting it. */ + initialCode?: string } let { updateOnBlur = true, placeholder = 'Write a JSON payload. The input schema will be inferred.

    Example:

    {
      "foo": "12"
    }', - selected = false + selected = false, + initialCode = '' }: Props = $props() - let pendingJson = $state('') + let pendingJson = $state(untrack(() => initialCode)) + // The last content this component wrote, kept only to skip a reseed that would replace the + // buffer with what it already holds — `setValue` resets the cursor and the undo stack. + let seededCode = untrack(() => initialCode) + // Latched from Monaco's own change event, never from `pendingJson`: that trails the buffer by + // SimpleEditor's debounce, a window in which typed text still looks like the seeded payload + // and a reseed lands on top of it. + let userEdited = false let simpleEditor: SimpleEditor | undefined = $state(undefined) let focusTrap: HTMLElement | undefined = $state() + $effect(() => { + const next = initialCode + untrack(() => { + if (next !== seededCode && !userEdited) { + seed(next) + } + }) + }) + + // `SimpleEditor.setCode` cancels the change burst its own `setValue` opens, so reseeding + // never dispatches `select` — the payload reaches `args` only when the user edits it. + function seed(code: string) { + seededCode = code + userEdited = false + pendingJson = code + simpleEditor?.setCode(code) + } + function updatePayloadFromJson(jsonInput: string) { if (jsonInput === undefined || jsonInput === null || jsonInput.trim() === '') { dispatch('select', undefined) @@ -33,8 +63,10 @@ } } + /** Authoritative overwrite: replaces the buffer whether or not it has been typed into, and + * re-establishes it as the content to keep following. */ export function setCode(code: string) { - simpleEditor?.setCode(code) + seed(code) } export function resetSelected(dispatchEvent?: boolean) { @@ -59,6 +91,7 @@
    (userEdited = true)} on:focus={() => { if (updateOnBlur) { dispatch('focus') diff --git a/frontend/src/lib/components/Login.svelte b/frontend/src/lib/components/Login.svelte index d1dc3ec7a1..10f996a33f 100644 --- a/frontend/src/lib/components/Login.svelte +++ b/frontend/src/lib/components/Login.svelte @@ -329,7 +329,11 @@ } else { goto(resolvedRd ?? '/') } - } else if (resolvedRd?.startsWith('/user/workspaces')) { + // See (root)/+layout.svelte for why /projects/import skips the picker. + } else if ( + resolvedRd?.startsWith('/user/workspaces') || + resolvedRd?.startsWith(`${base}/projects/import`) + ) { goto(resolvedRd) } else if (resolvedRd == '/#user-settings') { goto(`/user/workspaces#user-settings`) diff --git a/frontend/src/lib/components/ModuleTest.svelte b/frontend/src/lib/components/ModuleTest.svelte index 2adba428ee..50e915a4cb 100644 --- a/frontend/src/lib/components/ModuleTest.svelte +++ b/frontend/src/lib/components/ModuleTest.svelte @@ -50,14 +50,19 @@ let jobProgressReset: () => void = () => {} let stepHistoryLoader = getStepHistoryLoaderContext() + // Every explicit run re-evaluates the args with errors surfaced. The reactive evaluations + // that follow each flow edit stay quiet, so without this a failing expression is silently + // `undefined` in what the run is built from. Manually edited args are preserved across the + // refresh by `initializeFromSchema`. export function runTestWithStepArgs() { - const args = stepsInputArgs.getStepArgs(mod.id) - runTest(args) - } - - export function loadArgsAndRunTest() { - stepsInputArgs?.updateStepArgs(mod.id, flowStateStore.val, flowStore?.val, previewArgs?.val) - runTestWithStepArgs() + stepsInputArgs?.updateStepArgs( + mod.id, + flowStateStore.val, + flowStore?.val, + previewArgs?.val, + true + ) + runTest(stepsInputArgs.getStepArgs(mod.id)) } // A step's timeout is an InputTransform. Only a static numeric value can be applied diff --git a/frontend/src/lib/components/ProjectContentBadges.svelte b/frontend/src/lib/components/ProjectContentBadges.svelte new file mode 100644 index 0000000000..dfb8e0eb90 --- /dev/null +++ b/frontend/src/lib/components/ProjectContentBadges.svelte @@ -0,0 +1,73 @@ + + + + +
    + {#each shown as c (c.label)} + + {c.count} + {c.label}{c.count === 1 ? '' : 's'} + + {/each} +
    diff --git a/frontend/src/lib/components/ResourceEditorDrawer.svelte b/frontend/src/lib/components/ResourceEditorDrawer.svelte index eec4c7579b..296ab693a8 100644 --- a/frontend/src/lib/components/ResourceEditorDrawer.svelte +++ b/frontend/src/lib/components/ResourceEditorDrawer.svelte @@ -22,8 +22,16 @@ let { workspace = undefined, disableChatOffset = false, - onRestored = undefined - }: { workspace?: string; disableChatOffset?: boolean; onRestored?: () => void } = $props() + onRestored = undefined, + onSaved = undefined + }: { + workspace?: string + disableChatOffset?: boolean + onRestored?: () => void + /** Fires after Save has written, for a caller showing state derived from the + * resource — `onRestored` only covers restoring an old version. */ + onSaved?: () => void + } = $props() let drawer: Drawer | undefined = $state() let historyDrawer: Drawer | undefined = $state() @@ -149,9 +157,14 @@ variant="accent" unifiedSize="md" startIcon={{ icon: Save }} - on:click={() => { - resourceEditor?.save() + on:click={async () => { + // Closed before the write is awaited, the way it always was: `save()` toasts its + // own failures and never rejects, so waiting would only add visible lag to every + // caller of this drawer. `onSaved` still fires after the write lands. + const saved = resourceEditor?.save() drawer?.closeDrawer() + await saved + onSaved?.() }} disabled={!canSave} > diff --git a/frontend/src/lib/components/RunForm.svelte b/frontend/src/lib/components/RunForm.svelte index 89eaf8f7ab..a371f3eea9 100644 --- a/frontend/src/lib/components/RunForm.svelte +++ b/frontend/src/lib/components/RunForm.svelte @@ -19,6 +19,7 @@ import { page } from '$app/state' import { replaceState } from '$app/navigation' import JsonInputs from '$lib/components/JsonInputs.svelte' + import { argsToJsonPayload } from '$lib/schema' import { triggerableByAI } from '$lib/actions/triggerableByAI.svelte' import InputSelectedBadge from './schema/InputSelectedBadge.svelte' import { untrack } from 'svelte' @@ -53,6 +54,8 @@ args = scriptArgs psCommonParams = commonParams reloadArgs++ + // `reloadArgs` only keys the form; the JSON editor reads its payload once, at mount. + syncJsonEditor() } export async function run(overrideScheduledForStr?: string | undefined | null) { @@ -199,8 +202,10 @@ return result } - export function setCode(code: string) { - jsonEditor?.setCode(code) + /** Rewrite the open JSON editor from the current args. Only for args replaced from outside + * the editor: entering the JSON view already starts from whatever `args` holds. */ + export function syncJsonEditor() { + jsonEditor?.setCode(argsToJsonPayload(runnable?.schema, args)) } $effect(() => { overrideTag @@ -320,6 +325,7 @@ args = enforceDisabledDefaults(e.detail) } }} + initialCode={argsToJsonPayload(runnable.schema, args)} updateOnBlur={false} placeholder={`Write args as JSON.

    Example:

    {
      "foo": "12"
    }`} /> diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index 7bc4774e9f..fdab141d03 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -122,6 +122,7 @@ import { updateDelegateToGitRepoConfig, insertAdditionalInventories } from '$lib/ansibleUtils' import { copilotInfo } from '$lib/aiStore' import JsonInputs from '$lib/components/JsonInputs.svelte' + import { argsToJsonPayload } from '$lib/schema' import Toggle from './Toggle.svelte' import { deepEqual } from 'fast-equals' import { usePreparedAssetSqlQueries } from '$lib/infer.svelte' @@ -310,6 +311,10 @@ let moduleTestState: Record; schema: Schema }> = $state({}) let testPanelArgs: Record = $state({}) let testPanelSchema: Schema = $state(emptySchema()) + // Bumped whenever the args under test are replaced from outside the arg panel. Both arg + // views key off it: without a bump the JSON editor keeps showing, and on the next + // keystroke commits, the payload it was seeded with for the previous args. + let argsRender = $state(0) // editorCode is what the editor shows; code always holds the main script content let editorCode: string = $state(code) // Sync editorCode when code changes externally (template reset, copilot, @@ -329,7 +334,13 @@ }) function switchToModule(modulePath: string) { - if (activeModuleTab !== null && modules && activeModuleTab !== modulePath) { + // Re-clicking the tab you are already on is a no-op. Re-running the body would reset this + // module's test state whenever its inference is still pending or has failed (the catch + // leaves `moduleTestState` unwritten), losing both the filled-in args and the arg views. + if (activeModuleTab === modulePath) { + return + } + if (activeModuleTab !== null && modules) { // Switching from another module: save its content and test state modules[activeModuleTab] = { ...modules[activeModuleTab], content: editorCode } moduleTestState[activeModuleTab] = { args: testPanelArgs, schema: testPanelSchema } @@ -345,13 +356,20 @@ } else { testPanelArgs = {} testPanelSchema = emptySchema() + // Inference lands after the bump below, so the editor opens on `{}` and the arg + // views follow the schema in once it arrives. Remounting them again on arrival + // instead would discard anything typed while it was in flight. inferModuleSchema() } + argsRender++ } } function switchToMain() { - if (activeModuleTab !== null && modules) { + if (activeModuleTab === null) { + return + } + if (modules) { // Save current module content and test state modules[activeModuleTab] = { ...modules[activeModuleTab], content: editorCode } moduleTestState[activeModuleTab] = { args: testPanelArgs, schema: testPanelSchema } @@ -360,6 +378,7 @@ editorCode = code lastSyncedCode = code editor?.setCode(editorCode) + argsRender++ } // Whether the open file is tested as a runnable of its own. A `__mod` helper @@ -854,6 +873,7 @@ export function setArgs(nargs: Record) { args = nargs + argsRender++ } export async function runTest(opts?: { cascade?: boolean; skipDdlGuard?: boolean }) { @@ -1654,15 +1674,30 @@ $effect(() => { !hasPreprocessor && (selectedTab = 'main') }) + // `main` and `preprocessor` describe the same args under different schemas; every other tab + // (`diagram`) runs against main's schema, so it collapses into `main` here. + let lastSchemaTab = untrack(() => (selectedTab === 'preprocessor' ? 'preprocessor' : 'main')) $effect(() => { // Only depend on selectedTab (preprocessor ↔ main toggle). // Code changes are handled by the editor on:change handler and // explicit inferSchema calls (initContent, onMount), so we read // `code` inside untrack to avoid a redundant double-inference race. - selectedTab && untrack(() => code && inferSchema(code)) + selectedTab && + untrack(() => { + const schemaTab = selectedTab === 'preprocessor' ? 'preprocessor' : 'main' + const switched = schemaTab !== lastSchemaTab + lastSchemaTab = schemaTab + if (!code) return + // Bump on the switch itself, not on the inference it starts: the other tab's schema + // only lands once that resolves, and remounting the arg views then would discard + // anything typed while it was in flight. An untouched editor follows the schema in. + if (switched) { + argsRender++ + } + inferSchema(code) + }) }) - let argsRender = $state(0) export async function updateArgs(newArgs: Record) { if (Object.keys(newArgs).length > 0) { args = { ...newArgs } @@ -2299,19 +2334,24 @@ style="height: {!schemaHeight || schemaHeight < 600 ? 600 : schemaHeight}px" data-schema-picker > - { - if (e.detail) { - if (onModuleArgs) { - testPanelArgs = e.detail - } else { - args = e.detail + {#key argsRender} + { + if (e.detail) { + if (onModuleArgs) { + testPanelArgs = e.detail + } else { + args = e.detail + } } - } - }} - updateOnBlur={false} - placeholder={`Write args as JSON.

    Example:

    {
      "foo": "12"
    }`} - /> + }} + initialCode={onModuleArgs + ? argsToJsonPayload(testPanelSchema, testPanelArgs) + : argsToJsonPayload(schema, args)} + updateOnBlur={false} + placeholder={`Write args as JSON.

    Example:

    {
      "foo": "12"
    }`} + /> + {/key}
    {:else}
    diff --git a/frontend/src/lib/components/ServiceLogsInner.svelte b/frontend/src/lib/components/ServiceLogsInner.svelte index bad18661b6..16bdd7d2ad 100644 --- a/frontend/src/lib/components/ServiceLogsInner.svelte +++ b/frontend/src/lib/components/ServiceLogsInner.svelte @@ -2,7 +2,7 @@ import { createBubbler, preventDefault } from 'svelte/legacy' const bubble = createBubbler() - import { IndexSearchService, ServiceLogsService } from '$lib/gen' + import { IndexSearchService, ServiceLogsService, type LogSearchHit } from '$lib/gen' import TimeframeSelect, { serviceLogsTimeframes, @@ -253,39 +253,50 @@ try { let res = '' log.split('\n').forEach((line) => { + // A file can hold both formats: the ones written before the layer + // switched to JSON, and panics or subprocess output that was never + // JSON to begin with. Those lines pass through as they are rather + // than being dropped, which would render the file blank. + let obj: any = undefined if (line.startsWith('{') && line.endsWith('}')) { - let obj = JSON.parse(line) - if (typeof obj == 'object') { - let nl = '' - if (obj['timestamp']) { - nl += obj['timestamp'] + ' ' - } - if (obj['level']) { - let lvl = obj['level'] - if (lvl == 'ERROR') { - nl += '\x1b[31mERROR\x1b[0m ' - } else if (lvl == 'INFO') { - nl += '\x1b[32mINFO\x1b[0m ' - } else { - nl += obj['level'] + ' ' - } - } - if (obj['message']) { - nl += obj['message'] + ' ' - } - delete obj['timestamp'] - delete obj['level'] - delete obj['message'] - Object.keys(obj).forEach((key) => { - nl += - key + - '=' + - (typeof obj[key] == 'object' ? JSON.stringify(obj[key]) : obj[key]) + - ' ' - }) - res += nl + '\n' + try { + obj = JSON.parse(line) + } catch { + obj = undefined } } + if (obj === null || typeof obj !== 'object') { + res += line + '\n' + } else { + let nl = '' + if (obj['timestamp']) { + nl += obj['timestamp'] + ' ' + } + if (obj['level']) { + let lvl = obj['level'] + if (lvl == 'ERROR') { + nl += '\x1b[31mERROR\x1b[0m ' + } else if (lvl == 'INFO') { + nl += '\x1b[32mINFO\x1b[0m ' + } else { + nl += obj['level'] + ' ' + } + } + if (obj['message']) { + nl += obj['message'] + ' ' + } + delete obj['timestamp'] + delete obj['level'] + delete obj['message'] + Object.keys(obj).forEach((key) => { + nl += + key + + '=' + + (typeof obj[key] == 'object' ? JSON.stringify(obj[key]) : obj[key]) + + ' ' + }) + res += nl + '\n' + } }) return res @@ -294,6 +305,23 @@ } } + // A hit is one log line with its fields already separated, so rendering it is + // formatting rather than parsing — there is no JSON to prettify and no + // snippet to highlight. + function renderHit(hit: LogSearchHit): string { + const level = + hit.level === 'ERROR' + ? '\x1b[31mERROR\x1b[0m' + : hit.level === 'WARN' + ? '\x1b[33mWARN\x1b[0m' + : hit.level === 'INFO' + ? '\x1b[32mINFO\x1b[0m' + : hit.level + return [hit.ts, level, hit.message, hit.target ? `target=${hit.target}` : ''] + .filter(Boolean) + .join(' ') + } + let logs: any = $state() let debounceTimeout: number | undefined = undefined @@ -399,7 +427,9 @@ ) { const res = await ServiceLogsService.getLogFile({ path: `${hostname}/${path}` }) - content = processLogWithJsonFmt(ansi_up.ansi_to_html(res), jsonFmt) + // Prettify first: it emits its own ANSI for the level, which converting + // beforehand would leave in the output as literal escapes. + content = ansi_up.ansi_to_html(processLogWithJsonFmt(res, jsonFmt)) hitLineNumber = lineNumber logDrawerOpen = true @@ -687,23 +717,19 @@
    {:else if logs != undefined}
    - {#each logs.hits as { snippet_fragment, snippet_highlighted, document }} + + {#each logs.hits ?? [] as hit, i (`${i}:${hit.file_path}:${hit.line_no}`)} { - let logLineNumber = document.line_number[0] - let logFile = document.file_name[0] - let host = document.host[0] - let jsonFmt = document.json_fmt[0] - seeLogContext(logLineNumber, logFile, host, jsonFmt) - }} + content={renderHit(hit)} + highlighted={[]} + onClick={() => seeLogContext(hit.line_no, hit.file_path, hit.host, true)} /> {/each} - {#if logs.hits.length === 0} + {#if (logs.hits ?? []).length === 0}
    No logs
    {/if} - {#if logs.hits.length === 1000} + {#if (logs.hits ?? []).length === 1000}
    Older matches were truncated from this search, try refining your filters to get more precise results. diff --git a/frontend/src/lib/components/SimpleEditor.svelte b/frontend/src/lib/components/SimpleEditor.svelte index 2764b130e2..65713e19f7 100644 --- a/frontend/src/lib/components/SimpleEditor.svelte +++ b/frontend/src/lib/components/SimpleEditor.svelte @@ -64,6 +64,9 @@ const CHANGE_TIMEOUT = 200 let changeTimeoutId: number | undefined = undefined + // Monaco fires onDidChangeModelContent synchronously from within `setValue`, so without + // this an authoritative overwrite reads as a user edit on the `input` event. + let applyingCode = false let divEl: HTMLDivElement | null = null let editor = $state(null) @@ -74,6 +77,10 @@ let width = $state(0) let initialized = $state(false) let placeholderVisible = $state(false) + // Monaco's content origin. The placeholder is a plain overlay on the editor container, so + // without these it sits over the line-number gutter and off the line-1 baseline. + let contentLeft = $state(0) + let contentLineHeight = $state(0) let mounted = $state(false) let valueAfterDispose: string | undefined = undefined @@ -179,7 +186,12 @@ if (ncode != code) { code = ncode } - editor?.setValue(ncode) + applyingCode = true + try { + editor?.setValue(ncode) + } finally { + applyingCode = false + } // setValue emits a change event of its own; drop the burst it opens so an edit // made right after an authoritative overwrite still counts as a leading change. cancelPendingChanges() @@ -454,6 +466,12 @@ changeTimeoutId = undefined updateCode() }, CHANGE_TIMEOUT) + // `change` trails the buffer by CHANGE_TIMEOUT, too late for a consumer that has to + // know the moment the buffer stopped being the one it wrote. `input` says only that, + // carrying no value: read `getCode()` for what is on screen. + if (!applyingCode) { + dispatch('input') + } if (leading) { updateCode() } @@ -533,6 +551,13 @@ } if (placeholder) { + const syncPlaceholderOrigin = () => { + if (!editor) return + contentLeft = editor.getLayoutInfo().contentLeft + contentLineHeight = editor.getOption(meditor.EditorOption.lineHeight) + } + syncPlaceholderOrigin() + editor.onDidLayoutChange(syncPlaceholderOrigin) editor.onDidChangeModelContent(() => { if (!editor) return const value = editor.getValue() @@ -755,9 +780,10 @@ {#if placeholder}
    {@html placeholder}
    diff --git a/frontend/src/lib/components/TaggedTextInput.svelte b/frontend/src/lib/components/TaggedTextInput.svelte index 23bbc65e6b..9f19195b85 100644 --- a/frontend/src/lib/components/TaggedTextInput.svelte +++ b/frontend/src/lib/components/TaggedTextInput.svelte @@ -8,6 +8,7 @@ onTextSegmentAtCursorChange, onKeyDown, autofocus, + id, class: className = '' }: { tags: { regex: RegExp; id: string; onClear?: () => void }[] @@ -18,6 +19,7 @@ onTextSegmentAtCursorChange?: (segment: { text: string; start: number; end: number }) => void onKeyDown?: (e: KeyboardEvent) => void autofocus?: boolean + id?: string class?: string } = $props() @@ -337,7 +339,13 @@ function handleKeyDown(e: KeyboardEvent) { onKeyDown?.(e) - if (e.key === 'ArrowDown' || e.key === 'ArrowUp' || e.key === 'Enter') return + // Single-line filter input: block Enter's default newline insertion. Surrounding handlers + // (suggestion select, list open) still run on bubble; only the contenteditable break is gone. + if (e.key === 'Enter') { + e.preventDefault() + return + } + if (e.key === 'ArrowDown' || e.key === 'ArrowUp') return const cursorPos = getCursorPosition() const text = getTextContent() @@ -509,6 +517,7 @@
    - import { createGrid, type GridApi } from 'ag-grid-community' - import 'ag-grid-community/styles/ag-grid.css' - import 'ag-grid-community/styles/ag-theme-alpine.css' - import '$lib/components/apps/components/display/table/theme/windmill-theme.css' - import { transformColumnDefs } from '$lib/components/apps/components/display/table/utils' - import { multilineCellColDef } from '$lib/components/apps/components/display/table/multilineCellEditor' - import DarkModeObserver from '$lib/components/DarkModeObserver.svelte' + import DataTable from '$lib/components/table/DataTable.svelte' + import Head from '$lib/components/table/Head.svelte' + import Cell from '$lib/components/table/Cell.svelte' + import Row from '$lib/components/table/Row.svelte' + import EditableTextarea from '$lib/components/common/EditableTextarea.svelte' + import { Button, EmptyState } from '$lib/components/common' + import { ListPlus, Plus, Trash2 } from 'lucide-svelte' import { untrack } from 'svelte' import type { CaseDraft } from './evalUtils' let { cases = $bindable(), onRemove, - locked = false, - onEditingChange + onAdd, + focusCaseId = undefined, + locked = false }: { /** The drawer's working copy. Edits land here as they are made; the drawer writes them. */ cases: CaseDraft[] /** Asked before a row goes, since a stored case has runs that executed it. */ onRemove: (c: CaseDraft) => void + /** Adds a case. The empty table offers this itself, where the first row would be. */ + onAdd: () => void + /** A case to open for typing as soon as it appears — the one just added, so a new row is + * ready to be filled in rather than waiting to be clicked. */ + focusCaseId?: string /** The cases are being written: an edit made now is one the request already left behind. */ locked?: boolean - /** A cell opened or closed. Reported up so Save can be pressed for an edit the list does not - * hold yet: the press is what commits the cell, so it has to reach a live button. */ - onEditingChange?: (editing: boolean) => void } = $props() - type Row = { id: string; question: string; expected: string } + /** A dataset is capped at 1000 cases and every row here mounts two editors and a button, so the + * whole set at once is thousands of components for a table you read a screenful of. ag-grid + * virtualised its rows; `DataTable` paginates instead, which is the primitive already here. */ + const CASES_PER_PAGE = 25 + let currentPage = $state(1) + /** Bound to `DataTable`'s own per-page selector, so changing it there re-slices here rather + * than only relabelling the footer. */ + let perPage = $state(CASES_PER_PAGE) + let lastPage = $derived(Math.max(1, Math.ceil(cases.length / perPage))) + let pageCases = $derived(cases.slice((currentPage - 1) * perPage, currentPage * perPage)) + // A case removed from the last page leaves it empty; step back rather than show nothing. + $effect(() => { + if (currentPage > lastPage) currentPage = lastPage + }) + /** `expected` is whatever the case holds: a bare string, or a value shown as JSON. Read back + * the same way, so a case that held an object keeps holding one. */ function expectedToText(value: unknown): string { if (value == undefined) return '' return typeof value === 'string' ? value : JSON.stringify(value, null, 2) } - function toRow(c: CaseDraft): Row { - return { - id: c.id ?? '', - question: c.input?.user_message ?? '', - expected: expectedToText(c.expected) - } - } - - let api: GridApi | undefined = $state() - let eGui: HTMLDivElement | undefined = $state() - let darkMode = $state(false) - - const defaultColDef = { - flex: 1, - minWidth: 120, - ...multilineCellColDef - } - - $effect(() => eGui && untrack(() => mountGrid())) - function mountGrid() { - if (!eGui || api) return - createGrid(eGui, { - rowData: untrack(() => cases.map(toRow)), - columnDefs: columnDefs(), - defaultColDef: { ...defaultColDef, editable: untrack(() => !locked) }, - onCellValueChanged: (e) => { - const target = cases.find((c) => c.id === (e.data as Row).id) - if (!target) return - if (e.colDef.field === 'question') { - target.input = { ...target.input, user_message: e.newValue ?? '' } - } else if (e.colDef.field === 'expected') { - setExpected(target, e.newValue ?? '') - } - }, - onCellEditingStarted: () => onEditingChange?.(true), - onCellEditingStopped: () => onEditingChange?.(false), - suppressColumnMoveAnimation: true, - suppressDragLeaveHidesColumns: true, - onGridReady: (e) => (api = e.api) + /** The question editors, by case id, so a newly added row can be opened. */ + let questionEditors = $state>({}) + $effect(() => { + const id = focusCaseId + if (!id) return + untrack(() => { + // Onto the page the case is on before reaching for its editor: paginated, a case added to + // a full page is not rendered yet, and there would be nothing to open. + const at = cases.findIndex((c) => c.id === id) + if (at < 0) return + currentPage = Math.floor(at / perPage) + 1 + // After the row has been rendered and registered itself. + requestAnimationFrame(() => questionEditors[id]?.edit()) }) - } + }) - /** Text that parses as JSON is stored as JSON, so a structured answer can be written by hand. */ function setExpected(c: CaseDraft, text: string) { + // Cleared means the case has no expected answer, which is not the same as expecting the + // empty string: a scorer reads `undefined` as "nothing to measure here" and leaves the case + // out of its mean, where `''` scores it a hard zero. const trimmed = text.trim() if (!trimmed) { c.expected = undefined @@ -88,45 +82,93 @@ c.expected = text } } - - function columnDefs() { - return transformColumnDefs({ - columnDefs: [ - { field: 'question', headerName: 'Question', flex: 3 }, - { field: 'expected', headerName: 'Expected', flex: 2 } - ] as any, - onDelete: (values) => { - if (locked) return - const target = cases.find((c) => c.id === (values as Row).id) - if (target) onRemove(target) - } - }) - } - - // Keyed on which cases are in the list rather than on what is in them: pushing rows back on - // each keystroke would reset the cell being typed in. - let rowKey = $derived(cases.map((c) => c.id).join(',')) - $effect(() => { - rowKey - untrack(() => api?.updateGridOptions({ rowData: cases.map(toRow) })) - }) - - $effect(() => { - const editable = !locked - untrack(() => api?.updateGridOptions({ defaultColDef: { ...defaultColDef, editable } })) - }) - - /** Commit whatever cell is open into `cases`, and wait for it to land there: the drawer reads - * the list to save it, and a cell still being typed in is an edit that press is saving. */ - export async function flush() { - api?.stopEditing() - // The commit reaches `cases` through the grid's own event queue, a macrotask away: reading in - // the same turn, or after a microtask, reads the list as it was before the cell was typed in. - await new Promise((resolve) => setTimeout(resolve, 0)) - } - - -
    +{#if cases.length === 0} + +{:else} + perPage} + bind:currentPage + bind:perPage + rowCount={cases.length} + hasMore={currentPage < lastPage} + showPrev={currentPage > 1} + on:next={() => (currentPage = Math.min(currentPage + 1, lastPage))} + on:previous={() => (currentPage = Math.max(currentPage - 1, 1))} + > + + + + + + + + Question + Expected + + + + + {#each pageCases as c (c.id)} + + + + (c.input = { ...(c.input ?? {}), user_message: v })} + /> + + + setExpected(c, v)} + /> + + +
    -
    + +
    Cases {workingCases.length} @@ -369,13 +372,13 @@ Add a case
    -
    +
    (casesEditing = v)} />
    @@ -397,7 +400,7 @@ unifiedSize="md" variant="accent" loading={saving} - disabled={writing || !path || !!pathError || (nothingToSave && !casesEditing)} + disabled={writing || !path || !!pathError || nothingToSave} onclick={saveDataset} > Save @@ -408,7 +411,8 @@ variant="accent" startIcon={{ icon: Plus }} loading={creating} - disabled={creating || !path || !!pathError} + disabled={creating || !path || !!pathError || noCases} + title={noCasesTitle} onclick={createDataset} > Create dataset diff --git a/frontend/src/lib/components/aiEvals/EvalRunDialog.svelte b/frontend/src/lib/components/aiEvals/EvalRunDialog.svelte index d4d1714fd0..63fe366628 100644 --- a/frontend/src/lib/components/aiEvals/EvalRunDialog.svelte +++ b/frontend/src/lib/components/aiEvals/EvalRunDialog.svelte @@ -47,16 +47,6 @@ let hasDraft = $derived(editedConfig !== undefined) let dataset = $state(undefined) let hoveringDataset = $state(false) - /** Set while the dialog stands aside for the dataset drawer, holding the pane's dataset as it - * was on the way out: coming back onto a different one is the detour's answer, and adopted. */ - let steppedAside = $state<{ dataset: string | undefined } | undefined>(undefined) - - /** Hands the screen to the dataset drawer, to be given back when it closes. */ - function stepAside(go: () => void) { - steppedAside = { dataset: defaultDataset } - open = false - go() - } /** The agent's versions, for pinning one. Loaded when the dialog opens rather than held: a * version list goes stale the moment the agent is saved again. */ @@ -76,21 +66,26 @@ } } + /** Whether the dialog was already open on the previous run of the effect below, which reads it + * to tell an open from the pane's dataset moving underneath. */ + let wasOpen = false + $effect(() => { - if (!open) return + const isOpen = open + const pane = defaultDataset untrack(() => { - loadVersions() - const aside = steppedAside - steppedAside = undefined - if (aside) { - // Back from the drawer: a dataset created or edited there moves the pane onto it, and - // anything else leaves the field as it was left. - if (defaultDataset !== aside.dataset) dataset = defaultDataset + if (!isOpen) { + wasOpen = false return } - // Seeded on every open: the dataset last worked in, and the state of the agent there is - // most reason to measure. - dataset = defaultDataset + // Followed while the dialog stands, not only read at open: the dataset drawer opens over + // this dialog rather than in place of it, so creating, renaming or deleting a dataset + // there moves the pane's selection with the dialog still up. Nothing else moves it then. + dataset = pane + if (wasOpen) return + wasOpen = true + loadVersions() + // The state of the agent there is most reason to measure. choice = hasDraft ? 'draft' : 'deployed' }) }) @@ -184,7 +179,7 @@ unifiedSize="sm" variant="default" startIcon={{ icon: Plus }} - onclick={() => stepAside(onNewDataset)} + onclick={onNewDataset} > New dataset @@ -215,7 +210,7 @@ title="Edit this dataset" on:click={() => { close() - stepAside(() => onEditDataset(item.value ?? '')) + onEditDataset(item.value ?? '') }} /> {/snippet} @@ -228,7 +223,7 @@ btnClasses="w-full !h-auto !justify-start !rounded-none flex items-center gap-2 px-3 py-2 text-xs !font-normal text-secondary hover:bg-surface-hover" onClick={() => { close() - stepAside(onNewDataset) + onNewDataset() }} > @@ -247,7 +242,7 @@ startIcon={{ icon: Pencil }} iconOnly title="Edit this dataset" - on:click={() => stepAside(() => onEditDataset(dataset ?? ''))} + on:click={() => onEditDataset(dataset ?? '')} />
    {/if} diff --git a/frontend/src/lib/components/aiEvals/EvalRunsList.svelte b/frontend/src/lib/components/aiEvals/EvalRunsList.svelte index 5d84ef88ce..b544781902 100644 --- a/frontend/src/lib/components/aiEvals/EvalRunsList.svelte +++ b/frontend/src/lib/components/aiEvals/EvalRunsList.svelte @@ -7,17 +7,21 @@ import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte' import TimeAgo from '$lib/components/TimeAgo.svelte' import { Button } from '$lib/components/common' - import { Bot, Code2, Loader2, Plus } from 'lucide-svelte' + import { Bot, ChevronRight, Code2, Loader2, Plus } from 'lucide-svelte' + import { overlayHostActive, topmostSurface } from '$lib/components/common/overlayHost.svelte' import type { EvalDataset, EvalExperiment, ExperimentScore } from '$lib/gen' import { datasetSummary, experimentName, formatScore, subjectLabel } from './evalUtils' let { experiments, datasets, + caseProgress, loaded, + active = false, deployedHash = undefined, currentVersion = undefined, onOpen, + onHighlight, onEditDataset, onNew }: { @@ -25,17 +29,87 @@ experiments: EvalExperiment[] /** Whether the list has been read: an empty table is a statement about the agent. */ loaded: boolean + /** Whether this list is the page on screen. The keyboard is only answered while it is: the + * run page keeps its own rows, and both would otherwise move on one press. */ + active?: boolean /** The workspace's datasets, for naming the one a run is of by what it is for. */ datasets: EvalDataset[] + /** How many cases each still-running run has finished, keyed by run id. A run the flow has + * not been read for yet is at none of them rather than absent: the count is on the row from + * the moment it appears, so it never arrives late and shifts the column. */ + caseProgress: Record /** What the agent hashes to as deployed, and the version it is on: they resolve a run of * edits that were later saved, so a run is labelled here as the run picker labels it. */ deployedHash?: string currentVersion?: number onOpen: (experiment: EvalExperiment) => void + /** The highlighted run, reported up so the surface can act on it — arrowing into the run + * page opens the run under the highlight rather than whichever was opened last. */ + onHighlight?: (id: string | undefined) => void onEditDataset: (dataset: string) => void onNew: () => void } = $props() + /** The highlighted run, by id. One state for both the pointer and the keyboard, as a melt menu + * does it: hovering a row moves the highlight to it, so the arrows carry on from wherever the + * pointer left off instead of running a second, invisible cursor of their own. It says where + * the highlight is, not what is chosen — a run is not opened until Enter. + * + * By id and not by index: the list is newest-first and the poll prepends to it, so an index + * would quietly come to mean a different run and Enter would open the wrong one. */ + let cursorId = $state(undefined) + let cursor = $derived( + cursorId === undefined ? -1 : experiments.findIndex((e) => e.id === cursorId) + ) + let body: HTMLTableSectionElement | undefined = $state() + + // A window listener answers keys aimed anywhere, so it has to ask two questions the DOM cannot: + // is my host the visible one — session preview tabs stay mounted when hidden — and is my surface + // still the one on top, rather than under a drawer or a dialog opened since. + const hostActive = overlayHostActive() + const onTop = topmostSurface() + const listening = () => hostActive() && onTop() + + $effect(() => { + onHighlight?.(cursorId) + }) + + // A highlight on a run that has since gone, and the highlight itself when the list is not the + // page on screen. + $effect(() => { + if (!active || (cursorId !== undefined && cursor < 0)) cursorId = undefined + }) + + function move(by: number) { + if (experiments.length === 0) return + const from = cursor < 0 ? (by > 0 ? -1 : experiments.length) : cursor + const at = Math.max(0, Math.min(experiments.length - 1, from + by)) + cursorId = experiments[at]?.id + // `nearest`, so arrowing through a long list scrolls by a row rather than jumping the table. + requestAnimationFrame(() => + body?.querySelectorAll('tr')[at]?.scrollIntoView({ block: 'nearest' }) + ) + } + + function onKeydown(event: KeyboardEvent) { + if (!active || !listening() || event.metaKey || event.ctrlKey || event.altKey) return + const el = event.target as HTMLElement | null + if (el?.closest?.('input, textarea, select, [contenteditable="true"], [role="listbox"]')) return + if (event.key === 'ArrowDown') { + event.preventDefault() + move(1) + } else if (event.key === 'ArrowUp') { + event.preventDefault() + move(-1) + } else if (event.key === 'Enter' && experiments[cursor]) { + // Enter belongs to whatever is focused if that thing does something with it. A highlighted + // row is not a reason to swallow the press on `New evaluation` or a row's dataset button. + if (el?.closest?.('button, a[href], [role="button"], summary')) return + event.preventDefault() + onOpen(experiments[cursor]) + } + } + /** The one number a column reports: a pass rate where it has a line to pass, the mean where it * does not. */ function headline(score: ExperimentScore): string | undefined { @@ -44,13 +118,16 @@ } + + - + + @@ -58,12 +135,20 @@ Dataset Cases Scores - When + When + - - {#each experiments as experiment (experiment.id)} - onOpen(experiment)}> + + {#each experiments as experiment, i (experiment.id)} + + onOpen(experiment)} + on:hover={(e) => e.detail && (cursorId = experiment.id)} + >
    @@ -99,7 +184,18 @@ - {experiment.case_count} + {#if experiment.running} + + + + {caseProgress[experiment.id] ?? 0}/{experiment.case_count} + + {:else} + {experiment.case_count} + {/if}
    @@ -117,8 +213,6 @@ {value} {:else if score.failed > 0} failed - {:else if experiment.running} - {:else} — {/if} @@ -126,33 +220,34 @@ {/each} {#if (experiment.scores ?? []).length === 0} - {#if experiment.running} - - - scoring - - {:else} - not scored - {/if} + + not scored {/if}
    - + + + + {/each} {#if experiments.length === 0 && !loaded} - + {:else if experiments.length === 0} - +
    No runs yet diff --git a/frontend/src/lib/components/aiEvals/EvalScorers.svelte b/frontend/src/lib/components/aiEvals/EvalScorers.svelte index 68287b0fbc..d00e52d77e 100644 --- a/frontend/src/lib/components/aiEvals/EvalScorers.svelte +++ b/frontend/src/lib/components/aiEvals/EvalScorers.svelte @@ -196,26 +196,31 @@
    -
    +
    {#if scorers.length === 0} -
    +
    A scorer reads one run and returns a number. Every run of this dataset is measured by all of them, which is what makes two runs comparable.
    {:else} -
    + +
    {#each scorers as scorer (scorer.id)} -
    +
    {#if scorer.kind === 'agent'} - + {:else} - + {/if}
    - + {scorerLabel(scorer)} - {scorer.path} + {scorer.path}
    {#if scorer.pass_if != undefined} diff --git a/frontend/src/lib/components/aiEvals/EvalsPane.svelte b/frontend/src/lib/components/aiEvals/EvalsPane.svelte index d9e5b16fe4..55792ba8e2 100644 --- a/frontend/src/lib/components/aiEvals/EvalsPane.svelte +++ b/frontend/src/lib/components/aiEvals/EvalsPane.svelte @@ -9,9 +9,11 @@ import Label from '$lib/components/Label.svelte' import Popover from '$lib/components/Popover.svelte' import { Splitpanes, Pane } from 'svelte-splitpanes' + import AnimatedPane from '$lib/components/splitPanes/AnimatedPane.svelte' import { type AgentDraft, AiEvalsService, + JobService, type EvalCase, type EvalDataset, type EvalExperiment, @@ -35,9 +37,11 @@ Code2, ExternalLink } from 'lucide-svelte' + import PagedContent from '$lib/components/common/modal/PagedContent.svelte' import EvalDatasetDrawer from './EvalDatasetDrawer.svelte' import EvalRunsList from './EvalRunsList.svelte' import EvalRunDialog from './EvalRunDialog.svelte' + import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte' import GfmMarkdown from '$lib/components/GfmMarkdown.svelte' import { caseLabel, @@ -52,6 +56,10 @@ /** A dataset is capped at this many cases, so one page holds the whole set. */ const CASE_PAGE_SIZE = 1000 + /** The id the run's flow gives the loop over its cases (`CASES_NODE_ID` in `ai_evals/run.rs`). + * Looked up by id rather than by position: the flow has a step after the loop too. */ + const CASES_MODULE_ID = 'cases' + let { agentPath, opWorkspace = undefined, @@ -91,12 +99,56 @@ /** What the agent hashes to as deployed: a run of edits carrying it ran what was then saved. */ let deployedHash = $state(undefined) let running = $state(false) - let scorers = $derived(dataset?.scorers ?? []) + /** The run on screen belongs to a dataset still being read. Until it arrives the rows and the + * scorer columns would both be built from the *previous* dataset, so both are held back. */ + let datasetLoading = $state(false) + let scorers = $derived(datasetLoading ? [] : (dataset?.scorers ?? [])) let selectedCaseId = $state(undefined) let datasetDrawer: EvalDatasetDrawer | undefined = $state() let runDialogOpen = $state(false) - let resumeRunDialog = $state(false) + /** The run the list has highlighted, so arrowing into the run page opens that one. Without it + * the arrow could only fall back to whichever run was opened last, which on a dialog just + * opened is none at all. */ + let highlightedRunId = $state(undefined) + + /** How many cases each still-running run has finished, keyed by run id. Read from the flow + * executing the run: the list carries the case total, and counting the finished ones there + * would be a per-case query for every run listed. The flow already records it — one slot per + * case in `flow_jobs_success`, null until that case's iteration is over. */ + let caseProgress = $state>({}) + + async function readCaseProgress() { + const workspace = ws + const live = experiments.filter((e) => e.running) + if (!workspace || live.length === 0) { + if (Object.keys(caseProgress).length > 0) caseProgress = {} + return + } + const read = await Promise.all( + live.map(async (e) => { + try { + const update = await JobService.getJobUpdates({ + workspace, + id: e.run_job_id, + running: true, + noLogs: true + }) + const cases = update.flow_status?.modules?.find((m) => m.id === CASES_MODULE_ID) + if (!cases) return undefined + return [ + e.id, + (cases.flow_jobs_success ?? []).filter((s) => s != undefined).length + ] as const + } catch { + // Left out of the map, so the row reads `0/total` until a later poll answers. A + // flow that cannot be read is already the list's problem to report, not this one's. + return undefined + } + }) + ) + caseProgress = Object.fromEntries(read.filter((e) => e !== undefined)) + } let experiment = $derived(experiments.find((e) => e.id === experimentId)) @@ -120,6 +172,7 @@ runsLoadError = false try { experiments = await listSubjectExperiments() + await readCaseProgress() } catch (e) { runsLoadError = true sendUserToast(`Failed to load the runs: ${e}`, true) @@ -183,6 +236,8 @@ // Switching datasets leaves the previous request in flight; only the newest may write, or a // slow response for the dataset you just left replaces the one you are looking at. let loadGeneration = 0 + /** Which run the pane is opening; only the newest may clear `datasetLoading`. */ + let openGeneration = 0 async function loadDataset(path: string | undefined): Promise { const generation = ++loadGeneration @@ -345,6 +400,7 @@ await loadResults() } else { experiments = await listSubjectExperiments() + await readCaseProgress() } } finally { refreshing = false @@ -354,27 +410,46 @@ /** Opens a run, bringing its dataset with it and offering the run before it as the baseline. * Reading the cells is left to the effect on the selection, so every way in opens one alike. */ async function openRun(id: string) { - // Against the dataset that is loaded, not the one that is selected: skipping on the selection - // alone would leave a run open over a dataset whose cases and scorers were never read. const target = experiments.find((e) => e.id === id) - if (target && target.dataset !== dataset?.path) { - await useDataset(target.dataset) + // Only when the run itself changes: re-showing the one already open — arrowing back into it + // from the list — must keep whatever comparison the user picked. + if (id !== experimentId) { + const index = experiments.findIndex((e) => e.id === id) + // The run before it *of the same dataset*: the list spans datasets, and a run of another + // set of cases is not a baseline for this one. + baselineId = experiments.slice(index + 1).find((e) => e.dataset === target?.dataset)?.id } - const index = experiments.findIndex((e) => e.id === id) - // The run before it *of the same dataset*: the list spans datasets, and a run of another set - // of cases is not a baseline for this one. - baselineId = experiments.slice(index + 1).find((e) => e.dataset === target?.dataset)?.id experimentId = id - viewingRun = true selectedCaseId = undefined + // Opened first, read second: the dataset is a request, and waiting on it here is a click + // that does nothing at all until the network answers. The page carries the wait instead. + // + // Against what is *selected* as well as what is loaded. `selectedDataset` moves the moment a + // read starts, so a load still in flight for another dataset shows up here: without that + // test, opening a run of the dataset already committed would skip `useDataset` entirely and + // leave the in-flight one free to commit its cases under this run. + const needsDataset = + !!target && (target.dataset !== dataset?.path || target.dataset !== selectedDataset) + datasetLoading = needsDataset + viewingRun = true + if (needsDataset) { + // Numbered like `loadDataset`'s own read, and for the same reason: opening a second run + // while the first is still loading leaves two `finally`s racing, and the loser clearing + // the flag would uncover the table with neither dataset in hand. + const generation = ++openGeneration + try { + await useDataset(target!.dataset) + } finally { + if (generation === openGeneration) datasetLoading = false + } + } } async function runAll(runSubject: EvalSubject, path: string): Promise { if (!ws || !path) return false running = true - let id: string try { - id = await AiEvalsService.runExperiment({ + await AiEvalsService.runExperiment({ workspace: ws, requestBody: { dataset: path, subject: runSubject } }) @@ -386,9 +461,10 @@ // From here the run exists and is billing: what can still fail is reading it back, and // saying "failed to run" to that invites a second, duplicate run. try { + // Onto the list rather than into the run: a run that has just started has no answers and + // no scores, and the list already fills its row in as they land. Reading it is a click. if (path !== dataset?.path) await useDataset(path) await loadRuns() - await openRun(id) } catch (e) { sendUserToast( `The run started but could not be read back: ${e}. Reload the runs list to see it.`, @@ -407,8 +483,6 @@ /** The dataset is gone and every run of it with it: back to the list, on no dataset. */ async function datasetDeleted(path: string) { - // A run dialog waiting behind the drawer has nothing to come back to. - resumeRunDialog = false if (selectedDataset === path) { viewingRun = false selectedCaseId = undefined @@ -442,6 +516,13 @@ } let selectedRow = $derived(displayRows.find((row) => row.case_id === selectedCaseId)) + /** The case the side panel is showing. Held rather than read straight off the selection: the + * pane animates shut over a few hundred milliseconds, and the selection is gone on the first + * of them, which would empty the panel before it had finished closing. */ + let openRow = $state(undefined) + $effect(() => { + if (selectedRow) openRow = selectedRow + }) $effect(() => { if (!ws) return @@ -570,34 +651,52 @@
    + {#if loaded && loadError} +
    + Could not load evals + + The datasets or runs could not be read. Check your access to this agent and reload. + +
    + {:else} + + { + // Right opens the run under the highlight, falling back to whichever was open before; + // left is the way back, the same as the breadcrumb. + if (key === 'run') { + // Both branches go through `openRun`: it is what brings the run's own dataset back, + // and the fallback run may be of a dataset the list has since moved off. + const id = highlightedRunId ?? experimentId + if (id) openRun(id) + } else if (key === 'list') { + viewingRun = false + selectedCaseId = undefined + } + }} + pages={[ + { key: 'list', content: listPage }, + { key: 'run', content: runPage } + ]} + /> + {/if} +
    + +{#snippet listPage()} + +

    + Each run answers a dataset of cases with this agent and scores the answers, so runs can be + compared. +

    - {#if viewingRun} - - {/if}
    - {#if viewingRun && experiment?.run_job_id} - - Open the job - - - {/if} - {#if !viewingRun && loaded && datasets.length > 0} + {#if loaded && datasets.length > 0} -
    - {:else if !viewingRun || !loaded} - openRun(e.id)} - onEditDataset={async (path) => { - if (await useDataset(path)) datasetDrawer?.openDrawer('edit') - }} - onNew={() => (runDialogOpen = true)} - /> - {:else} - - - - - {#each scorers as scorer (scorer.id)} - - {/each} - - - - Case - Answer - {#each scorers as scorer, index (scorer.id)} - {@const mean = means.find((m) => m.scorer_id === scorer.id)} - {@const headline = columnHeadline(scorer, mean)} - - -
    - - {#if scorer.kind === 'agent'} - - {:else} - - {/if} - {scorerLabel(scorer)} - - - {#if headline} - - {headline.value} + + + + + {#each scorers as scorer (scorer.id)} + + {/each} + + + + Case + Answer + {#each scorers as scorer, index (scorer.id)} + {@const mean = means.find((m) => m.scorer_id === scorer.id)} + {@const headline = columnHeadline(scorer, mean)} + + +
    + + {#if scorer.kind === 'agent'} + + {:else} + + {/if} + {scorerLabel(scorer)} + + + {#if headline} + + {headline.value} + + {#if headline.delta && headline.direction !== 0} + 0 ? 'text-green-500' : headline.direction < 0 ? 'text-red-500' : 'text-tertiary'}`} + > + {headline.delta} - {#if headline.delta && headline.direction !== 0} - 0 ? 'text-green-500' : headline.direction < 0 ? 'text-red-500' : 'text-tertiary'}`} - > - {headline.delta} - - {/if} {/if} - -
    -     - {/each} + {/if} +     +
    +     + {/each} + + + + {#if datasetLoading} + + + + + - - + {:else} {#each displayRows as row (row.case_id)} {@const status = statusOf(row.status)} {/each} - -     - {/if} + {/if} + +
    - {#if selectedRow} - {@const openRow = selectedRow} - + + + {#if openRow}
    -
    - + +
    + {openRow.input?.user_message ?? caseLabel(openRow)} -
    - {#if openRow.job_id} - - Open the case job - - - {/if} -
    -
    + +
    {#if openRow.expected != undefined && openRow.expected !== ''} - +
    +
    + Expected +
    +
    + + {typeof openRow.expected === 'string' + ? openRow.expected + : JSON.stringify(openRow.expected, null, 2)} + +
    +
    {/if} {#if scorers.length > 0 && openRow.scores.length > 0} -
    -
    +{/snippet} { - if (await useDataset(path)) { - resumeRunDialog = true - datasetDrawer?.openDrawer('edit') - } - }} - onNewDataset={() => { - resumeRunDialog = true - datasetDrawer?.openDrawer('new') + if (await useDataset(path)) datasetDrawer?.openDrawer('edit') }} + onNewDataset={() => datasetDrawer?.openDrawer('new')} /> { - if (!resumeRunDialog) return - resumeRunDialog = false - // On the dataset the drawer was just in: the dialog opens on the pane's own, which - // creating or editing one has already moved to it. - runDialogOpen = true - }} /> + + diff --git a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.css b/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.css deleted file mode 100644 index 1ac9f8b31c..0000000000 --- a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.css +++ /dev/null @@ -1,26 +0,0 @@ -/* MultilineCellEditor: a popup positioned over the cell, so it has to paint the cell's own frame - rather than inherit it. */ -.ag-theme-alpine .wm-multiline-cell-editor, -.ag-theme-alpine-dark .wm-multiline-cell-editor { - background-color: var(--ag-background-color); -} -.ag-theme-alpine .wm-multiline-cell-editor textarea, -.ag-theme-alpine-dark .wm-multiline-cell-editor textarea { - display: block; - box-sizing: border-box; - /* Horizontal only: the vertical padding is set by the editor, which knows the height of the row - it is replacing. `line-height` here is what it computes against. */ - padding: 0 calc(var(--ag-cell-horizontal-padding) - 1px); - border: 1px solid var(--ag-input-focus-border-color); - border-radius: 3px; - outline: none; - resize: none; - /* Past this it scrolls rather than growing. */ - max-height: 40vh; - overflow-y: auto; - background-color: var(--ag-background-color); - color: var(--ag-foreground-color); - font: inherit; - line-height: 20px; - white-space: pre-wrap; -} diff --git a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.ts b/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.ts deleted file mode 100644 index 00975955c9..0000000000 --- a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.ts +++ /dev/null @@ -1,108 +0,0 @@ -import type { ColDef, ICellEditorComp, ICellEditorParams } from 'ag-grid-community' -// Beside the editor rather than in the AgGrid theme: that file is the vendored theme, and a rule -// added to it is one the next copy of it drops. -import './multilineCellEditor.css' - -/** Kept in step with the `line-height` the stylesheet gives the textarea. */ -const LINE_HEIGHT = 20 - -/** - * A text cell editor that starts the height of the cell and grows as lines are added, for columns - * holding prose rather than a value. Enter commits, Shift+Enter adds a line, Escape cancels. - * - * Rendered as a popup positioned over the cell: an in-cell editor is clipped to the row height, so - * growing is only visible if the editor is allowed to paint outside it. - */ -export class MultilineCellEditor implements ICellEditorComp { - private eGui!: HTMLDivElement - private textarea!: HTMLTextAreaElement - private params!: ICellEditorParams - private wasEmpty = false - - init(params: ICellEditorParams) { - this.params = params - this.eGui = document.createElement('div') - this.eGui.className = 'wm-multiline-cell-editor' - - this.wasEmpty = params.value == undefined - - this.textarea = document.createElement('textarea') - this.textarea.rows = 1 - // A keystroke that opened the edit replaces the value, as it does in every other cell; F2 - // and double-click keep it to be edited. - this.textarea.value = params.eventKey?.length === 1 ? params.eventKey : (params.value ?? '') - this.textarea.style.width = `${params.column.getActualWidth() - 2}px` - // Padded so one line fills the cell it replaces and a second costs a line rather than a row. - // From the row rather than from `--ag-row-height`, which is the theme's figure and not - // necessarily this grid's. - const rowHeight = params.node.rowHeight ?? 28 - const padding = Math.max(0, (rowHeight - LINE_HEIGHT - 2) / 2) - this.textarea.style.paddingTop = `${padding}px` - this.textarea.style.paddingBottom = `${padding}px` - - this.textarea.addEventListener('input', () => this.resize()) - this.textarea.addEventListener('keydown', (e) => { - if (e.key === 'Escape') { - // Kept from whatever is around the grid: a grid in a drawer or a dialog is under a - // surface that closes on Escape, and leaving an edit is not asking to leave that. - e.preventDefault() - e.stopPropagation() - this.params.api.stopEditing(true) - return - } - if (e.key !== 'Enter' || e.isComposing) return - // Both branches keep the key from the grid, which ends the edit on Enter whether or not - // Shift is held: Shift+Enter falls through to the textarea's own newline, and plain Enter - // ends the edit here instead. - e.stopPropagation() - if (!e.shiftKey) { - e.preventDefault() - this.params.stopEditing() - } - }) - this.eGui.appendChild(this.textarea) - } - - private resize() { - this.textarea.style.height = 'auto' - this.textarea.style.height = `${this.textarea.scrollHeight}px` - } - - getGui() { - return this.eGui - } - - afterGuiAttached() { - this.resize() - this.textarea.focus() - // At the end rather than selected: a selection is a keystroke away from erasing the cell. - const end = this.textarea.value.length - this.textarea.setSelectionRange(end, end) - } - - getValue() { - // Nothing typed into a cell that held nothing is not an edit: returning '' here would write - // an empty string over a null, which the grid would see as a change and commit. - if (this.wasEmpty && this.textarea.value === '') return this.params.value - return this.textarea.value - } - - isPopup() { - return true - } - - getPopupPosition(): 'over' | 'under' { - return 'over' - } -} - -/** - * What a column of prose needs, ready to spread into a colDef. `suppressKeyboardEvent` as well as - * the editor: the grid ends an edit on Enter from a handler a popup editor's DOM does not sit - * under, so the editor cannot keep Shift+Enter for itself on its own. - */ -export const multilineCellColDef: Pick = { - cellEditor: MultilineCellEditor, - suppressKeyboardEvent: (p) => - p.editing && (p.event as KeyboardEvent).key === 'Enter' && (p.event as KeyboardEvent).shiftKey -} diff --git a/frontend/src/lib/components/apps/components/helpers/InputValue.svelte b/frontend/src/lib/components/apps/components/helpers/InputValue.svelte index aba0569228..7ec2336d3b 100644 --- a/frontend/src/lib/components/apps/components/helpers/InputValue.svelte +++ b/frontend/src/lib/components/apps/components/helpers/InputValue.svelte @@ -19,6 +19,7 @@ import { computeGlobalContext, eval_like } from './eval' import { deepEqual } from 'fast-equals' import { deepMergeWithPriority, isCodeInjection, readFieldsRecursively } from '$lib/utils' + import { escapeTemplateBackticks } from '$lib/utils/templateLiteral' import sum from 'hash-sum' import { createDispatcherIfMounted } from '$lib/createDispatcherIfMounted' @@ -271,7 +272,7 @@ if ((input.type === 'template' || input.type == 'templatev2') && isCodeInjection(input.eval)) { try { const r = await eval_like( - '`' + input.eval.replaceAll('`', '\\`') + '`', + '`' + escapeTemplateBackticks(input.eval) + '`', computeGlobalContext($worldStore, id, fullContext), $stateStore, $mode == 'dnd', diff --git a/frontend/src/lib/components/common/EditableTextarea.svelte b/frontend/src/lib/components/common/EditableTextarea.svelte new file mode 100644 index 0000000000..26a6574027 --- /dev/null +++ b/frontend/src/lib/components/common/EditableTextarea.svelte @@ -0,0 +1,173 @@ + + + +{#if editing} + + +{:else} + + +{/if} diff --git a/frontend/src/lib/components/common/drawer/Drawer.svelte b/frontend/src/lib/components/common/drawer/Drawer.svelte index d375bf506e..6bb5358940 100644 --- a/frontend/src/lib/components/common/drawer/Drawer.svelte +++ b/frontend/src/lib/components/common/drawer/Drawer.svelte @@ -8,6 +8,7 @@ import { onMount, createEventDispatcher, setContext, untrack } from 'svelte' import { BROWSER } from 'esm-env' import Disposable from './Disposable.svelte' + import { setTopmostSurface } from '$lib/components/common/overlayHost.svelte' import ConditionalPortal from './ConditionalPortal.svelte' import { chatState } from '$lib/components/copilot/chat/sharedChatState.svelte' import { useReducedMotion } from '$lib/svelte5Utils.svelte' @@ -51,6 +52,11 @@ let disposable: Disposable | undefined = $state(undefined) + // A drawer stacks like a dialog does, so content inside it gets the same answer about whether + // its keys are meant for it. Without this, a drawer opened over a dialog would inherit the + // dialog's answer — false, because the drawer itself is now on top — and go deaf. + setTopmostSurface(() => disposable?.isTopmost() ?? true) + let reducedMotion = useReducedMotion() let duration = $derived(reducedMotion.val ? 0 : _duration) let durationMs = $derived(duration * 1000) diff --git a/frontend/src/lib/components/common/emptyState/EmptyState.svelte b/frontend/src/lib/components/common/emptyState/EmptyState.svelte index 8715065ad0..49c3cd2b77 100644 --- a/frontend/src/lib/components/common/emptyState/EmptyState.svelte +++ b/frontend/src/lib/components/common/emptyState/EmptyState.svelte @@ -10,6 +10,16 @@ label: string icon?: any onClick: () => void + /** + * `default` unless the surface has no other live call to action. Accent is for the case + * where this button is the only thing to press — a form whose submit is disabled until + * this is done, say — so it is not competing with one. + */ + variant?: 'default' | 'accent' + /** Same write lock the surface's other controls take. An empty state is still a live + * control: without this it stays clickable while a request that has already read the + * empty list is in flight, and whatever it adds is discarded when that request lands. */ + disabled?: boolean aiId?: string aiDescription?: string } @@ -32,11 +42,13 @@ {/if}
    {#if action} - +
    diff --git a/frontend/src/lib/components/common/modal/PagedContent.svelte b/frontend/src/lib/components/common/modal/PagedContent.svelte new file mode 100644 index 0000000000..430a53e66f --- /dev/null +++ b/frontend/src/lib/components/common/modal/PagedContent.svelte @@ -0,0 +1,202 @@ + + + + + + + +
    + {#each pages as page, i (page.key)} + {#if visited.includes(page.key) || warmed} + + +
    + {@render page.content()} +
    + {/if} + {/each} +
    + + diff --git a/frontend/src/lib/components/common/overlayHost.svelte.ts b/frontend/src/lib/components/common/overlayHost.svelte.ts index 0e76709cbf..7c4374115b 100644 --- a/frontend/src/lib/components/common/overlayHost.svelte.ts +++ b/frontend/src/lib/components/common/overlayHost.svelte.ts @@ -72,3 +72,23 @@ export function overlayHostActive(): () => boolean { const host = getOverlayHost() return () => host?.active() ?? true } + +const TOPMOST_SURFACE_KEY = 'topmostSurface' + +/** + * Declare whether the surface enclosing this subtree is the one on top. Set by whatever owns the + * stacking — a dialog, a drawer — so content inside it can tell a key meant for itself from one + * meant for something opened over it. + */ +export function setTopmostSurface(isTopmost: () => boolean) { + setContext(TOPMOST_SURFACE_KEY, isTopmost) +} + +/** + * Whether the enclosing surface is on top. True when nothing declared otherwise, so content that + * is not inside such a surface is not silently made deaf. + */ +export function topmostSurface(): () => boolean { + const isTopmost = getContext<(() => boolean) | undefined>(TOPMOST_SURFACE_KEY) + return () => isTopmost?.() ?? true +} diff --git a/frontend/src/lib/components/common/radioCard/RadioCard.svelte b/frontend/src/lib/components/common/radioCard/RadioCard.svelte index 6cbb9c78a0..f9b7f675a4 100644 --- a/frontend/src/lib/components/common/radioCard/RadioCard.svelte +++ b/frontend/src/lib/components/common/radioCard/RadioCard.svelte @@ -10,12 +10,14 @@ onSelect, disabled = false, icon = undefined, + showRadio = true, class: className = '' }: { /** Title shown in bold at the top of the card */ label: string - /** Optional supporting line under the label */ - description?: string + /** Optional supporting line under the label. A snippet when it needs markup + * of its own — an emphasised name, a count — rather than plain text. */ + description?: string | Snippet /** Whether this card is the selected option */ selected?: boolean /** Called when the card is clicked */ @@ -23,35 +25,55 @@ disabled?: boolean /** Optional leading icon, rendered after the radio */ icon?: Snippet + /** Draw the radio glyph. Turn it off where the card itself is the only + * control and the border and tint already say which one is picked — the dot + * is then a second, redundant answer to the same question. The group still + * reads as radios to a screen reader, which is what `role` carries. */ + showRadio?: boolean class?: string } = $props() + + // A snippet is a function; a description string is not. Checked rather than + // requiring callers to pick between two props. + const describedBySnippet = $derived(typeof description === 'function') +
    + +
    +{/snippet} + +{#snippet freeTierUsageBanner()} +
    + + {freeTierUsedPct}% of your free Windmill AI used + + +
    +{/snippet} +
    script editor to modify selected lines. {/if} + {#if freeTierExhausted} +
    + {@render freeTierExhaustedBanner()} +
    + {/if} {/if} {#if messages.length > 0} @@ -699,6 +756,9 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> isLast={messageIndex === messages.length - 1} /> {/each} + {#if freeTierExhausted} + {@render freeTierExhaustedBanner()} + {/if} {#if showTypingIndicator}
    {#if inputPreface} {@render inputPreface()} {/if} + {#if showFreeTierUsage} + {@render freeTierUsageBanner()} + {/if} = 80) + let capability = $derived( getReasoningCapability(providerModel.provider as AIProvider, providerModel.model) ) @@ -312,6 +319,13 @@ {#if effortLabel} · {effortLabel} {/if} + {#if freeTier && !freeTier.exhausted} + Free + {/if}
    diff --git a/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte b/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte index ffa06ca5df..96b452120b 100644 --- a/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte +++ b/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte @@ -3,7 +3,7 @@ import { getKnownModelContextWindow, getModelContextWindow } from '../modelConfig' import { getAiChatManager } from './aiChatManagerContext' import { AIMode } from './AIChatManager.svelte' - import Tooltip from '$lib/components/meltComponents/Tooltip.svelte' + import UsageMeter from './UsageMeter.svelte' import { formatTokenCount } from './tokenUsage' const aiChatManager = getAiChatManager() @@ -49,25 +49,11 @@ {#if visible} - - -
    -
    -
    -
    -
    - {#snippet text()} + + + {#snippet tooltip()}

    Context usage

    @@ -85,5 +71,5 @@ {/if}

    {/snippet} -     + {/if} diff --git a/frontend/src/lib/components/copilot/chat/UsageMeter.svelte b/frontend/src/lib/components/copilot/chat/UsageMeter.svelte new file mode 100644 index 0000000000..26e4758953 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/UsageMeter.svelte @@ -0,0 +1,39 @@ + + + +
    +
    +
    +
    +
    + {#snippet text()} + {@render tooltip()} + {/snippet} +     diff --git a/frontend/src/lib/components/copilot/chat/workspaceToolsZod.gen.ts b/frontend/src/lib/components/copilot/chat/workspaceToolsZod.gen.ts index 47237a0ae1..42773873c2 100644 --- a/frontend/src/lib/components/copilot/chat/workspaceToolsZod.gen.ts +++ b/frontend/src/lib/components/copilot/chat/workspaceToolsZod.gen.ts @@ -69,7 +69,7 @@ export const httpTriggerRequestSchema = z.object({ "wrap_body": z.boolean().describe("If true, wraps the request body in a 'body' parameter").optional(), "mode": z.enum(["enabled", "disabled", "suspended"]).describe("job trigger mode").optional(), "raw_string": z.boolean().describe("If true, passes the request body as a raw string instead of parsing as JSON").optional(), - "error_handler_path": z.string().describe("Path to a script or flow to run when the triggered job fails").optional(), + "error_handler_path": z.string().describe("Path to a script to run when the triggered job fails. A bare path, without the script/ or flow/ prefix a schedule error handler takes; it cannot be a flow.").optional(), "error_handler_args": z.record(z.string(), z.any()).describe("Arguments to pass to the error handler").optional(), "retry": z.object({ "constant": z.object({ @@ -128,7 +128,7 @@ export const websocketTriggerRequestSchema = z.object({ "message": z.string().describe("Message to send as heartbeat. Use {{state}} as a placeholder for a value extracted from incoming messages (see state_field)."), "state_field": z.string().describe("Optional. Top-level JSON field to extract from incoming messages. The extracted value replaces {{state}} in the heartbeat message.").optional() }).describe("Optional periodic heartbeat message configuration").nullable().optional(), - "error_handler_path": z.string().describe("Path to a script or flow to run when the triggered job fails").optional(), + "error_handler_path": z.string().describe("Path to a script to run when the triggered job fails. A bare path, without the script/ or flow/ prefix a schedule error handler takes; it cannot be a flow.").optional(), "error_handler_args": z.record(z.string(), z.any()).describe("Arguments to pass to the error handler").optional(), "retry": z.object({ "constant": z.object({ @@ -174,7 +174,7 @@ export const kafkaTriggerRequestSchema = z.object({ "auto_offset_reset": z.enum(["latest", "earliest"]).describe("Initial offset behavior when consumer group has no committed offset.").default("latest").optional(), "auto_commit": z.boolean().describe("When true (default), offsets are committed automatically after receiving each message. When false, you must manually commit offsets using the commit_offsets endpoint.").default(true).optional(), "mode": z.enum(["enabled", "disabled", "suspended"]).describe("job trigger mode").optional(), - "error_handler_path": z.string().describe("Path to a script or flow to run when the triggered job fails").optional(), + "error_handler_path": z.string().describe("Path to a script to run when the triggered job fails. A bare path, without the script/ or flow/ prefix a schedule error handler takes; it cannot be a flow.").optional(), "error_handler_args": z.record(z.string(), z.any()).describe("Arguments to pass to the error handler").optional(), "retry": z.object({ "constant": z.object({ @@ -206,7 +206,7 @@ export const natsTriggerRequestSchema = z.object({ "consumer_name": z.string().describe("JetStream consumer name (required when use_jetstream is true)").nullable().optional(), "subjects": z.array(z.string()).describe("Array of NATS subjects to subscribe to"), "mode": z.enum(["enabled", "disabled", "suspended"]).describe("job trigger mode").optional(), - "error_handler_path": z.string().describe("Path to a script or flow to run when the triggered job fails").optional(), + "error_handler_path": z.string().describe("Path to a script to run when the triggered job fails. A bare path, without the script/ or flow/ prefix a schedule error handler takes; it cannot be a flow.").optional(), "error_handler_args": z.record(z.string(), z.any()).describe("Arguments to pass to the error handler").optional(), "retry": z.object({ "constant": z.object({ @@ -247,7 +247,7 @@ export const postgresTriggerRequestSchema = z.object({ })).optional(), "transaction_to_track": z.array(z.string()) }).describe("Configuration for creating/managing the publication (tables, operations)").optional(), - "error_handler_path": z.string().describe("Path to a script or flow to run when the triggered job fails").optional(), + "error_handler_path": z.string().describe("Path to a script to run when the triggered job fails. A bare path, without the script/ or flow/ prefix a schedule error handler takes; it cannot be a flow.").optional(), "error_handler_args": z.record(z.string(), z.any()).describe("Arguments to pass to the error handler").optional(), "retry": z.object({ "constant": z.object({ @@ -289,7 +289,7 @@ export const mqttTriggerRequestSchema = z.object({ "script_path": z.string().describe("Path to the script or flow to execute when a message is received"), "is_flow": z.boolean().describe("True if script_path points to a flow, false if it points to a script"), "mode": z.enum(["enabled", "disabled", "suspended"]).describe("job trigger mode").optional(), - "error_handler_path": z.string().describe("Path to a script or flow to run when the triggered job fails").optional(), + "error_handler_path": z.string().describe("Path to a script to run when the triggered job fails. A bare path, without the script/ or flow/ prefix a schedule error handler takes; it cannot be a flow.").optional(), "error_handler_args": z.record(z.string(), z.any()).describe("Arguments to pass to the error handler").optional(), "retry": z.object({ "constant": z.object({ @@ -326,7 +326,7 @@ export const amqpTriggerRequestSchema = z.object({ "script_path": z.string().describe("Path to the script or flow to execute when a message is received"), "is_flow": z.boolean().describe("True if script_path points to a flow, false if it points to a script"), "mode": z.enum(["enabled", "disabled", "suspended"]).describe("job trigger mode").optional(), - "error_handler_path": z.string().describe("Path to a script or flow to run when the triggered job fails").optional(), + "error_handler_path": z.string().describe("Path to a script to run when the triggered job fails. A bare path, without the script/ or flow/ prefix a schedule error handler takes; it cannot be a flow.").optional(), "error_handler_args": z.record(z.string(), z.any()).describe("Arguments to pass to the error handler").optional(), "retry": z.object({ "constant": z.object({ @@ -357,7 +357,7 @@ export const sqsTriggerRequestSchema = z.object({ "script_path": z.string().describe("Path to the script or flow to execute when a message is received"), "is_flow": z.boolean().describe("True if script_path points to a flow, false if it points to a script"), "mode": z.enum(["enabled", "disabled", "suspended"]).describe("job trigger mode").optional(), - "error_handler_path": z.string().describe("Path to a script or flow to run when the triggered job fails").optional(), + "error_handler_path": z.string().describe("Path to a script to run when the triggered job fails. A bare path, without the script/ or flow/ prefix a schedule error handler takes; it cannot be a flow.").optional(), "error_handler_args": z.record(z.string(), z.any()).describe("Arguments to pass to the error handler").optional(), "retry": z.object({ "constant": z.object({ diff --git a/frontend/src/lib/components/copilot/loadCopilot.ts b/frontend/src/lib/components/copilot/loadCopilot.ts index 4634833b0b..496fb89fdc 100644 --- a/frontend/src/lib/components/copilot/loadCopilot.ts +++ b/frontend/src/lib/components/copilot/loadCopilot.ts @@ -1,6 +1,14 @@ import { WorkspaceService } from '$lib/gen' import { copilotWorkspace, setCopilotInfo } from '$lib/aiStore' import { workspaceAIClients } from './lib' +import { writable } from 'svelte/store' + +// The workspace of the most recent loadCopilot *request*, set synchronously before the +// await — as opposed to `copilotWorkspace`, which only updates once a load resolves. A +// background refresh (e.g. free-tier usage) compares against this so it can't supersede an +// in-flight load for a newer workspace (which would otherwise win the token and restore +// stale state). +export const copilotWorkspaceRequested = writable(undefined) // Lives here, not in $lib/aiStore, purely so that module needs no import of the AI // client — it is the one thing that wanted both. Moving it back recreates the @@ -20,6 +28,7 @@ let loadCopilotToken = 0 let inFlight: { workspace: string; promise: Promise } | undefined export function loadCopilot(workspace: string): Promise { + copilotWorkspaceRequested.set(workspace) if (inFlight?.workspace === workspace) { return inFlight.promise } diff --git a/frontend/src/lib/components/copilot/stepInputFillTelemetry.ts b/frontend/src/lib/components/copilot/stepInputFillTelemetry.ts new file mode 100644 index 0000000000..19efc2b932 --- /dev/null +++ b/frontend/src/lib/components/copilot/stepInputFillTelemetry.ts @@ -0,0 +1,13 @@ +import { logFeatureUsage } from '$lib/utils/featureUsage' + +// Anonymous counters for the AI filling of a step's inputs. Same rules as every other +// `logFeatureUsage` caller: aggregated counts only, and the two keys below are the whole +// vocabulary — no argument name, expression or step id ever reaches here. + +/** Which filler the user reached for: the per-field one, or the one above the whole form. */ +export type StepInputFillScope = 'single' | 'all' + +/** Counted where the user asks for a suggestion, not where one is accepted. */ +export function logStepInputFill(scope: StepInputFillScope): void { + logFeatureUsage('flow_step', 'ai_fill', { key: scope }) +} diff --git a/frontend/src/lib/components/flows/agentTelemetry.ts b/frontend/src/lib/components/flows/agentTelemetry.ts new file mode 100644 index 0000000000..fcfad440ff --- /dev/null +++ b/frontend/src/lib/components/flows/agentTelemetry.ts @@ -0,0 +1,19 @@ +import { logFeatureUsage } from '$lib/utils/featureUsage' + +// Anonymous counters for the reusable-agent lifecycle (`docs/reusable-ai-agents.md`). Same rules +// as every other `logFeatureUsage` caller: aggregated counts only, and the four keys below are +// the whole vocabulary — no agent path, prompt, model or tool ever reaches here. + +export type ReusableAgentEvent = + /** A step was saved as a new reusable agent. */ + | 'saved' + /** Edits to a linked agent were written back, propagating to every flow using it. */ + | 'updated' + /** A saved agent was picked into a new step. */ + | 'linked' + /** A linked step was forked back into a standalone agent. */ + | 'unlinked' + +export function logReusableAgentUsage(event: ReusableAgentEvent): void { + logFeatureUsage('ai_agent', 'reusable', { key: event }) +} diff --git a/frontend/src/lib/components/flows/content/AgentResourceBar.svelte b/frontend/src/lib/components/flows/content/AgentResourceBar.svelte index 687dc9d98c..f5d5ec19a1 100644 --- a/frontend/src/lib/components/flows/content/AgentResourceBar.svelte +++ b/frontend/src/lib/components/flows/content/AgentResourceBar.svelte @@ -31,6 +31,7 @@ linkedToolsScope } from '../linkedAgentToolsStore.svelte' import { getAgentEdit, getAgentEditingPath, setAgentEditingPath } from '../agentEditStore.svelte' + import { logReusableAgentUsage } from '../agentTelemetry' import { claimLinkedToolsFetch } from '../flowState' import type { AgentTool as AgentToolStrict } from '../agentToolUtils' import { resource } from 'runed' @@ -338,6 +339,7 @@ const linked = await persist(newPath, description) saveDrawer?.closeDrawer() if (linked) { + logReusableAgentUsage(updating ? 'updated' : 'saved') sendUserToast(updating ? `Updated agent ${newPath}` : `Saved reusable agent ${newPath}`) } } catch (e) { @@ -356,6 +358,7 @@ const path = editingPath try { if (await persist(path)) { + logReusableAgentUsage('updated') sendUserToast(`Updated agent ${path}`) } } catch (e) { @@ -419,6 +422,7 @@ const fork = await forkFromResource(true) if (fork) { setAgentEditingPath(tools, undefined) + logReusableAgentUsage('unlinked') sendUserToast('Forked agent. Its configuration was copied into this step') } else { sendUserToast('The step changed while loading the agent, so nothing was unlinked', true) diff --git a/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte b/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte index 6aeaa14f35..e6d7223fcb 100644 --- a/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte +++ b/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte @@ -464,7 +464,7 @@ btnClasses="px-1 py-1.5 bg-surface" on:click={() => { outputPicker?.toggleOpen(true) - moduleTest?.loadArgsAndRunTest() + moduleTest?.runTestWithStepArgs() }} dropdownItems={[ { diff --git a/frontend/src/lib/components/flows/map/InsertModuleInner.svelte b/frontend/src/lib/components/flows/map/InsertModuleInner.svelte index 3316463bbb..d1e65de1ea 100644 --- a/frontend/src/lib/components/flows/map/InsertModuleInner.svelte +++ b/frontend/src/lib/components/flows/map/InsertModuleInner.svelte @@ -14,6 +14,7 @@ import { ResourceService } from '$lib/gen' import { workspaceStore } from '$lib/stores' import type { FlowEditorContext } from '../types' + import { logReusableAgentUsage } from '../agentTelemetry' import { BotIcon, Loader2, Plus } from 'lucide-svelte' const dispatch = createEventDispatcher() @@ -271,6 +272,7 @@ +
    + +
    +
    +
    + {/if} + +
    + {#if showComposer} +
    + {#each homeAIExamples as example (example.label)} + + {/each} +
    + {:else} +
    + {/if} + + +
    + + {#if !$userStore?.operator && HOME_SHOW_HUB} + + {/if} +
    +
    + {#if showComposer && disabled} +
    +

    + {#if $aiUserDisabled} + Windmill AI is disabled in your account settings + {:else if freeTierExhausted} + You have used all of your free Windmill AI tokens + {:else} + No AI provider is configured + {/if} +

    + {#if $aiUserDisabled} + + + {:else} + + {/if} +
    + {/if} +
    +
    + + diff --git a/frontend/src/lib/components/home/ItemsList.svelte b/frontend/src/lib/components/home/ItemsList.svelte index d7717d1e04..0f0cb6a6c0 100644 --- a/frontend/src/lib/components/home/ItemsList.svelte +++ b/frontend/src/lib/components/home/ItemsList.svelte @@ -1,7 +1,7 @@ @@ -125,16 +127,8 @@ {runnable} isSelected={selectedRunnable === id} isEditing={editingId === id} - onSelect={() => { - selectedRunnable = id - onSelect?.(id) - }} - onDelete={() => { - delete runnables[id] - if (selectedRunnable === id) { - selectedRunnable = undefined - } - }} + onSelect={() => onSelect?.(id)} + onDelete={() => onDelete(id)} onRename={(newId) => renameRunnable(id, newId)} onRequestEdit={() => (editingId = id)} onCancelEdit={() => (editingId = undefined)} diff --git a/frontend/src/lib/components/raw_apps/RawAppSidebar.svelte b/frontend/src/lib/components/raw_apps/RawAppSidebar.svelte index b8d6e33c87..fbe02e1dc9 100644 --- a/frontend/src/lib/components/raw_apps/RawAppSidebar.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppSidebar.svelte @@ -3,6 +3,7 @@ SUBTLE_PANEL_TITLE } from '../apps/editor/settingsPanel/common/PanelSection.svelte' import type { Runnable } from '../apps/inputType' + import { WMILL_TS_PATH } from './utils' import RawAppInlineScriptPanelList from './RawAppInlineScriptPanelList.svelte' import FileExplorer from '../FileExplorer.svelte' import { Plus, File, Folder, Camera } from 'lucide-svelte' @@ -18,10 +19,14 @@ interface Props { runnables: Record + /** Read-only; the editor switches selection through `onSelectRunnable`. */ selectedRunnable: string | undefined files: Record modules?: Modules - onSelectFile?: (path: string) => void + onSelectRunnable?: (key: string) => void + onDeleteRunnable: (key: string) => void + onSelectPath?: (path: string) => void + /** Read-only; the editor switches selection through `onSelectPath`. */ selectedDocument: string | undefined historyManager?: RawAppHistoryManager historySelectedId?: number | undefined @@ -39,11 +44,13 @@ let { runnables, - selectedRunnable = $bindable(), + selectedRunnable, files = $bindable({}), modules, - onSelectFile, - selectedDocument = $bindable(), + onSelectRunnable, + onDeleteRunnable, + onSelectPath, + selectedDocument, historyManager, historySelectedId, onHistorySelect, @@ -79,13 +86,6 @@ } let fileExplorer: FileExplorer | undefined = $state() - - function handleSelectPath(path: string) { - selectedDocument = path - if (!path.endsWith('/')) { - onSelectFile?.(path) - } - }
    + {/if} {#if !hasFirstUserMessage} {/if} @@ -329,9 +382,7 @@ and reconcile would re-archive a workspace-archived one anyway. When the workspace is unavailable the SessionChangesBar below shows the move/discard banner instead (its actions are the real recovery path). --> -
    +
    This session is archived diff --git a/frontend/src/lib/components/sessions/sessionRecoveryNotice.svelte.ts b/frontend/src/lib/components/sessions/sessionRecoveryNotice.svelte.ts new file mode 100644 index 0000000000..635bd68601 --- /dev/null +++ b/frontend/src/lib/components/sessions/sessionRecoveryNotice.svelte.ts @@ -0,0 +1,18 @@ +import { SvelteSet } from 'svelte/reactivity' + +// Session ids opened as a stand-in for a `session_name` this browser doesn't +// hold. Kept in memory rather than on the Session record: persisted, the notice +// would replay on every reload of a session the user has since made their own. +const recovered = new SvelteSet() + +export function markSessionRecovered(id: string): void { + recovered.add(id) +} + +export function isSessionRecovered(id: string): boolean { + return recovered.has(id) +} + +export function clearSessionRecovered(id: string): void { + recovered.delete(id) +} diff --git a/frontend/src/lib/components/sessions/sessionState.svelte.ts b/frontend/src/lib/components/sessions/sessionState.svelte.ts index d8ef56b273..505af7a43a 100644 --- a/frontend/src/lib/components/sessions/sessionState.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionState.svelte.ts @@ -21,6 +21,7 @@ import { import { getLocalSetting, storeLocalSetting } from '$lib/utils' import { logFeatureUsage } from '$lib/utils/featureUsage' import { workspaceRootId } from './sessionScope.svelte' +import { clearSessionRecovered } from './sessionRecoveryNotice.svelte' import { type DBSchema, type IDBPDatabase } from 'idb' import { userScopedDb } from '$lib/userScopedDb' import { deleteItemsForSession } from '../copilot/chat/files/attachedFilesDB' @@ -343,8 +344,8 @@ export function setSessionDraftPrompt(sessionId: string, text: string): void { if ((s.draftPrompt ?? '') === text) return // Keep `transient` (means "in-memory only") set until the flush persists the // draft, so hydrateSessions preserves it across a reconcile inside this window; - // isReusableBlank, not `transient`, is what stops createSession reusing a typed - // draft. Only the IndexedDB write is debounced. + // isDiscardableDraft, not `transient`, is what stops createSession reusing a + // typed draft. Only the IndexedDB write is debounced. s.draftPrompt = text clearTimeout(draftPromptFlushHandles.get(sessionId)) draftPromptFlushHandles.set( @@ -748,30 +749,52 @@ export function requestComposerFocus(): void { composerFocusRequest.nonce++ } -// An untouched in-memory blank that `+` may reuse/discard. `draftPrompt === -// undefined` (never edited), not falsiness: a draft typed then erased to '' still -// has a pending flush and is a real session, so it must survive both. Every other +// An untouched in-memory blank that `+` may reuse and createSession may silently +// drop. `draftPrompt === undefined` (never edited), not falsiness: a draft typed +// then erased to '' still has a pending flush and is a real session. Every other // touch clears `transient` synchronously, so only the draft prompt needs checking. -function isReusableBlank(s: Session): boolean { +function isDiscardableDraft(s: Session): boolean { return !!s.transient && s.draftPrompt === undefined } +// Somewhere empty to put the user, for a URL naming a session this browser +// doesn't hold. Only `transient` makes "empty" trustworthy: chat seeding and +// attached-file persistence both key off `!transient` and leave every field +// below untouched, so a persisted session can hold a conversation regardless. +export function findEmptyLandingSession(): Session | undefined { + return sessionState.sessions.find( + (s) => + !!s.transient && + !s.archived && + !s.workspace_id && + // Falsiness, not `=== undefined`: we only navigate into the session, so a + // draft erased back to '' is still an empty composer to land on. + !s.draftPrompt?.trim() && + !s.pending_fork && + sessionInCurrentFamily(s) + ) +} + export function createSession(): Session { // Reuse an existing untouched draft from the active family rather than pile a // blank entry on every `+`, so several pending sessions can still be built up // in parallel, one touch at a time. A cross-family leftover blank is dropped // instead of reused (reusing it would act on that family). const reusable = sessionState.sessions.find( - (s) => isReusableBlank(s) && sessionInCurrentFamily(s) + (s) => isDiscardableDraft(s) && sessionInCurrentFamily(s) ) if (reusable) { sessionState.currentSessionId = reusable.id + // The blank recovery just landed on is exactly what this reuses, so `+` + // would otherwise hand back a session still carrying the recovery notice: + // asking for a new session must not be answered with "we couldn't find it". + clearSessionRecovered(reusable.id) // Reusing an already-active draft doesn't change currentSessionId, so ask // the composer to focus explicitly — the caller still navigates/redirects. requestComposerFocus() return reusable } - sessionState.sessions = sessionState.sessions.filter((s) => !isReusableBlank(s)) + sessionState.sessions = sessionState.sessions.filter((s) => !isDiscardableDraft(s)) const existingNumbers = sessionState.sessions .map((s) => /^session-(\d+)$/.exec(s.name)?.[1]) .map((n) => (n ? parseInt(n, 10) : 0)) @@ -1087,6 +1110,24 @@ export function setSessionArchived(id: string, archived: boolean) { persistTouched(s) } +// A counter rather than a flag: an inner teardown finishing must not reopen the +// gate while an outer one is still running. Released in a finally, so a delete +// that throws can't wedge it shut. +let openSessionTeardowns = $state(0) + +export function isTearingDownOpenSession(): boolean { + return openSessionTeardowns > 0 +} + +export async function withOpenSessionTeardown(run: () => Promise): Promise { + openSessionTeardowns++ + try { + return await run() + } finally { + openSessionTeardowns-- + } +} + export function deleteSession(id: string) { const s = sessionState.sessions.find((x) => x.id === id) if (!s) return diff --git a/frontend/src/lib/components/sessions/sessionState.test.ts b/frontend/src/lib/components/sessions/sessionState.test.ts index 7732c44746..1a54c86e78 100644 --- a/frontend/src/lib/components/sessions/sessionState.test.ts +++ b/frontend/src/lib/components/sessions/sessionState.test.ts @@ -4,14 +4,22 @@ import { commitSessionWorkspace, createSession, decideSessionLifecycle, + findEmptyLandingSession, isForkSession, + isTearingDownOpenSession, renameSession, sessionInCurrentFamily, setGeneratedSessionSummary, setSessionDraftPrompt, sessionState, + withOpenSessionTeardown, type Session } from './sessionState.svelte' +import { + clearSessionRecovered, + isSessionRecovered, + markSessionRecovered +} from './sessionRecoveryNotice.svelte' import { enterpriseLicense, usersWorkspaceStore, @@ -362,6 +370,28 @@ describe('createSession — reuses an untouched draft, family-scoped', () => { } }) + it('clears the recovery notice off the draft it reuses, so `+` is not answered with "not found"', () => { + const restore = withTwoFamilies('rootA') + const prevCurrent = sessionState.currentSessionId + const landed = session({ + id: 'recovered-blank', + name: 'session-903', + pending_workspace_id: 'forkA', + transient: true + }) + sessionState.sessions.push(landed) + markSessionRecovered(landed.id) + try { + expect(createSession().id).toBe('recovered-blank') + expect(isSessionRecovered('recovered-blank')).toBe(false) + } finally { + clearSessionRecovered('recovered-blank') + sessionState.sessions = sessionState.sessions.filter((s) => s.id !== 'recovered-blank') + sessionState.currentSessionId = prevCurrent + restore() + } + }) + it('drops an untouched draft left over from another family and starts in the active workspace', () => { const restore = withTwoFamilies('rootB') const prevCurrent = sessionState.currentSessionId @@ -488,3 +518,78 @@ describe('createSession — reuses an untouched draft, family-scoped', () => { } }) }) + +describe('findEmptyLandingSession — where an unresolvable session link lands', () => { + it('takes an untouched draft', () => { + const restore = withTwoFamilies('forkA') + const blank = session({ + id: 'landing-blank', + name: 'session-910', + pending_workspace_id: 'forkA', + transient: true + }) + sessionState.sessions.push(blank) + try { + expect(findEmptyLandingSession()?.id).toBe('landing-blank') + } finally { + sessionState.sessions = sessionState.sessions.filter((s) => s.id !== 'landing-blank') + restore() + } + }) + + it('passes over a persisted session, onto which chat seeding can graft a conversation', () => { + const restore = withTwoFamilies('forkA') + // ensureChatIdsSeeded assigns untagged legacy chats to `!transient` sessions + // and initRuntime loads them, without touching a field checked here. + const abandoned = session({ + id: 'landing-abandoned', + name: 'session-910', + pending_workspace_id: 'forkA' + }) + const others = sessionState.sessions + sessionState.sessions = [abandoned] + try { + expect(findEmptyLandingSession()).toBeUndefined() + } finally { + sessionState.sessions = others + restore() + } + }) + + it('passes over a session that has been sent, so recovery never reopens a conversation', () => { + const restore = withTwoFamilies('forkA') + const sent = session({ id: 'landing-sent', name: 'session-911', workspace_id: 'forkA' }) + // Sole candidate, so `undefined` pins the exclusion: `not.toBe` would also + // pass on any unrelated session the shared module state happens to hold. + const others = sessionState.sessions + sessionState.sessions = [sent] + try { + expect(findEmptyLandingSession()).toBeUndefined() + } finally { + sessionState.sessions = others + restore() + } + }) +}) + +describe('withOpenSessionTeardown — the gate that holds recovery off during a delete', () => { + it('stays shut until the outermost teardown finishes', async () => { + let innerDone = false + await withOpenSessionTeardown(async () => { + await withOpenSessionTeardown(async () => {}) + innerDone = true + expect(isTearingDownOpenSession()).toBe(true) + }) + expect(innerDone).toBe(true) + expect(isTearingDownOpenSession()).toBe(false) + }) + + it('reopens when the teardown throws, so a failed delete cannot wedge recovery shut', async () => { + await expect( + withOpenSessionTeardown(async () => { + throw new Error('fork deletion failed') + }) + ).rejects.toThrow('fork deletion failed') + expect(isTearingDownOpenSession()).toBe(false) + }) +}) diff --git a/frontend/src/lib/components/sessions/sessionSwitch.svelte.ts b/frontend/src/lib/components/sessions/sessionSwitch.svelte.ts index 0d30088767..aded390394 100644 --- a/frontend/src/lib/components/sessions/sessionSwitch.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionSwitch.svelte.ts @@ -83,7 +83,11 @@ export async function openEditorInSession( previewParams?: Record, opts?: { seedPrompt?: string; autoSend?: boolean } ): Promise { - await openInSession(withPreviewParams(sessionTargetHref(target), previewParams), workspaceId, opts) + await openInSession( + withPreviewParams(sessionTargetHref(target), previewParams), + workspaceId, + opts + ) } // Open a fresh AI session showing a workspace page (Runs, a trigger list) in its diff --git a/frontend/src/lib/components/settings/PremiumInfo.svelte b/frontend/src/lib/components/settings/PremiumInfo.svelte index a1036a0ff6..42bfef9d65 100644 --- a/frontend/src/lib/components/settings/PremiumInfo.svelte +++ b/frontend/src/lib/components/settings/PremiumInfo.svelte @@ -1,11 +1,11 @@ @@ -154,17 +145,15 @@
    {#snippet actions()} - - - + {/snippet} @@ -260,8 +249,8 @@
    Developers

    - Calculated on the MAXIMUM number of users in a given billing - period, see the Customer Portal for more info. + Calculated on the MAXIMUM number of users in a given billing period, see the + Customer Portal for more info.

    @@ -276,8 +265,8 @@
    Operators

    - Calculated on the MAXIMUM number of operators in a given - billing period, see the Customer Portal for more info. + Calculated on the MAXIMUM number of operators in a given billing period, see + the Customer Portal for more info.

    @@ -295,8 +284,9 @@ 1 developer = 1 seat, 2 operators = 1 seat.

    - u = ceil({formatNumber(premiumInfo.developerNb)} + {formatNumber(premiumInfo.operatorNb)}/2) - = {formatNumber(premiumInfo.seatsFromUsers)} + u = ceil({formatNumber(premiumInfo.developerNb)} + {formatNumber( + premiumInfo.operatorNb + )}/2) = {formatNumber(premiumInfo.seatsFromUsers)}

    @@ -311,9 +301,8 @@
    Executions this month

    - One execution equals one job - up to 1 second on a worker with 2GB of memory, with each additional - second counting as an extra execution. + One execution equals one job up to 1 second on a worker with 2GB of memory, + with each additional second counting as an extra execution.

    @@ -365,8 +354,8 @@ Used seats (billed)

    - Highest between seats from 'Developers + Operators' and 'Seats from executions'. - This is the number of seats used for billing this month. + Highest between seats from 'Developers + Operators' and 'Seats from + executions'. This is the number of seats used for billing this month.

    u + c = {formatNumber(premiumInfo.usedSeats)} @@ -398,8 +387,8 @@

    Estimate your monthly cost

    - This is a rough estimate based on your expected team size and workload. Actual billing is based - on the maximum number of users and executions in a given month. + This is a rough estimate based on your expected team size and workload. Actual billing is + based on the maximum number of users and executions in a given month.

    @@ -420,13 +409,13 @@
    Operators
    - {estimatedOps} operator{estimatedOps === 1 ? '' : 's'} + {estimatedOps} operator{estimatedOps === 1 + ? '' + : 's'}
    -

    - 2 operators = 1 seat -

    +

    2 operators = 1 seat

    @@ -434,8 +423,8 @@
    Monthly executions
    - One execution equals one job up to 1 second on a virtual CPU with 2 GB of memory, with - each additional second counting as an extra execution. + One execution equals one job up to 1 second on a virtual CPU with 2 GB of memory, + with each additional second counting as an extra execution.
    @@ -449,9 +438,7 @@ format={(v) => `${v * 10}k`} hideInput /> -

    - Each seat includes 10k executions per month. -

    +

    Each seat includes 10k executions per month.

    @@ -558,8 +545,8 @@
  • Every seat includes 10 000 executions - One execution equals one job up to 1 second on a virtual CPU with 2 GB of memory, with - each additional second counting as an extra execution. + One execution equals one job up to 1 second on a virtual CPU with 2 GB of memory, + with each additional second counting as an extra execution.
    • {:else} @@ -586,9 +573,7 @@ {/if} {:else} -
    - Workspace is on the team plan -
    +
    Workspace is on the team plan
    {/if} {:else if planTitle == 'Enterprise'} {#if plan != 'enterprise'} @@ -601,9 +586,7 @@ See more {:else} -
    - Workspace is on enterprise plan -
    +
    Workspace is on enterprise plan
    {/if} {:else if planTitle === 'Free'} {#if plan} @@ -611,9 +594,7 @@ Cancel your plan in the Customer Portal to downgrade to the free plan {:else} -
    - Workspace is on the free plan -
    +
    Workspace is on the free plan
    {/if} {/if} diff --git a/frontend/src/lib/components/sidebar/SidebarUsage.svelte b/frontend/src/lib/components/sidebar/SidebarUsage.svelte index e62cd02565..1625ed83e2 100644 --- a/frontend/src/lib/components/sidebar/SidebarUsage.svelte +++ b/frontend/src/lib/components/sidebar/SidebarUsage.svelte @@ -2,21 +2,18 @@ import { resource } from 'runed' import { goto } from '$lib/navigation' import { isCloudHosted } from '$lib/cloud' - import { UserService } from '$lib/gen' + import { WorkspaceService } from '$lib/gen' import { isPremiumStore, usageStore, userStore, - userWorkspaces, workspaceMembershipVersion, workspaceStore, - workspaceUsageStore, - type UserWorkspace + workspaceUsageStore } from '$lib/stores' import { refreshExecutions } from '$lib/usage.svelte' import { logFeatureUsage } from '$lib/utils/featureUsage' import { scopedValue, tagged } from '$lib/utils/scopedValue' - import { findWorkspaceAncestors } from '$lib/utils/workspaceHierarchy' import { Button } from '$lib/components/common' import Modal from '$lib/components/common/modal/Modal.svelte' import { Tooltip } from '$lib/components/meltComponents' @@ -30,51 +27,30 @@ let open = $state(false) - // A fork's usage and tier resolve to its billing root while its member list is a - // subset of the root's, so seats must come from the root or the cap is fork-sized - // against root usage. `undefined` when the root isn't visible from here: the cap - // is then unknowable, and the caller hides the meter rather than guessing. - function billingRoot(workspace: string, all: UserWorkspace[]): string | undefined { - const self = all.find((w) => w.id === workspace) - if (!self) return undefined - if (!self.parent_workspace_id) return workspace - const top = findWorkspaceAncestors(workspace, all).at(-1) - return top && !top.parent_workspace_id ? top.id : undefined - } + // Seat count for a paid workspace, the basis of its included executions. The server + // resolves a fork to the workspace its plan is billed on and counts the seats there, + // because neither is answerable from here: a fork's member list is a subset of that + // root's, and a fork member need not be a member of the root at all. + const fetchSeats = tagged( + async (workspace: string) => (await WorkspaceService.getBillableSeats({ workspace })).seats + ) - // Seat count for a paid workspace, the basis of its included executions. Only the - // user list is needed: `premium_info` carries the same usage number as - // `workspaceUsageStore` but requires admin and only exists when Stripe is - // configured, so it would leave regular members with no block at all. - const fetchSeats = tagged(async (root: string) => { - // Throws for a fork member with no seat in the root, which is the same answer as - // an unresolvable root: leave the paid meter hidden. - const users = await UserService.listUsers({ workspace: root }) - // Same basis as the backend's `count_paid_seats`: disabled members and service - // accounts are not billed, so counting them inflates the cap and hides a real - // overage. 1 developer = 1 seat, 2 operators = 1 seat. - const billable = users.filter((u) => !u.disabled && !u.is_service_account) - const developers = billable.filter((u) => !u.operator).length - const operators = billable.length - developers - return Math.ceil(developers + operators / 2) - }) - - const billingRootId = $derived.by(() => { - const workspace = $workspaceStore - if (!isCloudHosted() || !$isPremiumStore || !workspace) return undefined - return billingRoot(workspace, $userWorkspaces ?? []) - }) + const meteredWorkspace = $derived( + isCloudHosted() && $isPremiumStore ? $workspaceStore : undefined + ) // The membership version is in the key so a change re-resolves the cap, but not in // the tag: tagging by it would blank the bar on every change. const seatsResource = resource( () => - billingRootId ? { root: billingRootId, version: $workspaceMembershipVersion } : undefined, - async (key) => (key ? await fetchSeats(key.root) : undefined) + meteredWorkspace + ? { workspace: meteredWorkspace, version: $workspaceMembershipVersion } + : undefined, + async (key) => (key ? await fetchSeats(key.workspace) : undefined) ) const scopedSeats = scopedValue() - const seats = $derived(scopedSeats(billingRootId, seatsResource.current)) + const seats = $derived(scopedSeats(meteredWorkspace, seatsResource.current)) type QuotaKey = 'user' | 'workspace' diff --git a/frontend/src/lib/components/text_input/TextInput.svelte b/frontend/src/lib/components/text_input/TextInput.svelte index c57f57120a..03fcedbd8e 100644 --- a/frontend/src/lib/components/text_input/TextInput.svelte +++ b/frontend/src/lib/components/text_input/TextInput.svelte @@ -81,6 +81,11 @@ size?: ButtonType.UnifiedSize unifiedHeight?: boolean underlyingInputEl?: UnderlyingInputElT + /** + * Passed to the `autosize` action on the `textarea` variant. Chiefly `minHeight: 0`, for a + * field that hugs one line instead of reserving the action's 30px floor. + */ + autosizeParams?: import('$lib/autosize').AutosizeParams } export function focus() { @@ -108,7 +113,8 @@ error, size = 'md', unifiedHeight = true, - underlyingInputEl: _underlyingInputEl + underlyingInputEl: _underlyingInputEl, + autosizeParams }: Props = $props() let underlyingInputEl = $derived(_underlyingInputEl ?? ('input' as const)) @@ -152,7 +158,7 @@ onpointerdown={(e) => e.stopImmediatePropagation()} bind:this={inputEl} bind:value - use:autosize + use:autosize={autosizeParams} > {:else if underlyingInputEl === 'input'} {#if step.substeps?.length} -
    - +
    +
    {/if}
    diff --git a/frontend/src/lib/components/workspaceSettings/AddDataTableWizard.svelte b/frontend/src/lib/components/workspaceSettings/AddDataTableWizard.svelte index 2e19cbea82..3495bdbee0 100644 --- a/frontend/src/lib/components/workspaceSettings/AddDataTableWizard.svelte +++ b/frontend/src/lib/components/workspaceSettings/AddDataTableWizard.svelte @@ -21,6 +21,8 @@ FolderService, OauthService, ResourceService, + UserService, + type User, VariableService, WorkspaceService } from '$lib/gen' @@ -85,6 +87,38 @@ customInstanceDbs: ResourceReturn confirmationModal: ConfirmationModalHandle defaultInstanceDbName: () => string + /** Name to open with, when the caller needs a table of a particular name rather + * than whatever the user picks — the import wizard configures the one a project's + * migrations target. + * + * Locked when `onFinishAlso` is also given, because that work targets this name and + * nothing carries an edit through to it: renaming `main` to `other` would create + * `other`, then run the migrations against `main`, fail, and leave a data table + * nobody asked for. Editable without one, where the name is only a name. */ + initialName?: string + /** Where the dialog portals to. `#content` is the app shell's scroll container, + * which only exists inside the `(logged)` layout; a page reparented out of it + * (the hub import wizard) has to say `body` or the portal finds nothing and the + * dialog never appears. */ + modalTarget?: string + /** What the caller does once the table exists, named on the final button so the + * user is told before pressing it — the import wizard runs the project's + * migrations, which is otherwise invisible until it has already happened. */ + finishAlso?: string + /** The work `finishAlso` names. Run as the last checklist step, so it reports + * where the rest of the run does instead of starting after the dialog closes. + * Throwing marks that step failed; the table itself is already made either way. */ + onFinishAlso?: () => Promise + /** The workspace everything here is created in and checked against. + * + * Defaults to `$workspaceStore`, which is right for the settings page — it is the + * workspace being looked at. The import wizard is the exception: its page is + * reparented out of `(logged)`, so nothing re-runs the layout's workspace + * persistence, and after a reload the store still names whatever workspace the + * user came from while the plan in the URL names the destination. Left ambient, + * this would create the data table in one workspace and run the project's + * migrations in the other. */ + workspace?: string } let { @@ -95,9 +129,62 @@ onDone, customInstanceDbs, confirmationModal, - defaultInstanceDbName + defaultInstanceDbName, + initialName, + modalTarget = '#content', + finishAlso, + onFinishAlso, + workspace: workspaceProp }: Props = $props() + /** + * The caller needs this exact table, and has follow-up work bound to its name. + * + * Captured when the dialog opens rather than read live: `initialName` is the caller's + * `wizardFor`, which it clears from `onDone` — and that fires after a *failed* run too, + * while the dialog stays up offering Back. Reading it live releases the lock exactly when + * the user is most likely to edit, which is the divergence the lock exists to stop. + */ + let nameLocked = $state(false) + + /** Every write and every check goes through this, never `$workspaceStore` directly. */ + const targetWorkspace = $derived(workspaceProp ?? $workspaceStore ?? '') + + /** + * Who the caller is *in the destination*, which is not who `$userStore` describes. + * + * `$userStore` is the membership of the workspace the app is in. Routing the API calls + * elsewhere without routing this leaves the username behind: after a reload on the import + * wizard's step 4 it names the workspace the user came from, and a resource path built + * from it lands on `u/` inside the destination — failing an ownership check, + * or for an admin, quietly putting database credentials in another member's namespace. + */ + let targetUser = $state(undefined) + const aimedElsewhere = $derived(!!workspaceProp && workspaceProp !== $workspaceStore) + const ambientUsername = $derived($userStore?.username ?? '') + const targetUsername = $derived(aimedElsewhere ? (targetUser?.username ?? '') : ambientUsername) + /** The destination's membership could not be read, so nothing here knows who the user is. */ + let membershipFailed = $state(false) + + async function loadTargetUser(): Promise { + const ws = workspaceProp + if (!ws || ws === $workspaceStore) { + targetUser = undefined + membershipFailed = false + return + } + try { + targetUser = await UserService.whoami({ workspace: ws }) + membershipFailed = false + } catch { + // Recorded rather than swallowed: an unknown username silently becomes `admin` in + // the default path, which is the wrong namespace to write credentials into. Setup + // is blocked instead. + targetUser = undefined + membershipFailed = true + } + } + const STEPS = ['Choose a database', 'Set it up', 'Review'] let wiz: WizardState = $state( @@ -146,7 +233,7 @@ } clearTimeout(variableCheck) variableCheck = setTimeout(async () => { - const taken = await VariableService.existsVariable({ workspace: $workspaceStore!, path }) + const taken = await VariableService.existsVariable({ workspace: targetWorkspace, path }) // Two checks can be in flight at once and resolve out of order. A `false` for a path // nobody is on any more would clear the error guarding the one about to be written; // a `true` would disable Finish over a path this run stopped caring about. @@ -163,7 +250,7 @@ * in flight when Finish is pressed. */ async function pathConflictMessage(path: string): Promise { - const workspace = $workspaceStore! + const workspace = targetWorkspace // Each namespace answers to its own claim. Holding the secret says nothing about who owns // the resource beside it, so one claim must not wave the other's check through. const [variable, resource] = await Promise.all([ @@ -178,11 +265,14 @@ let maxStep = $state(1) function defaultProjectName(): string { - return `windmill-${$workspaceStore ?? 'workspace'}` + return `windmill-${targetWorkspace || 'workspace'}` } function defaultTableName(): string { - return existingNames.includes('main') ? `${$workspaceStore ?? 'data'}_datatable` : 'main' + // A caller that needs a specific name wins over the usual "main, unless taken": + // the import wizard's migrations only apply to a table of the name they target. + if (initialName) return initialName + return existingNames.includes('main') ? `${targetWorkspace || 'data'}_datatable` : 'main' } // Takes the list rather than reading it, so the fetch that loads it can seed off its own @@ -190,7 +280,7 @@ function defaultFolder(list: string[] = folders): string { // The first folder this admin can write to, so the resource lands somewhere the team // can find and repair. A workspace with no folders falls back to the personal space. - return list.length ? `f/${list[0]}` : `u/${$userStore?.username ?? 'admin'}` + return list.length ? `f/${list[0]}` : `u/${targetUsername || 'admin'}` } // A row this run wrote and could not take back out is still its own: `removeRow` reports @@ -236,7 +326,7 @@ ) const pgResources = resource( - () => (opened && wiz.provider === 'resource' ? ($workspaceStore ?? '') : ''), + () => (opened && wiz.provider === 'resource' ? targetWorkspace : ''), async (workspace) => { if (!workspace) return undefined const list = await ResourceService.listResource({ workspace, resourceType: 'postgresql' }) @@ -330,7 +420,7 @@ ) const folderNames = resource( - () => (opened ? ($workspaceStore ?? '') : ''), + () => (opened ? targetWorkspace : ''), async (workspace) => { if (!workspace) return [] const all = await FolderService.listFolderNames({ workspace }) @@ -374,7 +464,11 @@ let resumedPath = $state(undefined) function reset(from: WizardResume | undefined) { + nameLocked = !!initialName && !!onFinishAlso resumedPath = from?.resourcePath + // A pending confirmation that never settled leaves `dismissing` true, and `finally` + // cannot clear what never resolves — so a fresh open always starts dismissable. + dismissing = false wiz = newWizardState({ name: from?.name || defaultTableName(), projectName: from?.projectName || defaultProjectName(), @@ -391,6 +485,7 @@ createdProjects = [] nameConflictFor = undefined lastFailure = '' + finishAlsoFailed = false pathTakenError = '' poolerUnavailable = undefined if (from) { @@ -424,7 +519,11 @@ * what keeps the restore independent of when the prop it was assigned to reaches this * component. */ - export function open(parked?: WizardResume) { + export async function open(parked?: WizardResume) { + // Awaited before `reset`, which seeds the resource path from `defaultFolder()` and so + // needs the destination's username. Seeding first and correcting later loses whenever + // the folder list resolves first, and never corrects at all if `whoami` fails. + await loadTargetUser() reset(parked ?? resume) opened = true } @@ -472,12 +571,14 @@ // Also retires any check still in flight, so its answer cannot land on the edited value. probeToken++ clearProbe(wiz) - // Read off one attempt against one project; the review step would otherwise warn about - // a limitation that no longer applies while claiming session pooling right above it. + // Read off one attempt against one project, so it does not survive a change of inputs: + // the review step would otherwise warn about a limitation that does not apply to what + // it is describing, while claiming session pooling right above it. poolerUnavailable = undefined // Same for the failure carried back to the review step: it names inputs that have since // been edited, so it would describe a run nobody can still act on. lastFailure = '' + finishAlsoFailed = false if (maxStep > wiz.step) maxStep = wiz.step } @@ -527,7 +628,7 @@ settle({ checking: false, report: undefined, error: undefined }) return } - const report = await probeDatatableConnection($workspaceStore!, database) + const report = await probeDatatableConnection(targetWorkspace, database) settle({ checking: false, report, error: undefined }) } catch (err: any) { settle({ @@ -579,6 +680,13 @@ ) /** Why the last run failed, kept on the review step after the checklist is dropped. */ let lastFailure = $state('') + /** + * The appended `onFinishAlso` step failed while `runSetup` itself succeeded. Tracked apart + * from `run.result`, which stays the setup's own verdict: the data table really was + * created, so a retry must re-run only this last step. Re-running the setup would ask for + * the table name it has just taken, and be refused as a duplicate. + */ + let finishAlsoFailed = $state(false) /** * A refused pre-flight means nothing ran, so the checklist from a previous attempt has to @@ -631,7 +739,7 @@ const name = wiz.review.name.trim() try { if (claimedName !== name) { - const settings = await WorkspaceService.getSettings({ workspace: $workspaceStore! }) + const settings = await WorkspaceService.getSettings({ workspace: targetWorkspace }) if (settings.datatable?.datatables?.[name]) { nameConflictFor = { name, @@ -666,6 +774,7 @@ } run = { steps: planSteps(wiz), running: true } lastFailure = '' + finishAlsoFailed = false // The database is registered by the call whatever it answers, so asking for one is // already leaving something behind. if (wiz.provider === 'instance' && wiz.instance.mode === 'create') { @@ -675,7 +784,7 @@ let result: RunResult | undefined = undefined try { result = await runSetup(wiz, { - workspace: $workspaceStore!, + workspace: targetWorkspace, supabaseToken: supaOauth.token, onInstanceDbsChanged: async () => { await customInstanceDbs.refetch() @@ -684,9 +793,27 @@ onPoolerUnavailable: (reason) => (poolerUnavailable = reason), createdProjects, claims, - username: $userStore?.username ?? '' + username: targetUsername }) } finally { + // The caller's own finishing work, appended to the same checklist. It only runs + // on a clean setup: there is no table for it to act on otherwise. + if (result?.ok && onFinishAlso && finishAlso) { + const title = finishAlso.charAt(0).toUpperCase() + finishAlso.slice(1) + run.steps = [...run.steps, { title, status: 'running' }] + try { + await onFinishAlso() + run.steps = run.steps.map((s, i) => + i === run.steps.length - 1 ? { ...s, status: 'done' as const } : s + ) + } catch (err: any) { + const description = err?.body ?? err?.message ?? String(err) + run.steps = run.steps.map((s, i) => + i === run.steps.length - 1 ? { ...s, status: 'failed' as const, description } : s + ) + finishAlsoFailed = true + } + } // `runSetup` catches per step, but anything escaping it would otherwise leave the // button spinning with a page reload the only way out. // Kept, not replaced: what an earlier attempt wrote is still out there, so a later @@ -713,7 +840,10 @@ /** * Whether closing would throw away work. A failed run counts: its inputs are still editable * and it may have left something behind. A run in flight cannot be closed at all, and one - * that succeeded has nothing left to lose. + * that made its data table has nothing left to lose — including when `onFinishAlso` failed + * afterwards, because the table is real and working and the caller owns what is left. The + * import step, the only caller that passes one, shows that failure on its own row with a + * way to run it again and will not let Finish through while it stands. */ function hasUnfinishedIntent(): boolean { return wiz.provider !== undefined && !run.running && !run.result?.ok @@ -730,19 +860,52 @@ return } dismissing = true - const confirmed = await confirmationModal.ask({ - title: 'Leave without adding a data table?', - // A run that failed and was sent back to be edited leaves whatever it got through - // behind it, so promising otherwise would be a lie exactly when it matters most. - children: leftBehind - ? 'The setup that already ran left what it created behind, and what you have filled in here will be lost.' - : 'Nothing has been created yet, and what you have filled in here will be lost.', - confirmationText: 'Discard' - }) - dismissing = false - // Re-read rather than trust the entry check: a run can start while the dialog is up, and - // answering Discard would otherwise tear the modal down in the middle of it. - if (confirmed && !preventClose) close() + // `finally`, because the flag is what blocks a second attempt: an `ask` that throws + // would otherwise leave the dialog permanently undismissable — the backdrop, Escape + // and the close button all return early here, so the only way out would be a reload. + try { + const confirmed = await confirmationModal.ask({ + title: 'Leave without adding a data table?', + // A run that failed and was sent back to be edited leaves whatever it got through + // behind it, so promising otherwise would be a lie exactly when it matters most. + children: leftBehind + ? 'The setup that already ran left what it created behind, and what you have filled in here will be lost.' + : 'Nothing has been created yet, and what you have filled in here will be lost.', + confirmationText: 'Discard' + }) + // Re-read rather than trust the entry check: a run can start while the dialog is up, and + // answering Discard would otherwise tear the modal down in the middle of it. + if (confirmed && !preventClose) close() + } finally { + dismissing = false + } + } + + /** Re-runs only the appended step, which is the only thing that failed. */ + async function retryFinishAlso() { + if (!onFinishAlso || !finishAlso) return + const title = finishAlso.charAt(0).toUpperCase() + finishAlso.slice(1) + finishAlsoFailed = false + run = { + ...run, + running: true, + steps: [...run.steps.slice(0, -1), { title, status: 'running' as const }] + } + try { + await onFinishAlso() + run.steps = run.steps.map((s, i) => + i === run.steps.length - 1 ? { ...s, status: 'done' as const } : s + ) + } catch (err: any) { + const description = err?.body ?? err?.message ?? String(err) + run.steps = run.steps.map((s, i) => + i === run.steps.length - 1 ? { ...s, status: 'failed' as const, description } : s + ) + finishAlsoFailed = true + } finally { + run = { ...run, running: false } + onDone() + } } function close() { @@ -752,10 +915,18 @@ // The single primary action. Its label says what it is about to do, and doing it is what // moves the wizard on. let primary = $derived.by(() => { + // Ahead of everything: without the destination's membership the resource path would be + // guessed, and a guess here writes database credentials into somebody else's namespace. + if (aimedElsewhere && membershipFailed) + return { label: 'Cannot read your access to this workspace', disabled: true } if (submitting && !run.running) return { label: 'Setting things up', disabled: true, busy: true } if (run.steps.length) { if (run.running) return { label: 'Setting things up', disabled: true, busy: true } + // Before the `ok` check: the setup succeeded and the step after it did not, so + // "Done" would be offered over a failed row. + if (finishAlsoFailed) + return { label: 'Try again', disabled: false, act: retryFinishAlso } if (run.result?.ok) return { label: 'Done', disabled: false, act: close } // A run that died because the Supabase token expired would retry into the same 401 // forever; authorizing again is the only thing that can move it on. @@ -809,11 +980,12 @@ act: enterReview } } + const created = + wiz.provider === 'supabase' && wiz.supabase.mode === 'create' + ? 'Create project and data table' + : 'Create data table' return { - label: - wiz.provider === 'supabase' && wiz.supabase.mode === 'create' - ? 'Create project and data table' - : 'Create data table', + label: finishAlso ? `${created} and ${finishAlso}` : created, disabled: // Guards the way back as well as the way forward: the stepper can return to step 2, // and not every control there invalidates the review it just made stale. @@ -842,7 +1014,7 @@ else opened = v } } - target="#content" + target={modalTarget} formStyling title="Add a data table" contentClasses="flex flex-col" @@ -1034,7 +1206,7 @@ {#if wiz.instance.mode === 'existing'} {@const shared = ( customInstanceDbs.current?.[wiz.instance.dbName ?? '']?.used_by_workspaces ?? [] - ).filter((w) => w !== $workspaceStore)} + ).filter((w) => w !== targetWorkspace)} {#if shared.length} @@ -1047,7 +1219,7 @@
    {#each instanceDbs as { name, db } (name)} {@const selected = wiz.instance.dbName === name} - {@const others = (db.used_by_workspaces ?? []).filter((w) => w !== $workspaceStore)} + {@const others = (db.used_by_workspaces ?? []).filter((w) => w !== targetWorkspace)} {:else if s.phase === 'draft'} + {/if} + {/if} + {#if s.liveOnHub && s.phase === 'draft'} + {/if}
    @@ -270,8 +319,10 @@ {#if s.phase === 'predeploy'}
    - Bundling creates a draft project on the Hub from the selected scripts, flows and - apps of {s.selectedFolder}/. + Bundling creates {s.liveOnHub + ? 'an update to your Hub project' + : 'a draft project on the Hub'} from the selected scripts, flows and apps of + {s.selectedFolder}/. {s.selectedItems.length} of {s.filteredWorkspaceItems.length} items selected.
    @@ -351,6 +402,31 @@ {/if} {/if} + {#if s.liveOnHub && s.phase !== 'live' && s.phase !== 'under_review'} + + Visitors keep seeing the published version, with its stars, forks and comments, + until this update is approved. Approving replaces it in place; discarding leaves it + exactly as it is. + + {/if} + {#if s.pipelineReplayMayBeStale && s.phase === 'draft'} + + This update carries the cascade recorded for the version that is live, and at least + one item has changed since. Record it again below, or visitors will replay the old + run as though it were this version. + + {/if} + {#if s.rejectionReason && s.phase === 'draft'} + + {s.rejectionReason} + + {/if} {#if s.phase === 'draft'}
    @@ -477,18 +553,21 @@
    {/if} {#if s.phase === 'under_review'} -
    - -
    - Locked while under review - - The Windmill team is reviewing this submission. Editing, recording, and sharing - actions are disabled. Estimated turnaround: 1-2 business days. - -
    -
    + The Windmill team is reviewing your project. Submission is locked until they answer + — no new version can be sent to the Hub, and no recording added to this one. + Estimated turnaround: 1-2 business days{#if s.hubSupportsUpdates}; cancel the + submission to get back to it sooner{/if}.{#if s.liveOnHub} + Visitors keep seeing the published version meanwhile, with its stars, forks and + comments; approving replaces it in place.{/if} Your folder itself is untouched — keep + editing your scripts and flows as usual. + {/if} {#if s.phase === 'draft'} {@const recordedCount = s.recordableItems.filter((i) => i.rec === 'recorded').length} @@ -668,7 +747,11 @@ Waiting for the Windmill team to review the submission. {:else} - Iterate further by starting a new draft. + + {s.liveOnHub + ? 'Publish an update to change it — this stays live until the update is approved.' + : 'Iterate further by starting a new draft.'} + {/if} {/snippet} diff --git a/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte b/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte index 1e6c990819..a2ccf0b958 100644 --- a/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte @@ -44,6 +44,14 @@ onDiscard?: () => void } = $props() + const creatableStorageTypes = [ + { value: 's3', label: 'S3' }, + { value: 'azure_blob', label: 'Azure Blob' }, + { value: 's3_aws_oidc', label: 'AWS OIDC' }, + { value: 'azure_workload_identity', label: 'Azure Workload Identity' }, + { value: 'gcloud_storage', label: 'Google Cloud Storage' } + ] + let advancedPermissionModalState: | { open: false } | { open: true; storage: S3ResourceSettingsItem } = $state({ open: false }) @@ -294,31 +302,38 @@
    - {#if tableRow[1].resourceType === 'filesystem'} - +
    + - {/if} + {#if tableRow[1].resourceType === 'filesystem'} + + Filesystem storage points the workspace at a directory on the server's own + disk. Only development builds of Windmill accept it — switch this storage to + S3, Azure Blob or Google Cloud Storage to configure it here. + + {/if} +
    {#if tableRow[1].resourceType === 'filesystem'} diff --git a/frontend/src/lib/components/workspaceSettings/deployToHubSession.svelte.ts b/frontend/src/lib/components/workspaceSettings/deployToHubSession.svelte.ts index 5cc7f0d1d7..c421553a7c 100644 --- a/frontend/src/lib/components/workspaceSettings/deployToHubSession.svelte.ts +++ b/frontend/src/lib/components/workspaceSettings/deployToHubSession.svelte.ts @@ -198,8 +198,31 @@ export class DeployToHubSession { // Whether the Hub currently has a custom logo for this project (from // rehydration) — drives the "Remove current logo" affordance. hubHasRemoteLogo = $state(false) + // A pipeline recording is attached on the Hub. An update inherits the published + // one, which is only a demo of the new version if nothing it runs changed. + hubHasPipelineRecording = $state(false) + // The Hub's own verdict: this update runs different content from the published + // version. False when there is no update in flight. + hubItemsChanged = $state(false) + // The attached pipeline recording is the published version's, copied when this + // update started, rather than one recorded for it. Authoritative across reloads, + // unlike `pipelineRecorded`, which only remembers this session. + hubPipelineRecordingInherited = $state(false) effectiveSlug = $state('') hubItemIds = $state>({}) + // Set once the project is published: everything the wizard shows from here on + // describes an update to it, and the published version keeps serving until that + // update is approved. `phase` is the update's own status, not the project's. + liveOnHub = $state(false) + /** This Hub knows about pending updates — it answers rehydration with a `live` + * key. An older one takes a project offline to republish and has neither the + * withdraw nor the discard endpoint, so the actions built on them stay hidden. */ + hubSupportsUpdates = $state(false) + // A reviewer's verdict on the current draft, shown so the publisher knows what + // to fix before resubmitting. + rejectionReason = $state(undefined) + discardingUpdate = $state(false) + withdrawing = $state(false) // Best-effort data table migrations for the bundle, editable in the drawer and // pushed on deploy. Regenerated when the bundle drawer opens. @@ -228,6 +251,10 @@ export class DeployToHubSession { submitting = $state(false) syncing = $state(false) + // Set from the Hub's answer to the draft request: this push went into an update + // rather than over the published project. + #publishedAsUpdate = false + // Intra-session tokens: latest call wins among competing calls on this session. #triggerLoadTok = 0 #recordRunTok = 0 @@ -316,6 +343,18 @@ export class DeployToHubSession { ) pipelineScriptPathSet = $derived(new Set(this.pipelineScriptPaths)) isPipelineProject = $derived(this.pipelineScriptPaths.length > 0) + /** The pipeline replay this update carries came from the published version, and + * something it runs has changed since — so it is a recording of another version. + * `hubItemsChanged` is the Hub comparing content, not a guess from which items + * carry recordings: an item nobody ever recorded has not changed. */ + pipelineReplayMayBeStale = $derived( + this.liveOnHub && + this.isPipelineProject && + this.hubHasPipelineRecording && + this.hubPipelineRecordingInherited && + this.hubItemsChanged && + !this.pipelineRecorded + ) hubSlug = $derived(this.effectiveSlug || sanitizeSlug(this.hubName)) relevantTriggers = $derived.by(() => { @@ -573,6 +612,17 @@ export class DeployToHubSession { this.hubSummary = p.summary ?? '' this.hubReadme = p.readme ?? '' this.hubHasRemoteLogo = p.has_logo === true + this.hubHasPipelineRecording = p.has_pipeline_recording === true + this.hubItemsChanged = p.items_changed === true + this.hubPipelineRecordingInherited = p.pipeline_recording_inherited === true + this.rejectionReason = p.rejection_reason ?? undefined + // `live` is a key this Hub always sends — null unless an update is in + // flight, in which case the fields above describe that update and the + // project itself is still published. Its absence means a Hub old enough to + // still take a project offline while it re-publishes, so the wizard must + // not promise otherwise. + this.hubSupportsUpdates = 'live' in p + this.liveOnHub = this.hubSupportsUpdates && (p.live?.approved === true || p.status === 'live') this.phase = p.status === 'live' ? 'live' : p.status === 'under_review' ? 'under_review' : 'draft' const ids: Record = {} @@ -898,6 +948,9 @@ export class DeployToHubSession { try { const parsed = JSON.parse(text) if (typeof parsed?.slug === 'string') returnedSlug = parsed.slug + // The Hub decides this: publishing over an approved project goes into a + // pending update instead, and the project keeps serving meanwhile. + this.#publishedAsUpdate = parsed?.pending_revision === true } catch {} if (!returnedSlug) { sendUserToast(`Hub did not return a slug. Aborting publish to avoid path drift.`, true) @@ -1252,8 +1305,13 @@ export class DeployToHubSession { // UI stuck in `predeploy`; rehydrate then upgrades to authoritative state. this.draftItems = itemsSnapshot.map((i) => ({ ...i, rec: 'none' })) this.phase = 'draft' + const asUpdate = this.#publishedAsUpdate await this.rehydrateFromHub() - sendUserToast(`Draft created on the Hub. Add recordings before submitting for review.`) + sendUserToast( + asUpdate + ? `Update ready on the Hub. Your published project stays live until it is approved.` + : `Draft created on the Hub. Add recordings before submitting for review.` + ) } finally { this.deploying = false } @@ -1313,12 +1371,82 @@ export class DeployToHubSession { } } + /** Go back to picking items, to publish again. Local only — nothing reaches the + * Hub until the bundle is confirmed, and where the Hub supports updates the + * published version keeps serving even then. */ startNewDraft = () => { this.draftItems = [] this.recordings = {} + this.rejectionReason = undefined + // All of it belongs to the update just finished, not the one starting. The + // captured cascade especially: left in place, the next update could save a + // replay of the version it replaces. Bumping the token first abandons a run + // still in flight, which would otherwise write its result back over this. + this.#pipelineRunTok++ + this.pipelineRecorded = false + this.pipelineRecordingResult = undefined + this.pipelineRunState = 'idle' + this.pipelineRunError = undefined this.phase = 'predeploy' } + /** Take the submission back out of review. Everything pushed for it is kept, so + * it can be fixed and submitted again. */ + cancelSubmission = async () => { + if (this.withdrawing) return + const slug = this.effectiveSlug + if (!slug) return + this.withdrawing = true + try { + const res = await fetch( + `/api/w/${this.workspace}/hub/projects/${encodeURIComponent(slug)}/withdraw${this.#folderQs()}`, + { method: 'POST', credentials: 'include' } + ) + if (!res.ok) { + sendUserToast(`Could not cancel the submission: ${await res.text()}`, true) + return + } + if (this.#disposed) return + this.phase = 'draft' + await this.rehydrateFromHub() + sendUserToast(`Submission cancelled. Everything you pushed is still here.`) + } catch (e: any) { + sendUserToast(`Could not cancel the submission: ${e?.message ?? e}`, true) + } finally { + this.withdrawing = false + } + } + + /** Throw away an update in progress and go back to what is published. */ + discardUpdate = async () => { + if (this.discardingUpdate) return + const slug = this.effectiveSlug + if (!slug) return + this.discardingUpdate = true + try { + const res = await fetch( + `/api/w/${this.workspace}/hub/projects/${encodeURIComponent(slug)}/discard_update${this.#folderQs()}`, + { method: 'POST', credentials: 'include' } + ) + if (!res.ok) { + sendUserToast(`Could not discard the update: ${await res.text()}`, true) + return + } + if (this.#disposed) return + this.draftItems = [] + this.recordings = {} + this.deploymentStatus = {} + this.rejectionReason = undefined + this.phase = 'live' + await this.rehydrateFromHub() + sendUserToast(`Update discarded. The published project is unchanged.`) + } catch (e: any) { + sendUserToast(`Could not discard the update: ${e?.message ?? e}`, true) + } finally { + this.discardingUpdate = false + } + } + /** Reset record-drawer state and load the target's schema. */ async openRecord(it: DeployItem) { const tok = ++this.#recordRunTok diff --git a/frontend/src/lib/components/workspaceSettings/projectInstall.ts b/frontend/src/lib/components/workspaceSettings/projectInstall.ts index abc48a3ad6..6fd4bdd728 100644 --- a/frontend/src/lib/components/workspaceSettings/projectInstall.ts +++ b/frontend/src/lib/components/workspaceSettings/projectInstall.ts @@ -44,6 +44,11 @@ export interface InstallResult { path: string ok: boolean error?: string + /** + * Already in the destination, so nothing was written. Not a failure and not an import — + * reporting it as either would be a lie, and the difference is what a retry is for. + */ + skipped?: boolean } // Guarding an item's own path is not enough: the `$res:`/script/flow refs baked @@ -201,7 +206,11 @@ async function importApp(workspace: string, a: ExportItem): Promise { // Apply one migration to the target data table. If the data table opted into // migrations, record it (datatable_migrations + _wm_migrations, run only this // version); otherwise run the SQL once as a preview job (unrecorded). -async function applyOneMigration( +// +// Exported because the import can leave migrations unapplied: a data table the +// project needs may not be configured in the destination yet, and the wizard's setup +// step runs them once it is. +export async function applyOneMigration( workspace: string, projectSlug: string, m: ProjectMigration @@ -254,15 +263,73 @@ async function applyOneMigration( * reviewed) migrations. Each item's outcome is reported through `onResult`; * failures never abort the remaining items. */ +/** + * The kinds an import writes that carry a path and can therefore already be there. + * + * Triggers carry their own kind too (`trigger:schedule`, `trigger:http`, … — the values of + * `WorkspaceTriggerKind`): each trigger kind is a separate table keyed on + * `(path, workspace_id)`, so one workspace can hold a schedule and an HTTP trigger both + * called `f/cal/sync`. Flattening them to `trigger` would let whichever exists answer for + * the other. + */ +export type ImportedKind = + | 'script' + | 'flow' + | 'app' + | 'resource' + | `trigger:${string}` + +/** + * The key `alreadyPresent` is built and read with. Kind and path together, because the kinds + * share one path namespace and a bare path cannot say which of them is already there. + */ +export function presenceKey(kind: ImportedKind, path: string): string { + return `${kind}:${path}` +} + export async function installProject(args: { workspace: string exportData: ProjectExport folder: string migrations: ProjectMigration[] + /** Called once, before the reviewed migrations are applied, when there are any. Lets a + * caller show them as their own step rather than folding them into the item import. */ + onMigrationsStart?: () => void + /** + * Asked before each write. Returning true stops the run where it is — the writes already + * made stay, the rest never start. Nothing here can cancel a request already in flight, + * so this is the granularity available without threading an `AbortSignal` through every + * service call: the import wizard uses it when the user confirms leaving mid-run. + */ + stopped?: () => boolean + /** + * What is already in the destination, as `presenceKey` keys — so a retry writes only what + * is missing instead of replaying the bundle into a wall of "already exists". Built from + * retargeted paths, because that is what these items will actually be called. + * + * Keyed by kind and not by path alone: the five kinds share one `f//` namespace, so + * a trigger and a script may legitimately both be called `f/cal/sync`. A flat path set + * would let either one mask the other and silently skip an item that was never imported. + * + * Never a way to *replace* anything: an item that is there is left exactly as it is, + * which is the same promise `updateIfExists: false` makes for a resource whose value + * someone has since filled in. + */ + alreadyPresent?: Set hasEeLicense: boolean onResult: (r: InstallResult) => void }): Promise { - const { workspace, exportData, folder, migrations, hasEeLicense, onResult } = args + const { + workspace, + exportData, + folder, + migrations, + hasEeLicense, + onResult, + onMigrationsStart, + stopped, + alreadyPresent + } = args const record = (path: string, p: Promise): Promise => p.then( @@ -270,6 +337,20 @@ export async function installProject(args: { (e: any) => onResult({ path, ok: false, error: errorMessage(e) }) ) + /** Every write goes through here, so one check covers items, variables and migrations. */ + const halted = () => stopped?.() === true + + /** + * True when the destination already has this path, so the write is not attempted. Reported + * rather than dropped: the checklist has to account for every item the project ships, and + * "already there" is a different thing from "imported". + */ + const present = (kind: ImportedKind, path: string): boolean => { + if (!alreadyPresent?.has(presenceKey(kind, path))) return false + onResult({ path, ok: true, skipped: true }) + return true + } + try { await FolderService.createFolder({ workspace, requestBody: { name: folder } }) } catch {} @@ -309,6 +390,8 @@ export async function installProject(args: { } for (const s of proj.scripts) { + if (halted()) return + if (present('script', s.path)) continue // `$var:` is resolved in job args (flow inputs, schedule args, trigger config), // not in script source, so there is no variable arg to contain here. await checkedItem(s.path, extractScriptRefs(s.content ?? ''), undefined, () => @@ -316,19 +399,26 @@ export async function installProject(args: { ) } for (const f of proj.flows) { + if (halted()) return + if (present('flow', f.path)) continue await checkedItem(f.path, extractFlowRefs(f.value), f.value, () => importFlow(workspace, f)) } for (const r of proj.resources) { + if (halted()) return + if (present('resource', r.path)) continue await checked(r.path, () => importResourceStub(workspace, r)) } // Placeholders for the project's internal `$var:`/`$jsonvar:` refs (retargeted // into this folder). External refs are rejected per-item, so only stub in-folder // ones; guard again in case an out-of-folder ref slipped through retargeting. for (const p of collectExportVarPaths(proj)) { + if (halted()) return if (!p.startsWith(prefix)) continue await record(`variable: ${p}`, importVariablePlaceholder(workspace, p)) } for (const a of proj.apps) { + if (halted()) return + if (present('app', a.path)) continue const isRaw = a.app_type === 'raw' const refs = isRaw ? extractRawAppRefs(a.value?.raw ?? '') : extractAppRefs(a.value) // Raw apps hold their runnables in the `value.raw` JSON string; parse it so the @@ -368,6 +458,8 @@ export async function installProject(args: { return varContainmentViolation(cfg, folder) } for (const t of proj.triggers) { + if (halted()) return + if (present(`trigger:${t.kind}`, String(t.path))) continue const violation = guard(t.path, t.runnable_path) ?? triggerConfigViolation(t) await record( String(t.path), @@ -389,7 +481,10 @@ export async function installProject(args: { } // Apply the reviewed data table migrations after items exist. + if (halted()) return + if (migrations.length) onMigrationsStart?.() for (const m of migrations) { + if (halted()) return await record( `data table: ${m.datatable_name}`, applyOneMigration(workspace, exportData.project.slug, m) diff --git a/frontend/src/lib/components/workspaceSettings/projectInstallPresence.test.ts b/frontend/src/lib/components/workspaceSettings/projectInstallPresence.test.ts new file mode 100644 index 0000000000..186d581d58 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectInstallPresence.test.ts @@ -0,0 +1,106 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +/** + * A retry must not re-write what the destination already has. The interesting kind is the + * trigger: it is the one item the import creates through an API that rejects an existing + * path outright, so replaying it turns "already there" into a reported failure. + */ + +const created = vi.hoisted(() => ({ triggers: [] as string[] })) + +vi.mock('$lib/gen', () => { + const nothing = vi.fn(async () => []) + return { + AppService: { createApp: vi.fn(), listApps: nothing }, + FlowService: { createFlow: vi.fn(), listFlows: nothing }, + FolderService: { createFolder: vi.fn() }, + ResourceService: { createResource: vi.fn(), listResource: nothing }, + ScriptService: { createScript: vi.fn(), listScripts: nothing }, + VariableService: { createVariable: vi.fn(), listVariable: nothing }, + WorkspaceService: { listDataTables: vi.fn(async () => []) } + } +}) + +vi.mock('../triggers/workspaceTriggersList', () => ({ + TRIGGER_KINDS: { + schedule: { badge: 'Schedule', resourceField: undefined }, + http: { badge: 'HTTP', resourceField: undefined } + }, + createWorkspaceTriggerDisabled: vi.fn(async (_ws: string, t: { path: string; kind: string }) => { + created.triggers.push(`${t.kind}:${t.path}`) + }), + triggerHandlerRefs: () => [] +})) + +import { installProject, presenceKey } from './projectInstall' + +const exportData = { + project: { slug: 'calendly', name: 'Calendly', summary: '', readme: null }, + scripts: [], + flows: [], + apps: [], + resources: [], + variables: [], + triggers: [ + { + kind: 'schedule', + path: 'f/calendly/nightly', + runnable_path: 'f/calendly/sync', + runnable_kind: 'script', + summary: null, + config: {} + } + ], + migrations: [] +} as any + +async function run(alreadyPresent?: Set) { + const results: any[] = [] + await installProject({ + workspace: 'w', + exportData, + folder: 'calendly', + migrations: [], + hasEeLicense: true, + alreadyPresent, + onResult: (r) => results.push(r) + }) + return results +} + +describe('installProject presence', () => { + beforeEach(() => { + created.triggers = [] + }) + + it('creates a trigger the destination does not have', async () => { + const results = await run(new Set()) + expect(created.triggers).toEqual(['schedule:f/calendly/nightly']) + expect(results).toContainEqual({ path: 'f/calendly/nightly', ok: true }) + }) + + // Without the skip the retry calls the create API again, which rejects the existing + // path, and the row reads as a failure for something that is already there. + it('skips a trigger that is already there instead of re-creating it', async () => { + const results = await run(new Set([presenceKey('trigger:schedule', 'f/calendly/nightly')])) + expect(created.triggers).toEqual([]) + expect(results).toContainEqual({ path: 'f/calendly/nightly', ok: true, skipped: true }) + }) + + // Kinds share one `f//` namespace, so the key has to carry the kind: a script of + // the same name is not this trigger and must not stand in for it. + it('does not let another kind at the same path mask the trigger', async () => { + const results = await run(new Set([presenceKey('script', 'f/calendly/nightly')])) + expect(created.triggers).toEqual(['schedule:f/calendly/nightly']) + expect(results).toContainEqual({ path: 'f/calendly/nightly', ok: true }) + }) + + // Each trigger kind is its own table keyed on (path, workspace_id), so a workspace can + // hold a schedule and an HTTP trigger both called `f/calendly/nightly`. The one that + // exists must not answer for the one that does not. + it('does not let another trigger kind at the same path mask this one', async () => { + const results = await run(new Set([presenceKey('trigger:http', 'f/calendly/nightly')])) + expect(created.triggers).toEqual(['schedule:f/calendly/nightly']) + expect(results).toContainEqual({ path: 'f/calendly/nightly', ok: true }) + }) +}) diff --git a/frontend/src/lib/hubProject.ts b/frontend/src/lib/hubProject.ts new file mode 100644 index 0000000000..b58b338f7d --- /dev/null +++ b/frontend/src/lib/hubProject.ts @@ -0,0 +1,87 @@ +import type { Component } from 'svelte' +import { appIconComponent } from '$lib/components/icons' +import { SettingService } from '$lib/gen' +import { DEFAULT_HUB_BASE_URL } from '$lib/hub' +import type { ImportProjectSummary } from '$lib/components/ImportProjectCard.svelte' + +/** + * The browser-reachable hub. `hub_accessible_url` exists precisely for this: on a + * private instance `hub_base_url` may be an address only the server can resolve. + * The (logged) layout does the same lookup, but the import wizard renders outside + * it, so it has to ask for itself. + */ +export async function hubBrowserUrl(): Promise { + try { + const accessible = (await SettingService.getGlobal({ key: 'hub_accessible_url' })) as string + if (accessible) return accessible.replace(/\/+$/, '') + const base = (await SettingService.getGlobal({ key: 'hub_base_url' })) as string + if (base) return base.replace(/\/+$/, '') + } catch { + // Unset or unreadable — the public hub is the right default either way. + } + return DEFAULT_HUB_BASE_URL.replace(/\/+$/, '') +} + +/** Shape of `GET /projects/` — the hub's own summary endpoint. */ +interface HubProject { + slug: string + name: string + summary: string + author: string + apps: string[] + logoApp: string | null + hasLogo: boolean + counts: { scripts: number; flows: number; apps: number; resources: number; total: number } +} + +/** + * Fetches one project's presentation straight from the hub. Cross-origin and + * unauthenticated by design: this runs before the wizard has a workspace, so the + * workspace-scoped `/api/w//hub/...` proxy is not available yet. The endpoint + * is public and sends `Access-Control-Allow-Origin: *`. + */ +export async function fetchHubProject(slug: string): Promise { + const hub = await hubBrowserUrl() + const res = await fetch(`${hub}/projects/${encodeURIComponent(slug)}`, { + headers: { accept: 'application/json' } + }) + if (!res.ok) throw new Error(`hub returned ${res.status}`) + const p = (await res.json()) as HubProject + return { + slug: p.slug, + name: p.name, + summary: p.summary, + author: p.author, + apps: p.apps ?? [], + // A project with an uploaded logo shows that; otherwise the icon of the + // integration it is filed under, otherwise its first integration. + logoUrl: p.hasLogo ? `${hub}/projects/${encodeURIComponent(p.slug)}/logo` : undefined, + iconApps: [p.logoApp, ...(p.apps ?? [])].filter( + (a, i, all): a is string => !!a && all.indexOf(a) === i + ), + counts: { + apps: p.counts?.apps ?? 0, + flows: p.counts?.flows ?? 0, + scripts: p.counts?.scripts ?? 0, + resources: p.counts?.resources ?? 0 + } + } +} + +/** + * The icon for a hub integration slug, resolved from the icons Windmill already bundles. + * + * Not fetched from the hub: the hub renders these out of `@windmill-labs/components`, which + * is this frontend's own package, so asking it over HTTP is a round trip to get our own + * assets back — and it made the card depend on a cross-origin request that an `API_SECRET` + * hub refuses anyway. + * + * The alias exists because the two repos disagree on one slug: the hub files Postgres scripts + * under `postgres`, the icon set ships the mark as `postgresql`. The hub bridges it in + * `aliasApp`; this is the same bridge on the consuming side. + */ +const HUB_APP_ICON_ALIAS: Record = { postgres: 'postgresql' } + +export function hubAppIcon(app: string): Component | undefined { + return appIconComponent(HUB_APP_ICON_ALIAS[app] ?? app) +} diff --git a/frontend/src/lib/importWizard/abandon.test.ts b/frontend/src/lib/importWizard/abandon.test.ts new file mode 100644 index 0000000000..abf0ff61b9 --- /dev/null +++ b/frontend/src/lib/importWizard/abandon.test.ts @@ -0,0 +1,237 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +// Everything the executor reaches over the network, stubbed. The two behaviours under test +// are decisions it makes around those calls, not the calls themselves. +vi.mock('$lib/gen', () => ({ + WorkspaceService: { + createWorkspace: vi.fn(), + listDataTables: vi.fn(async () => []), + // No workspace of ours at that id: the existing-workspace plan these tests use never + // reaches the create, and an empty list is the honest answer for a fresh instance. + listWorkspaces: vi.fn(async () => []) + }, + UserService: { + whoami: vi.fn(async () => ({ username: 'u' })), + globalWhoami: vi.fn(async () => ({ email: 'u@example.com' })) + } +})) +vi.mock('$lib/storeUtils', () => ({ switchWorkspace: vi.fn() })) +// What the destination already holds. A test sets this to stand in for a workspace that has +// some of the bundle in it — a half-finished run, or an existing workspace. +const present = vi.hoisted(() => ({ + paths: new Set(), + /** Fires when the probe is entered, so a test can abandon while it is in flight. */ + onProbe: undefined as (() => void) | undefined +})) +vi.mock('./probe', async (orig) => ({ + ...(await orig()), + probeWorkspace: vi.fn(async () => ({ exists: false, ours: false })), + probeImportedPaths: vi.fn(async () => { + present.onProbe?.() + return present.paths + }) +})) +vi.mock('$lib/user', () => ({ getUserExt: vi.fn(async () => ({ username: 'u' })) })) +// Let a test abandon *during* a write loop, which is the only way it happens for real: +// `run()` clears the flag on entry so a retry can proceed. Two hooks, because the item and +// migration phases stop in different places and the second is what pins the migrate row. +const hooks = vi.hoisted(() => ({ + afterFirstItem: undefined as (() => void) | undefined, + afterMigrationsStart: undefined as (() => void) | undefined +})) + +vi.mock('$lib/components/workspaceSettings/projectInstall', () => ({ + installProject: vi.fn(async (args: any) => { + // Ordered as the real one is: every item loop, then `onMigrationsStart`, then the + // migrations — and `stopped` checked before each write, returning the same way it + // returns on success. + for (const path of ['a', 'b', 'c']) { + if (args.stopped?.() === true) return + // Keyed exactly as the real `installProject` keys it, so this stand-in cannot drift + // into testing a contract the production code does not have. + if (args.alreadyPresent?.has(`script:${path}`)) { + args.onResult({ path, ok: true, skipped: true }) + continue + } + args.onResult({ path, ok: true }) + hooks.afterFirstItem?.() + hooks.afterFirstItem = undefined + } + if (args.stopped?.() === true) return + if (args.migrations?.length) { + args.onMigrationsStart?.() + hooks.afterMigrationsStart?.() + hooks.afterMigrationsStart = undefined + for (const m of args.migrations) { + if (args.stopped?.() === true) return + args.onResult({ path: `data table: ${m.datatable_name}`, ok: true }) + } + } + }) +})) + +// The export the run fetches from the hub proxy. Two items so an abandoned run can stop +// partway through, which is the case under test. +const EXPORT = { + project: { slug: 'calendly', name: 'Calendly', summary: '', readme: null }, + scripts: [], + flows: [], + apps: [], + resources: [], + triggers: [], + migrations: [] +} +vi.stubGlobal( + 'fetch', + vi.fn(async () => ({ ok: true, status: 200, text: async () => JSON.stringify(EXPORT) })) +) + +import { ImportExecution } from './execution.svelte' + +const PLAN = { slug: 'calendly', destination: { kind: 'existing' as const, workspaceId: 'ws-a' } } +const deps = { reviewMigrations: async () => [], hasEeLicense: false } + +/** A project that ships one, so `#import` appends the `migrate` row at all. */ +const MIGRATION = { + datatable_name: 'main', + sql: 'CREATE TABLE IF NOT EXISTS "calendly"."config" (id int)', + sql_down: '', + enabled: true +} +const depsWithMigration = { reviewMigrations: async () => [MIGRATION], hasEeLicense: false } + +describe('planTag', () => { + // A run handed back after a remount is checked against the plan being rendered. Tagging + // it with that plan instead of its own would make the check pass by construction. + it('identifies the plan the run belongs to', () => { + const a = new ImportExecution(PLAN, deps) + const b = new ImportExecution( + { ...PLAN, destination: { kind: 'existing', workspaceId: 'ws-b' } }, + deps + ) + expect(a.planTag).not.toBe(b.planTag) + }) + + it('ignores the folder, which stays editable after the run is made', () => { + const a = new ImportExecution(PLAN, deps) + const tag = a.planTag + a.setFolder('somewhere-else') + expect(a.planTag).toBe(tag) + }) + + it('separates two projects going to the same workspace', () => { + const a = new ImportExecution(PLAN, deps) + const b = new ImportExecution({ ...PLAN, slug: 'bitly' }, deps) + expect(a.planTag).not.toBe(b.planTag) + }) +}) + +describe('abandoning mid-import', () => { + beforeEach(() => { + hooks.afterFirstItem = undefined + hooks.afterMigrationsStart = undefined + present.paths = new Set() + }) + + it('does not report done, so the resumed step offers Retry rather than Continue', async () => { + const run = new ImportExecution(PLAN, deps) + hooks.afterFirstItem = () => run.abandon() + await run.run() + // `installProject` returned early exactly as it does on success; calling that done + // would report a clean import over items that never started. + expect(run.done).toBe(false) + expect(run.error).toMatch(/stopped/i) + }) + + it('reports done when nothing abandoned it', async () => { + const run = new ImportExecution(PLAN, deps) + await run.run() + expect(run.done).toBe(true) + expect(run.itemResults.length).toBe(3) + }) + + it('stops the migrate row spinning when it is abandoned mid-migration', async () => { + const run = new ImportExecution(PLAN, depsWithMigration) + // After `onMigrationsStart`, which is where the row is actually set to running — + // the real one fires it at the head of the migration loop, past every item loop. + hooks.afterMigrationsStart = () => run.abandon() + await run.run() + const migrate = run.tasks.find((t) => t.key === 'migrate') + // Guards the test itself: without a migration in the export there is no row, and the + // assertion below would pass over a branch that never ran. + expect(migrate).toBeDefined() + // A row left on `running` reads as work still in progress on a run that has stopped. + expect(migrate?.status).not.toBe('running') + expect(run.done).toBe(false) + }) +}) + +/** + * A retry must write only what is missing. Resending the whole bundle turns every item that + * already landed into an "already exists" failure — a wall of red over work that succeeded. + * What is already there is skipped, and reported as skipped rather than as imported, because + * the checklist has to account for every item the project ships. + */ +describe('retrying over what is already there', () => { + beforeEach(() => { + present.paths = new Set() + }) + + it('writes nothing for a path the destination already holds', async () => { + present.paths = new Set(['script:a', 'script:b']) + const run = new ImportExecution(PLAN, deps) + await run.run() + const byPath = new Map(run.itemResults.map((r) => [r.path, r])) + expect(byPath.get('a')?.skipped).toBe(true) + expect(byPath.get('b')?.skipped).toBe(true) + expect(byPath.get('c')?.skipped).toBeUndefined() + }) + + it('still accounts for every item, so the checklist stays complete', async () => { + present.paths = new Set(['script:a', 'script:b']) + const run = new ImportExecution(PLAN, deps) + await run.run() + expect(run.itemResults.length).toBe(3) + expect(run.done).toBe(true) + }) + + it('says what it did rather than claiming to have imported all of it', async () => { + present.paths = new Set(['script:a', 'script:b']) + const run = new ImportExecution(PLAN, deps) + await run.run() + const importRow = run.tasks.find((t) => t.key === 'import') + expect(importRow?.detail).toMatch(/1 imported/) + expect(importRow?.detail).toMatch(/2 already there/) + }) + + it('imports everything when the destination is empty', async () => { + const run = new ImportExecution(PLAN, deps) + await run.run() + expect(run.itemResults.every((r) => !r.skipped)).toBe(true) + expect(run.tasks.find((t) => t.key === 'import')?.detail).toMatch(/3 imported/) + }) +}) + +describe('abandoning while the presence probe is in flight', () => { + beforeEach(() => { + present.paths = new Set() + present.onProbe = undefined + }) + + /** + * `import` goes `running` before the probe is asked, so returning straight out of an + * abandon here leaves a spinner on a run that has stopped — next to an enabled Retry and + * with no explanation of why it stopped. + */ + it('leaves no task running', async () => { + const run = new ImportExecution(PLAN, deps as any) + // Abandoned from inside the probe: `import` is already `running` by then, and the + // executor's next look at the flag is the early return under test. + present.onProbe = () => run.abandon() + await run.run() + expect(run.tasks.find((t) => t.status === 'running')).toBeUndefined() + expect(run.tasks.find((t) => t.key === 'import')?.status).toBe('failed') + expect(run.error).toBeTruthy() + expect(run.done).toBe(false) + }) +}) diff --git a/frontend/src/lib/importWizard/execution.svelte.ts b/frontend/src/lib/importWizard/execution.svelte.ts new file mode 100644 index 0000000000..06c33f3222 --- /dev/null +++ b/frontend/src/lib/importWizard/execution.svelte.ts @@ -0,0 +1,517 @@ +import { UserService, WorkspaceService } from '$lib/gen' +import { switchWorkspace } from '$lib/storeUtils' +import { userStore, workspaceStore } from '$lib/stores' +import { getUserExt } from '$lib/user' +import { get } from 'svelte/store' +import { enterNewWorkspace, refreshWorkspaceList } from '$lib/workspaceCreation' +import { + installProject, + type InstallResult +} from '$lib/components/workspaceSettings/projectInstall' +import type { + ProjectExport, + ProjectMigration +} from '$lib/components/workspaceSettings/projectBundle' +import { planWorkspaceId, type ImportPlan } from './plan' +import { probeImportedPaths, probeWorkspace } from './probe' + +/** + * The only thing in the wizard that changes anything. It takes a finished plan and + * runs it as an ordered, observable list of tasks, so the last step can show what + * is happening and exactly where it stopped. + * + * Everything the run needs from the outside — reviewing data table migrations, + * the EE licence — is injected, so the wizard's UI decisions stay in the wizard + * and this file stays testable without a browser. + */ + +/** + * Whether an import is mid-flight, readable by anything that can navigate away + * from it. The run outlives no component: leaving the last step unmounts the + * migration review the executor is awaiting, so a run in progress has to block + * the stepper and the browser rather than be silently detached. + */ +const runState = $state({ active: false }) +export function importIsRunning(): boolean { + return runState.active +} + +export type TaskStatus = 'pending' | 'running' | 'done' | 'failed' | 'skipped' + +export interface TaskView { + key: string + label: string + status: TaskStatus + detail?: string +} + +/** + * What a plan will do, as the same task list the run reports against. Exported so the + * last step can show it before the run starts: the checklist is what the step says it + * is going to do, and the run then fills in the same rows rather than replacing them. + * + * Derived from the plan alone — no network — so it is safe to call while rendering. + */ +export function plannedTasks(plan: ImportPlan): TaskView[] { + const d = plan.destination + const tasks: TaskView[] = [] + if (d?.kind === 'new') { + tasks.push({ key: 'create', label: `Create workspace ${d.id}`, status: 'pending' }) + } + tasks.push({ key: 'fetch', label: 'Fetch the project from the hub', status: 'pending' }) + // The destination rides on this row when nothing creates it, so the list still says + // where the items are going in the existing-workspace case. + tasks.push({ + key: 'import', + label: d?.kind === 'existing' ? `Import the items into ${d.workspaceId}` : 'Import the items', + status: 'pending' + }) + return tasks +} + +export interface ExecutionDeps { + /** + * Chooses which data table migrations to run. Returns the migrations to apply, + * or null to abort the whole import (the user backed out at the warning). + */ + reviewMigrations: ( + workspace: string, + migrations: ProjectMigration[] + ) => Promise + hasEeLicense: boolean +} + +export class ImportExecution { + #plan: ImportPlan + #deps: ExecutionDeps + + /** Carried between attempts so a retry does not redo finished work. */ + // The hub's summary endpoint counts scripts/flows/apps/resources; only the export + // carries triggers and data table migrations. Surfaced so the last step can say + // what it is about to create — the warning below it talks about triggers, and the + // page this replaced did show both. + #export = $state(undefined) + + /** + * Data tables the project's migrations target. The wizard compares these with the + * destination's configured tables to decide whether a setup step is needed — + * `installProject` skips a migration whose data table does not exist. + */ + get datatableNames(): string[] { + const e = this.#export + if (!e) return [] + return [ + ...new Set( + (e.migrations ?? []) + .filter((m) => m.enabled && (m.sql ?? '').trim() !== '') + .map((m) => m.datatable_name) + ) + ] + } + + /** + * How many resources the project shipped. Every one arrives as an empty stub — + * the hub never publishes resource values — so a non-zero count means the setup + * step has something to offer. + */ + get resourceCount(): number { + return this.#export?.resources?.length ?? 0 + } + + get extraCounts(): { triggers: number; migrations: number } | undefined { + const e = this.#export + if (!e) return undefined + return { + triggers: e.triggers?.length ?? 0, + migrations: (e.migrations ?? []).filter((m) => m.enabled && (m.sql ?? '').trim() !== '') + .length + } + } + // $state, not a plain field: the UI offers to delete the workspace this run + // created, and a plain field would never re-render that button. + #workspaceCreated = $state(false) + + tasks = $state([]) + results = $state([]) + running = $state(false) + /** Set when a run stopped early; cleared when a retry starts. */ + error = $state(undefined) + done = $state(false) + + // Where the app pointed before this run switched away from it, so undoing the run + // can put it back. Captured at construction rather than at switch time: by then + // `$workspaceStore` already holds the workspace being entered. + #priorWorkspace = get(workspaceStore) + + constructor(plan: ImportPlan, deps: ExecutionDeps) { + this.#plan = plan + this.#deps = deps + this.tasks = this.#initialTasks() + } + + /** + * Identifies the plan this run belongs to — destination and project, not the folder, + * which stays editable on the last step and is pushed onto the run instead. A caller + * handing a run back after a remount compares this against the plan it is rendering; + * computing the tag from *that* plan would make the check pass by construction. + */ + get planTag(): string { + return JSON.stringify(this.#plan.destination) + this.#plan.slug + } + + get workspaceId(): string | undefined { + return planWorkspaceId(this.#plan) + } + + /** + * True once this run created a workspace — the only case where deleting is ours to offer. + * + * Deliberately not satisfied by having *adopted* one. `workspace.owner` is enough to know + * a create can be skipped — the id is one this user made — but not enough to offer to + * delete it, because `owner` is an identity, not a run: a second import by the same person + * into the same id looks identical. Skipping a create wrongly is recoverable; deleting a + * workspace is not, so an adopted run finishes the import and leaves the undo to the run + * that did the creating. + */ + get createdWorkspace(): boolean { + return this.#workspaceCreated + } + + get failedCount(): number { + return this.results.filter((r) => !r.ok).length + } + + /** `installProject` reports migrations through the same channel as items, tagged by this + * prefix. Split so the import row counts what it imported and the migrate row counts + * what it migrated — one failure should not be attributed to both. */ + static readonly MIGRATION_PREFIX = 'data table: ' + get itemResults(): InstallResult[] { + return this.results.filter((r) => !r.path.startsWith(ImportExecution.MIGRATION_PREFIX)) + } + get migrationResults(): InstallResult[] { + return this.results.filter((r) => r.path.startsWith(ImportExecution.MIGRATION_PREFIX)) + } + + #initialTasks(): TaskView[] { + return plannedTasks(this.#plan) + } + + #set(key: string, status: TaskStatus, detail?: string) { + this.tasks = this.tasks.map((t) => (t.key === key ? { ...t, status, detail } : t)) + } + + /** + * Set when the user confirms leaving mid-run. Nothing here can abort a request already + * in flight — `installProject` takes no signal — so this stops the run at the next phase + * boundary instead, which is as far as "stops where it is" can honestly go. + * + * The workspace it created stays, and the run stays resumable: coming back to the link + * re-probes the instance, finds the workspace, and carries on rather than trying to create + * it a second time. + */ + #abandoned = false + + /** The user has left. Stop at the next phase boundary and leave the run resumable. */ + abandon() { + this.#abandoned = true + } + + /** + * Runs every task that has not already succeeded. Safe to call again after a failure: the + * destination is asked what it already holds, so a workspace that exists is entered rather + * than recreated and an item that landed is skipped rather than rewritten. What a second + * run costs is the reads, not the writes. + */ + async run(): Promise { + if (this.running) return + this.#abandoned = false + this.running = true + runState.active = true + this.error = undefined + try { + const workspace = await this.#ensureWorkspace() + if (!workspace || this.#abandoned) return + const exportData = await this.#ensureExport(workspace) + if (!exportData || this.#abandoned) return + await this.#import(workspace, exportData) + } finally { + this.running = false + runState.active = false + } + } + + /** + * The folder is the one part of the plan still editable on the last step, so a + * retry after changing it must import where the field now says — not where the + * first attempt was told to. + */ + setFolder(folder: string) { + this.#plan = { ...this.#plan, folder } + } + + async #ensureWorkspace(): Promise { + const d = this.#plan.destination + if (!d) { + this.error = 'No destination' + return undefined + } + if (d.kind === 'existing') { + if (!d.workspaceId) { + this.error = 'No destination workspace' + return undefined + } + switchWorkspace(d.workspaceId) + await this.#adoptUser(d.workspaceId) + return d.workspaceId + } + // Asked of the instance, not remembered. A retry after entering it failed must not + // run the create again — that would only report the id as taken by the workspace this + // run just made — and after a reload the field is false again while the workspace is + // still there. `ours` is what makes adopting it safe: an id that exists but belongs to + // someone else is not this run's work, and importing into it would be importing into + // a stranger's workspace. + const already = await probeWorkspace(d.id, await this.#email()) + if (!this.#workspaceCreated && !(already.exists && already.ours)) { + this.#set('create', 'running') + try { + await WorkspaceService.createWorkspace({ + requestBody: { id: d.id, name: d.name, username: d.username } + }) + } catch (e: any) { + const detail = e?.body?.toString?.() ?? String(e) + this.#set('create', 'failed', detail) + this.error = `Could not create the workspace: ${detail}` + return undefined + } + this.#workspaceCreated = true + } + try { + await enterNewWorkspace(d.id) + await this.#adoptUser(d.id) + } catch (e: any) { + const detail = e?.body?.toString?.() ?? String(e) + this.#set('create', 'failed', `created, but could not be entered: ${detail}`) + this.error = `Created ${d.id}, but could not enter it: ${detail}` + return undefined + } + this.#set('create', 'done') + return d.id + } + + async #ensureExport(workspace: string): Promise { + if (this.#export) { + this.#set('fetch', 'done') + return this.#export + } + this.#set('fetch', 'running') + try { + // Workspace-scoped on purpose: this is the same proxy the rest of the app + // uses, so a private hub reachable only from the server still works. + const res = await fetch( + `/api/w/${encodeURIComponent(workspace)}/hub/projects/${encodeURIComponent(this.#plan.slug)}/export`, + { credentials: 'include', headers: { accept: 'application/json' } } + ) + const text = await res.text() + if (!res.ok) throw new Error(`export ${res.status}: ${text}`) + this.#export = JSON.parse(text) as ProjectExport + this.#set('fetch', 'done', `${itemCount(this.#export)} items`) + return this.#export + } catch (e: any) { + const detail = e?.message ?? String(e) + this.#set('fetch', 'failed', detail) + this.error = `Could not read the project: ${detail}` + return undefined + } + } + + /** + * Leave the checklist saying what actually happened, from wherever the run stopped. + * + * Reached from every point after `import` goes `running`, so nothing is left spinning on a + * run that has ended. A partial import is failed rather than done: calling it done reports + * a clean import over items that never started, and the resumed step offers Continue where + * it should offer Retry. + */ + #settleAbandoned() { + const landed = this.itemResults.length + this.#set('import', 'failed', `stopped after ${landed} item${landed === 1 ? '' : 's'}`) + // The migrate row is appended once the review settles and set running by + // `onMigrationsStart`. Stopping before its loop leaves it spinning forever, which + // reads as work still in progress on a run that has stopped. + if (this.tasks.some((t) => t.key === 'migrate' && t.status === 'running')) { + this.#set('migrate', 'pending') + } + this.error = 'Import stopped. Retry to import what is left.' + } + + async #import(workspace: string, exportData: ProjectExport): Promise { + this.#set('import', 'running') + const folder = this.#plan.folder?.trim() || exportData.project.slug + + let migrations: ProjectMigration[] | null + try { + migrations = await this.#deps.reviewMigrations(workspace, exportData.migrations ?? []) + } catch (e: any) { + this.#set('import', 'failed', String(e)) + this.error = `Could not plan the data table migrations: ${e}` + return + } + if (migrations === null) { + // The user backed out at the missing-data-table warning. + this.#set('import', 'skipped', 'cancelled at the data table warning') + this.error = 'Import cancelled.' + return + } + + // Appended only once the review has settled: until then nothing knows whether any + // migration is runnable here, and a row that might not apply is worse than none. + if (migrations.length && !this.tasks.some((t) => t.key === 'migrate')) { + const n = migrations.length + this.tasks = [ + ...this.tasks, + { + key: 'migrate', + label: `Run ${n} data table migration${n === 1 ? '' : 's'}`, + status: 'pending' + } + ] + } + + this.results = [] + // Asked every run, not only on a retry: the destination may be a workspace that already + // holds some of these paths, and a run interrupted halfway is indistinguishable from + // one that never started. On a workspace this run just created the answer is empty and + // nothing is skipped. + const alreadyPresent = await probeImportedPaths(workspace, folder, { + triggers: exportData.triggers.length > 0, + hasEeLicense: this.#deps.hasEeLicense + }) + // Settled, not just returned: `import` has been `running` since before the probe, and + // leaving it there shows a spinner on a run that has stopped, next to a Retry button. + if (this.#abandoned) { + this.#settleAbandoned() + return + } + try { + await installProject({ + alreadyPresent, + workspace, + exportData, + folder, + migrations, + hasEeLicense: this.#deps.hasEeLicense, + onResult: (r) => (this.results = [...this.results, r]), + onMigrationsStart: () => this.#set('migrate', 'running'), + // Checked before every write, so leaving mid-run stops the remaining items + // rather than only the phases. What already landed stays and is listed. + stopped: () => this.#abandoned + }) + } catch (e: any) { + this.#set('import', 'failed', String(e)) + this.error = `The import stopped: ${e}` + return + } + + // `installProject` returns early when `stopped` goes true, and it returns the same way + // it does on success — so the tail has to ask why. An abandoned run has written only + // what it got through; calling that `done` reports a clean import over items that + // never started, and the resumed step would offer Continue instead of Retry. + if (this.#abandoned) { + this.#settleAbandoned() + return + } + + const items = this.itemResults + const failed = items.filter((r) => !r.ok).length + const skipped = items.filter((r) => r.skipped).length + // Three outcomes, so the row says which: written, left alone because it was already + // there, and failed. Rolling the second into the first would report an import that + // did not happen. + const wrote = items.length - failed - skipped + const parts: string[] = [] + if (wrote > 0 || (failed === 0 && skipped === 0)) parts.push(`${wrote} imported`) + if (skipped > 0) parts.push(`${skipped} already there`) + if (failed > 0) parts.push(`${failed} failed`) + this.#set('import', failed > 0 ? 'failed' : 'done', parts.join(', ')) + + const migrated = this.migrationResults + const badMigrations = migrated.filter((r) => !r.ok).length + if (migrated.length) { + const badly = migrated.filter((r) => !r.ok) + this.#set( + 'migrate', + badly.length ? 'failed' : 'done', + badly.length ? badly.map((r) => r.error).join('; ') : undefined + ) + } + // A partial import is finished, not broken: the items that landed are real, + // and the failures are listed. Only a hard stop leaves `done` false. + this.done = true + // Both kinds of failure, because `error` is what offers Retry. A migration that fails + // against an existing data table is as retryable as a failed item; leaving it out here + // would present the run as a clean finish with no way to run it again. + const problems: string[] = [] + if (failed > 0) problems.push(`${failed} item${failed === 1 ? '' : 's'} failed to import`) + if (badMigrations > 0) { + problems.push( + `${badMigrations} data table migration${badMigrations === 1 ? '' : 's'} failed` + ) + } + if (problems.length) this.error = `${problems.join(', ')}.` + } + + /** + * Load the membership for the workspace this run just entered. + * + * The wizard's page is reparented out of `(logged)`, so it never gets that + * layout's `getUserExt` call and `$userStore` stays undefined. Anything deciding + * what the user may do then reads "no user" and refuses: `canWrite` returns false + * without one, which renders every field of the resource editor disabled — the + * setup step could show the credentials to fill and then not let anyone fill them. + */ + async #adoptUser(workspace: string): Promise { + try { + userStore.set(await getUserExt(workspace)) + } catch { + // Leave it unset; the step degrades to read-only rather than failing the run. + } + } + + /** + * Who we are, for the ownership check. `$userStore` is workspace-scoped and this page is + * reparented out of `(logged)`, so it is unset until a run adopts one — `globalWhoami` is + * the identity that exists before any workspace does. + */ + async #email(): Promise { + const known = get(userStore)?.email + if (known) return known + try { + return (await UserService.globalWhoami()).email + } catch { + return undefined + } + } + + /** Undoes the one thing this run created, when the user asks for it. */ + async deleteCreatedWorkspace(): Promise { + const d = this.#plan.destination + if (!this.#workspaceCreated || d?.kind !== 'new') return + await WorkspaceService.deleteWorkspace({ workspace: d.id }) + // Leave the app pointing somewhere that exists. The run switched into the + // workspace it created; without this the store keeps the deleted id, the + // layout persists it to local/sessionStorage, and the next full page load + // fails `getUserExt` and logs the user out. + switchWorkspace(this.#priorWorkspace) + await refreshWorkspaceList() + this.#workspaceCreated = false + // The id is free again; the next `#ensureWorkspace` asks the instance and finds it + // gone, so a retry creates rather than adopts. + this.#set('create', 'pending') + this.done = false + this.results = [] + } +} + +function itemCount(e: ProjectExport): number { + return e.scripts.length + e.flows.length + e.apps.length + e.resources.length +} diff --git a/frontend/src/lib/importWizard/migrationOutcome.test.ts b/frontend/src/lib/importWizard/migrationOutcome.test.ts new file mode 100644 index 0000000000..87a23cc8ef --- /dev/null +++ b/frontend/src/lib/importWizard/migrationOutcome.test.ts @@ -0,0 +1,94 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +/** + * A run that imported every item but failed a data table migration is not a clean finish. + * `error` is what offers Retry, so this is about whether the user can act on the failure — + * the `migrate` row already says it happened. + */ + +vi.mock('$lib/gen', () => ({ + WorkspaceService: { + createWorkspace: vi.fn(), + listDataTables: vi.fn(async () => []), + listWorkspaces: vi.fn(async () => []) + }, + UserService: { + whoami: vi.fn(async () => ({ username: 'u' })), + globalWhoami: vi.fn(async () => ({ email: 'u@example.com' })) + } +})) +vi.mock('$lib/storeUtils', () => ({ switchWorkspace: vi.fn() })) +vi.mock('./probe', async (orig) => ({ + ...(await orig()), + probeWorkspace: vi.fn(async () => ({ exists: false, ours: false })), + probeImportedPaths: vi.fn(async () => new Set()) +})) +vi.mock('$lib/user', () => ({ getUserExt: vi.fn(async () => ({ username: 'u' })) })) + +/** Whether the migration this run applies succeeds. The item writes always do. */ +const outcome = vi.hoisted(() => ({ migrationOk: true })) + +vi.mock('$lib/components/workspaceSettings/projectInstall', () => ({ + installProject: vi.fn(async (args: any) => { + args.onResult({ path: 'f/calendly/one', ok: true }) + if (args.migrations?.length) { + args.onMigrationsStart?.() + for (const m of args.migrations) { + args.onResult({ + path: `data table: ${m.datatable_name}`, + ok: outcome.migrationOk, + error: outcome.migrationOk ? undefined : 'relation already exists' + }) + } + } + }) +})) + +const EXPORT = { + project: { slug: 'calendly', name: 'Calendly', summary: '', readme: null }, + scripts: [], + flows: [], + apps: [], + resources: [], + triggers: [], + migrations: [] +} +vi.stubGlobal( + 'fetch', + vi.fn(async () => ({ ok: true, status: 200, text: async () => JSON.stringify(EXPORT) })) +) + +import { ImportExecution } from './execution.svelte' + +const PLAN = { slug: 'calendly', destination: { kind: 'existing' as const, workspaceId: 'ws-a' } } +const MIGRATION = { + datatable_name: 'main', + sql: 'CREATE TABLE IF NOT EXISTS "calendly"."config" (id int)', + sql_down: '', + enabled: true +} +const deps = { reviewMigrations: async () => [MIGRATION], hasEeLicense: false } + +describe('a failed migration', () => { + beforeEach(() => { + outcome.migrationOk = true + }) + + it('marks the migrate row failed and leaves the run retryable', async () => { + outcome.migrationOk = false + const run = new ImportExecution(PLAN, deps as any) + await run.run() + expect(run.tasks.find((t) => t.key === 'migrate')?.status).toBe('failed') + // `error` is what the page reads to offer Retry: a `migrate` row saying failed with + // `error` unset sends it down the finished-run path with no way to run it again. + expect(run.error).toBeTruthy() + expect(run.error).toContain('migration') + }) + + it('says nothing went wrong when the migration succeeds', async () => { + const run = new ImportExecution(PLAN, deps as any) + await run.run() + expect(run.tasks.find((t) => t.key === 'migrate')?.status).toBe('done') + expect(run.error).toBeFalsy() + }) +}) diff --git a/frontend/src/lib/importWizard/plan.test.ts b/frontend/src/lib/importWizard/plan.test.ts new file mode 100644 index 0000000000..c81ed700bc --- /dev/null +++ b/frontend/src/lib/importWizard/plan.test.ts @@ -0,0 +1,172 @@ +import { describe, expect, it } from 'vitest' + +import { + planProblem, + planToSearch, + planWorkspaceId, + readPlan, + type ImportPlan, + type WizardStep +} from './plan' + +// The wizard keeps no state of its own — the plan *is* the URL. Back, forward, the +// stepper and a pasted link are all the same operation, which only holds if this +// round trip is lossless. + +const read = (search: string) => readPlan(new URL(`https://x.dev/projects/import${search}`)) +const roundTrip = (plan: ImportPlan, step: WizardStep) => read(planToSearch(plan, step)) + +describe('readPlan / planToSearch round trip', () => { + const cases: Array<[string, ImportPlan, WizardStep]> = [ + ['bare slug', { slug: 'support-automation' }, 1], + ['new destination', { slug: 's', destination: { kind: 'new', id: 'w', name: 'W' } }, 2], + [ + 'new with username', + { slug: 's', destination: { kind: 'new', id: 'w', name: 'W', username: 'ada' } }, + 2 + ], + [ + 'existing destination', + { slug: 's', destination: { kind: 'existing', workspaceId: 'admins' } }, + 3 + ], + [ + 'folder carried', + { slug: 's', destination: { kind: 'existing', workspaceId: 'admins' }, folder: 'finance' }, + 3 + ] + ] + + for (const [name, plan, step] of cases) { + it(name, () => { + expect(roundTrip(plan, step)).toEqual({ plan, step }) + }) + } + + it('survives characters that need encoding', () => { + const plan: ImportPlan = { + slug: 'a b&c=d', + destination: { kind: 'new', id: 'w', name: 'Name & Co = 100%' } + } + expect(roundTrip(plan, 2)).toEqual({ plan, step: 2 }) + }) +}) + +describe('readPlan', () => { + it('reads legacy links that carry only a workspace', () => { + expect(read('?hub=s&workspace=admins').plan.destination).toEqual({ + kind: 'existing', + workspaceId: 'admins' + }) + }) + + it('reads legacy links that carry only new_workspace_id, falling the name back to the id', () => { + expect(read('?hub=s&new_workspace_id=w').plan.destination).toEqual({ + kind: 'new', + id: 'w', + name: 'w', + username: undefined + }) + }) + + it('clamps and rounds the step', () => { + expect(read('?hub=s&step=0').step).toBe(1) + // 4 is the optional setup step, so that is the ceiling. + expect(read('?hub=s&step=9').step).toBe(4) + expect(read('?hub=s&step=nope').step).toBe(1) + // A fractional step would match neither `=== 2` nor `=== 3`. + expect(read('?hub=s&step=2.5').step).toBe(3) + expect(read('?hub=s&step=2.4').step).toBe(2) + }) + + it('has no destination when nothing names one', () => { + expect(read('?hub=s').plan.destination).toBeUndefined() + }) +}) + +/** + * `create_workspace` takes the username it is given and writes it to `usr.username` without + * checking it — `Some("")` passes its only guard — so a blank or malformed one becomes a + * workspace whose owner has no usable name. The wizard is the last thing that can refuse it. + */ +describe('planProblem — the new-workspace username', () => { + const dest = (username?: string) => ({ + slug: 'calendly', + destination: { kind: 'new' as const, id: 'calendly', name: 'Calendly', username } + }) + + it('asks for nothing when the instance derives the username', () => { + expect(planProblem(dest(undefined))).toBeUndefined() + }) + + it('refuses a blank or whitespace-only username', () => { + expect(planProblem(dest(''))).toMatch(/needs a username/i) + expect(planProblem(dest(' '))).toMatch(/needs a username/i) + }) + + it('refuses one the backend would store verbatim but never accept elsewhere', () => { + expect(planProblem(dest('1bad'))).toMatch(/letters and numbers/i) + expect(planProblem(dest('a b'))).toMatch(/letters and numbers/i) + }) + + it('accepts a valid one', () => { + expect(planProblem(dest('guilhem'))).toBeUndefined() + }) +}) + +/** `validate_workspace_name` refuses > 50 chars, and only at creation — two steps later. */ +describe('planProblem — the new-workspace name length', () => { + const named = (name: string) => ({ + slug: 'calendly', + destination: { kind: 'new' as const, id: 'calendly', name } + }) + + it('accepts the longest name the backend takes', () => { + expect(planProblem(named('x'.repeat(50)))).toBeUndefined() + }) + + it('refuses one character more, rather than failing at create', () => { + expect(planProblem(named('x'.repeat(51)))).toMatch(/too long/i) + }) +}) + +describe('planProblem', () => { + it('names what is missing, in the order the wizard asks for it', () => { + expect(planProblem({ slug: '' })).toMatch(/No project/) + expect(planProblem({ slug: 's' })).toMatch(/Pick a destination/) + expect(planProblem({ slug: 's', destination: { kind: 'new', id: 'w', name: '' } })).toMatch( + /needs a name/ + ) + expect(planProblem({ slug: 's', destination: { kind: 'existing' } })).toMatch( + /Pick the workspace/ + ) + }) + + it('validates the id of either destination kind', () => { + expect( + planProblem({ slug: 's', destination: { kind: 'new', id: 'not valid', name: 'W' } }) + ).toMatch(/letters, numbers and dashes/) + // An existing id arrives from the URL just as a new one does. + expect( + planProblem({ slug: 's', destination: { kind: 'existing', workspaceId: '../admins' } }) + ).toMatch(/not a valid workspace id/) + }) + + it('rejects a folder name the import could not create', () => { + const base: ImportPlan = { slug: 's', destination: { kind: 'existing', workspaceId: 'admins' } } + expect(planProblem({ ...base, folder: 'ok_folder-1' })).toBeUndefined() + expect(planProblem({ ...base, folder: 'not ok' })).toMatch(/Folder/) + }) +}) + +describe('planWorkspaceId', () => { + it('is the id either kind of destination will end up in', () => { + expect(planWorkspaceId({ slug: 's', destination: { kind: 'new', id: 'w', name: 'W' } })).toBe( + 'w' + ) + expect( + planWorkspaceId({ slug: 's', destination: { kind: 'existing', workspaceId: 'admins' } }) + ).toBe('admins') + expect(planWorkspaceId({ slug: 's' })).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/importWizard/plan.ts b/frontend/src/lib/importWizard/plan.ts new file mode 100644 index 0000000000..f209041787 --- /dev/null +++ b/frontend/src/lib/importWizard/plan.ts @@ -0,0 +1,131 @@ +/** + * The import wizard's state, and the only thing the first two steps produce. + * + * Nothing is created, switched or written while the wizard is being filled in: + * steps 1 and 2 only ever describe *what should happen*, and the plan travels in + * the URL. That is what makes the back button and the stepper safe — going back + * is a URL change and cannot leave a half-made workspace behind, because there is + * no state anywhere else to unwind. + * + * `execution.svelte.ts` is the only thing that acts on a plan, and only when the + * user asks it to on the last step. + */ + +import { validateUsername } from '$lib/utils' +import { validateWorkspaceId, WORKSPACE_NAME_MAX_LENGTH } from '$lib/utils/workspaceId' + +/** + * `existing` carries no workspace until one is picked: step 1 answers *which kind* + * of destination and step 2 answers *which one*, and the plan has to be able to + * hold the state in between. Encoding that gap as an absent destination would make + * "chose an existing workspace, has not picked it" and "answered nothing yet" the + * same URL. + */ +export type ImportDestination = + | { kind: 'new'; name: string; id: string; username?: string } + | { kind: 'existing'; workspaceId?: string } + +export interface ImportPlan { + /** The hub project being imported. */ + slug: string + /** Undefined until step 1 has been answered. */ + destination?: ImportDestination + /** Folder the items land in; defaults to the project slug at execution time. */ + folder?: string +} + +/** 4 is the optional setup step, reached only when the import leaves work to do. */ +export type WizardStep = 1 | 2 | 3 | 4 + +export const FOLDER_NAME_RE = /^[a-zA-Z_0-9-]+$/ + +export function readPlan(url: URL): { plan: ImportPlan; step: WizardStep } { + const params = url.searchParams + const slug = params.get('hub') ?? '' + + const newId = params.get('new_workspace_id') + const existing = params.get('workspace') + // `destination` records the step 1 answer on its own; the older links that only + // carried `workspace` or `new_workspace_id` still read as the kind they imply. + const kind = params.get('destination') + const destination: ImportDestination | undefined = + newId || kind === 'new' + ? { + kind: 'new', + id: newId ?? '', + name: params.get('new_workspace_name') || newId || '', + username: params.get('username') || undefined + } + : existing || kind === 'existing' + ? { kind: 'existing', workspaceId: existing || undefined } + : undefined + + // Rounded as well as clamped: the steps are compared with `>` and `===`, so a + // fractional `?step=2.5` would clamp to 2.5 and match neither. + const raw = Number(params.get('step') ?? 1) + const step = (Number.isFinite(raw) ? Math.min(4, Math.max(1, Math.round(raw))) : 1) as WizardStep + + return { plan: { slug, destination, folder: params.get('folder') || undefined }, step } +} + +export function planToSearch(plan: ImportPlan, step: WizardStep): string { + const params = new URLSearchParams({ hub: plan.slug }) + if (step !== 1) params.set('step', String(step)) + if (plan.destination) params.set('destination', plan.destination.kind) + if (plan.destination?.kind === 'new') { + params.set('new_workspace_id', plan.destination.id) + params.set('new_workspace_name', plan.destination.name) + if (plan.destination.username) params.set('username', plan.destination.username) + } else if (plan.destination?.workspaceId) { + params.set('workspace', plan.destination.workspaceId) + } + if (plan.folder) params.set('folder', plan.folder) + return `?${params}` +} + +/** + * Whether the plan can be executed. Returned as a reason rather than a boolean so + * the button can say why it is disabled instead of just being grey. + */ +export function planProblem(plan: ImportPlan): string | undefined { + if (!plan.slug) return 'No project to import' + const d = plan.destination + if (!d) return 'Pick a destination first' + if (d.kind === 'new') { + if (!d.name.trim()) return 'The new workspace needs a name' + // The backend refuses a longer one (`validate_workspace_name`), and only at + // creation — by then the wizard has already walked the user through two more steps. + if (d.name.trim().length > WORKSPACE_NAME_MAX_LENGTH) { + return `The name is too long (${d.name.trim().length} chars). Maximum is ${WORKSPACE_NAME_MAX_LENGTH}.` + } + const idProblem = validateWorkspaceId(d.id) + if (idProblem) return idProblem + // Only asked for when the instance does not derive it. `create_workspace` takes + // whatever it is given here — `Some("")` passes its only check — so a blank or + // malformed username is written to `usr.username` verbatim rather than refused. + // The sibling creator validates it; this is the same check. + if (d.username !== undefined) { + if (!d.username.trim()) return 'The new workspace needs a username' + const bad = validateUsername(d.username.trim()) + if (bad) return bad + } + } else if (!d.workspaceId) { + return 'Pick the workspace to import into' + } else if (validateWorkspaceId(d.workspaceId)) { + // The id arrives from the URL exactly as the new-workspace one does, so it gets + // the same check. Downstream it is interpolated into a credentialed same-origin + // API path and pushed into `workspaceStore`; an id that cannot name a workspace + // has no business reaching either. + return 'That is not a valid workspace id' + } + if (plan.folder && !FOLDER_NAME_RE.test(plan.folder)) { + return 'Folder: letters, digits, dashes and underscores' + } + return undefined +} + +/** The workspace the plan lands in, once one has been named or picked. */ +export function planWorkspaceId(plan: ImportPlan): string | undefined { + const d = plan.destination + return (d?.kind === 'new' ? d.id : d?.workspaceId) || undefined +} diff --git a/frontend/src/lib/importWizard/probe.test.ts b/frontend/src/lib/importWizard/probe.test.ts new file mode 100644 index 0000000000..1d66a0e14b --- /dev/null +++ b/frontend/src/lib/importWizard/probe.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from 'vitest' + +import { expectedTables } from './probe' + +/** + * `expectedTables` is inference over what the export happens to say, so it is the part of the + * probe that can be wrong while everything still type-checks. The network reads around it + * either answer or throw. + */ + +describe('expectedTables', () => { + // The shape `datatableSchemaSql.ts` emits, and what every published project carries. + const REAL = `BEGIN; +CREATE SCHEMA IF NOT EXISTS "calendly"; +CREATE TABLE IF NOT EXISTS "calendly"."config" ( + "id" SERIAL NOT NULL, + "host_name" text NOT NULL DEFAULT ''::text +); +CREATE TABLE IF NOT EXISTS "calendly"."bookings" ( "id" SERIAL NOT NULL ); +COMMIT;` + + it('reads every table a migration creates', () => { + expect(expectedTables(REAL)).toEqual(['calendly.config', 'calendly.bookings']) + }) + + it('does not mistake the schema for a table', () => { + expect(expectedTables(REAL)).not.toContain('calendly') + }) + + it('reads the form without IF NOT EXISTS', () => { + expect(expectedTables('CREATE TABLE "bitly"."links" (id int)')).toEqual(['bitly.links']) + }) + + it('is case- and whitespace-insensitive the way SQL is', () => { + expect(expectedTables('create table\n "a" . "b" (x int)')).toEqual(['a.b']) + }) + + it('reports each table once, however many times it is named', () => { + const sql = 'CREATE TABLE "a"."b" (x int); CREATE TABLE IF NOT EXISTS "a"."b" (x int);' + expect(expectedTables(sql)).toEqual(['a.b']) + }) + + /** + * The answer that keeps a caller honest. An unquoted or unqualified `CREATE TABLE` is + * something this cannot resolve — the schema would come from `search_path` at run time — + * so it reads as nothing expected, and the caller treats that as "cannot tell" rather than + * as "no tables, so the migration must have run". + */ + it('claims nothing about SQL it cannot resolve', () => { + expect(expectedTables('CREATE TABLE links (id int)')).toEqual([]) + expect(expectedTables('CREATE TABLE bitly.links (id int)')).toEqual([]) + expect(expectedTables('')).toEqual([]) + }) +}) diff --git a/frontend/src/lib/importWizard/probe.ts b/frontend/src/lib/importWizard/probe.ts new file mode 100644 index 0000000000..0df5173536 --- /dev/null +++ b/frontend/src/lib/importWizard/probe.ts @@ -0,0 +1,181 @@ +/** + * What is already true in the destination, read from the destination itself. + * + * Nothing here is remembered between runs, and nothing may be: a note saying "this run + * created workspace X" outlives the reload it was written for, but it also outlives the + * workspace it names, and the two are indistinguishable when it is read back. The plan in + * the URL says what should exist; these functions ask the instance what does. + * + * The one thing the instance cannot answer is *which tables a migration was supposed to + * create*. That is inferred from the SQL the project ships (`expectedTables`), because the + * export states only what to run, never what running it should produce. + */ + +import { ResourceService, ScriptService, FlowService, AppService, WorkspaceService } from '$lib/gen' +import type { ProjectMigration } from '$lib/components/workspaceSettings/projectBundle' +import { + presenceKey, + type ImportedKind +} from '$lib/components/workspaceSettings/projectInstall' +import { listAllWorkspaceTriggers } from '$lib/components/triggers/workspaceTriggersList' + +/** + * The tables a migration creates, as `schema.table`, read off its `CREATE TABLE` statements. + * + * Inference, not a contract: the export ships SQL and nothing else, so this is the only way + * to check a migration's work without a record of it having run. It deliberately reads only + * the shape this project's generator emits (`datatableSchemaSql.ts` always writes the + * schema-qualified, quoted form) — anything hand-edited into a different shape simply reads + * as no expected tables, which makes the caller fall back to "cannot tell" rather than to a + * confident wrong answer. + */ +export function expectedTables(sql: string): string[] { + const out: string[] = [] + const re = /CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?"([^"]+)"\s*\.\s*"([^"]+)"/gi + let m: RegExpExecArray | null + while ((m = re.exec(sql)) !== null) out.push(`${m[1]}.${m[2]}`) + return [...new Set(out)] +} + +export interface WorkspaceState { + /** The user is a member of a workspace with this id. */ + exists: boolean + /** …and it is one they own, so a run of theirs is what made it. */ + ours: boolean +} + +/** + * Whether the destination workspace is there, and whether it is the caller's. + * + * `listWorkspaces` answers both in one call: it returns only workspaces the caller is a + * member of, each carrying the `owner` email set at creation. Ownership is what makes + * skipping a create safe — an id that exists but belongs to someone else is not a workspace + * this run made, and importing into it would be importing into a stranger's. + */ +export async function probeWorkspace( + id: string, + email: string | undefined +): Promise { + try { + const mine = await WorkspaceService.listWorkspaces() + const found = mine.find((w) => w.id === id) + if (!found) return { exists: false, ours: false } + return { exists: true, ours: !!email && found.owner === email } + } catch { + // Cannot tell. Reported as absent so the caller creates rather than adopts: a failed + // create is a clear error, adopting the wrong workspace is a silent one. + return { exists: false, ours: false } + } +} + +const PROBE_PAGE_SIZE = 100 +/** + * A stop, not a limit on what may be imported: 100 pages is 10,000 items in one folder, far + * past any project, and a paginating endpoint that never returns a short page would otherwise + * loop forever. Hitting it under-reports, which only ever means "still to do". + */ +const MAX_PROBE_PAGES = 100 + +/** + * Which of the items the import would write are already there, as `presenceKey` keys. + * + * Scoped by `pathStart` to the import's own folder, so the four path-bearing kinds are four + * small reads rather than a workspace scan. Presence is not provenance — importing into an + * existing workspace that already held a path reads the same as having imported it — so + * callers use this to decide what is left to do, never to claim credit for what is there. + * + * Triggers are asked for separately and only when the project ships some: they have no + * prefix-filtered list endpoint, so answering for them means one call per trigger kind, and a + * project without triggers should not pay for that. + */ +export async function probeImportedPaths( + workspace: string, + folder: string, + opts?: { triggers?: boolean; hasEeLicense?: boolean } +): Promise> { + const pathStart = `f/${folder}/` + const found = new Set() + /** + * Every page, not the first one. These endpoints paginate and default to 30 rows, so a + * single call answers for a small project and quietly under-reports a large one — leaving + * everything past the first page to be created again, and rejected as already existing. + */ + const collectAll = async ( + kind: ImportedKind, + list: (page: number) => Promise + ): Promise => { + for (let page = 1; page <= MAX_PROBE_PAGES; page++) { + const rows = ((await list(page)) as { path?: string }[] | undefined) ?? [] + for (const r of rows) if (r.path) found.add(presenceKey(kind, r.path)) + if (rows.length < PROBE_PAGE_SIZE) return + } + } + const calls: Promise[] = [ + collectAll('script', (page) => + ScriptService.listScripts({ workspace, pathStart, page, perPage: PROBE_PAGE_SIZE }) + ), + collectAll('flow', (page) => + FlowService.listFlows({ workspace, pathStart, page, perPage: PROBE_PAGE_SIZE }) + ), + collectAll('app', (page) => + AppService.listApps({ workspace, pathStart, page, perPage: PROBE_PAGE_SIZE }) + ), + collectAll('resource', (page) => + ResourceService.listResource({ workspace, pathStart, page, perPage: PROBE_PAGE_SIZE }) + ) + ] + if (opts?.triggers) { + // `failedKinds` is deliberately ignored: a kind that could not be listed leaves its + // triggers out of the set, and a missing key only ever means "still to do". + calls.push( + listAllWorkspaceTriggers(workspace, { + includeEeOnly: opts.hasEeLicense === true + }).then(({ triggers }) => { + for (const t of triggers) { + if (t.path?.startsWith(pathStart)) found.add(presenceKey(`trigger:${t.kind}`, t.path)) + } + }) + ) + } + // One kind failing should narrow the answer, not lose the others: a missing key only ever + // means "still to do", which is the safe direction. + await Promise.allSettled(calls) + return found +} + +/** + * Whether every table a data table's migrations create is in it. + * + * The ground truth for "did this run", and the only one that covers both paths + * `applyOneMigration` takes: it records a migration when the data table has migrations + * enabled, and otherwise runs the SQL once as a job that nothing remembers. The tables + * outlive both. + * + * Every migration for one data table at once: they all target the same schema, so asking + * per migration would introspect the same database N times for one answer. + * + * `undefined` means the question could not be answered — the schema was unreadable, or the + * SQL named no tables this can recognise. Distinct from `false`, because "not there" invites + * a caller to run the migration and "cannot tell" does not. + */ +export async function probeMigrationsApplied( + workspace: string, + datatableName: string, + migrations: ProjectMigration[] +): Promise { + const wanted = [...new Set(migrations.flatMap((m) => expectedTables(m.sql ?? '')))] + if (wanted.length === 0) return undefined + try { + const schema = (await WorkspaceService.getDatatableFullSchema({ + workspace, + requestBody: { source: `datatable://${datatableName}` } + })) as Record> + const present = new Set() + for (const [schemaName, tables] of Object.entries(schema ?? {})) { + for (const table of Object.keys(tables ?? {})) present.add(`${schemaName}.${table}`) + } + return wanted.every((t) => present.has(t)) + } catch { + return undefined + } +} diff --git a/frontend/src/lib/importWizard/probePaging.test.ts b/frontend/src/lib/importWizard/probePaging.test.ts new file mode 100644 index 0000000000..17e7fb9a11 --- /dev/null +++ b/frontend/src/lib/importWizard/probePaging.test.ts @@ -0,0 +1,75 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +/** + * The list endpoints paginate. A probe that reads one page answers correctly for a small + * project and under-reports a large one, which puts every item past the first page back + * through a create call that rejects it. + */ + +const calls = vi.hoisted(() => ({ script: [] as { page?: number; perPage?: number }[] })) +/** What the destination's trigger listing answers with, per test. */ +const triggerRows = vi.hoisted(() => ({ rows: [] as { kind: string; path: string }[] })) + +/** 150 scripts in the folder — more than one page at any page size the probe might pick. */ +const ALL = Array.from({ length: 150 }, (_, i) => ({ path: `f/calendly/s${i}` })) + +vi.mock('$lib/gen', () => ({ + ScriptService: { + listScripts: vi.fn(async (args: any) => { + calls.script.push({ page: args.page, perPage: args.perPage }) + const per = args.perPage ?? 30 + return ALL.slice((args.page - 1) * per, args.page * per) + }) + }, + FlowService: { listFlows: vi.fn(async () => []) }, + AppService: { listApps: vi.fn(async () => []) }, + ResourceService: { listResource: vi.fn(async () => []) }, + WorkspaceService: { listWorkspaces: vi.fn(async () => []), getDatatableFullSchema: vi.fn() } +})) +vi.mock('$lib/components/triggers/workspaceTriggersList', () => ({ + listAllWorkspaceTriggers: vi.fn(async () => ({ triggers: triggerRows.rows, failedKinds: [] })) +})) + +import { probeImportedPaths } from './probe' +import { presenceKey } from '$lib/components/workspaceSettings/projectInstall' + +describe('probeImportedPaths paging', () => { + beforeEach(() => { + calls.script = [] + }) + + it('reads every page, not just the first', async () => { + const found = await probeImportedPaths('w', 'calendly') + expect(found.size).toBe(150) + expect(found.has(presenceKey('script', 'f/calendly/s0'))).toBe(true) + // The one that a single-page probe misses, and would then try to create again. + expect(found.has(presenceKey('script', 'f/calendly/s149'))).toBe(true) + }) + + it('stops on the first short page rather than asking forever', async () => { + await probeImportedPaths('w', 'calendly') + const pages = calls.script.map((c) => c.page) + expect(pages).toEqual([1, 2]) + expect(new Set(calls.script.map((c) => c.perPage))).toEqual(new Set([100])) + }) +}) + +describe('trigger presence keys', () => { + beforeEach(() => { + triggerRows.rows = [] + }) + + /** + * Each trigger kind is its own table keyed on `(path, workspace_id)`, so a workspace can + * hold a schedule and an HTTP trigger both called `f/calendly/sync`. The probe builds the + * keys and `installProject` reads them, so they have to agree that those are two things — + * key on the path alone and the one that exists reports the other as already imported. + */ + it('keys a trigger by its kind, so one kind cannot answer for another', async () => { + triggerRows.rows = [{ kind: 'http', path: 'f/calendly/sync' }] + const found = await probeImportedPaths('w', 'calendly', { triggers: true }) + expect(found.has(presenceKey('trigger:http', 'f/calendly/sync'))).toBe(true) + // The schedule the project also ships at that path has not been imported. + expect(found.has(presenceKey('trigger:schedule', 'f/calendly/sync'))).toBe(false) + }) +}) diff --git a/frontend/src/lib/schema.test.ts b/frontend/src/lib/schema.test.ts new file mode 100644 index 0000000000..195a7b9478 --- /dev/null +++ b/frontend/src/lib/schema.test.ts @@ -0,0 +1,62 @@ +import { describe, it, expect } from 'vitest' +import { argsToJsonPayload } from './schema' +import type { Schema } from './common' + +const schemaOf = (...names: string[]): Schema => + ({ + $schema: undefined, + type: 'object', + properties: Object.fromEntries(names.map((n) => [n, { type: 'string' }])), + required: [] + }) as Schema + +describe('argsToJsonPayload', () => { + it('spells out every schema property in schema order, unset ones as null', () => { + // `0`, `false` and `''` are values, not gaps: only a missing arg becomes `null`. + expect(argsToJsonPayload(schemaOf('a', 'b', 'c'), { c: 0, a: false })).toBe( + JSON.stringify({ a: false, b: null, c: 0 }, null, '\t') + ) + }) + + it('keeps args the schema does not declare, after the declared ones', () => { + expect(argsToJsonPayload(schemaOf('a'), { z: 9, a: 1 })).toBe( + JSON.stringify({ a: 1, z: 9 }, null, '\t') + ) + }) + + it('falls back to the schema default for an absent arg, but not over an explicit null', () => { + // `args` only carries defaults once a `SchemaForm` has mounted for that schema, and the + // JSON view alone never mounts one — seeding `null` there would commit `null` over the + // argument's default on the first keystroke. + const schema = schemaOf('a', 'b') + schema.properties.a.default = 'hi' + schema.properties.b.default = 42 + expect(argsToJsonPayload(schema, {})).toBe(JSON.stringify({ a: 'hi', b: 42 }, null, '\t')) + expect(argsToJsonPayload(schema, { a: null })).toBe( + JSON.stringify({ a: null, b: 42 }, null, '\t') + ) + }) + + it('keeps declared args named after Object.prototype members', () => { + // A plain `nargs[key]` read returns the inherited function for an unset `constructor`, + // and `JSON.stringify` drops function-valued properties — the argument would vanish. + expect(argsToJsonPayload(schemaOf('constructor', 'toString', 'ok'), {})).toBe( + JSON.stringify({ constructor: null, toString: null, ok: null }, null, '\t') + ) + }) + + it('keeps undeclared args named after Object.prototype members', () => { + // On a plain `{}` accumulator, `'constructor' in payload` is true before anything is + // assigned to it. + expect(argsToJsonPayload(undefined, { constructor: 'x', toString: 'y', ok: 1 })).toBe( + JSON.stringify({ constructor: 'x', toString: 'y', ok: 1 }, null, '\t') + ) + }) + + it('handles a missing schema or missing args', () => { + expect(argsToJsonPayload(undefined, undefined)).toBe('{}') + expect(argsToJsonPayload(schemaOf('a'), undefined)).toBe( + JSON.stringify({ a: null }, null, '\t') + ) + }) +}) diff --git a/frontend/src/lib/schema.ts b/frontend/src/lib/schema.ts index c96b635202..36faa8a590 100644 --- a/frontend/src/lib/schema.ts +++ b/frontend/src/lib/schema.ts @@ -52,3 +52,33 @@ export function schemaToObject(schema: Schema, args: Record): Objec }) return object } + +/** Args as the JSON payload the JSON editor starts from. Every schema property is spelled out, + * so an argument with no value yet still shows its name; args the schema does not declare are + * kept, since what the editor holds replaces the args wholesale on the next keystroke. */ +export function argsToJsonPayload( + schema: Schema | undefined, + args: Record | undefined +): string { + const nargs = args ?? {} + // Null prototype: an arg named after an `Object.prototype` member (`constructor`, + // `toString`) has to be an own key here, or the `in` check below reads it as already + // present and its value never reaches the payload. + const payload: Record = Object.create(null) + const props = schema?.properties ?? {} + // Schema order first, so the payload reads like the form it replaces. + for (const key of Object.keys(props)) { + // Own-property read: an arg named after an `Object.prototype` member (`constructor`, + // `toString`) would otherwise come back as the inherited function, which `JSON.stringify` + // drops. An arg that is merely absent falls back to the schema default — `args` only + // carries defaults once a `SchemaForm` has mounted, which the JSON view alone never does. + payload[key] = + (Object.prototype.hasOwnProperty.call(nargs, key) ? nargs[key] : props[key]?.default) ?? null + } + for (const [key, value] of Object.entries(nargs)) { + if (!(key in payload)) { + payload[key] = value + } + } + return JSON.stringify(payload, null, '\t') +} diff --git a/frontend/src/lib/utils/templateLiteral.test.ts b/frontend/src/lib/utils/templateLiteral.test.ts new file mode 100644 index 0000000000..3aa16ecf38 --- /dev/null +++ b/frontend/src/lib/utils/templateLiteral.test.ts @@ -0,0 +1,133 @@ +import { describe, it, expect } from 'vitest' +import { escapeTemplateBackticks, unescapeTemplateBackticks } from './templateLiteral' + +// Template mode stores its value as a JS template literal, so backticks in the text have to be +// escaped. Escaping them inside `${...}` too is what broke nested template literals: a backslash +// is a syntax error in expression position. +describe('escapeTemplateBackticks', () => { + const nested = + '--input ${flow_input.iter.value}${results.config ? ` --config ${results.config}` : ""} --host ${flow_input.hostname}' + + it('leaves a nested template literal inside an interpolation intact', () => { + const expr = '`' + escapeTemplateBackticks(nested) + '`' + expect(expr).not.toContain('\\`') + expect( + new Function('flow_input', 'results', 'return ' + expr)( + { iter: { value: 'data.csv' }, hostname: 'host1' }, + { config: '/tmp/cfg.json' } + ) + ).toBe('--input data.csv --config /tmp/cfg.json --host host1') + }) + + it('still escapes a backtick in the literal text', () => { + expect(escapeTemplateBackticks('a ` b')).toBe('a \\` b') + expect(escapeTemplateBackticks('a ` ${x} ` b')).toBe('a \\` ${x} \\` b') + }) + + it('leaves an escaped interpolation as literal text', () => { + // `\\${...}` is escaped in the template source, so the backticks inside it are literal + // text and still need escaping. + expect(escapeTemplateBackticks('\\${foo `bar`}')).toBe('\\${foo \\`bar\\`}') + expect(unescapeTemplateBackticks('\\${foo \\`bar\\`}')).toBe('\\${foo `bar`}') + expect( + () => new Function('return `' + escapeTemplateBackticks('\\${foo `bar`}') + '`') + ).not.toThrow() + }) + + // Braces, quotes, regex literals and comments all hide backticks and braces from anything + // short of a real lexer, which is why the parser decides. + it('handles text only a lexer can read correctly', () => { + const inputs = [ + '${ x["}"] } `', + "${ f({ a: '`' }) } `", + "${ x.replace(/'/g, '') } `", + '${ /* ` */ x } `', + '{"match": ${/{/.test(flow_input.x)}, "literal": "`x`"}' + ] + for (const v of inputs) { + expect(() => new Function('return `' + escapeTemplateBackticks(v) + '`')).not.toThrow() + expect(unescapeTemplateBackticks(escapeTemplateBackticks(v))).toBe(v) + } + }) + + // The failure that matters most is not a syntax error but literal text quietly becoming code: + // here the author's `+ flow_input.y +` must stay text rather than being evaluated. + it('never lets literal text escape into the expression', () => { + const v = '{"match": ${/{/.test(flow_input.x)}, "literal": "` + flow_input.y + `"}' + const evaluated = new Function('flow_input', 'return `' + escapeTemplateBackticks(v) + '`')({ + x: 'x', + y: 'LEAKED' + }) + expect(evaluated).not.toContain('LEAKED') + expect(evaluated).toContain('` + flow_input.y + `') + }) + + // An expression the old blanket rule broke — it escaped backticks inside `${...}`, which + // does not parse — comes back as the author typed it, instead of showing the backslashes + // and escaping them one deeper on every save. + it('heals an expression the old rule broke', () => { + const broken = '-p ${a}${b ? \\` --x ${c}\\` : ""}' + const clean = '-p ${a}${b ? ` --x ${c}` : ""}' + expect(unescapeTemplateBackticks(broken)).toBe(clean) + expect(escapeTemplateBackticks(clean)).toBe(clean) + }) + + // A text that already parses is left alone: an over-escaped legacy value and a backslash the + // author wrote are the same bytes, so healing on looks alone would drop a real character. + it('leaves an expression that already parses alone', () => { + const run = (body: string) => new Function('return `' + body + '`')() + for (const stored of ['${"\\`"}', '${"a\\\\`"}']) { + expect(unescapeTemplateBackticks(stored)).toBe(stored) + expect(run(escapeTemplateBackticks(unescapeTemplateBackticks(stored)))).toBe(run(stored)) + } + }) + + // ...but a backtick escaped inside a nested template belongs there and must survive. + it('leaves an escaped backtick that is inside a nested template alone', () => { + const stored = '${cond ? `a\\`b` : ""}' + expect(unescapeTemplateBackticks(stored)).toBe(stored) + }) + + // Escapes the author wrote inside a nested template are not the old rule's doing, and + // stripping them changes what the expression means — here into chained tagged templates, + // which throw. Only a text whose backticks are *all* escaped came from the old rule. + it('leaves escapes that belong to a nested template alone', () => { + const run = (body: string) => new Function('flag', 'value', 'return `' + body + '`')(true, 'X') + for (const stored of ['${flag ? `\\`\\`${value}\\`\\`` : ""}', '${flag ? `a\\`b` : ""}']) { + expect(unescapeTemplateBackticks(stored)).toBe(stored) + expect(escapeTemplateBackticks(unescapeTemplateBackticks(stored))).toBe(stored) + expect(run(stored)).toBe(run(escapeTemplateBackticks(unescapeTemplateBackticks(stored)))) + } + }) + + it('round-trips through unescapeTemplateBackticks', () => { + for (const v of [ + nested, + 'a ` b', + '${ x["}"] } `', + 'plain', + '${a}${b}', + '\\${a}', + '${cond ? `a\\`b` : ""}' + ]) { + expect(unescapeTemplateBackticks(escapeTemplateBackticks(v))).toBe(v) + } + }) + // The guarantee that matters: opening a flow in the editor and saving it back must not change + // the stored expression, including for a value that only the all-or-nothing fallback can + // handle. + it('never rewrites the stored expression on a view/save cycle', () => { + const inputs = [ + '{"match": ${/{/.test(flow_input.x)}, "literal": "`x`"}', + '${cond ? `a\\`b` : ""}', + "${ x.replace(/'/g, '') } `", + 'a ` b', + '${ /* ` */ x } `' + ] + for (const v of inputs) { + const stored = escapeTemplateBackticks(v) + expect(() => new Function('return `' + stored + '`')).not.toThrow() + expect(escapeTemplateBackticks(unescapeTemplateBackticks(stored))).toBe(stored) + } + }) +}) diff --git a/frontend/src/lib/utils/templateLiteral.ts b/frontend/src/lib/utils/templateLiteral.ts new file mode 100644 index 0000000000..388169ea95 --- /dev/null +++ b/frontend/src/lib/utils/templateLiteral.ts @@ -0,0 +1,59 @@ +import { parseExpressionAt } from 'acorn' + +/** + * Template mode stores what the author typed as a JS template literal, so the text is spliced + * between backticks. A backtick in the literal part has to be escaped or it ends the literal + * early — but one inside a `${...}` must not be, since a backslash is a syntax error in + * expression position and a nested template literal there is legitimate. + * + * Telling those apart means knowing where each `${...}` ends, which needs a real JS lexer: + * regex literals, comments and nested templates all hide braces and backticks from anything + * simpler. So rather than escaping selectively, ask the parser whether the text already reads as + * one template literal. If it does, it needs no escaping at all; if it does not, escape every + * backtick, which is what this did before nested templates were supported. + * + * Known limitation: a value mixing a bare literal backtick with a nested template cannot be + * expressed either way, and gets the all-or-nothing fallback. Escaping the literal one by hand + * makes the whole value parse and it is then kept verbatim. + */ +function isCompleteTemplateBody(text: string): boolean { + const source = '`' + text + '`' + try { + const node = parseExpressionAt(source, 0, { ecmaVersion: 'latest' }) + // The type is what rejects a body that closes its own literal early: `` ` + evil() + ` `` + // parses, but as a concatenation, and would evaluate the author's literal text. The span + // rejects a body that stops short, like `` `a` x ``. + return node.type === 'TemplateLiteral' && node.start === 0 && node.end === source.length + } catch { + return false + } +} + +/** Escape `text` so it can be wrapped in backticks and mean what the author typed. */ +export function escapeTemplateBackticks(text: string): string { + // No backtick means nothing to escape and nothing to decide, which is every ordinary value. + if (!text.includes('`')) { + return text + } + return isCompleteTemplateBody(text) ? text : text.replaceAll('`', '\\`') +} + +/** Inverse of {@link escapeTemplateBackticks}, for turning an expression back into a template. */ +export function unescapeTemplateBackticks(text: string): string { + if (!text.includes('`')) { + return text + } + const unescaped = text.replaceAll('\\`', '`') + if (escapeTemplateBackticks(unescaped) === text) { + return unescaped + } + // Only an expression the old blanket rule *broke* is healed: it escaped backticks inside + // `${...}` too, which does not parse, while the unescaped form does. A text that already + // parses is left alone even if it looks over-escaped, because the two are indistinguishable + // from the text alone and guessing changes what the expression means — `${"a\\\\`"}` is a + // backslash the author wrote, not one the old rule added. + if (!isCompleteTemplateBody(text) && isCompleteTemplateBody(unescaped)) { + return unescaped + } + return text +} diff --git a/frontend/src/lib/utils/workspaceId.test.ts b/frontend/src/lib/utils/workspaceId.test.ts new file mode 100644 index 0000000000..8972498138 --- /dev/null +++ b/frontend/src/lib/utils/workspaceId.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, it } from 'vitest' + +import { WORKSPACE_ID_MAX_LENGTH, toWorkspaceId, validateWorkspaceId } from './workspaceId' + +// This module was extracted so the import wizard and the create-workspace form +// could not drift on what a workspace id is. These pin the contract they share. + +describe('validateWorkspaceId', () => { + it('accepts dash-separated groups of word characters', () => { + for (const id of ['a', 'prod', 'my_ws', 'a-b-c', 'a1-2b', 'wm-fork-x']) { + expect(validateWorkspaceId(id), id).toBeUndefined() + } + }) + + it('rejects leading, trailing and doubled dashes, and anything outside \\w', () => { + for (const id of ['-a', 'a-', 'a--b', '', 'a b', 'a.b', 'a/b', 'é']) { + expect(validateWorkspaceId(id), id).toMatch(/letters, numbers and dashes/) + } + }) + + it('measures length against effectiveId, but the character rule against what was typed', () => { + const typed = 'a'.repeat(45) + // The typed id is fine on its own... + expect(validateWorkspaceId(typed)).toBeUndefined() + // ...but a fork submits it prefixed, and that is what the backend stores. + const prefixed = `wm-fork-${typed}` + expect(prefixed.length).toBeGreaterThan(WORKSPACE_ID_MAX_LENGTH) + expect(validateWorkspaceId(typed, prefixed)).toMatch(/too long/) + }) + + it('reports the character problem before the length one', () => { + expect(validateWorkspaceId('a b'.padEnd(80, 'c'))).toMatch(/letters, numbers and dashes/) + }) + + it('allows exactly the maximum length', () => { + const id = 'a'.repeat(WORKSPACE_ID_MAX_LENGTH) + expect(validateWorkspaceId(id)).toBeUndefined() + expect(validateWorkspaceId(id + 'a')).toMatch(/too long/) + }) +}) + +describe('toWorkspaceId', () => { + it('produces something validateWorkspaceId accepts', () => { + for (const raw of [ + 'Support automation', + ' GitHub Release Dashboard ', + 'a//b__c', + 'Ünïcødé nåme', + '---leading and trailing---', + 'MiXeD CaSe' + ]) { + const id = toWorkspaceId(raw) + expect(validateWorkspaceId(id), `${raw} -> ${id}`).toBeUndefined() + } + }) + + it('lowercases, collapses runs into single dashes, and trims them', () => { + expect(toWorkspaceId('Support automation')).toBe('support-automation') + expect(toWorkspaceId('a//b')).toBe('a-b') + expect(toWorkspaceId(' spaced out ')).toBe('spaced-out') + }) + + it('clips to the maximum length without leaving a trailing dash', () => { + // Slicing at 50 lands mid-separator here; the result must still be valid. + const raw = `${'a'.repeat(WORKSPACE_ID_MAX_LENGTH - 1)} tail` + const id = toWorkspaceId(raw) + expect(id.length).toBeLessThanOrEqual(WORKSPACE_ID_MAX_LENGTH) + expect(id.endsWith('-')).toBe(false) + expect(validateWorkspaceId(id)).toBeUndefined() + }) +}) + +describe('validateWorkspaceId — the reserved id', () => { + // `check_w_id_conflict` refuses it, and `existsWorkspace` reports it free, so without + // this the wizard walks the user to the last step before the create fails. + it('refuses `global`', () => { + expect(validateWorkspaceId('global')).toMatch(/not allowed/i) + }) + + it('refuses it as the effective id too', () => { + expect(validateWorkspaceId('wm-fork-x', 'global')).toMatch(/not allowed/i) + }) + + it('allows a fork named `global`, which reaches the backend as `wm-fork-global`', () => { + expect(validateWorkspaceId('global', 'wm-fork-global')).toBeUndefined() + }) + + it('still accepts ids that merely contain it', () => { + expect(validateWorkspaceId('global-ops')).toBeUndefined() + expect(validateWorkspaceId('my-global')).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/utils/workspaceId.ts b/frontend/src/lib/utils/workspaceId.ts new file mode 100644 index 0000000000..492a00d391 --- /dev/null +++ b/frontend/src/lib/utils/workspaceId.ts @@ -0,0 +1,53 @@ +/** + * What the backend accepts as a workspace id, in one place. Every screen that lets + * someone name a workspace validates against this — a laxer copy elsewhere only + * moves the rejection from the form to the create call, after the user has + * finished the whole flow. + */ + +/** Letters, digits and underscores in dash-separated groups: no leading, trailing or doubled dash. */ +export const WORKSPACE_ID_RE = /^\w+(-\w+)*$/ +/** The DB column and the git branch name derived from it both stop here. */ +export const WORKSPACE_ID_MAX_LENGTH = 50 + +/** `validate_workspace_name` (windmill-common/src/workspaces.rs:246) refuses a longer name. */ +export const WORKSPACE_NAME_MAX_LENGTH = 50 + +/** `check_w_id_conflict` (windmill-api-workspaces/src/workspaces.rs:5111) rejects this id. */ +const RESERVED_WORKSPACE_ID = 'global' + +/** + * The reason `id` is not a usable workspace id, or undefined when it is. + * + * `effectiveId` is what actually reaches the backend: a fork's id is submitted + * with a `wm-fork-` prefix, so the length limit applies to the prefixed form while + * the character rule still applies to what the user typed. + */ +export function validateWorkspaceId(id: string, effectiveId: string = id): string | undefined { + if (!WORKSPACE_ID_RE.test(id)) { + return 'ID can only contain letters, numbers and dashes and must not finish by a dash' + } + // `check_w_id_conflict` refuses it outright, and `existsWorkspace` reports it free — + // so without this the wizard walks the user to the last step before the create fails. + // Only the effective id, which is what the backend receives: it defaults to the raw one, + // so a plain `global` is still caught, while a fork named `global` — submitted as + // `wm-fork-global`, which the backend accepts — is not. + if (effectiveId === RESERVED_WORKSPACE_ID) { + return `'${RESERVED_WORKSPACE_ID}' is not allowed as a workspace ID` + } + if (effectiveId.length > WORKSPACE_ID_MAX_LENGTH) { + return `ID '${effectiveId}' is too long (${effectiveId.length} chars). Maximum is ${WORKSPACE_ID_MAX_LENGTH}.` + } + return undefined +} + +/** Slugifies free text into something `validateWorkspaceId` accepts, for a prefill. */ +export function toWorkspaceId(raw: string): string { + return raw + .toLowerCase() + .replace(/[^a-z0-9-]+/g, '-') + .replace(/-{2,}/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, WORKSPACE_ID_MAX_LENGTH) + .replace(/-+$/, '') +} diff --git a/frontend/src/lib/workspaceCreation.ts b/frontend/src/lib/workspaceCreation.ts new file mode 100644 index 0000000000..49e4bdbea3 --- /dev/null +++ b/frontend/src/lib/workspaceCreation.ts @@ -0,0 +1,80 @@ +/** + * The instance policy around creating a workspace, and what the client has to do + * once one exists. Shared by every screen that creates workspaces — the workspace + * settings creator and the hub import wizard — so the two cannot drift apart on + * who is allowed to create, what username the backend expects, or what state the + * client is left in afterwards. + * + * Id validation lives next door in `$lib/utils/workspaceId`, which has no imports + * and so can be used from anywhere. + */ + +import { SettingService, UserService, WorkspaceService } from '$lib/gen' +import { usersWorkspaceStore } from '$lib/stores' +import { switchWorkspace } from '$lib/storeUtils' +import { isCloudHosted } from '$lib/cloud' +import { base } from '$lib/base' + +/** + * Whether this user may create a workspace at all. Self-hosted instances default + * `CREATE_WORKSPACE_REQUIRE_SUPERADMIN` to true, so offering the choice to + * everyone ends in a 403 at the last step. + * + * Superadmin arrives asynchronously in most callers, so pass the current value and + * call again when it flips. When the gate is on, the server is asked directly + * rather than trusting that value: `refreshSuperadmin` skips its fetch once the + * store holds anything, so a page loaded logged out leaves it `false` for the rest + * of the session — including right after signing in. + */ +export async function canCreateWorkspace(isSuperadmin: boolean): Promise { + if (isSuperadmin || isCloudHosted()) return true + try { + const r = await fetch(base + '/api/workspaces/create_workspace_require_superadmin') + if ((await r.text()) != 'true') return true + return !!(await UserService.globalWhoami()).super_admin + } catch { + return false + } +} + +export interface UsernamePolicy { + /** When true the backend derives the username and rejects one sent explicitly. */ + automate: boolean + /** A username to prefill with, when the caller has to ask for one. */ + suggested?: string +} + +/** + * `createWorkspace` rejects a username when the instance automates them and + * requires one when it does not, so the field only exists in the second case. + */ +export async function loadUsernamePolicy(): Promise { + const automate = + ((await SettingService.getGlobal({ + key: 'automate_username_creation' + })) as boolean | null) ?? true + if (automate) return { automate: true } + try { + const me = await UserService.globalWhoami() + const from = me.name ? me.name.split(' ')[0] : me.email.split('@')[0] + return { automate: false, suggested: from.replace(/\./g, '').toLowerCase() } + } catch { + return { automate: false } + } +} + +/** + * Re-reads the workspaces this user belongs to. Anything that creates or deletes a + * workspace owes the client this call: `usersWorkspaceStore` is what the picker and + * every derived workspace list read from, and nothing else refreshes it until a + * full page load. + */ +export async function refreshWorkspaceList(): Promise { + usersWorkspaceStore.set(await WorkspaceService.listUserWorkspaces()) +} + +/** Enters a workspace that was just created, with the list refreshed to match. */ +export async function enterNewWorkspace(id: string): Promise { + await refreshWorkspaceList() + switchWorkspace(id) +} diff --git a/frontend/src/routes/(root)/(logged)/+page.svelte b/frontend/src/routes/(root)/(logged)/+page.svelte index c26dfdf2d0..89bdd41e1c 100644 --- a/frontend/src/routes/(root)/(logged)/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/+page.svelte @@ -5,22 +5,12 @@ import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte' import FlowIcon from '$lib/components/home/FlowIcon.svelte' - import CreateActionsMenu from '$lib/components/home/CreateActionsMenu.svelte' import { getScriptByPath } from '$lib/scripts' import type { HubItem } from '$lib/components/flows/pickers/model' import PickHubScript from '$lib/components/flows/pickers/PickHubScript.svelte' import PickHubFlow from '$lib/components/flows/pickers/PickHubFlow.svelte' import HighlightCode from '$lib/components/HighlightCode.svelte' - import HomeConnectDrawer from '$lib/components/home/HomeConnectDrawer.svelte' - import { - ExternalLink, - GitFork, - Globe2, - Loader2, - Code, - LayoutDashboard, - PlugZap - } from 'lucide-svelte' + import { ExternalLink, GitFork, Globe2, Loader2, Code, LayoutDashboard } from 'lucide-svelte' import { hubBaseUrlStore } from '$lib/stores' import { base } from '$lib/base' @@ -28,7 +18,6 @@ import PickHubApp from '$lib/components/flows/pickers/PickHubApp.svelte' import { writable } from 'svelte/store' import type { EditorBreakpoint } from '$lib/components/apps/types' - import { HOME_SHOW_HUB } from '$lib/consts' import { setQuery } from '$lib/navigation' import { page } from '$app/state' import { goto, replaceState } from '$app/navigation' @@ -42,6 +31,8 @@ import NoDirectDeployAlert from '$lib/components/NoDirectDeployAlert.svelte' import { useSearchParams } from '$lib/svelte5UtilsKit.svelte' import { z } from 'zod' + import HomeAIChat from '$lib/components/home/HomeAIChat.svelte' + import { isGlobalAiEnabled } from '$lib/components/copilot/chat/global/gate' type Tab = 'hub' | 'workspace' @@ -97,7 +88,6 @@ } let workspaceTutorials: WorkspaceTutorials | undefined = $state(undefined) - let homeConnectDrawer: HomeConnectDrawer | undefined = $state(undefined) // Provide workspaceTutorials to child components via a reactive wrapper let workspaceTutorialsContext = $derived(workspaceTutorials) @@ -272,49 +262,23 @@ > -
    - {#if $workspaceStore == 'admins'} -
    +
    + + {#if isGlobalAiEnabled()} +
    + +
    + {/if} + {#if $workspaceStore == 'admins'} The Admins workspace is for admins only and contains scripts whose purpose is to manage your Windmill instance, such as keeping resource types up to date. +
    {/if} -
    -

    - Home -

    -
    - {#if !$userStore?.operator && HOME_SHOW_HUB} - - {/if} - - {#if !$userStore?.operator && showCreateButtons} -
    - -
    - {/if} -
    -
    @@ -396,9 +360,8 @@
    {#if tab == 'workspace'} - + {/if}
    - diff --git a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte index 96f0c4f2f6..6d2a0a5ca4 100644 --- a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte @@ -732,9 +732,6 @@ rightTooltip: 'Fill args from JSON' }} lightMode - on:change={(e) => { - runForm?.setCode(JSON.stringify(args ?? {}, null, '\t')) - }} /> {/if}
    @@ -817,7 +814,7 @@ const nargs = JSON.parse(JSON.stringify(e.detail)) args = nargs if (jsonView) { - runForm?.setCode(JSON.stringify(args ?? {}, null, '\t')) + runForm?.syncJsonEditor() } }} /> diff --git a/frontend/src/routes/(root)/(logged)/projects/import/+page@(root).svelte b/frontend/src/routes/(root)/(logged)/projects/import/+page@(root).svelte new file mode 100644 index 0000000000..a1bc2ab970 --- /dev/null +++ b/frontend/src/routes/(root)/(logged)/projects/import/+page@(root).svelte @@ -0,0 +1,550 @@ + + +{#if leaving} + +
    + + Taking you to {planWorkspaceId(plan) ?? 'your workspace'}… +
    +     +{:else if !slug} + +

    + This page needs a ?hub=<slug> to know which project to import. + Open it from a project on the hub. +

    +     +{:else} + + + + {#snippet subtitleSnippet()} + + Connected as {$usersWorkspaceStore?.email} + {#if step === 1} + · + + Switch account + + {/if} + + {/snippet} + + + + {#if step === 1} +
    + {#if project} + + {:else if projectError} +

    + Could not read {slug} from the hub. You can still choose a + destination — the project is fetched again before it is imported. +

    + {:else} +
    + Loading {slug}… +
    + {/if} + +
    +

    Where should it go?

    + + +
    + {#if canCreate} + (chosen = 'new')} + > + {#snippet icon()} + + {/snippet} + {#snippet description()} + Creates {name || slug} and imports + {itemsLabel} into it. + {/snippet} + + {/if} + + (chosen = 'existing')} + > + {#snippet icon()} + + {/snippet} + +
    + +
    + +
    +
    +
    + {:else if step === 2} +
    + {#if !choiceIsExisting} +
    +

    Name the new workspace

    +
    + +
    + + +
    + + {#if !automateUsername} + + {/if} + {:else} +
    +

    Pick a workspace

    +

    The project is imported into this one.

    +
    + + {#if workspaceList.loading} +
    + Loading your workspaces… +
    + {:else if workspaceList.error} +

    + Could not list your workspaces. Reload the page, or go back and create a new one. +

    + {:else if workspaces.length === 0} +

    + You are not a member of any workspace yet. Go back and create one, or ask an admin to + invite you. +

    + {:else} + + {#if workspaces.length > 1} +
    +
    + + +
    + {#if hasForks} + + {/if} +
    + {/if} + + + + {/if} + {/if} + +
    + + {#if !choiceIsExisting} + + {/if} +
    +
    + {:else if step === 3} + go({ folder }, 3, { replace: true })} + onFinish={() => + setupNeeded + ? // Replaces rather than pushes: after a reload on step 4 the run is gone, and + // a step-3 entry in history is a browser-Back route to the same fresh import + // the stepper is now blocked from reaching. + go({}, 4, { replace: true }) + : finish()} + onBack={() => go({}, 2)} + onExecution={(e) => (execution = e)} + resume={execution} + /> + {:else} + go({}, 3) : undefined} + /> + {/if} +     +{/if} diff --git a/frontend/src/routes/(root)/(logged)/projects/install/+page.svelte b/frontend/src/routes/(root)/(logged)/projects/install/+page.svelte deleted file mode 100644 index 06f2344356..0000000000 --- a/frontend/src/routes/(root)/(logged)/projects/install/+page.svelte +++ /dev/null @@ -1,374 +0,0 @@ - - -
    - {#if !slug} -

    Missing ?hub=<slug>.

    - {:else if loading} -
    - Loading project… -
    - {:else if loadError} -

    Failed to load project: {loadError}

    - {:else if data} -

    Add “{data.project.name}” to workspace

    -

    {data.project.summary}

    - -
    -

    - Folder in {workspace} -

    - -

    - Items import under f/{folderName.trim() || data.project.slug}/. -

    -
    - -
    - {counts?.scripts} scripts - {counts?.flows} flows - {counts?.apps} apps - {counts?.resources} resources - {counts?.triggers} triggers - {#if counts && counts.migrations > 0} - {counts.migrations} data table migrations - {/if} -
    - -
    - Resources are imported as empty stubs — set their values after import; a resource whose path - already exists is reported as failed (existing values are never overwritten). Trigger kinds - are recreated disabled, except GCP and Azure triggers, which manage cloud subscriptions at - creation and must be re-created manually after filling their resource. Kafka, NATS, SQS, GCP - and Azure triggers all require Enterprise. Triggers that reference a resource depend on stubs - imported empty, so fill in the resource value before re-enabling the trigger. -
    - -
    - - {#if done} - - {/if} -
    - - {#if results.length} -
      - {#each results as r} -
    • - {r.ok ? '✓' : '✗'} - {r.path} - {#if !r.ok}— {r.error}{/if} -
      • - {/each} -
    - {/if} - {/if} -
    - - - - - - closeMigrationReview(false)}> - closeMigrationReview(false)}> -
    -

    - This project ships migrations that recreate the data tables it uses. Review and edit the - SQL, then choose which to run. A migration runs against the data table of the same name in - {workspace}; if that data table has migrations enabled it is - recorded, otherwise it runs once as a preview job. -

    - {#each reviewList as m (m.datatable_name)} -
    -
    - {m.datatable_name} - -
    - {#if m.run} - - {/if} -
    - {/each} -
    - {#snippet actions()} - - - {/snippet} -     -     diff --git a/frontend/src/routes/(root)/(logged)/projects/install/+page.ts b/frontend/src/routes/(root)/(logged)/projects/install/+page.ts new file mode 100644 index 0000000000..718b624bd4 --- /dev/null +++ b/frontend/src/routes/(root)/(logged)/projects/install/+page.ts @@ -0,0 +1,13 @@ +import { redirect } from '@sveltejs/kit' +import { base } from '$app/paths' +import type { PageLoad } from './$types' + +/** + * `/projects/install?hub=` was where the hub's "Add to workspace" button + * pointed before the import wizard existed. Hubs upgrade on their own schedule — + * a self-hosted one may keep sending people here for a long time — so the old + * entry point forwards to the wizard rather than 404ing, query string intact. + */ +export const load: PageLoad = ({ url }) => { + redirect(307, `${base}/projects/import${url.search}`) +} diff --git a/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte b/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte index 6b3b0e5c1f..f0023ddb49 100644 --- a/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte @@ -388,7 +388,7 @@ if (!held || !current || block?.label !== 'retry' || block['dbt_retry_job']) return args = { ...current, command: { ...block, dbt_retry_job: held } } if (jsonView) { - runForm?.setCode(JSON.stringify(args, null, '\t')) + runForm?.syncJsonEditor() } }) .catch(() => {}) @@ -430,7 +430,7 @@ } } if (jsonView) { - runForm?.setCode(JSON.stringify(args, null, '\t')) + runForm?.syncJsonEditor() } } @@ -934,9 +934,6 @@ rightTooltip: 'Fill args from JSON' }} lightMode - on:change={(e) => { - runForm?.setCode(JSON.stringify(args ?? {}, null, '\t')) - }} /> {/if}
    @@ -1051,7 +1048,7 @@ const nargs = JSON.parse(JSON.stringify(e.detail)) args = nargs if (jsonView) { - runForm?.setCode(JSON.stringify(args ?? {}, null, '\t')) + runForm?.syncJsonEditor() } }} /> diff --git a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte index b6f5029616..fbf045fe19 100644 --- a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte @@ -1,5 +1,5 @@ diff --git a/frontend/src/routes/(root)/(logged)/user/(user)/workspaces/+page.svelte b/frontend/src/routes/(root)/(logged)/user/(user)/workspaces/+page.svelte index c4d2168d26..ed9bf2b11e 100644 --- a/frontend/src/routes/(root)/(logged)/user/(user)/workspaces/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/user/(user)/workspaces/+page.svelte @@ -26,6 +26,7 @@ import { switchWorkspace } from '$lib/storeUtils' import { GitFork, Settings, User, Search, ChevronsDownUp, ChevronsUpDown } from 'lucide-svelte' import { isCloudHosted } from '$lib/cloud' + import { canCreateWorkspace } from '$lib/workspaceCreation' import AnimatedButton from '$lib/components/common/button/AnimatedButton.svelte' import { emptyString } from '$lib/utils' import { getUserExt } from '$lib/user' @@ -104,9 +105,7 @@ let onlyAdminsWorkspace = $derived(allWorkspaces.length === 1 && allWorkspaces[0].id === 'admins') async function getCreateWorkspaceRequireSuperadmin() { - const r = await fetch(base + '/api/workspaces/create_workspace_require_superadmin') - const t = await r.text() - createWorkspace = t != 'true' + createWorkspace = await canCreateWorkspace(false) } let createWorkspace = $state($superadmin || isCloudHosted()) diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index 782da1bb2c..8dd2937273 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -41,7 +41,7 @@ import { sendUserToast } from '$lib/toast' import { clone, emptyString, encodeState, hasUnsavedChanges } from '$lib/utils' import { downloadViaClient, shouldDownloadViaClient } from '$lib/utils/downloadFile' - import { Slack, Target } from 'lucide-svelte' + import { ExternalLink, Slack, Target } from 'lucide-svelte' import SidebarNavigation from '$lib/components/common/sidebar/SidebarNavigation.svelte' import PremiumInfo from '$lib/components/settings/PremiumInfo.svelte' @@ -1472,9 +1472,30 @@ This workspace is a fork of {currentWorkspace.parent_workspace_id}. It runs on the parent's plan and its executions count toward the parent's usage and bill, - so there is no separate subscription here. Manage billing, seats, and quotas from - the parent workspace's settings. + so it is never invoiced separately. Manage billing, seats, and quotas from the + parent workspace's settings. + {#if plan} +
    + + It is on a paid plan that is billed on its own, so this workspace is paid for + twice. Cancel that subscription in the customer portal to keep only + {currentWorkspace.parent_workspace_id}'s plan. This workspace keeps + running either way, on the parent's plan. + {#if customer_id} +
    + +
    + {/if} +     +
    + {/if} {:else} {/if} diff --git a/frontend/src/routes/(root)/+layout.svelte b/frontend/src/routes/(root)/+layout.svelte index 1f7a813eac..0e404afbcc 100644 --- a/frontend/src/routes/(root)/+layout.svelte +++ b/frontend/src/routes/(root)/+layout.svelte @@ -1,4 +1,5 @@