diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 5dc2004c6f..f4fac9d2e0 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.804.0" + ".": "1.805.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 4acfb0edad..f431b9fd96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,30 @@ # Changelog +## [1.805.0](https://github.com/windmill-labs/windmill/compare/v1.804.0...v1.805.0) (2026-09-07) + + +### Features + +* **git-sync:** sync extra_perms for variables ([#11004](https://github.com/windmill-labs/windmill/issues/11004)) ([ee9e550](https://github.com/windmill-labs/windmill/commit/ee9e550a484fda286eeab43b7db5f314b8b2d0d9)) +* go to referenced row from foreign-keyed cells in the database manager ([#10998](https://github.com/windmill-labs/windmill/issues/10998)) ([e2b63d1](https://github.com/windmill-labs/windmill/commit/e2b63d177ae4e5c980cb5da34154540c90771b63)) +* let `// materialize` declare a `dbt://` warehouse-relation write ([#10978](https://github.com/windmill-labs/windmill/issues/10978)) ([c6e0302](https://github.com/windmill-labs/windmill/commit/c6e0302d7c1c60147f19d55a3923be8b1aa99c9c)) +* report resource type picks to the hub and rank pickers by popularity ([#10982](https://github.com/windmill-labs/windmill/issues/10982)) ([48a5615](https://github.com/windmill-labs/windmill/commit/48a56158c135c3b13a02f73b7b8438bc691f85b4)) +* run a linked AI agent's draft when testing a flow, and offer to deploy it ([#10993](https://github.com/windmill-labs/windmill/issues/10993)) ([7feaf61](https://github.com/windmill-labs/windmill/commit/7feaf619cf0ec021d66be14ef535cf2149bec58a)) +* show the new-tab icon on a chat path pill while the modifier is held ([#10976](https://github.com/windmill-labs/windmill/issues/10976)) ([5da4ea4](https://github.com/windmill-labs/windmill/commit/5da4ea43fbd01e43aa14e75dc597d7ce5d8797ab)) + + +### Bug Fixes + +* **cli:** keep permissioned_as on single-item push, as sync push does ([#11000](https://github.com/windmill-labs/windmill/issues/11000)) ([5f3f99b](https://github.com/windmill-labs/windmill/commit/5f3f99ba6915b7c5df663a30b35f4cd02050e728)) +* **cli:** say which workspace id is targeted, and when wmill.yaml is bypassed ([#11006](https://github.com/windmill-labs/windmill/issues/11006)) ([7643e9b](https://github.com/windmill-labs/windmill/commit/7643e9bd77c56f72596b8dca50801baf58984198)) +* **frontend:** no phantom draft when opening a CLI-pushed script ([#10997](https://github.com/windmill-labs/windmill/issues/10997)) ([1be390a](https://github.com/windmill-labs/windmill/commit/1be390aa878e15a58f530f3a878e8f9caeb89c43)) +* **frontend:** stop hover flicker on asset nodes shared with an overflow popover ([#10996](https://github.com/windmill-labs/windmill/issues/10996)) ([519a5c8](https://github.com/windmill-labs/windmill/commit/519a5c8bc70b44a7417e83c26c7b9c58b2c4fb9c)) +* let a draft-only schedule, trigger or resource be deleted ([#11010](https://github.com/windmill-labs/windmill/issues/11010)) ([8d0f475](https://github.com/windmill-labs/windmill/commit/8d0f4754e4e0c78696ee0c97ff2de3016ece3bac)) +* point the app viewer's edit button at the editor for the app's kind ([#11009](https://github.com/windmill-labs/windmill/issues/11009)) ([8f553ea](https://github.com/windmill-labs/windmill/commit/8f553eab353103fd8a28a00532e1766f133590de)) +* seed runs page filter defaults through the url so they survive sync ([#11005](https://github.com/windmill-labs/windmill/issues/11005)) ([f381acd](https://github.com/windmill-labs/windmill/commit/f381acdb37f66f5e272bc37938e69f734987d53f)) +* stop an untouched item's form from saving a draft nobody wrote ([#10964](https://github.com/windmill-labs/windmill/issues/10964)) ([c3f7f8a](https://github.com/windmill-labs/windmill/commit/c3f7f8a45830fb548aa628ebf6e2b6c95c6de67f)) +* write and read python job files as utf-8, not the platform locale ([#10994](https://github.com/windmill-labs/windmill/issues/10994)) ([670404f](https://github.com/windmill-labs/windmill/commit/670404ffe27fedc3858b46b0c6b3312fbe175e13)) + ## [1.804.0](https://github.com/windmill-labs/windmill/compare/v1.803.0...v1.804.0) (2026-09-05) diff --git a/backend/.sqlx/query-1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6.json b/backend/.sqlx/query-1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6.json new file mode 100644 index 0000000000..b60dd43fdf --- /dev/null +++ b/backend/.sqlx/query-1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings\n SET dbt_warehouses = '{\"main\": {\"resource_path\": \"u/test-user/wh\"}}'::jsonb\n WHERE workspace_id = 'test-workspace'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6" +} diff --git a/backend/.sqlx/query-1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704.json b/backend/.sqlx/query-1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704.json new file mode 100644 index 0000000000..77b00d9daa --- /dev/null +++ b/backend/.sqlx/query-1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind)\n VALUES ('test-workspace', 'main/analytics/marts', 'dbt', 'w', 'u/test-user/project',\n 'script')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704" +} diff --git a/backend/.sqlx/query-33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db.json b/backend/.sqlx/query-33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db.json new file mode 100644 index 0000000000..fbfab14017 --- /dev/null +++ b/backend/.sqlx/query-33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM asset WHERE workspace_id = 'test-workspace' AND kind = 'dbt' AND usage_path = 'u/test-user/ingest' AND usage_access_type = 'w'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db" +} diff --git a/backend/.sqlx/query-49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e.json b/backend/.sqlx/query-49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e.json new file mode 100644 index 0000000000..24e563d9ad --- /dev/null +++ b/backend/.sqlx/query-49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH producer AS (\n SELECT 'dbt://' || a.path AS trigger_ref, a.usage_path, s.language\n FROM asset a\n JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path\n AND s.archived = false AND s.deleted = false\n WHERE a.workspace_id = $1 AND a.kind = 'dbt'\n AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw')\n AND 'dbt://' || a.path = ANY($2)\n )\n SELECT DISTINCT st.trigger_ref || ' → ' || st.runnable_path AS \"edge!\"\n FROM script_trigger st\n WHERE st.workspace_id = $1 AND st.trigger_kind = 'asset'\n AND st.trigger_ref = ANY($2)\n AND EXISTS (SELECT 1 FROM producer p\n WHERE p.trigger_ref = st.trigger_ref\n AND p.usage_path <> st.runnable_path\n AND p.language = 'dbt')\n AND NOT EXISTS (SELECT 1 FROM producer p\n WHERE p.trigger_ref = st.trigger_ref\n AND p.usage_path <> st.runnable_path\n AND p.language <> 'dbt')\n ORDER BY 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "edge!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e" +} diff --git a/backend/.sqlx/query-6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7.json b/backend/.sqlx/query-6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7.json new file mode 100644 index 0000000000..07c923620a --- /dev/null +++ b/backend/.sqlx/query-6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by,\n language)\n VALUES ('test-workspace', 1, 'u/test-user/project', '', '', '', 'test-user', 'dbt')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7" +} diff --git a/backend/.sqlx/query-69c9ac0e5761c00a9e9a7ff96431e5ebfce96786448fb5273d49f13978df65c0.json b/backend/.sqlx/query-69c9ac0e5761c00a9e9a7ff96431e5ebfce96786448fb5273d49f13978df65c0.json new file mode 100644 index 0000000000..b1b92ecc75 --- /dev/null +++ b/backend/.sqlx/query-69c9ac0e5761c00a9e9a7ff96431e5ebfce96786448fb5273d49f13978df65c0.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT resource_type, count(*) as \"count!\" FROM resource WHERE workspace_id = $1 GROUP BY resource_type", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "resource_type", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "69c9ac0e5761c00a9e9a7ff96431e5ebfce96786448fb5273d49f13978df65c0" +} diff --git a/backend/.sqlx/query-ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf.json b/backend/.sqlx/query-ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf.json new file mode 100644 index 0000000000..2e0e072867 --- /dev/null +++ b/backend/.sqlx/query-ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT trigger_ref FROM script_trigger WHERE workspace_id = 'test-workspace' AND runnable_path = 'u/test-user/consumer' AND trigger_kind = 'asset'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "trigger_ref", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf" +} diff --git a/backend/.sqlx/query-b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946.json b/backend/.sqlx/query-b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946.json new file mode 100644 index 0000000000..6b2ec01b23 --- /dev/null +++ b/backend/.sqlx/query-b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946.json @@ -0,0 +1,63 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT s.path AS \"path!\", s.language AS \"language!: ScriptLang\"\n FROM asset a\n JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path\n AND s.archived = false AND s.deleted = false\n WHERE a.workspace_id = $1 AND a.kind = 'dbt' AND a.path = $2\n AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw')\n AND a.usage_path <> ALL($3)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "language!: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby", + "rlang", + "dbt" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946" +} diff --git a/backend/.sqlx/query-e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570.json b/backend/.sqlx/query-e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570.json new file mode 100644 index 0000000000..46a9c8e505 --- /dev/null +++ b/backend/.sqlx/query-e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind)\n VALUES ('test-workspace', 'main/analytics/orders', 'dbt', 'w', 'u/test-user/project',\n 'script')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 6563951281..5a19b73d1c 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -2763,18 +2763,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.16" +version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -2782,27 +2782,27 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.20" +version = "0.9.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" +checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.22" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" [[package]] name = "crossterm_winapi" @@ -6640,9 +6640,9 @@ dependencies = [ [[package]] name = "io-uring" -version = "0.7.14" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d64d8ca234d152948ceaede1f419b6a83983a5ecccaac05fb337a809c96d3aa6" +checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb" dependencies = [ "bitflags 2.13.1", "cfg-if", @@ -6664,9 +6664,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.12.1" +version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" [[package]] name = "ipnetwork" @@ -9077,9 +9077,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a07a60cc7a4d00c91f95c685609d1d2f79050e6804b70ebedd7650f0b839bcf" +checksum = "6d45aeb61b4bf818e12d4205f2466f8c4748f85f4fce0146d1c03d69d753f0ad" dependencies = [ "memchr", "ucd-trie", @@ -9087,9 +9087,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3a83744a5c8455b8b3e0dc5031362780a347c878bdd11584d1a8984228cc88d" +checksum = "89cc5a242e25ed4e7704d0be240f2cfbe20a8c27e7e252d94835be93d92dc39f" dependencies = [ "pest", "pest_generator", @@ -9097,9 +9097,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd3451aa3de60d4b9a1e736885e4dea6b31617598026f12256ad566d63304a" +checksum = "7abf21475cc3820fe4b2ca2dc2142902f67a02189f3b5b3a229f4febc01a43e5" dependencies = [ "pest", "pest_meta", @@ -9110,9 +9110,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e04d3a0849e241d7dfce834c83b1c5edc8622009e8dd51a12ba1927c32f05496" +checksum = "adba4db388f687393c18c51348d44a41d870ca9df71a2c98172ea3035dc6936e" dependencies = [ "pest", ] @@ -14747,7 +14747,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-nats", @@ -14835,7 +14835,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.804.0" +version = "1.805.0" dependencies = [ "async-stream", "async-trait", @@ -14868,7 +14868,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14881,7 +14881,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "argon2", @@ -15021,7 +15021,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15044,7 +15044,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15061,7 +15061,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15087,7 +15087,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.804.0" +version = "1.805.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15097,7 +15097,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15114,7 +15114,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -15136,7 +15136,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15159,7 +15159,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15175,7 +15175,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15197,7 +15197,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15218,7 +15218,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15232,7 +15232,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-nats", @@ -15267,7 +15267,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15292,7 +15292,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15320,7 +15320,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15342,7 +15342,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15362,7 +15362,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15400,7 +15400,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15428,7 +15428,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.804.0" +version = "1.805.0" dependencies = [ "lazy_static", "serde", @@ -15440,7 +15440,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.804.0" +version = "1.805.0" dependencies = [ "argon2", "axum 0.8.9", @@ -15464,7 +15464,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15478,7 +15478,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15513,7 +15513,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.804.0" +version = "1.805.0" dependencies = [ "chrono", "lazy_static", @@ -15527,7 +15527,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15546,7 +15546,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.804.0" +version = "1.805.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15652,7 +15652,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.804.0" +version = "1.805.0" dependencies = [ "chrono", "futures", @@ -15672,7 +15672,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.804.0" +version = "1.805.0" dependencies = [ "regex", "serde", @@ -15687,7 +15687,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15714,7 +15714,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "futures", @@ -15731,7 +15731,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.804.0" +version = "1.805.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15747,7 +15747,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -15768,7 +15768,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -15799,7 +15799,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "arc-swap", @@ -15824,7 +15824,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-stream", @@ -15858,7 +15858,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "futures", @@ -15876,7 +15876,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.804.0" +version = "1.805.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15885,7 +15885,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -15897,7 +15897,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde_json", @@ -15909,7 +15909,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "gosyn", @@ -15921,7 +15921,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -15933,7 +15933,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde_json", @@ -15945,7 +15945,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "nu-parser", @@ -15956,7 +15956,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15967,7 +15967,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15979,7 +15979,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15990,7 +15990,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-recursion", @@ -16012,7 +16012,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde_json", @@ -16024,7 +16024,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -16038,7 +16038,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16055,7 +16055,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -16068,7 +16068,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde", @@ -16080,7 +16080,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -16098,7 +16098,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16114,7 +16114,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16130,7 +16130,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -16144,7 +16144,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-recursion", @@ -16183,7 +16183,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "const_format", @@ -16223,7 +16223,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.804.0" +version = "1.805.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16234,7 +16234,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-recursion", @@ -16269,7 +16269,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16293,7 +16293,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16326,7 +16326,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16353,7 +16353,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16386,7 +16386,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16406,7 +16406,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16440,7 +16440,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16476,7 +16476,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16499,7 +16499,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16523,7 +16523,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-nats", @@ -16547,7 +16547,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16582,7 +16582,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16610,7 +16610,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16635,7 +16635,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "bitflags 2.13.1", @@ -16654,7 +16654,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-once-cell", @@ -16772,7 +16772,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.804.0" +version = "1.805.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index b7ea007f78..9ef1d099ee 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.804.0" +version = "1.805.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.804.0" +version = "1.805.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 9ed900214c..13597b06c9 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.804.0" +version = "1.805.0" dependencies = [ "aho-corasick", "anyhow", @@ -6274,7 +6274,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.804.0" +version = "1.805.0" dependencies = [ "proc-macro2", "quote", @@ -6286,7 +6286,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.804.0" +version = "1.805.0" dependencies = [ "convert_case", "serde", @@ -6295,7 +6295,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -6307,7 +6307,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde_json", @@ -6319,7 +6319,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "gosyn", @@ -6331,7 +6331,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -6343,7 +6343,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde_json", @@ -6355,7 +6355,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "nu-parser", @@ -6366,7 +6366,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6377,7 +6377,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6389,7 +6389,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6400,7 +6400,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-recursion", @@ -6422,7 +6422,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde_json", @@ -6434,7 +6434,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -6448,7 +6448,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "convert_case", @@ -6465,7 +6465,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -6478,7 +6478,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde", @@ -6490,7 +6490,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -6508,7 +6508,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6524,7 +6524,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6540,7 +6540,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6572,7 +6572,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -6586,7 +6586,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index cbb7102820..7de860c637 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.804.0" +version = "1.805.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 843a83a310..30c6047255 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -29,10 +29,10 @@ pub enum AssetKind { Ducklake, DataTable, Volume, - /// A warehouse relation a dbt project builds or reads, - /// `dbt:////`, the warehouse named as the - /// workspace configures it. The scheme names the producer, the path stays - /// the relation — see `windmill_types::AssetKind::Dbt`. + /// A warehouse relation, `dbt:////`, the warehouse + /// named as the workspace configures it. The scheme names the namespace dbt + /// made — a script in any language but dbt's own can declare a write to one — + /// and the path stays the relation. See `windmill_types::AssetKind::Dbt`. Dbt, } @@ -288,11 +288,13 @@ pub struct RetrySpec { } // `// materialize [manual] [append] [key=] [history] [track=]` -// — declares that this script produces a *managed* materialization of `` -// (a `ducklake://` table). By default the runtime generates the write DDL around +// — declares that this script produces ``. A `ducklake://` table is +// materialized *managed* by default: the runtime generates the write DDL around // the script's single trailing `SELECT` and owns idempotency, partition-state // and snapshot capture. `manual` is the escape hatch: the script writes its own -// DDL and the runtime only records state (track-only). The reconciliation +// DDL and the runtime only records state (track-only) — and it is the only mode a +// `dbt://` warehouse relation has, since nothing generates warehouse DDL (deploy +// enforces that; see docs/dbt-runtime.md). The reconciliation // strategy options apply to managed mode: none → DELETE-by-partition + INSERT // (replace); `key=` → MERGE (dedup within slice, SCD type 1); `append` → // INSERT-only. `append` wins if both are given (deploy-time warning). @@ -804,6 +806,18 @@ pub fn canonicalize_table_asset_path(path: &str) -> String { ) } +/// Whether a `dbt://` path names a whole relation, `//`. +/// +/// Every producer spells one that way — the manifest ingest derives it from +/// `relation_name`, a `// materialize` target is checked against it — so anything +/// else can be produced by nothing and read by nothing. Both sides of the deploy +/// ask here rather than counting segments themselves: a subscription and a write +/// that disagreed on the shape would refuse and accept the same string. +pub fn is_full_relation_path(path: &str) -> bool { + let mut segments = path.split('/'); + segments.clone().count() == 3 && !segments.any(str::is_empty) +} + /// A doubled delimiter inside a quoted identifier is that delimiter, literally — /// the same rule the worker's `split_relation` applies to `relation_name`. Both /// have to decode it or one spelling of a table becomes two graph nodes: the dbt @@ -1740,10 +1754,7 @@ mod pipeline_annotation_tests { // just stop being the same node and the cross-boundary cascade never fires. #[test] fn table_paths_from_every_spelling_canonicalize_to_one_key() { - let canonical = Some(( - AssetKind::Dbt, - Cow::Owned("main/analytics/orders".into()), - )); + let canonical = Some((AssetKind::Dbt, Cow::Owned("main/analytics/orders".into()))); for spelling in [ // Hand-written annotation. "dbt://main/analytics/orders", @@ -1765,6 +1776,17 @@ mod pipeline_annotation_tests { } } + /// The shape both halves of the deploy check against: a subscription and a + /// write that disagreed on it would refuse and accept the same string. + #[test] + fn a_whole_relation_is_three_non_empty_segments() { + assert!(is_full_relation_path("main/analytics/orders")); + assert!(is_full_relation_path("main/archive.sales/orders")); + for partial in ["main", "main/analytics", "main/analytics/orders/x", "", "main//orders"] { + assert!(!is_full_relation_path(partial), "{partial} is not a relation"); + } + } + // A relation that overrode its database carries `.` in // one segment, and each half can be quoted independently. Stripping only // the outer pair leaves a key the manifest ingest never produces, so the @@ -1791,10 +1813,7 @@ mod pipeline_annotation_tests { // database qualifier. assert_eq!( parse_asset_syntax("dbt://main/\"sales.v2\"/orders", false), - Some(( - AssetKind::Dbt, - Cow::Owned("main/sales.v2/orders".into()) - )) + Some((AssetKind::Dbt, Cow::Owned("main/sales.v2/orders".into()))) ); } @@ -1813,14 +1832,8 @@ mod pipeline_annotation_tests { "dbt://main/analytics/\"order\"\"s\"", "main/analytics/order\"s", ), - ( - "dbt://main/`da``ta`/`orders`", - "main/da`ta/orders", - ), - ( - "dbt://main/[my]]schema]/[orders]", - "main/my]schema/orders", - ), + ("dbt://main/`da``ta`/`orders`", "main/da`ta/orders"), + ("dbt://main/[my]]schema]/[orders]", "main/my]schema/orders"), // And in one half of a database-qualified segment. ( "dbt://main/\"arch\"\"ive\".\"sales\"/orders", @@ -1839,8 +1852,14 @@ mod pipeline_annotation_tests { // apart. A lone delimiter treated as opening a quote would be dropped — // `sa"les` filed as `sales` — and the two derivations would split. for (decoded, spelled) in [ - ("dbt://main/sa\"les/orders", "dbt://main/\"sa\"\"les\"/orders"), - ("dbt://main/analytics/order\"s", "dbt://main/analytics/\"order\"\"s\""), + ( + "dbt://main/sa\"les/orders", + "dbt://main/\"sa\"\"les\"/orders", + ), + ( + "dbt://main/analytics/order\"s", + "dbt://main/analytics/\"order\"\"s\"", + ), ( "dbt://main/arch\"ive.sales/orders", "dbt://main/\"arch\"\"ive\".\"sales\"/orders", diff --git a/backend/windmill-api-groups/src/granular_acls.rs b/backend/windmill-api-groups/src/granular_acls.rs index c6f88dace5..07b8004d83 100644 --- a/backend/windmill-api-groups/src/granular_acls.rs +++ b/backend/windmill-api-groups/src/granular_acls.rs @@ -318,6 +318,19 @@ async fn add_granular_acl( ) .await? } + "variable" => { + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Variable { path: path.to_string(), parent_path: None }, + Some(format!("Variable '{}' changed permissions", path)), + true, + None, + ) + .await? + } _ => (), } @@ -528,6 +541,19 @@ async fn remove_granular_acl( ) .await? } + "variable" => { + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Variable { path: path.to_string(), parent_path: None }, + Some(format!("Variable '{}' changed permissions", path)), + true, + None, + ) + .await? + } _ => (), } } diff --git a/backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs b/backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs new file mode 100644 index 0000000000..60f846ca1f --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs @@ -0,0 +1,250 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +async fn deploy(port: u16, path: &str, content: &str) -> reqwest::Response { + authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&json!({ + "path": path, + "summary": "", + "description": "", + "content": content, + "language": "deno", + "schema": { "type": "object", "properties": {}, "required": [] } + })) + .send() + .await + .unwrap() +} + +/// A `dbt://` relation is one graph node only while every side spells it the same +/// way, and three sides derive that spelling independently: the `// materialize` +/// target becomes an `asset.path`, a `// on` ref becomes a `script_trigger`, and +/// the deploy-time refusal joins the two. The unit tests on `sole_dbt_producer` +/// prove the predicate; only a deploy proves the handler feeds it the key the +/// table actually holds — so a canonicalization that drifted on one side would +/// pass those and split the node here. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_dbt_materialize_target_deploy_contract(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + sqlx::query!( + r#"UPDATE workspace_settings + SET dbt_warehouses = '{"main": {"resource_path": "u/test-user/wh"}}'::jsonb + WHERE workspace_id = 'test-workspace'"# + ) + .execute(&db) + .await?; + + // Nothing generates warehouse DDL, so a managed target is refused rather than + // degraded into the track-only mode it would silently become. + let resp = deploy( + port, + "u/test-user/managed", + "// materialize dbt://main/analytics/orders\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("must be `manual`")); + + // The warehouse segment is the identity a dbt model keys on; a name the + // workspace does not configure strands the write on an unreachable node. + let resp = deploy( + port, + "u/test-user/unknown_wh", + "// materialize manual dbt://nope/analytics/orders\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("does not configure")); + + // Only the DuckDB executor runs `// data_test` probes, and it runs them around + // a managed write — so a declarer in another language would deploy green with + // its assertions silently never executed. + let resp = deploy( + port, + "u/test-user/tested", + "// materialize manual dbt://main/analytics/orders\n// data_test not_null id\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp + .text() + .await? + .contains("`// data_test` is not supported")); + + // Both halves are held to the same relation: every producer is a whole + // `//` under a configured warehouse, so a + // subscription to anything else names something nothing can ever write. + for (path, ref_, expected) in [ + ( + "u/test-user/partial_sub", + "dbt://main/analytics", + "not a whole warehouse relation", + ), + ( + "u/test-user/unknown_wh_sub", + "dbt://nope/analytics/orders", + "does not configure", + ), + ] { + let resp = deploy( + port, + path, + &format!("// on {ref_}\nexport async function main() {{}}"), + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains(expected)); + } + + // Past `asset.path`'s column, where the manifest ingest drops the relation and + // no producer row can exist on either side — computed from the bound so it + // cannot drift under it. + let overlong = format!( + "main/analytics/{}", + "o".repeat(windmill_common::dbt_manifest::MAX_ASSET_PATH_LEN) + ); + let resp = deploy( + port, + "u/test-user/overlong_sub", + &format!("// on dbt://{overlong}\nexport async function main() {{}}"), + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("characters an asset path holds")); + + // Any language may declare the write — the DuckLake write engine is DuckDB's, + // this declaration is not — and the target is canonicalized on the way into + // `asset`, so a hand-written mixed-case spelling lands on the model's key. + let resp = deploy( + port, + "u/test-user/ingest", + "// materialize manual dbt://main/ANALYTICS/Orders\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 201); + // The create response, not `{:x}` over the stored i64: `ScriptHash` decodes + // hex and demands 8 bytes, while `LowerHex` drops leading zeros, so a hash + // under 2^60 would 422 the rename below instead of reaching the refusal. + let ingest_hash = resp.text().await?; + let write = sqlx::query_scalar!( + "SELECT path FROM asset WHERE workspace_id = 'test-workspace' AND kind = 'dbt' \ + AND usage_path = 'u/test-user/ingest' AND usage_access_type = 'w'" + ) + .fetch_one(&db) + .await?; + assert_eq!(write, "main/analytics/orders"); + + // That producer is native, so subscribing to what it writes is accepted — and + // the `// on` ref has to canonicalize identically, or the row it stores names + // a relation nothing produces. + let resp = deploy( + port, + "u/test-user/consumer", + "// on dbt://main/\"Analytics\"/\"Orders\"\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 201); + let trigger_ref = sqlx::query_scalar!( + "SELECT trigger_ref FROM script_trigger WHERE workspace_id = 'test-workspace' \ + AND runnable_path = 'u/test-user/consumer' AND trigger_kind = 'asset'" + ) + .fetch_one(&db) + .await?; + assert_eq!(trigger_ref, "dbt://main/analytics/orders"); + + // With dbt as the only producer the same subscription can never be woken — a + // dbt run does not dispatch — so the deploy refuses it and names the project. + sqlx::query!( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by, + language) + VALUES ('test-workspace', 1, 'u/test-user/project', '', '', '', 'test-user', 'dbt')" + ) + .execute(&db) + .await?; + sqlx::query!( + "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) + VALUES ('test-workspace', 'main/analytics/marts', 'dbt', 'w', 'u/test-user/project', + 'script')" + ) + .execute(&db) + .await?; + let resp = deploy( + port, + "u/test-user/mart_consumer", + "// on dbt://main/analytics/MARTS\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("u/test-user/project")); + + // A rename is the other half of that: the producer's write still sits at the + // OLD path in the committed snapshot this deploy reads, while the same + // transaction removes it — so it must not count as the producer that would + // wake the subscription the rename adds. + sqlx::query!( + "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) + VALUES ('test-workspace', 'main/analytics/orders', 'dbt', 'w', 'u/test-user/project', + 'script')" + ) + .execute(&db) + .await?; + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&json!({ + "path": "u/test-user/ingest_renamed", + "parent_hash": ingest_hash, + "summary": "", + "description": "", + "content": "// on dbt://main/analytics/orders\nexport async function main() {}", + "language": "deno", + "schema": { "type": "object", "properties": {}, "required": [] } + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("u/test-user/project")); + + // Neither annotation is accepted on a dbt script: the graph ingest + // republishes that path's asset and trigger rows wholesale, so either would + // deploy something the dependency job then silently removes. + for content in [ + "# materialize manual dbt://main/analytics/orders\nprofile:\n warehouse: main\n", + "# on dbt://main/analytics/orders\nprofile:\n warehouse: main\n", + ] { + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&json!({ + "path": "u/test-user/dbt_project", + "summary": "", + "description": "", + "content": content, + "language": "dbt", + "modules": { "dbt_project.yml": { "content": "name: p\n", "language": "dbt" } }, + "schema": { "type": "object", "properties": {}, "required": [] } + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("a dbt script cannot")); + } + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/protection_rules.rs b/backend/windmill-api-integration-tests/tests/protection_rules.rs index 9469554999..e032acef7b 100644 --- a/backend/windmill-api-integration-tests/tests/protection_rules.rs +++ b/backend/windmill-api-integration-tests/tests/protection_rules.rs @@ -123,6 +123,72 @@ async fn test_protection_rules(db: Pool) -> anyhow::Result<()> { .await?; assert!(!resp.status().is_success(), "Non-admin should be blocked from flows: {}", resp.status()); + // ======================================== + // 4b. ...but a draft-only resource stays deletable: nothing is deployed at + // its path, so its DELETE is a draft discard rather than a deployment. + // ======================================== + + let draft_only_path = "u/test-user-2/draft_only_resource"; + let resp = authed( + client().post(format!("{base}/drafts/update/resource/{draft_only_path}")), + "SECRET_TOKEN_2", + ) + .json(&json!({ "value": { + "path": draft_only_path, + "value": { "a": 1 }, + "resource_type": "c_test", + "description": "" + }})) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "Non-admin should save a draft: {}", + resp.text().await? + ); + + let resp = authed( + client().delete(format!("{base}/resources/delete/{draft_only_path}")), + "SECRET_TOKEN_2", + ) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "Draft-only delete should not be gated by the deploy rules: {}", + resp.text().await? + ); + + let remaining: i64 = sqlx::query_scalar( + "SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND path = $1 \ + AND typ = 'resource'::DRAFT_KIND", + ) + .bind(draft_only_path) + .fetch_one(&db) + .await?; + assert_eq!(remaining, 0, "the draft should be gone"); + + // The gate itself is still there for a DEPLOYED resource at the same path. + let resp = authed( + client().post(format!("{base}/resources/create")), + "SECRET_TOKEN_2", + ) + .json(&json!({ + "path": draft_only_path, + "value": { "a": 1 }, + "resource_type": "c_test", + "description": "" + })) + .send() + .await?; + assert!( + !resp.status().is_success(), + "Non-admin should still be blocked from creating a resource: {}", + resp.status() + ); + // ======================================== // 5. Admin bypasses protection rule // ======================================== diff --git a/backend/windmill-api-integration-tests/tests/schedules.rs b/backend/windmill-api-integration-tests/tests/schedules.rs index 89ba0ec2cc..881f11ff43 100644 --- a/backend/windmill-api-integration-tests/tests/schedules.rs +++ b/backend/windmill-api-integration-tests/tests/schedules.rs @@ -113,7 +113,9 @@ async fn test_schedule_endpoints(db: Pool) -> anyhow::Result<()> { "expected at least 2 schedules, got {}", list.len() ); - assert!(list.iter().any(|s| s["path"] == "u/test-user/test_schedule")); + assert!(list + .iter() + .any(|s| s["path"] == "u/test-user/test_schedule")); // --- list_with_jobs --- let resp = authed(client().get(format!("{base}/list_with_jobs"))) @@ -125,18 +127,14 @@ async fn test_schedule_endpoints(db: Pool) -> anyhow::Result<()> { assert!(!list.is_empty()); // --- update --- - let resp = authed(client().post(schedule_url( - port, - "update", - "u/test-user/test_schedule", - ))) - .json(&json!({ - "schedule": "0 0 */12 * * *", - "timezone": "Europe/Paris" - })) - .send() - .await - .unwrap(); + let resp = authed(client().post(schedule_url(port, "update", "u/test-user/test_schedule"))) + .json(&json!({ + "schedule": "0 0 */12 * * *", + "timezone": "Europe/Paris" + })) + .send() + .await + .unwrap(); assert_eq!(resp.status(), 200, "update: {}", resp.text().await?); // verify update @@ -204,14 +202,11 @@ async fn test_schedule_endpoints(db: Pool) -> anyhow::Result<()> { assert_eq!(resp.status(), 200); // --- delete --- - let resp = authed(client().delete(schedule_url( - port, - "delete", - "u/test-user/another_schedule", - ))) - .send() - .await - .unwrap(); + let resp = + authed(client().delete(schedule_url(port, "delete", "u/test-user/another_schedule"))) + .send() + .await + .unwrap(); assert_eq!(resp.status(), 200); let resp = authed_get(port, "exists", "u/test-user/another_schedule").await; @@ -220,17 +215,108 @@ async fn test_schedule_endpoints(db: Pool) -> anyhow::Result<()> { // ===== Global endpoints ===== // --- preview --- - let resp = authed(client().post(format!( - "http://localhost:{port}/api/schedules/preview" - ))) - .json(&json!({ - "schedule": "0 0 */6 * * *", - "timezone": "UTC" - })) - .send() - .await - .unwrap(); + let resp = authed(client().post(format!("http://localhost:{port}/api/schedules/preview"))) + .json(&json!({ + "schedule": "0 0 */6 * * *", + "timezone": "UTC" + })) + .send() + .await + .unwrap(); assert_eq!(resp.status(), 200, "preview: {}", resp.text().await?); Ok(()) } + +/// A schedule with no `schedule` row is listed from the `draft` table, so its +/// DELETE drops that draft, then 404s once nothing is left at the path. A legacy +/// (`email IS NULL`) draft is owned by nobody and stays put. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_delete_draft_only_schedule(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let path = "u/test-user/draft_only_schedule"; + + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/update/trigger_schedule/{path}" + ))) + .json(&json!({ "value": { + "path": path, + "schedule": "0 0 */6 * * *", + "timezone": "UTC", + "script_path": "u/test-user/never_deployed", + "is_flow": false, + }})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "save draft: {}", resp.text().await?); + + let resp = authed(client().get(format!( + "http://localhost:{port}/api/w/test-workspace/schedules/list?include_draft_only=true" + ))) + .send() + .await + .unwrap(); + let listed: Vec = resp.json().await?; + assert!( + listed + .iter() + .any(|s| s["path"] == path && s["draft_only"] == json!(true)), + "draft-only schedule should be listed: {listed:?}" + ); + + let resp = authed(client().delete(schedule_url(port, "delete", path))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "delete: {}", resp.text().await?); + + let remaining: i64 = sqlx::query_scalar( + "SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND path = $1 \ + AND typ = 'trigger_schedule'::DRAFT_KIND", + ) + .bind(path) + .fetch_one(&db) + .await?; + assert_eq!(remaining, 0, "the draft should be gone"); + + let resp = authed(client().delete(schedule_url(port, "delete", path))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 404, "nothing left at the path"); + + // A legacy (email IS NULL) draft is owned by nobody and keeps the write gate + // on the drafts routes, so this one must not become a second door to it. + let legacy_path = "u/test-user/legacy_draft_only_schedule"; + sqlx::query( + "INSERT INTO draft (workspace_id, email, path, typ, value) \ + VALUES ('test-workspace', NULL, $1, 'trigger_schedule'::DRAFT_KIND, '{}'::json)", + ) + .bind(legacy_path) + .execute(&db) + .await?; + + let resp = authed(client().delete(schedule_url(port, "delete", legacy_path))) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 404, + "legacy draft is not this route's to delete" + ); + + let legacy_remaining: i64 = sqlx::query_scalar( + "SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND path = $1 \ + AND typ = 'trigger_schedule'::DRAFT_KIND", + ) + .bind(legacy_path) + .fetch_one(&db) + .await?; + assert_eq!(legacy_remaining, 1, "the legacy draft should survive"); + + Ok(()) +} diff --git a/backend/windmill-api-schedule/src/lib.rs b/backend/windmill-api-schedule/src/lib.rs index 6ae902e984..1b6b49a136 100644 --- a/backend/windmill-api-schedule/src/lib.rs +++ b/backend/windmill-api-schedule/src/lib.rs @@ -31,8 +31,8 @@ use windmill_common::{ self, TriggerHistoryEvent, TriggerOperation, TriggerSource, SCHEDULE_TRIGGER_KIND, }, user_drafts::{ - delete_all_drafts_for_path, fetch_draft_only_list_rows, overlay_or_draft_only, - UserDraftItemKind, WithDraftOverlay, WithDraftQuery, + delete_all_drafts_for_path, delete_draft_only_for_path, fetch_draft_only_list_rows, + overlay_or_draft_only, UserDraftItemKind, WithDraftOverlay, WithDraftQuery, }, utils::{ escape_ilike_pattern, not_found_if_none, paginate, Pagination, ScheduleType, StripPath, @@ -1331,6 +1331,18 @@ async fn delete_schedule( .flatten(); if exists.is_none() { + drop(tx); + if delete_draft_only_for_path( + &db, + &w_id, + UserDraftItemKind::TriggerSchedule, + path, + &authed.email, + ) + .await? + { + return Ok(format!("Draft-only schedule {} deleted", path)); + } return Err(windmill_common::error::Error::NotFound(format!( "Schedule {} not found", path diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index ccc5815aa7..d24b3c3a5c 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -39,8 +39,8 @@ use sqlx::{FromRow, Postgres, Transaction}; use std::{collections::HashMap, sync::Arc}; use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; use windmill_audit::ActionKind; -use windmill_dep_map::{lock_hash::record_lock_hashes, process_relative_imports}; use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap; +use windmill_dep_map::{lock_hash::record_lock_hashes, process_relative_imports}; use windmill_common::{ assets::{ @@ -1081,6 +1081,57 @@ fn lock_hash_entry(path: &str, lock: &str) -> [(String, i64); 1] { [(path.to_string(), hash_script(lock))] } +/// The `dbt://` relation both halves of a deploy have to agree on: a whole +/// `//`, under a warehouse this workspace configures. +/// +/// Every producer is held to exactly this — a `// materialize` target here, a +/// descriptor's `profile.warehouse` in the worker — so a subscription to anything +/// else names a relation nothing can ever write. No later deploy fixes that and +/// no dormant-edge warning reports it, since the warning fires on a dbt project's +/// ingest and no project can claim a relation under a warehouse that isn't there. +/// Asking here rather than at each site is what keeps the two from drifting into +/// refusing and accepting the same string. +async fn validate_dbt_relation( + db: &sqlx::Pool, + w_id: &str, + relation: &str, + what: &str, +) -> Result<()> { + if !windmill_parser::asset_parser::is_full_relation_path(relation) { + return Err(Error::BadRequest(format!( + "{what} `dbt://{relation}` is not a whole warehouse relation \ + (`dbt:////`)." + ))); + } + // `asset.path` is VARCHAR(255) and the manifest ingest drops a relation that + // outgrows it rather than failing the whole graph, so past the column no + // producer row can exist on either side — a write would be rejected by + // Postgres mid-deploy, and `script_trigger.trigger_ref` is unbounded text + // that would take the subscription and keep it dormant for good. + let max = windmill_common::dbt_manifest::MAX_ASSET_PATH_LEN; + if relation.chars().count() > max { + return Err(Error::BadRequest(format!( + "{what} `dbt://{relation}` is longer than the {max} characters an asset path \ + holds, so it cannot be recorded." + ))); + } + let warehouse = relation.split('/').next().unwrap_or_default(); + // Only the resolver's own "no such warehouse" is the annotation's fault. Its + // other failures — the query, and a setting entry with no `resource_path` — + // keep their own error: blaming the warehouse name for those misdescribes + // them, and flattening the malformed-setting one to a 400 hides a server + // fault behind a client one. + windmill_common::workspaces::dbt_warehouse_exists(db, w_id, warehouse) + .await + .map_err(|e| match e { + Error::NotFound(_) => Error::BadRequest(format!( + "{what} `dbt://{relation}` names a warehouse this workspace does not \ + configure: {e}" + )), + other => other, + }) +} + async fn create_script_internal<'c>( mut ns: NewScript, w_id: String, @@ -1565,34 +1616,90 @@ async fn create_script_internal<'c>( // membership; parsed writes tell us what is produced (we don't record // them in auto_kind itself). let pipeline_annotations = parse_pipeline_annotations(&ns.content); - // `// materialize` materializes a `ducklake:///` target from a - // DuckDB script. These two constraints hold for *both* modes: a non-DuckLake - // target would otherwise deploy, register a producer in the asset graph, then - // silently no-op at run time (`build_materialized_query` returns `Ok(None)`), - // and a non-DuckDB script never reaches the executor that records state. The - // managed-only checks (single trailing SELECT, no SQL args) come after — a - // `manual` script owns its DDL and skips them. + // `// materialize` names what this script produces. Two target kinds, and the + // runtime behind each is what constrains the annotation: + // • `ducklake:///
` — the DuckDB executor generates the write + // (or, in `manual` mode, records the state the script wrote itself), so + // the script has to be a DuckDB one and the target has to name a table. + // A non-DuckDB script never reaches that executor. + // • `dbt:////` — a warehouse relation. Nothing + // generates warehouse DDL, so the declaration is track-only (`manual`) + // and any language but dbt's own may make it: the script writes the + // relation, the worker records the materialization, and the relation's + // asset node is shared with whatever dbt model reads it. A dbt project's + // own writes are read from its manifest, so it may not declare one. + // Any other kind would deploy, register a producer in the asset graph, then + // silently no-op at run time (`build_materialized_query` returns `Ok(None)`). + // The managed-only checks (single trailing SELECT, no SQL args) come after — + // a `manual` script owns its DDL and skips them. if let Some(m) = pipeline_annotations.materialize.as_ref() { - if ns.language != ScriptLang::DuckDb { - return Err(Error::BadRequest(format!( - "`// materialize` is only supported for DuckDB scripts, not {}. Use the \ - wmll.ducklake helpers to materialize from other languages.", - ns.language.as_str() - ))); - } - if m.target_kind != windmill_parser::asset_parser::AssetKind::Ducklake { + use windmill_parser::asset_parser::AssetKind as PAssetKind; + // The producer half of the rule the trigger loop below applies to `// on`: + // a dbt project's writes come from its manifest, and the graph ingest + // republishes this path's asset rows wholesale, so a declared one would be + // wiped by the very deploy that accepted it while its runs kept stamping + // the relation. + if ns.language == ScriptLang::Dbt { return Err(Error::BadRequest( - "`// materialize` only supports a DuckLake target \ - (`ducklake:///
`); other asset kinds aren't materializable." + "a dbt script cannot declare `// materialize`: what a project builds is read \ + from its manifest and published by the graph ingest, not annotated." .to_string(), )); } - if !m.target_path.contains('/') { - return Err(Error::BadRequest(format!( - "`// materialize` needs a table in the target: \ - `ducklake://{0}/
` (got `ducklake://{0}`).", - m.target_path - ))); + match m.target_kind { + PAssetKind::Ducklake => { + if ns.language != ScriptLang::DuckDb { + return Err(Error::BadRequest(format!( + "`// materialize` is only supported for DuckDB scripts, not {}. Use the \ + wmll.ducklake helpers to materialize from other languages, or declare a \ + warehouse relation with `// materialize manual dbt://…`.", + ns.language.as_str() + ))); + } + if !m.target_path.contains('/') { + return Err(Error::BadRequest(format!( + "`// materialize` needs a table in the target: \ + `ducklake://{0}/
` (got `ducklake://{0}`).", + m.target_path + ))); + } + } + PAssetKind::Dbt => { + // `// data_test` runs as verifier probes the DuckDB executor + // splices around a MANAGED write. Nothing generates a warehouse + // write, so nothing would run them — and unlike the DuckLake + // `manual` case, which at least fails loudly in that executor, a + // declarer in another language would deploy green with its + // data-quality assertions silently never executed. + if !pipeline_annotations.data_tests.is_empty() { + return Err(Error::BadRequest( + "`// data_test` is not supported with a `dbt://` target: the checks run \ + against a managed materialization, and a warehouse relation is \ + written by the script itself. Assert on the relation with a dbt \ + test in the project that reads it." + .to_string(), + )); + } + if !m.manual { + return Err(Error::BadRequest( + "`// materialize dbt://…` must be `manual`: Windmill generates no \ + warehouse DDL, so the script issues its own write and only the outcome \ + is recorded. Write \ + `// materialize manual dbt:////`." + .to_string(), + )); + } + validate_dbt_relation(&db, &w_id, &m.target_path, "`// materialize` target") + .await?; + } + _ => { + return Err(Error::BadRequest( + "`// materialize` only supports a DuckLake (`ducklake:///
`) or \ + warehouse-relation (`dbt:////`) target; other asset \ + kinds aren't materializable." + .to_string(), + )); + } } if !m.manual { if let Err(e) = windmill_parser::sql_materialize::classify_wrap(&ns.content) { @@ -2375,16 +2482,43 @@ async fn create_script_internal<'c>( let Some((trigger_kind, trigger_ref)) = trigger_spec_to_row(spec) else { continue; }; - // A `dbt://` subscription can never fire: dbt is the only producer of a - // warehouse relation (`// materialize` takes DuckLake targets only) and a - // dbt run does not dispatch. Refusing beats persisting a row that draws a - // cascade arrow on the canvas and then never wakes anything. - if trigger_ref.starts_with("dbt://") { - return Err(Error::BadRequest(format!( - "`{trigger_ref}` cannot be subscribed to: a dbt run does not trigger downstream \ - runs, and nothing else writes a warehouse relation. Declare the read without \ - `on` to keep the lineage edge, or schedule this script." - ))); + // A `dbt://` subscription fires only when a NON-dbt job materialized the + // relation: `// materialize manual dbt://…` declares such a write, while a + // dbt run records its models and does not dispatch. So refuse exactly the + // edge that cannot fire — one whose relation is already claimed by dbt and + // by nothing else — rather than every `dbt://` edge (`sole_dbt_producer`, + // which takes the workspace pool: under RLS an unreadable native producer + // would refuse a live subscription). + if let Some(relation) = trigger_ref.strip_prefix("dbt://") { + // The subscriber side of the same rule: a dbt project is not woken by + // the asset cascade. Its graph ingest clears these rows for its own + // path, so accepting one here would deploy an edge the dependency job + // then silently removes. + if ns.language == ScriptLang::Dbt { + return Err(Error::BadRequest(format!( + "a dbt script cannot subscribe to `{trigger_ref}`: dbt orders its own DAG \ + and a project is run on its schedule, not woken by an asset cascade." + ))); + } + validate_dbt_relation(&db, &w_id, relation, "subscription target").await?; + // Both paths under a rename: the old one's committed write row is + // still there and this transaction is about to remove it. + let deploying_paths = match p_path_opt.as_deref().filter(|old| *old != ns.path) { + Some(old) => vec![ns.path.clone(), old.to_string()], + None => vec![ns.path.clone()], + }; + if let Some(dbt_owner) = + windmill_common::assets::sole_dbt_producer(&db, &w_id, relation, &deploying_paths) + .await? + { + return Err(Error::BadRequest(format!( + "`{trigger_ref}` cannot be subscribed to: it is built by the dbt project at \ + `{dbt_owner}`, and a dbt run does not trigger downstream runs. Declare the \ + read without `on` to keep the lineage edge, or schedule this script. A \ + relation written by a `// materialize manual {trigger_ref}` script can be \ + subscribed to." + ))); + } } // Effective debounce for this edge: per-`// on debounce=` wins, // else the script-level `// debounce` default. Debounce only diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 0630b3e262..f6960126b1 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.804.0 + version: 1.805.0 title: Windmill API contact: @@ -8528,6 +8528,84 @@ paths: items: type: string + /w/{workspace}/resources/type/resource_counts: + get: + summary: count the workspace's resources per resource type + operationId: listResourceCountsByType + tags: + - resource + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: resource count per resource_type + content: + application/json: + schema: + type: array + items: + type: object + properties: + resource_type: + type: string + count: + type: integer + required: + - resource_type + - count + + /w/{workspace}/resources/type/hub/info: + get: + summary: list what the hub knows about its resource types + operationId: listHubResourceTypeInfo + tags: + - resource + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: each hub resource type with its integration and pick count, empty if the hub answers neither read + content: + application/json: + schema: + type: array + items: + type: object + properties: + name: + type: string + app: + description: the integration the resource type belongs to, which is not always its own name + type: string + picks: + type: integer + required: + - name + - app + - picks + + /w/{workspace}/resources/type/hub/pick/{name}: + post: + summary: record a hub resource type pick + operationId: pickHubResourceType + tags: + - resource + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Name" + responses: + "200": + description: whether the hub recorded the pick + content: + application/json: + schema: + type: object + properties: + success: + type: boolean + required: + - success + /w/{workspace}/npm_proxy/config: get: summary: get npm proxy configuration @@ -8777,6 +8855,9 @@ paths: properties: name: type: string + picks: + description: how often the integration has been picked, absent on a hub that does not count picks + type: integer required: - name diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index a674e6c38a..36b6eab0bb 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -5825,6 +5825,9 @@ mod embed_token_tests { "GET", ), ("/api/w/test/resources/list_search", "GET"), + // The metadata allowlist matches `resources/type/` by prefix, so a route + // added under it that is not a read must be denied by its method. + ("/api/w/test/resources/type/hub/pick/slack", "POST"), // Workspace-wide job enumeration/export must NOT be reachable — an app // reads only jobs it launched, by id (blocked via the app_embed sentinel). ("/api/w/test/jobs/list", "GET"), diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 05cd354cd4..1c22981ee0 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -346,7 +346,7 @@ pub(crate) struct ArchiveQueryParams { default_ts: Option, /// Settings format version: "v1" (default) returns legacy flat format, "v2" returns grouped format settings_version: Option, - /// Opt-in: include `extra_perms` on flow / script / app rows. Default `false` + /// Opt-in: include `extra_perms` on script / flow / app / variable rows. Default `false` /// so cross-workspace tarball imports do not carry over ACLs referring to /// identities that may not exist in the target workspace. `wmill sync pull` /// passes `true` to surface ACLs in the git-tracked yaml. @@ -365,8 +365,8 @@ pub(crate) struct ArchiveQueryParams { /// pre-existing serialization for folders and groups so /// no customer sees a one-time noisy diff on upgrade. /// * `KeepIfNonEmpty` — keep when there is at least one entry, drop when `{}` -/// or null. New surface for flow / script / app, which -/// never carried ACLs in source before this change. +/// or null. New surface for script / flow / app / variable, +/// which never carried ACLs in source before this change. #[derive(Clone, Copy)] pub enum ExtraPermsBehavior { Drop, @@ -665,7 +665,7 @@ pub(crate) async fn tarball_workspace( check_scopes(&authed, || "variables:read".to_string())?; } - // Opt-in behavior for surfacing per-resource ACLs on flow/app rows. + // Opt-in behavior for surfacing per-resource ACLs on script/flow/app/variable rows. // Folder and group rows have always carried `extra_perms` in source and // continue to do so unconditionally (`KeepEvenEmpty`) so existing // customer git repos see no one-time noisy diff. @@ -1002,8 +1002,7 @@ pub(crate) async fn tarball_workspace( Error::internal_err(format!("Error decrypting variable {}: {}", var.path, e)) })?); } - let var_str = - &to_string_without_metadata(&var, ExtraPermsBehavior::Drop, None).unwrap(); + let var_str = &to_string_without_metadata(&var, new_kinds_extra_perms, None).unwrap(); archive .write_to_archive(&var_str, &format!("{}.variable.json", var.path)) .await?; diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index 117b41632a..4d5523524c 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -175,10 +175,12 @@ fn is_write_access(access: Option) -> bool { /// producers). Resource / datatable / volume reads stay explicit-`// on`: /// a config/lookup read cascading is more often surprising than wanted. fn is_auto_trigger_kind(kind: AssetKind) -> bool { - // `Dbt` is deliberately NOT here. dbt is the only thing that can produce a - // warehouse relation (`// materialize` takes DuckLake targets only) and a dbt - // run does not dispatch, so a derived `dbt://` edge could never fire — it - // would draw a cascade arrow into a script nothing can wake. + // `Dbt` is deliberately NOT here. A warehouse relation is usually built by + // the dbt project that reads it, and a dbt run does not dispatch, so deriving + // an edge from every `dbt://` read would draw cascade arrows that mostly never + // fire. The relations a native `// materialize manual dbt://…` script writes + // do wake subscribers, but only through an explicit `// on`, which is where + // the author states that this particular relation has such a producer. matches!(kind, AssetKind::Ducklake | AssetKind::S3Object) } @@ -235,6 +237,134 @@ pub fn derive_pipeline_asset_trigger_refs( out } +/// A dbt script that builds the `dbt://` relation at `asset_path`, when dbt is +/// its ONLY producer. +/// +/// That is the one shape in which subscribing to a warehouse relation can never +/// be woken: a dbt run records the models it built and does not dispatch +/// (`asset_dispatch` returns early for `ScriptLang::Dbt`), while a script that +/// declares `// materialize manual dbt://…` fans out on the ordinary path. +/// +/// `None` covers both "some non-dbt script materializes it" and "nothing +/// produces it yet" — the second is the ordinary deploy-order case, identical to +/// every other asset kind, not a dormant edge. +/// +/// **Give it the workspace pool, not an RLS-scoped transaction.** `script` +/// carries RLS while `asset` does not, so a scoped executor hides producers, and +/// the hidden ones fail in the harmful direction: a native producer the deployer +/// cannot read leaves a dbt-only set behind and refuses a subscription that would +/// have fired. What it discloses in exchange is the path of a dbt script building +/// a relation the caller already named, which the workspace asset graph hands out +/// for every `dbt://` node anyway (the source that script wrote stays gated). +/// Callers must therefore already be scoped to `workspace_id`. +/// +/// `deploying_paths` is excluded from the producer set, and has to be: reading +/// committed rows means the deploying script's own are the version being +/// replaced, so one that just dropped its `// materialize` would still count as a +/// producer and let a now-dormant subscription through. Pass every path this +/// deploy is rewriting — under a rename that is the old path as well as the new +/// one, whose committed write row the transaction is about to remove. Excluding +/// them is free of the opposite error, because a script never wakes its own +/// subscription — the dispatcher skips that as a self-loop. +/// +/// A producer another deploy is committing concurrently is invisible either way, +/// and the outcome depends on which side it is. An uncommitted NATIVE producer +/// leaves a dbt-only set and refuses, with a message the user can retry past. An +/// uncommitted DBT one leaves an empty set and accepts — and if that ingest then +/// commits and runs [`dormant_dbt_subscriptions`] before this deploy's trigger +/// row lands, neither side reports the edge it left dormant. Serializing the two +/// is not worth it: they would have to share a per-relation lock, and the ingest +/// takes `script … FOR UPDATE` before its own advisory lock, so a deploy holding +/// relation locks first inverts that order into a deadlock across the two +/// subsystems. The next deploy of that project warns (docs/dbt-runtime.md). +pub async fn sole_dbt_producer<'e>( + executor: impl PgExecutor<'e>, + workspace_id: &str, + asset_path: &str, + deploying_paths: &[String], +) -> error::Result> { + use crate::scripts::ScriptLang; + let producers = sqlx::query!( + r#"SELECT s.path AS "path!", s.language AS "language!: ScriptLang" + FROM asset a + JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path + AND s.archived = false AND s.deleted = false + WHERE a.workspace_id = $1 AND a.kind = 'dbt' AND a.path = $2 + AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw') + AND a.usage_path <> ALL($3)"#, + workspace_id, + asset_path, + deploying_paths + ) + .fetch_all(executor) + .await?; + if producers + .iter() + .any(|p| !matches!(p.language, ScriptLang::Dbt)) + { + return Ok(None); + } + Ok(producers.into_iter().next().map(|p| p.path)) +} + +/// The set form of [`sole_dbt_producer`], for asking about many relations at +/// once: every `// on dbt://` edge among `relations` whose producers +/// are all dbt scripts, rendered as `dbt:// → `. +/// +/// A dbt deploy asks this about the relations it just ingested, because that +/// ingest is what can retroactively leave a subscription accepted earlier — when +/// nothing produced the relation — with dbt as its only producer. +/// +/// Spells the predicate the same way its singular sibling does, per subscriber: +/// the producer set excludes the subscriber's own path (a script never wakes +/// itself) and has to be non-empty (nothing produces it yet is deploy order, not +/// a dormant edge). Two "is dbt the sole producer" rules that drifted apart would +/// silence this warning with nothing failing. +/// +/// Same disclosure and executor contract as [`sole_dbt_producer`]: workspace +/// pool, caller already scoped to `workspace_id`. +pub async fn dormant_dbt_subscriptions<'e>( + executor: impl PgExecutor<'e>, + workspace_id: &str, + relations: &[String], +) -> error::Result> { + if relations.is_empty() { + return Ok(vec![]); + } + let refs = relations + .iter() + .map(|r| format!("dbt://{r}")) + .collect::>(); + Ok(sqlx::query_scalar!( + r#"WITH producer AS ( + SELECT 'dbt://' || a.path AS trigger_ref, a.usage_path, s.language + FROM asset a + JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path + AND s.archived = false AND s.deleted = false + WHERE a.workspace_id = $1 AND a.kind = 'dbt' + AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw') + AND 'dbt://' || a.path = ANY($2) + ) + SELECT DISTINCT st.trigger_ref || ' → ' || st.runnable_path AS "edge!" + FROM script_trigger st + WHERE st.workspace_id = $1 AND st.trigger_kind = 'asset' + AND st.trigger_ref = ANY($2) + AND EXISTS (SELECT 1 FROM producer p + WHERE p.trigger_ref = st.trigger_ref + AND p.usage_path <> st.runnable_path + AND p.language = 'dbt') + AND NOT EXISTS (SELECT 1 FROM producer p + WHERE p.trigger_ref = st.trigger_ref + AND p.usage_path <> st.runnable_path + AND p.language <> 'dbt') + ORDER BY 1"#, + workspace_id, + &refs + ) + .fetch_all(executor) + .await?) +} + /// Clear and reinsert the full static-asset usage set of a script in one tx, /// invalidating the producer-writes cache at most once and only on a real /// change. The cache (asset_dispatch::ASSET_PRODUCER_WRITES_CACHE) keys a diff --git a/backend/windmill-common/src/dbt_manifest.rs b/backend/windmill-common/src/dbt_manifest.rs index a81faf80c3..e96fd7bc78 100644 --- a/backend/windmill-common/src/dbt_manifest.rs +++ b/backend/windmill-common/src/dbt_manifest.rs @@ -8,9 +8,11 @@ //! Two things this module is deliberate about: //! //! * **Asset identity is the physical relation.** A model becomes -//! `dbt:////`: the scheme names dbt, which is the -//! only thing that creates one, but the PATH is the relation and never dbt's -//! own `unique_id`. Two projects meet at a handoff — one materializes a mart, +//! `dbt:////`: the scheme names the namespace dbt +//! made — it is the only thing that DERIVES one, while any other language can +//! declare a write to one (decision 25) — but the PATH is the relation and +//! never dbt's own `unique_id`. Two projects meet at a handoff — one +//! materializes a mart, //! the next declares it a `source` — where `model.a.orders` and //! `source.b.analytics.orders` differ but the relation does not, so keying on //! the node id would leave each project an island; a native script reading the diff --git a/backend/windmill-common/src/user_drafts.rs b/backend/windmill-common/src/user_drafts.rs index 115dae34e7..aed9b5c5de 100644 --- a/backend/windmill-common/src/user_drafts.rs +++ b/backend/windmill-common/src/user_drafts.rs @@ -401,6 +401,45 @@ pub async fn fetch_draft_only_list_rows( Ok(rows) } +/// Delete the caller's OWN draft at a path with no deployed row, for the DELETE +/// route of a kind whose list synthesizes such rows via +/// `fetch_draft_only_list_rows`. The `NOT EXISTS` leaves a deployed row's draft +/// alone, so a route may call this on its not-found branch whatever the reason +/// for the miss. `Ok(false)` means nothing matched: the caller reports its own error. +/// +/// Takes no permission check and callers must not add one: an email-scoped row +/// belongs to the caller, who can always discard it, as `update_draft`'s +/// own-discard does. Legacy (`email IS NULL`) rows are owned by nobody and keep +/// their write gate, so discarding one stays on the `update_draft` route. +pub async fn delete_draft_only_for_path( + db: &DB, + w_id: &str, + kind: UserDraftItemKind, + path: &str, + email: &str, +) -> Result { + let Some(table) = kind.deployed_table() else { + return Ok(false); + }; + // `table` is from the closed `deployed_table()` enum, never user input. + let sql = format!( + "DELETE FROM draft \ + WHERE workspace_id = $1 AND typ = $2::text::DRAFT_KIND AND path = $3 \ + AND email = $4 \ + AND NOT EXISTS (SELECT 1 FROM {table} t \ + WHERE t.workspace_id = draft.workspace_id AND t.path = draft.path)" + ); + let deleted = sqlx::query(&sql) + .bind(w_id) + .bind(kind.as_str()) + .bind(path) + .bind(email) + .execute(db) + .await? + .rows_affected(); + Ok(deleted > 0) +} + /// The get-by-path draft choreography, shared by every entity's "get by path" /// route. Given the deployed entity as an `Option` (caller maps its own "not /// found" to `None`): diff --git a/backend/windmill-common/tests/dbt_producer_rules.rs b/backend/windmill-common/tests/dbt_producer_rules.rs new file mode 100644 index 0000000000..f63e0879b5 --- /dev/null +++ b/backend/windmill-common/tests/dbt_producer_rules.rs @@ -0,0 +1,196 @@ +/*! + * One rule, two spellings: "is dbt the sole producer of this warehouse + * relation". `sole_dbt_producer` decides whether a `// on dbt://…` subscription + * is refused at deploy; `dormant_dbt_subscriptions` names the edges a dbt deploy + * retroactively leaves unwakeable. Both must answer "yes, dormant" only when + * every script writing the relation is a dbt one — a dbt run does not dispatch — + * and "no" both when a native `// materialize manual dbt://…` producer exists and + * when nothing produces the relation yet, which is the ordinary deploy-order + * case. Every way of getting this wrong is silent: a dormant edge on the canvas, + * a refused deploy of a valid pipeline, or a warning that stops appearing. + */ + +use sqlx::{Pool, Postgres}; +use windmill_common::assets::{dormant_dbt_subscriptions, sole_dbt_producer}; + +const WS: &str = "test-workspace"; +const RELATION: &str = "main/analytics/orders"; +const SUBSCRIBER: &str = "u/test-user/consumer"; + +async fn plant_producer(db: &Pool, path: &str, language: &str, hash: i64) { + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by, + language) + VALUES ($1, $2, $3, '', '', '', 'test-user', $4::text::script_lang)", + ) + .bind(WS) + .bind(hash) + .bind(path) + .bind(language) + .execute(db) + .await + .expect("insert script"); + sqlx::query( + "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) + VALUES ($1, $2, 'dbt', 'w', $3, 'script') ON CONFLICT DO NOTHING", + ) + .bind(WS) + .bind(RELATION) + .bind(path) + .execute(db) + .await + .expect("insert asset"); +} + +async fn plant_subscriber(db: &Pool, path: &str) { + sqlx::query( + "INSERT INTO script_trigger (workspace_id, runnable_kind, runnable_path, trigger_kind, + trigger_ref) + VALUES ($1, 'script', $2, 'asset', 'dbt://' || $3)", + ) + .bind(WS) + .bind(path) + .bind(RELATION) + .execute(db) + .await + .expect("insert script_trigger"); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn no_producer_is_not_dormant(db: Pool) { + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + None, + "a relation nothing produces yet must not refuse the subscription" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn dbt_only_producer_is_dormant(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + Some("u/test-user/project".to_string()) + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_native_producer_beside_dbt_is_not_dormant(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, "u/test-user/ingest", "postgresql", 2).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + None + ); +} + +/// `asset` is keyed by path while `script` holds every version of it, so the +/// language has to be read off the live one: a path converted to dbt still has +/// its old native versions sitting in `script`. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_superseded_native_version_does_not_count(db: Pool) { + plant_producer(&db, "u/test-user/project", "postgresql", 1).await; + sqlx::query("UPDATE script SET archived = true WHERE hash = 1") + .execute(&db) + .await + .expect("archive the old version"); + plant_producer(&db, "u/test-user/project", "dbt", 2).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + Some("u/test-user/project".to_string()) + ); +} + +/// The rows of the script being deployed describe the version it replaces, so a +/// script dropping its `// materialize` while adding a subscription would +/// otherwise count itself as the producer that wakes it — and commit a dormant +/// edge. It can never be that producer anyway: the dispatcher skips self-loops. +/// Under a rename that write sits at the OLD path, which the deploy is removing +/// in the same uncommitted transaction, so both paths have to be excluded. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn the_subscriber_is_never_its_own_producer(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, SUBSCRIBER, "postgresql", 2).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + Some("u/test-user/project".to_string()) + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_renamed_producer_is_excluded_too(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, "u/test-user/old_ingest", "postgresql", 2).await; + assert_eq!( + sole_dbt_producer( + &db, + WS, + RELATION, + &[SUBSCRIBER.to_string(), "u/test-user/old_ingest".to_string()] + ) + .await + .unwrap(), + Some("u/test-user/project".to_string()), + "the write this deploy is moving off the old path cannot wake the subscription" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn the_set_form_agrees_with_the_singular_one(db: Pool) { + let relations = vec![RELATION.to_string()]; + plant_subscriber(&db, "u/test-user/consumer").await; + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + assert_eq!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap(), + vec![format!("dbt://{RELATION} → u/test-user/consumer")], + "dbt alone builds it, so the subscription can never be woken" + ); + + plant_producer(&db, "u/test-user/ingest", "postgresql", 2).await; + assert!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap() + .is_empty(), + "a native producer wakes it, so the edge is live" + ); +} + +/// The two ways the set form could stop meaning what the singular one means: a +/// relation nothing produces is deploy order rather than a dormant edge, and a +/// subscriber's own write is not a producer that can wake it — the dispatcher +/// skips self-loops, so that edge is dormant and has to be named. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn the_set_form_matches_on_the_edge_cases_too(db: Pool) { + let relations = vec![RELATION.to_string()]; + plant_subscriber(&db, SUBSCRIBER).await; + assert!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap() + .is_empty(), + "nothing produces it yet, so nothing is dormant" + ); + + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, SUBSCRIBER, "postgresql", 2).await; + assert_eq!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap(), + vec![format!("dbt://{RELATION} → {SUBSCRIBER}")], + "the subscriber's own write cannot wake it, so dbt is still the sole producer" + ); +} diff --git a/backend/windmill-queue/src/asset_dispatch.rs b/backend/windmill-queue/src/asset_dispatch.rs index af9ca957b9..a3f7154cda 100644 --- a/backend/windmill-queue/src/asset_dispatch.rs +++ b/backend/windmill-queue/src/asset_dispatch.rs @@ -248,11 +248,13 @@ async fn try_dispatch(db: &DB, job: &MiniCompletedJob) -> Result // A dbt run records the relations it builds, so it looks like a producer // here — but dbt does not trigger downstream runs. Its own DAG is dbt's to // order; the only thing a cascade would add is waking Windmill scripts that - // read a mart, and nothing outside dbt can declare a `dbt://` write, so - // that edge exists in one direction only. Cascading from a project whose - // per-run selection can build any subset of itself needs a per-run write set - // to be correct, which is a design worth doing deliberately rather than - // inferring. Until then dbt materializes and reports; it does not dispatch. + // read a mart. Cascading from a project whose per-run selection can build any + // subset of itself needs a per-run write set to be correct, which is a design + // worth doing deliberately rather than inferring: the deploy-time write set is + // not what ran, and the per-relation state table keeps one row per relation. + // Until then dbt materializes and reports; it does not dispatch. The opposite + // direction does: a native `// materialize manual dbt://…` script reaches the + // fan-out below on the ordinary path, on the strength of its own asset rows. if job.script_lang == Some(ScriptLang::Dbt) { return Ok(DispatchResult::default()); } diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index f9bfe2f4df..cf3c73f8da 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -50,9 +50,9 @@ use windmill_common::{ error::{self, Error, JsonResult, Result}, get_database_url, user_drafts::{ - delete_all_drafts_for_path, delete_own_draft_for_path, fetch_draft_only, - fetch_draft_only_list_rows, maybe_overlay_draft, UserDraftItemKind, WithDraftOverlay, - WithDraftQuery, + delete_all_drafts_for_path, delete_draft_only_for_path, delete_own_draft_for_path, + fetch_draft_only, fetch_draft_only_list_rows, maybe_overlay_draft, UserDraftItemKind, + WithDraftOverlay, WithDraftQuery, }, utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath}, variables, @@ -89,6 +89,9 @@ pub fn workspaced_service() -> Router { .route("/git_commit_hash/{*path}", get(get_git_commit_hash)) .route("/type/list", get(list_resource_types)) .route("/type/listnames", get(list_resource_types_names)) + .route("/type/resource_counts", get(list_resource_counts_by_type)) + .route("/type/hub/info", get(list_hub_resource_type_info)) + .route("/type/hub/pick/{name}", post(pick_hub_resource_type)) .route("/type/get/{name}", get(get_resource_type)) .route("/type/exists/{name}", get(exists_resource_type)) .route("/type/update/{name}", post(update_resource_type)) @@ -134,7 +137,10 @@ pub struct EditResourceType { /// `Option` conflates: an absent field leaves the extension alone, while an /// explicit `null` clears it. A hub pull relies on both — a type that stops /// being a file type has to stop being one locally too. - #[serde(default, deserialize_with = "windmill_common::more_serde::double_option")] + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option" + )] pub format_extension: Option>, } @@ -1309,6 +1315,17 @@ async fn delete_resource( let path = path.to_path(); check_scopes(&authed, || format!("resources:write:{}", path))?; + + // Ahead of the deploy rules: nothing is deployed at a draft-only path, so + // gating this discard on them would strand the row in a protected workspace. + // Ahead of the transaction too — the not-found branch other kinds hang this + // off is the `not_found_if_none` below, past the linked-variable cascade. + if delete_draft_only_for_path(&db, &w_id, UserDraftItemKind::Resource, path, &authed.email) + .await? + { + return Ok(format!("draft-only resource {} deleted", path)); + } + if let RuleCheckResult::Blocked(msg) = check_deploy_rules( &w_id, AuditAuthorable::username(&authed), @@ -1320,6 +1337,7 @@ async fn delete_resource( { return Err(Error::PermissionDenied(msg)); } + let mut tx = user_db.begin(&authed).await?; // Capture resource data for trashbin before deleting @@ -2566,6 +2584,352 @@ async fn list_resource_types_names( Ok(Json(rows)) } +#[derive(Serialize)] +struct ResourceTypeCount { + resource_type: String, + count: i64, +} + +/// How many resources of each type this workspace holds — how popular a type is *here*, +/// which is what the pickers rank on below the hub's own pick counts. +async fn list_resource_counts_by_type( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, +) -> JsonResult> { + // A count per type is aggregate, so there is no path to narrow it by: a token scoped + // to individual resources gets nothing rather than a total spanning paths it cannot + // read. Callers treat the refusal as "no local signal". + check_scopes(&authed, || "resources:read".to_string())?; + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query!( + "SELECT resource_type, count(*) as \"count!\" FROM resource WHERE workspace_id = $1 GROUP BY resource_type", + &w_id + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + + Ok(Json( + rows.into_iter() + .map(|r| ResourceTypeCount { resource_type: r.resource_type, count: r.count }) + .collect(), + )) +} + +/// A hub read, remembered with the hub it came from: `hub_base_url` is a live instance +/// setting, so a cache ignoring it would keep serving the previous hub's answers. +struct HubCached { + hub_base_url: String, + fetched_at: std::time::Instant, + value: T, +} + +/// The index changes only when a resource type is published to the hub; picks move slowly +/// and only reorder a list. Both are read on every drawer open, hence caching at all. +const HUB_RT_INDEX_TTL: std::time::Duration = std::time::Duration::from_secs(60 * 60); +const HUB_RT_PICKS_TTL: std::time::Duration = std::time::Duration::from_secs(5 * 60); + +/// How long a *failed* index read is remembered. Short, and deliberately not the hour a +/// success is good for: this read is on the path of every picker open, so an unreachable +/// hub must not cost an outbound timeout each time, while one blip must not silence pick +/// reporting for an hour. +const HUB_RT_INDEX_FAILURE_TTL: std::time::Duration = std::time::Duration::from_secs(60); + +static HUB_RT_INDEX: LazyLock< + std::sync::RwLock>>>>, +> = LazyLock::new(|| std::sync::RwLock::new(None)); +static HUB_RT_PICKS: LazyLock>>>> = + LazyLock::new(|| std::sync::RwLock::new(None)); + +fn hub_cache_get( + cache: &std::sync::RwLock>>, + hub_base_url: &str, + ttl: std::time::Duration, +) -> Option { + let guard = cache.read().ok()?; + let entry = guard.as_ref()?; + (entry.hub_base_url == hub_base_url && entry.fetched_at.elapsed() < ttl) + .then(|| entry.value.clone()) +} + +fn hub_cache_put(cache: &std::sync::RwLock>>, hub_base_url: &str, value: T) { + if let Ok(mut guard) = cache.write() { + *guard = Some(HubCached { + hub_base_url: hub_base_url.to_string(), + fetched_at: std::time::Instant::now(), + value, + }); + } +} + +#[derive(Deserialize)] +struct HubResourceTypeEntry { + id: i64, + name: String, + /// Optional so a hub that does not send it costs only the mapping. Required, it would + /// fail the whole parse and take pick reporting — which needs just the id — with it. + #[serde(default)] + app: Option, +} + +#[derive(Clone)] +struct HubResourceType { + id: i64, + /// The integration the type belongs to. Usually the type's own name, but not always: + /// `discord_webhook` and `discord_bot_configuration` are both `discord`, and only the + /// hub knows that. Without it a workspace holding a `discord_webhook` resource looks + /// like one that has never touched Discord. + app: String, +} + +/// Reads the index cache, choosing the TTL by what is stored: a failure expires far sooner +/// than a success. `None` is a miss, `Some(None)` a remembered failure. +fn hub_index_cached(hub_base_url: &str) -> Option>> { + let guard = HUB_RT_INDEX.read().ok()?; + let entry = guard.as_ref()?; + if entry.hub_base_url != hub_base_url { + return None; + } + let ttl = if entry.value.is_some() { + HUB_RT_INDEX_TTL + } else { + HUB_RT_INDEX_FAILURE_TTL + }; + (entry.fetched_at.elapsed() < ttl).then(|| entry.value.clone()) +} + +/// What the hub knows about every published resource type, keyed by the name Windmill +/// addresses it by. `None` when the hub cannot be reached or does not answer with a list. +async fn hub_resource_types( + db: &DB, + hub_base_url: &str, +) -> Option> { + if let Some(cached) = hub_index_cached(hub_base_url) { + return cached; + } + let index = async { + let response = windmill_common::utils::http_get_from_hub( + &windmill_common::utils::HTTP_CLIENT, + &format!("{hub_base_url}/resource_types/list"), + false, + None, + Some(db), + ) + .await + .ok()?; + if !response.status().is_success() { + return None; + } + // Only the id and the app are kept. That listing carries every type's schema — + // around a megabyte — and neither reporting a pick nor grouping types by + // integration needs it. + Some( + response + .json::>() + .await + .ok()? + .into_iter() + .map(|rt| { + let app = rt.app.unwrap_or_else(|| rt.name.clone()); + (rt.name, HubResourceType { id: rt.id, app }) + }) + .collect::>(), + ) + } + .await; + + hub_cache_put(&HUB_RT_INDEX, hub_base_url, index.clone()); + index +} + +#[derive(Serialize)] +struct PickHubResourceTypeResult { + success: bool, +} + +/// Tells the hub a resource type was taken into a workspace, which is the counter its +/// `/resource_types/picked` ranking reads. +/// +/// Never fails the caller: a hub predating the route, an unreachable one, and a type that +/// is local-only all mean the same thing — not counted — and the request that reaches here +/// has already saved the user's resource. +/// +/// POST, and scoped as a write, because it changes state on the hub under the instance's +/// own credentials. The sibling `/type/*` routes are metadata reads that a `resources:run` +/// app-embed token may make, and both the method and this check keep such a token — which +/// is untrusted app JavaScript — from driving hub counters through us. +async fn pick_hub_resource_type( + authed: ApiAuthed, + Extension(db): Extension, + Path((_w_id, name)): Path<(String, String)>, +) -> JsonResult { + check_scopes(&authed, || "resources:write".to_string())?; + let hub_base_url = (**windmill_common::HUB_BASE_URL.load()).clone(); + let success = async { + let id = hub_resource_types(&db, &hub_base_url).await?.get(&name)?.id; + let response = windmill_common::utils::http_get_from_hub( + &windmill_common::utils::HTTP_CLIENT, + &format!("{hub_base_url}/resource_types/{id}/pick"), + false, + None, + Some(&db), + ) + .await + .ok()?; + Some(response.status().is_success()) + } + .await + .unwrap_or(false); + + if !success { + tracing::debug!("hub did not record a pick for resource type {name}"); + } + Ok(Json(PickHubResourceTypeResult { success })) +} + +#[derive(Deserialize, Clone)] +struct HubResourceTypePicks { + name: String, + /// The hub counts picks in a bigint, which its driver serialises as a string. + #[serde(deserialize_with = "windmill_common::more_serde::maybe_number")] + picks: i64, +} + +#[derive(Deserialize)] +struct HubPickedResourceTypes { + resource_types: Vec, +} + +#[cfg(test)] +mod hub_picks_tests { + use super::*; + + /// Two properties of the index cache that a later edit could quietly drop: it is keyed on + /// the hub it came from, and a failure is forgotten long before a success is. + #[test] + fn the_index_cache_is_keyed_on_the_hub_and_forgets_failures_sooner() { + let index = || { + Some(HashMap::from([( + "slack".to_string(), + HubResourceType { id: 1, app: "slack".to_string() }, + )])) + }; + + hub_cache_put(&HUB_RT_INDEX, "https://hub.example", index()); + assert!(hub_index_cached("https://hub.example").is_some_and(|v| v.is_some())); + // Switching hubs must miss rather than serve the previous hub's mapping. + assert!(hub_index_cached("https://other.example").is_none()); + + // A remembered failure reads as a hit (so the hub is not re-attempted) carrying + // nothing, and only until the shorter of the two TTLs. + hub_cache_put(&HUB_RT_INDEX, "https://hub.example", None); + assert!(hub_index_cached("https://hub.example").is_some_and(|v| v.is_none())); + assert!(HUB_RT_INDEX_FAILURE_TTL < HUB_RT_INDEX_TTL); + + if let Ok(mut guard) = HUB_RT_INDEX.write() { + *guard = None; + } + } + + /// The hub counts picks in a bigint, which postgres.js serialises as a string. Typing + /// the field as a plain i64 fails the whole response, and the ranking silently empties. + #[test] + fn picks_decode_from_a_string_or_a_number() { + let parsed: HubPickedResourceTypes = serde_json::from_str( + r#"{"resource_types":[{"name":"slack","picks":"42"},{"name":"github","picks":7}]}"#, + ) + .unwrap(); + assert_eq!( + parsed + .resource_types + .iter() + .map(|rt| (rt.name.as_str(), rt.picks)) + .collect::>(), + vec![("slack", 42), ("github", 7)] + ); + } +} + +/// One hub resource type as the pickers need it. +#[derive(Serialize)] +struct HubResourceTypeInfo { + name: String, + /// The integration it belongs to, so a caller can total a workspace's resources per + /// integration rather than per type. + app: String, + picks: i64, +} + +/// What the hub knows about its resource types: which integration each belongs to, and how +/// often each has been picked. +/// +/// Empty rather than an error when the hub answers neither read, so the pickers treat an +/// older or private hub as "no hub signal" and fall back to what the workspace itself uses. +/// The two reads degrade independently: a hub that lists types but has no `picked` route +/// still supplies the type-to-integration mapping, which is what decides whether a +/// workspace's resources are recognised as belonging to an integration at all. +async fn list_hub_resource_type_info( + Extension(db): Extension, +) -> JsonResult> { + let hub_base_url = (**windmill_common::HUB_BASE_URL.load()).clone(); + let picks = match hub_cache_get(&HUB_RT_PICKS, &hub_base_url, HUB_RT_PICKS_TTL) { + Some(picks) => picks, + None => { + let fetched = async { + let response = windmill_common::utils::http_get_from_hub( + &windmill_common::utils::HTTP_CLIENT, + &format!("{hub_base_url}/resource_types/picked"), + false, + Some(vec![("limit", "200".to_string())]), + Some(&db), + ) + .await + .ok()?; + if !response.status().is_success() { + return None; + } + Some( + response + .json::() + .await + .ok()? + .resource_types, + ) + } + .await + .unwrap_or_default(); + hub_cache_put(&HUB_RT_PICKS, &hub_base_url, fetched.clone()); + fetched + } + }; + + let mut picks_by_name: HashMap = + picks.into_iter().map(|rt| (rt.name, rt.picks)).collect(); + let index = hub_resource_types(&db, &hub_base_url) + .await + .unwrap_or_default(); + + let mut info: Vec = index + .into_iter() + .map(|(name, rt)| HubResourceTypeInfo { + picks: picks_by_name.remove(&name).unwrap_or(0), + name, + app: rt.app, + }) + .collect(); + // What the index did not account for is a type the picks read knows and the listing does + // not — which is what a hub answering only the second read looks like. Its own name is + // the same guess a caller makes for any type the mapping misses. + info.extend( + picks_by_name + .into_iter() + .map(|(name, picks)| HubResourceTypeInfo { app: name.clone(), name, picks }), + ); + + Ok(Json(info)) +} + async fn get_resource_type( authed: ApiAuthed, Extension(user_db): Extension, diff --git a/backend/windmill-trigger/src/handler.rs b/backend/windmill-trigger/src/handler.rs index 8ef7c1ee39..953f3a571b 100644 --- a/backend/windmill-trigger/src/handler.rs +++ b/backend/windmill-trigger/src/handler.rs @@ -18,8 +18,9 @@ use windmill_common::{ error::{Error, JsonResult, Result}, trigger_history::{self, TriggerHistoryEvent, TriggerOperation, TriggerSource}, user_drafts::{ - delete_all_drafts_for_path, delete_own_draft_for_path, fetch_draft_only_list_rows, - overlay_or_draft_only, UserDraftItemKind, WithDraftOverlay, WithDraftQuery, + delete_all_drafts_for_path, delete_draft_only_for_path, delete_own_draft_for_path, + fetch_draft_only_list_rows, overlay_or_draft_only, UserDraftItemKind, WithDraftOverlay, + WithDraftQuery, }, utils::{paginate, Pagination, StripPath}, worker::CLOUD_HOSTED, @@ -990,6 +991,18 @@ async fn delete_trigger( .await?; if !deleted { + drop(tx); + if delete_draft_only_for_path( + &db, + &workspace_id, + T::user_draft_item_kind(), + path, + &authed.email, + ) + .await? + { + return Ok(format!("Draft-only trigger '{}' deleted", path)); + } return Err(Error::NotFound(format!( "Trigger not found at path: {}", path diff --git a/backend/windmill-types/src/assets.rs b/backend/windmill-types/src/assets.rs index 463a365b06..166406b632 100644 --- a/backend/windmill-types/src/assets.rs +++ b/backend/windmill-types/src/assets.rs @@ -14,17 +14,20 @@ pub enum AssetKind { Ducklake, DataTable, Volume, - /// A warehouse relation a dbt project builds or reads, - /// `dbt:////`, where `` is the name the - /// workspace configures it under. + /// A warehouse relation, `dbt:////`, where + /// `` is the name the workspace configures it under. /// - /// The SCHEME names the producer — dbt is the only thing that creates one — - /// while the PATH stays the physical relation, because that is what two - /// projects agree on: a mart one builds is a `source` the next reads, and - /// their dbt `unique_id`s differ (`model.a.orders` vs - /// `source.b.analytics.orders`) where the relation does not - /// (docs/dbt-runtime.md, decision 11). A dbt run does not trigger that - /// reader — the shared node is lineage, not a cascade edge. + /// The SCHEME names the namespace dbt made rather than an exclusive + /// producer: dbt is what derives these relations from a project, and a script + /// in any language but dbt's own can DECLARE one it writes + /// (`// materialize manual dbt://…`) — a project's writes are read from its + /// manifest, never annotated. The PATH stays the physical relation, + /// because that is what two producers agree on: a mart one builds is a + /// `source` the next reads, and their dbt `unique_id`s differ + /// (`model.a.orders` vs `source.b.analytics.orders`) where the relation does + /// not (docs/dbt-runtime.md, decision 11). A dbt run does not trigger the + /// readers of what it built — that shared node is lineage, not a cascade + /// edge — while a declared write does (decision 25). Dbt, } diff --git a/backend/windmill-worker/src/dbt_executor.rs b/backend/windmill-worker/src/dbt_executor.rs index 04376dd189..e09e155ec4 100644 --- a/backend/windmill-worker/src/dbt_executor.rs +++ b/backend/windmill-worker/src/dbt_executor.rs @@ -693,6 +693,7 @@ pub(crate) async fn dbt_dep( &conn, ) .await; + warn_dormant_subscribers(db, w_id, job_id, &ingested, &conn).await; } !published } else { @@ -3268,7 +3269,7 @@ async fn ingest_from_run( let snapshot_job = p.graph_refresh.snapshot_job(job.id); match conn { Connection::Sql(db) => { - persist_ingest( + let published = persist_ingest( db, &job.workspace_id, script_path, @@ -3281,6 +3282,13 @@ async fn ingest_from_run( p.graph_refresh.publishes_ownership(), ) .await?; + // Publishing ownership from a RUN makes this project the owner of + // those relations exactly as a deploy does, so it can be what leaves + // a subscription accepted while the relation had no producer with dbt + // as its only one. Same warning the deploy emits. + if published && p.graph_refresh.publishes_ownership() { + warn_dormant_subscribers(db, &job.workspace_id, &job.id, &ingested, conn).await; + } } // An agent worker reaches these tables only through the API. Publishing // is the whole of what it needs: a worker that can replace the graph @@ -3333,7 +3341,7 @@ enum GraphPublisher { /// Replace this script's graph, unless a newer version of it has been deployed. /// /// Write one ingest: the sidecar rows and the `asset` usages the manifest -/// implies. No subscriptions — a `dbt://` one could never fire. +/// implies. No subscriptions — a dbt project is not woken by the cascade. /// /// Returns whether this job was still the one entitled to the path-keyed half — /// false once a newer version has superseded it, or once the version is gone. @@ -3424,9 +3432,10 @@ async fn persist_ingest( &ingested.assets, ) .await?; - // A `dbt://` subscription can never fire, so none are derived from the - // manifest. The delete stays to clear what earlier versions wrote, which would - // otherwise keep drawing cascade arrows that wake nothing. + // A dbt project is not woken by the asset cascade (refused at deploy), so + // none are derived from the manifest either. The delete stays to clear what + // earlier versions wrote, which would otherwise keep drawing cascade arrows + // that wake nothing. sqlx::query!( "DELETE FROM script_trigger WHERE workspace_id = $1 AND runnable_kind = 'script' AND runnable_path = $2 @@ -3440,6 +3449,56 @@ async fn persist_ingest( Ok(true) } +/// Log the `// on dbt://…` subscriptions this project's relations leave dormant. +/// +/// Subscribing to a relation dbt already owns is refused at the subscriber's +/// deploy, but one deployed while nothing produced that relation is accepted — +/// as it is for every other asset kind — and an ingest is what can afterwards +/// make dbt its only producer. A dbt run does not dispatch, so such an edge is +/// drawn on the canvas and never fires; a job's own log is where that ordering is +/// visible. +/// +/// Called from both points that publish ownership, the deploy and a run whose +/// static descriptor found its profile moved — either can be the one that claims +/// the relation. +async fn warn_dormant_subscribers( + db: &sqlx::Pool, + w_id: &str, + job_id: &Uuid, + ingested: &windmill_common::dbt_manifest::IngestedManifest, + conn: &Connection, +) { + use windmill_common::assets::AssetUsageAccessType; + let relations: Vec = ingested + .assets + .iter() + .filter(|a| { + matches!( + a.access_type.or(a.alt_access_type), + Some(AssetUsageAccessType::W) | Some(AssetUsageAccessType::RW) + ) + }) + .map(|a| a.path.clone()) + .collect(); + match windmill_common::assets::dormant_dbt_subscriptions(db, w_id, &relations).await { + Ok(edges) if !edges.is_empty() => { + append_logs( + job_id, + w_id, + format!( + "\nThese subscriptions will not fire — a dbt run does not trigger downstream \ + runs, and nothing else writes their relation:\n {}\n", + edges.join("\n ") + ), + conn, + ) + .await; + } + Ok(_) => {} + Err(e) => tracing::warn!("listing dormant `dbt://` subscribers failed: {e:#}"), + } +} + /// Serialize publishers for one script path and confirm this job's version is /// still the newest. Both happen inside the caller's transaction, so a newer /// publisher either commits before this check sees it, or waits behind it and diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index 54457f81b0..b2e69d26d6 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -112,7 +112,9 @@ async fn fetch_custom_test_body(conn: &Connection, w_id: &str, path: &str) -> Re // the user's own ATTACH. `// data_test` lines append verifier probes that run // against the freshly-materialized target and raise (failing the run) on // violation. Returns `None` when there is no materialize annotation or the -// target isn't a ducklake (only ducklake is materialized in v1). +// target isn't a ducklake: only ducklake has a write engine, and a `dbt://` +// target is recorded by the generic job path instead (worker.rs, +// `record_declared_warehouse_write`). fn build_materialized_query( query: &str, partition_value: Option<&str>, diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 7e620439b0..8dc52ef936 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -5527,7 +5527,7 @@ async fn handle_code_execution_job( .await?; let language = language.clone(); - run_language_executor( + let result = run_language_executor( job, conn, client, @@ -5553,7 +5553,110 @@ async fn handle_code_execution_job( false, in_pipeline, ) - .await + .await; + record_declared_warehouse_write(job, conn, code, &result).await; + result +} + +/// Record the outcome of a `// materialize manual dbt:////` +/// declaration, the way the DuckDB executor records a DuckLake target. +/// +/// Nothing generates warehouse DDL, so the script issues its own write and this +/// is the only thing that turns it into a `materialized_partition` row — the +/// relation's last writer on the run page and the graph. Language-agnostic on +/// purpose — the DuckLake write engine is DuckDB's, this declaration is anyone's +/// — except dbt's own, which is refused at deploy. +/// +/// Best-effort, and it can be: the cascade fans out from the deploy-time `asset` +/// rows, not from this one, so a lost row costs the relation its last writer and +/// nothing else. It must not fail a job whose write already landed. +/// +/// Shares the reach of every other runtime pipeline annotation, which is this +/// function's caller: a job handed to a dedicated worker or a flow runner never +/// passes through it, so — exactly as `// partitioned` is not resolved there — +/// such a run performs its write and records no row. +async fn record_declared_warehouse_write( + job: &MiniPulledJob, + conn: &Connection, + code: &str, + result: &error::Result>, +) { + use windmill_common::materialization::{ + MaterializationStatus, RecordMaterializationRequest, UNPARTITIONED, + }; + // A DEPLOYED script only. The annotation is a deploy-time contract — `manual`, + // a three-segment relation, a configured warehouse — checked in + // `create_script_internal`, which also required write access to the path. A + // preview, hub or inline-flow body reaches this function without any of that, + // so honouring it there would let `jobs:run` alone restamp any relation's last + // writer from a script that never touched it. + if job.kind != JobKind::Script { + return; + } + // Cheap guard: the annotation scan is skipped for the overwhelming majority + // of jobs, which carry no `materialize` line at all. + if !code.contains("materialize") { + return; + } + let Some(m) = windmill_parser::asset_parser::parse_pipeline_annotations(code) + .materialize + .filter(|m| m.target_kind == windmill_parser::asset_parser::AssetKind::Dbt) + else { + return; + }; + // The slice this run wrote, resolved once upstream (`resolve_partition_for_job`) + // and carried in the args the cascade reads too, so a partitioned producer + // records the same identity everything else propagates. + let partition = job + .args + .as_ref() + .and_then(|a| a.0.get(windmill_common::partition::PARTITION_ARG)) + .and_then(|v| serde_json::from_str::(v.get()).ok()) + .unwrap_or_else(|| UNPARTITIONED.to_string()); + let (status, error) = match result { + Ok(_) => (MaterializationStatus::Materialized, None), + Err(e) => (MaterializationStatus::Failed, Some(e.to_string())), + }; + let recorded = match conn { + Connection::Sql(db) => windmill_common::materialization::record_materialization( + db, + &job.workspace_id, + windmill_common::assets::AssetKind::Dbt, + &m.target_path, + &partition, + status, + None, + None, + Some(job.id), + error.as_deref(), + ) + .await + .map_err(|e| anyhow::anyhow!("{e:#}")), + Connection::Http(client) => { + crate::agent_workers::record_materialization_from_agent_http( + client, + &job.workspace_id, + &RecordMaterializationRequest { + asset_kind: windmill_common::assets::AssetKind::Dbt, + asset_path: m.target_path.clone(), + partition, + status, + snapshot_id: None, + row_count: None, + job_id: Some(job.id), + error, + schema: None, + }, + ) + .await + } + }; + if let Err(e) = recorded { + tracing::warn!( + "recording the materialization of dbt://{} failed: {e:#}", + m.target_path + ); + } } /// True when `path` contains only `Normal`/`CurDir` components, i.e. it cannot diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 12930e8de4..2899ff694c 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.804.0"; +export const VERSION = "v1.805.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/commands/app/app.ts b/cli/src/commands/app/app.ts index 0f46ec76be..b35efc02a3 100644 --- a/cli/src/commands/app/app.ts +++ b/cli/src/commands/app/app.ts @@ -12,7 +12,11 @@ import * as wmill from "../../../gen/services.gen.ts"; import { ListableApp, Policy } from "../../../gen/types.gen.ts"; import { GlobalOptions, isSuperset } from "../../types.ts"; -import { getWmillYamlPath, mergeConfigWithConfigFile } from "../../core/conf.ts"; +import { + getWmillYamlPath, + mergeConfigWithConfigFile, + readEffectiveSyncBehavior, +} from "../../core/conf.ts"; import { readInlinePathSync } from "../../utils/utils.ts"; import devCommand from "./dev.ts"; import lintCommand from "./lint.ts"; @@ -21,6 +25,7 @@ import newCommand from "./new.ts"; import generateAgentsCommand from "./generate_agents.ts"; import { isVersionsGeq1585 } from "../sync/global.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; import { applyExtraPermsDiff } from "../../core/extra_perms.ts"; export interface AppFile { @@ -420,6 +425,8 @@ async function push( if (isRawAppByName || hasRawAppYaml) { const { pushRawApp } = await import("./raw_apps.ts"); const merged = await mergeConfigWithConfigFile(opts); + // Raw-app ownership preservation is not implemented on either push + // path: sync push hands pushRawApp no context either. await pushRawApp( workspace.workspaceId, remotePath, @@ -429,7 +436,16 @@ async function push( ); log.info(colors.bold.underline.green("Raw app pushed")); } else { - await pushApp(workspace.workspaceId, remotePath, absoluteFilePath); + await pushApp( + workspace.workspaceId, + remotePath, + absoluteFilePath, + undefined, + await buildPermissionedAsContext( + workspace.workspaceId, + await readEffectiveSyncBehavior(opts, workspace), + ), + ); log.info(colors.bold.underline.green("App pushed")); } } diff --git a/cli/src/commands/flow/flow.ts b/cli/src/commands/flow/flow.ts index 7a383fcfeb..863690ce7d 100644 --- a/cli/src/commands/flow/flow.ts +++ b/cli/src/commands/flow/flow.ts @@ -4,7 +4,7 @@ import { Command } from "@cliffy/command"; import { Confirm } from "@cliffy/prompt/confirm"; import { Table } from "@cliffy/table"; import * as log from "../../core/log.ts"; -import { dirname, sep as SEP } from "node:path"; +import { dirname, sep as SEP, resolve as pathResolve } from "node:path"; import { stringify as yamlStringify } from "yaml"; import { yamlParseFile } from "../../utils/yaml.ts"; import { readTextFile, validateRequiredArgs } from "../../utils/utils.ts"; @@ -21,11 +21,16 @@ import { } from "../../core/context.ts"; import { resolve, track_job, pollForJobResult } from "../script/script.ts"; import { defaultFlowDefinition } from "../../../bootstrap/flow_bootstrap.ts"; -import { SyncOptions, mergeConfigWithConfigFile } from "../../core/conf.ts"; +import { + SyncOptions, + mergeConfigWithConfigFile, + readEffectiveSyncBehavior, +} from "../../core/conf.ts"; import { FSFSElement, elementsToMap, ignoreF } from "../sync/sync.ts"; import { Flow } from "../../../gen/types.gen.ts"; import { applyExtraPermsDiff } from "../../core/extra_perms.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; import { collectPathScriptPaths, replaceInlineScripts, @@ -327,10 +332,20 @@ async function push(opts: Options & { message?: string }, filePath: string, remo if (!validatePath(remotePath)) { return; } + // Reading the config moves the cwd to the wmill.yaml root when it sits in a + // parent directory, so pin the file against the invocation cwd first. + filePath = pathResolve(filePath); const workspace = await resolveWorkspace(opts); await requireLogin(opts); + const syncBehavior = await readEffectiveSyncBehavior(opts, workspace); - await pushFlow(workspace.workspaceId, remotePath, filePath, opts.message); + await pushFlow( + workspace.workspaceId, + remotePath, + filePath, + opts.message, + await buildPermissionedAsContext(workspace.workspaceId, syncBehavior) + ); log.info(colors.bold.underline.green("Flow pushed")); } diff --git a/cli/src/commands/schedule/schedule.ts b/cli/src/commands/schedule/schedule.ts index 2eb6ba42b8..096615c757 100644 --- a/cli/src/commands/schedule/schedule.ts +++ b/cli/src/commands/schedule/schedule.ts @@ -6,13 +6,19 @@ import { Command } from "@cliffy/command"; import { Table } from "@cliffy/table"; import { colors } from "@cliffy/ansi/colors"; import * as log from "../../core/log.ts"; -import { sep as SEP } from "node:path"; +import { sep as SEP, resolve as pathResolve } from "node:path"; import { requireLogin } from "../../core/auth.ts"; import { resolveWorkspace, validatePath } from "../../core/context.ts"; -import { mergeConfigWithConfigFile } from "../../core/conf.ts"; +import { + mergeConfigWithConfigFile, + readEffectiveSyncBehavior, +} from "../../core/conf.ts"; import * as wmill from "../../../gen/services.gen.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; -import { lookupUsernameByEmail } from "../../core/permissioned_as.ts"; +import { + buildPermissionedAsContext, + lookupUsernameByEmail, +} from "../../core/permissioned_as.ts"; import { GlobalOptions, @@ -299,8 +305,12 @@ async function disable(opts: GlobalOptions, path: string) { } async function push(opts: GlobalOptions, filePath: string, remotePath: string) { + // Reading the config moves the cwd to the wmill.yaml root when it sits in a + // parent directory, so pin the file against the invocation cwd first. + filePath = pathResolve(filePath); const workspace = await resolveWorkspace(opts); await requireLogin(opts); + const syncBehavior = await readEffectiveSyncBehavior(opts, workspace); if (!validatePath(remotePath)) { return; @@ -317,7 +327,8 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) { workspace.workspaceId, remotePath, undefined, - parseFromFile(filePath) + parseFromFile(filePath), + await buildPermissionedAsContext(workspace.workspaceId, syncBehavior) ); console.log(colors.bold.underline.green("Schedule pushed")); } diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index fada434e0d..d2afdeeee3 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -7,6 +7,7 @@ import { validatePath, } from "../../core/context.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; import { applyExtraPermsDiff } from "../../core/extra_perms.ts"; import { writeFile, stat, mkdir } from "node:fs/promises"; import { Buffer } from "node:buffer"; @@ -58,6 +59,7 @@ import { SyncOptions, mergeConfigWithConfigFile, readConfigFile, + readEffectiveSyncBehavior, } from "../../core/conf.ts"; import { SyncCodebase, listSyncCodebases } from "../../utils/codebase.ts"; import { pollJobWithQueueLogging } from "../../utils/job_polling.ts"; @@ -231,7 +233,11 @@ async function push(opts: PushOptions, filePath: string) { opts.message, opts, await getRawWorkspaceDependencies(true), - codebases + codebases, + await buildPermissionedAsContext( + workspace.workspaceId, + await readEffectiveSyncBehavior(opts, workspace) + ) ); log.info(colors.bold.underline.green(`Script ${filePath} pushed`)); } diff --git a/cli/src/commands/sync/pull.ts b/cli/src/commands/sync/pull.ts index 2702e13714..d75dc81912 100644 --- a/cli/src/commands/sync/pull.ts +++ b/cli/src/commands/sync/pull.ts @@ -104,7 +104,7 @@ export async function downloadZip( // from v1 the on-behalf-of address is stripped below, so the tarball sends the // `has_on_behalf_of` marker instead and never resolves an address. // `preserve_extra_perms=true` opts the tarball into surfacing granular ACLs - // on flow / script / app rows. Default-off on the server protects cross- + // on script / flow / app / variable rows. Default-off on the server protects cross- // workspace tarball imports from carrying ACLs that reference identities // missing in the target workspace; the CLI sync flow explicitly wants them. const baseParams = `&plain_secret=${plainSecrets ?? false @@ -150,7 +150,18 @@ export async function downloadZip( } if (zipResponse.status === 404 || body.includes("no rows returned")) { - log.info(colors.red(`Workspace '${workspace.workspaceId}' not found on ${workspace.remote}. Please check your --workspace and try again.`)); + log.info( + colors.red( + `Workspace id '${workspace.workspaceId}' not found on ${workspace.remote}` + + (workspace.name !== workspace.workspaceId + ? ` (resolved from profile '${workspace.name}')` + : "") + + `.\n` + + `Note this is the workspace *id* sent to the API, which is not necessarily what you passed to --workspace:\n` + + ` - check 'wmill workspace list' (the 'workspace id' column)\n` + + ` - check the 'workspaces' block of wmill.yaml ('workspaceId' overrides the workspace name)` + ) + ); } else { log.info(colors.red(`Failed to request tarball from API: ${zipResponse.status} ${zipResponse.statusText}`)); if (body) log.info(colors.red(body)); diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index be5d9e5639..99a098f848 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -76,7 +76,8 @@ import { } from "../../utils/utils.ts"; import { getEffectiveSettings, - getWorkspaceNames, + inferWsNameFromProfile, + resolveWsNameForConfigFromFlags, mergeConfigWithConfigFile, parseSyncBehavior, SyncOptions, @@ -85,7 +86,10 @@ import { WorkspaceEntryConfig, } from "../../core/conf.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; -import { preCheckPermissionedAs } from "../../core/permissioned_as.ts"; +import { + buildPermissionedAsContext, + preCheckPermissionedAs, +} from "../../core/permissioned_as.ts"; import { fromWorkspaceSpecificPath, toWorkspaceSpecificPath, @@ -429,37 +433,6 @@ export function computeWsSpecificFlagOnlyPushes( return out; } -// Resolve workspace name from a --branch override (git branch → workspace name). -// Falls back to using the branch value as-is (backward compat: old key = branch name). -function resolveWsNameFromBranch( - opts: SyncOptions, - branchName: string, -): string { - const match = findWorkspaceByGitBranch(opts.workspaces, branchName); - return match ? match[0] : branchName; -} - -// Resolve wsNameForConfig from CLI flags. Prefers --branch → matching config key, -// then --workspace → matching config key (incl. when --base-url is set). Returns -// undefined when no flag-based resolution applies; callers then fall back to -// inferWsNameFromProfile on the resolved workspace profile. -export function resolveWsNameForConfigFromFlags( - opts: SyncOptions & { branch?: string; workspace?: string }, -): string | undefined { - if (opts.branch) { - return resolveWsNameFromBranch(opts, opts.branch); - } - if (opts.workspace) { - // Use getWorkspaceNames so reserved keys (e.g. commonSpecificItems) are filtered out, - // matching the behavior of findWorkspaceByGitBranch / inferWsNameFromProfile. - const validKeys = getWorkspaceNames(opts.workspaces); - if (validKeys.includes(opts.workspace)) { - return opts.workspace; - } - } - return undefined; -} - // Warn if --workspace overrides auto-detected branch or if workspace not in config. function warnWorkspaceOverride( opts: SyncOptions, @@ -507,33 +480,6 @@ function resolveWsNameForFiles(_opts: SyncOptions, wsName: string): string { return wsName; } -// After resolveWorkspace, infer the workspace config name from the resolved profile -// by matching baseUrl + workspaceId against the workspaces config entries. -function inferWsNameFromProfile( - opts: SyncOptions, - profile: { remote: string; workspaceId: string }, -): string | undefined { - if (!opts.workspaces) return undefined; - const wsNames = Object.keys(opts.workspaces).filter( - (k) => k !== "commonSpecificItems", - ); - for (const name of wsNames) { - const entry = (opts.workspaces as any)[name] as WorkspaceEntryConfig; - if (!entry?.baseUrl) continue; - try { - const entryUrl = new URL(entry.baseUrl).toString(); - const profileUrl = new URL(profile.remote).toString(); - const entryWsId = entry.workspaceId ?? name; - if (entryUrl === profileUrl && entryWsId === profile.workspaceId) { - return name; - } - } catch { - continue; - } - } - return undefined; -} - // Merge CLI options with effective settings, preserving CLI flags as overrides function mergeCliWithEffectiveOptions< T extends GlobalOptions & SyncOptions & { repository?: string }, @@ -5540,27 +5486,19 @@ export async function push( return; } - let permissionedAsContext: PermissionedAsContext | undefined = undefined; - if (parseSyncBehavior(opts.syncBehavior) >= 1) { - const user = await wmill.whoami({ workspace: workspace.workspaceId }); - const userIsAdminOrDeployer = - user.is_admin || (user.groups ?? []).includes("wm_deployers"); - log.debug( - `permissioned_as: user=${user.email}, is_admin=${user.is_admin}, groups=${JSON.stringify(user.groups)}, isAdminOrDeployer=${userIsAdminOrDeployer}`, + const permissionedAsContext: PermissionedAsContext | undefined = + await buildPermissionedAsContext( + workspace.workspaceId, + opts.syncBehavior, ); - permissionedAsContext = { - userCache: new Map(), - userIsAdminOrDeployer, - userEmail: user.email, - }; - + if (permissionedAsContext) { // ws_specific_flag changes have no content payload, so they don't // affect permissioned_as resolution — filter them out before the // pre-check (which expects only added/edited/deleted). await preCheckPermissionedAs( changes.filter((c) => c.name !== "ws_specific_flag"), - user.email, - userIsAdminOrDeployer, + permissionedAsContext.userEmail, + permissionedAsContext.userIsAdminOrDeployer, opts.acceptOverridingPermissionedAsWithSelf ?? false, !!process.stdin.isTTY, ); diff --git a/cli/src/commands/trigger/trigger.ts b/cli/src/commands/trigger/trigger.ts index 83b62f9eb4..b011a0f0b7 100644 --- a/cli/src/commands/trigger/trigger.ts +++ b/cli/src/commands/trigger/trigger.ts @@ -23,7 +23,7 @@ import { Command } from "@cliffy/command"; import { Table } from "@cliffy/table"; import { colors } from "@cliffy/ansi/colors"; import * as log from "../../core/log.ts"; -import { sep as SEP } from "node:path"; +import { sep as SEP, resolve as pathResolve } from "node:path"; import { GlobalOptions, isSuperset, @@ -41,6 +41,8 @@ import { getCurrentGitBranch } from "../../utils/git.ts"; import { requireLogin } from "../../core/auth.ts"; import { validatePath, resolveWorkspace } from "../../core/context.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; +import { readEffectiveSyncBehavior } from "../../core/conf.ts"; type Trigger = { http: HttpTrigger; @@ -620,8 +622,12 @@ async function extractTriggerKindFromPath(filePath: string): Promise; } /** @@ -152,37 +154,42 @@ export async function pushVariable( log.debug(`Variable ${remotePath} does not exist on remote`); } + // extra_perms is synced independently via /acls/* (see applyExtraPermsDiff) + // so a perm-only edit never rewrites the variable value. Strip the field from + // the body that goes to update_variable / create_variable and treat it as a + // separate step both for the up-to-date short-circuit and after the write. + const { extra_perms: localPerms, ...localVariableBody } = localVariable; + if (variable) { - if (isSuperset(localVariable, variable)) { + if (isSuperset(localVariableBody, variable)) { log.debug(`Variable ${remotePath} is up-to-date`); - return; - } + } else { + log.debug(`Variable ${remotePath} is not up-to-date, updating`); - log.debug(`Variable ${remotePath} is not up-to-date, updating`); - - // Apply is_secret only when it differs from the remote (the value is always - // sent, so the server allows the flag change). Upgrades (non-secret->secret) - // always apply; downgrades only when explicitly allowed (single-file push) — - // see allowSecretDowngrade. `undefined` leaves the flag untouched. - let nextIsSecret: boolean | undefined = undefined; - if (localVariable.is_secret !== variable.is_secret) { - if (localVariable.is_secret) { - nextIsSecret = true; - } else if (allowSecretDowngrade) { - nextIsSecret = false; + // Apply is_secret only when it differs from the remote (the value is always + // sent, so the server allows the flag change). Upgrades (non-secret->secret) + // always apply; downgrades only when explicitly allowed (single-file push) — + // see allowSecretDowngrade. `undefined` leaves the flag untouched. + let nextIsSecret: boolean | undefined = undefined; + if (localVariableBody.is_secret !== variable.is_secret) { + if (localVariableBody.is_secret) { + nextIsSecret = true; + } else if (allowSecretDowngrade) { + nextIsSecret = false; + } } - } - await wmill.updateVariable({ - workspace, - path: remotePath.replaceAll(SEP, "/"), - alreadyEncrypted: !plainSecrets, - requestBody: { - ...localVariable, - is_secret: nextIsSecret, - ...(wsSpecific !== undefined ? { ws_specific: wsSpecific } : {}), - }, - }); + await wmill.updateVariable({ + workspace, + path: remotePath.replaceAll(SEP, "/"), + alreadyEncrypted: !plainSecrets, + requestBody: { + ...localVariableBody, + is_secret: nextIsSecret, + ...(wsSpecific !== undefined ? { ws_specific: wsSpecific } : {}), + }, + }); + } } else { log.info(colors.yellow.bold(`Creating new variable ${remotePath}...`)); await wmill.createVariable({ @@ -190,11 +197,22 @@ export async function pushVariable( alreadyEncrypted: !plainSecrets, requestBody: { path: remotePath.replaceAll(SEP, "/"), - ...localVariable, + ...localVariableBody, ...(wsSpecific !== undefined ? { ws_specific: wsSpecific } : {}), }, }); } + + // Synced whether or not the body changed. No refetch: folder perms are never + // merged onto item.extra_perms, and the update/create body carries no + // extra_perms, so the value getVariable read above is still the remote one. + await applyExtraPermsDiff( + workspace, + "variable", + remotePath.replaceAll(SEP, "/"), + localPerms, + (variable as any)?.extra_perms, + ); } async function push( diff --git a/cli/src/core/conf.ts b/cli/src/core/conf.ts index 1fcce7a43f..0c746d2261 100644 --- a/cli/src/core/conf.ts +++ b/cli/src/core/conf.ts @@ -145,7 +145,9 @@ function getGitRepoRoot(): string | null { } export const GLOBAL_CONFIG_OPT = { noCdToRoot: false }; -function findWmillYaml(): string | null { + +// Pure upward search: no chdir, no logging. findWmillYaml() adds the chdir. +function locateWmillYaml(): string | null { const startDir = resolve(process.cwd()); const isInGitRepo = isGitRepository(); const gitRoot = isInGitRepo ? getGitRepoRoot() : null; @@ -176,6 +178,13 @@ function findWmillYaml(): string | null { currentDir = parentDir; } + return foundPath; +} + +function findWmillYaml(): string | null { + const startDir = resolve(process.cwd()); + const foundPath = locateWmillYaml(); + // If wmill.yaml was found in a parent directory, warn the user and change working directory if ( !GLOBAL_CONFIG_OPT.noCdToRoot && @@ -198,6 +207,37 @@ export function getWmillYamlPath(): string | null { return findWmillYaml(); } +/** + * Look up one `workspaces` entry, for diagnostics only. readConfigFile() must + * not be used for that: it chdirs to the config's directory, exits on an + * unsupported syncBehavior and throws on a malformed file. A diagnostic may + * never fail or relocate the command it is diagnosing. + */ +export async function peekWorkspaceEntry( + workspaceName: string +): Promise { + if (RESERVED_WORKSPACE_KEYS.has(workspaceName)) { + return undefined; + } + const wmillYamlPath = locateWmillYaml(); + if (!wmillYamlPath) { + return undefined; + } + try { + const conf = (await yamlParseFile(wmillYamlPath)) as SyncOptions; + const workspaces = + conf?.workspaces ?? + conf?.gitBranches ?? + conf?.environments ?? + conf?.git_branches; + const entry = (workspaces as any)?.[workspaceName]; + return typeof entry === "object" && entry !== null ? entry : undefined; + } catch (e) { + log.debug(`Failed to parse ${wmillYamlPath} for workspace lookup: ${e}`); + return undefined; + } +} + let legacyConfigWarned = false; export async function readConfigFile(opts?: { warnIfMissing?: boolean }): Promise { @@ -630,6 +670,83 @@ export async function getEffectiveSettings( return effective; } +// Resolve workspace name from a --branch override (git branch → workspace name). +// Falls back to using the branch value as-is (backward compat: old key = branch name). +function resolveWsNameFromBranch(opts: SyncOptions, branchName: string): string { + const match = findWorkspaceByGitBranch(opts.workspaces, branchName); + return match ? match[0] : branchName; +} + +// Resolve wsNameForConfig from CLI flags. Prefers --branch → matching config key, +// then --workspace → matching config key (incl. when --base-url is set). Returns +// undefined when no flag-based resolution applies; callers then fall back to +// inferWsNameFromProfile on the resolved workspace profile. +export function resolveWsNameForConfigFromFlags( + opts: SyncOptions & { branch?: string; workspace?: string } +): string | undefined { + if (opts.branch) { + return resolveWsNameFromBranch(opts, opts.branch); + } + if (opts.workspace) { + // Use getWorkspaceNames so reserved keys (e.g. commonSpecificItems) are filtered out, + // matching the behavior of findWorkspaceByGitBranch / inferWsNameFromProfile. + const validKeys = getWorkspaceNames(opts.workspaces); + if (validKeys.includes(opts.workspace)) { + return opts.workspace; + } + } + return undefined; +} + +/** + * Match a workspace config entry to a resolved workspace profile by remote + + * workspace id. The fallback for when no flag names the entry outright. + */ +export function inferWsNameFromProfile( + opts: SyncOptions, + profile: { remote: string; workspaceId: string } +): string | undefined { + if (!opts.workspaces) return undefined; + for (const name of getWorkspaceNames(opts.workspaces)) { + const entry = (opts.workspaces as any)[name] as WorkspaceEntryConfig; + if (!entry?.baseUrl) continue; + try { + const entryUrl = new URL(entry.baseUrl).toString(); + const profileUrl = new URL(profile.remote).toString(); + const entryWsId = entry.workspaceId ?? name; + if (entryUrl === profileUrl && entryWsId === profile.workspaceId) { + return name; + } + } catch { + continue; + } + } + return undefined; +} + +/** + * `syncBehavior` as the workspace being pushed to sees it. The top level alone + * misses a `workspaces..overrides.syncBehavior`, which is where a repo + * that varies settings per workspace puts it, and the entry to read is the one + * `--workspace` names — falling back to the profile, then to the git branch — + * the same order `sync push` resolves it in. + */ +export async function readEffectiveSyncBehavior( + opts: { workspace?: string }, + profile?: { remote: string; workspaceId: string } +): Promise { + const config = await readConfigFile({ warnIfMissing: false }); + const named = resolveWsNameForConfigFromFlags({ ...config, ...opts }); + const effective = await getEffectiveSettings( + config, + undefined, + false, + true, + named ?? (profile ? inferWsNameFromProfile(config, profile) : undefined) + ); + return effective.syncBehavior; +} + const RESERVED_WORKSPACE_KEYS = new Set(["commonSpecificItems"]); /** diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 35bd6bc2e7..b5494b112c 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.804.0"; +export const VERSION = "1.805.0"; diff --git a/cli/src/core/context.ts b/cli/src/core/context.ts index 502157f0a7..80b27ea52e 100644 --- a/cli/src/core/context.ts +++ b/cli/src/core/context.ts @@ -20,6 +20,7 @@ import { import { getLastUsedProfile, setLastUsedProfile } from "./branch-profiles.ts"; import { readConfigFile, + peekWorkspaceEntry, findWorkspaceByGitBranch, getEffectiveWorkspaceId, getWmillYamlPath, @@ -219,6 +220,9 @@ async function tryResolveWorkspace( // First try: look up workspace by name in wmill.yaml workspaces config const config = await readConfigFile({ warnIfMissing: false }); const wsEntry = config.workspaces?.[opts.workspace] as WorkspaceEntryConfig | undefined; + // What wmill.yaml said to target, kept for the fallback below: a profile + // found by name can silently point somewhere else entirely. + let configuredTarget: { workspaceId: string; baseUrl: string } | undefined; if (wsEntry?.baseUrl) { const workspaceId = getEffectiveWorkspaceId(opts.workspace, wsEntry); let normalizedBaseUrl: string; @@ -231,6 +235,8 @@ async function tryResolveWorkspace( }; } + configuredTarget = { workspaceId, baseUrl: normalizedBaseUrl }; + // Find matching profile by baseUrl + workspaceId const allProfs = await allWorkspaces(opts.configDir); const matching = allProfs.filter( @@ -283,6 +289,22 @@ async function tryResolveWorkspace( ), }; } + if ( + configuredTarget && + (e.workspaceId !== configuredTarget.workspaceId || + e.remote !== configuredTarget.baseUrl) + ) { + log.warnStderr( + colors.yellow( + `⚠️ Falling back to the local profile named '${opts.workspace}' (${e.workspaceId} on ${e.remote}), which does NOT match wmill.yaml:\n` + + ` wmill.yaml maps workspace '${opts.workspace}' to ${configuredTarget.workspaceId} on ${configuredTarget.baseUrl}, but no profile targets it.\n` + + ` Run: wmill workspace add ${configuredTarget.workspaceId} ${configuredTarget.baseUrl}` + ) + ); + } + log.infoStderr( + `Using local profile '${e.name}' → ${e.workspaceId} on ${e.remote}` + ); (opts as any).__secret_workspace = e; return { isError: false, value: e }; } @@ -486,6 +508,8 @@ export async function resolveWorkspace( return process.exit(-1); } + let resolved: Workspace | undefined; + // Try to find existing workspace profile by name, then by workspaceId + remote if (opts.workspace) { let existingWorkspace = await getWorkspaceByName( @@ -523,19 +547,45 @@ export async function resolveWorkspace( ); return process.exit(-1); } - return { + resolved = { ...existingWorkspace, token: opts.token, }; } } - return { + resolved ??= { remote: normalizedBaseUrl, workspaceId: opts.workspace, name: opts.workspace, token: opts.token, }; + + // --base-url pins the target, so wmill.yaml's `workspaces` block is never + // consulted and `--workspace` reaches the API as a workspace id. Name the + // id being sent, and the mapping being skipped, before the request 404s + // on an id the user never typed. + // Only an explicit `workspaceId:` is worth reporting: an entry without one + // maps the name to itself, leaving nothing to correct. + const yamlEntry = await peekWorkspaceEntry(opts.workspace); + const yamlWorkspaceId = yamlEntry?.workspaceId; + if (yamlWorkspaceId && yamlWorkspaceId !== resolved.workspaceId) { + log.warnStderr( + colors.yellow( + `⚠️ --base-url is set, so wmill.yaml is not consulted: workspace id '${resolved.workspaceId}' is sent to the API.\n` + + ` wmill.yaml maps workspace '${opts.workspace}' to workspace id '${yamlWorkspaceId}'${yamlEntry!.baseUrl ? ` on ${yamlEntry!.baseUrl}` : ""}.\n` + + ` Use '--workspace ${yamlWorkspaceId}', or drop --base-url/--token to resolve through wmill.yaml.` + ) + ); + } + log.infoStderr( + `Using workspace id '${resolved.workspaceId}' on ${normalizedBaseUrl} (--base-url given` + + (resolved.name !== resolved.workspaceId + ? `, profile '${resolved.name}')` + : ")") + ); + (opts as any).__secret_workspace = resolved; + return resolved; } else { log.infoStderr( colors.red( diff --git a/cli/src/core/permissioned_as.ts b/cli/src/core/permissioned_as.ts index 5ac48753d6..57570b3549 100644 --- a/cli/src/core/permissioned_as.ts +++ b/cli/src/core/permissioned_as.ts @@ -3,6 +3,7 @@ import * as log from "./log.ts"; import { colors } from "@cliffy/ansi/colors"; import { Confirm } from "@cliffy/prompt/confirm"; import { getTypeStrFromPath } from "../types.ts"; +import { parseSyncBehavior } from "./conf.ts"; export interface PermissionedAsContext { userCache: Map; @@ -10,6 +11,35 @@ export interface PermissionedAsContext { userEmail: string; } +/** + * The whole-tree `sync push` and the single-item `push` commands must resolve + * ownership the same way, so both build the context here: a push that leaves it + * undefined reassigns `permissioned_as` / `on_behalf_of` to whoever ran it. + * Undefined below syncBehavior v1, where that reassignment is the contract, and + * for a caller who is neither admin nor in `wm_deployers` the backend enforces + * it anyway — the flag on the context is what keeps the CLI from claiming + * otherwise. + */ +export async function buildPermissionedAsContext( + workspace: string, + syncBehavior: string | number | undefined +): Promise { + if (parseSyncBehavior(syncBehavior) < 1) { + return undefined; + } + const user = await wmill.whoami({ workspace }); + const userIsAdminOrDeployer = + user.is_admin || (user.groups ?? []).includes("wm_deployers"); + log.debug( + `permissioned_as: user=${user.email}, is_admin=${user.is_admin}, groups=${JSON.stringify(user.groups)}, isAdminOrDeployer=${userIsAdminOrDeployer}` + ); + return { + userCache: new Map(), + userIsAdminOrDeployer, + userEmail: user.email, + }; +} + async function ensureUserCache( workspace: string, cache: Map diff --git a/cli/test/base_url_workspace_resolution_unit.test.ts b/cli/test/base_url_workspace_resolution_unit.test.ts new file mode 100644 index 0000000000..66f618d113 --- /dev/null +++ b/cli/test/base_url_workspace_resolution_unit.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, test } from "bun:test"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { resolveWorkspace } from "../src/core/context.ts"; +import { getWorkspaceConfigFilePath } from "../windmill-utils-internal/src/config/config.ts"; +import type { GlobalOptions } from "../src/types.ts"; + +const BASE_URL = "http://localhost:9999/"; + +// --base-url pins the target: --workspace reaches the API as a workspace id and +// wmill.yaml is not consulted. The warning that says so may only peek at the +// file — readConfigFile() exits on an unsupported syncBehavior and throws on a +// malformed one, so resolving through it lets an unrelated config fail a +// command that never needed it. +async function withWmillYaml( + wmillYaml: string, + fn: (opts: GlobalOptions) => Promise +): Promise { + const repoDir = await mkdtemp(path.join(os.tmpdir(), "wmill_baseurl_repo_")); + const configDir = await mkdtemp(path.join(os.tmpdir(), "wmill_baseurl_conf_")); + const originalCwd = process.cwd(); + try { + await writeFile(path.join(repoDir, "wmill.yaml"), wmillYaml); + await writeFile(await getWorkspaceConfigFilePath(configDir), ""); + + process.chdir(repoDir); + await fn({ + configDir, + baseUrl: BASE_URL, + token: "sometoken", + workspace: "staging", + } as GlobalOptions); + } finally { + process.chdir(originalCwd); + await rm(repoDir, { recursive: true, force: true }); + await rm(configDir, { recursive: true, force: true }); + } +} + +describe("--base-url workspace resolution", () => { + const rejectedConfigs: [string, string][] = [ + ["an unsupported syncBehavior", "syncBehavior: v2\n"], + ["a malformed file", 'workspaces:\n staging:\n baseUrl: "unterminated\n'], + ]; + + for (const [label, wmillYaml] of rejectedConfigs) { + test(`resolves despite ${label}`, async () => { + await withWmillYaml(wmillYaml, async (opts) => { + const workspace = await resolveWorkspace(opts); + expect(workspace.workspaceId).toBe("staging"); + expect(workspace.remote).toBe(BASE_URL); + }); + }); + } + + test("a workspaces mapping never overrides the explicit workspace id", async () => { + await withWmillYaml( + "workspaces:\n staging:\n baseUrl: http://elsewhere.example/\n workspaceId: admins\n", + async (opts) => { + const workspace = await resolveWorkspace(opts); + expect(workspace.workspaceId).toBe("staging"); + expect(workspace.remote).toBe(BASE_URL); + } + ); + }); +}); diff --git a/cli/test/schedule_push_permissioned_as_unit.test.ts b/cli/test/schedule_push_permissioned_as_unit.test.ts new file mode 100644 index 0000000000..941415409d --- /dev/null +++ b/cli/test/schedule_push_permissioned_as_unit.test.ts @@ -0,0 +1,125 @@ +/** + * Regression guard: the standalone `wmill schedule push` must resolve ownership + * the same way `wmill sync push` does. It only preserves the remote's + * `permissioned_as` when the command hands `pushSchedule` a context, so a push + * that builds none silently reassigns the schedule to whoever ran it. + */ + +import { expect, test, describe, beforeEach, mock } from "bun:test"; +import { mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let updateScheduleCalls: any[] = []; +let remotePermissionedAs: string | undefined = "u/svc"; + +const REMOTE_SCHEDULE = () => ({ + path: "u/admin/sched", + schedule: "0 0 */6 * * *", + timezone: "Etc/UTC", + script_path: "u/admin/script", + is_flow: false, + args: {}, + enabled: false, + summary: "before", + permissioned_as: remotePermissionedAs, +}); + +mock.module("../gen/services.gen.ts", () => ({ + getSchedule: async () => REMOTE_SCHEDULE(), + updateSchedule: async (a: unknown) => { + updateScheduleCalls.push(a); + }, + whoami: async () => ({ + email: "deployer@windmill.dev", + username: "deployer", + is_admin: true, + groups: [], + }), +})); + +const realContext = await import("../src/core/context.ts"); +mock.module("../src/core/context.ts", () => ({ + ...realContext, + resolveWorkspace: async () => ({ + workspaceId: "w", + name: "w", + remote: "http://localhost/", + token: "t", + }), +})); + +const realAuth = await import("../src/core/auth.ts"); +mock.module("../src/core/auth.ts", () => ({ + ...realAuth, + requireLogin: async () => ({}), +})); + +const scheduleCommand = (await import("../src/commands/schedule/schedule.ts")) + .default; + +async function pushIn(wmillYamlTail: string): Promise { + const dir = await mkdtemp(join(tmpdir(), "windmill_sched_push_")); + await writeFile( + join(dir, "wmill.yaml"), + `defaultTs: bun\nincludeSchedules: true\n${wmillYamlTail}`, + "utf-8" + ); + await writeFile( + join(dir, "sched.schedule.yaml"), + `schedule: "0 0 */6 * * *"\ntimezone: Etc/UTC\nscript_path: u/admin/script\nis_flow: false\nargs: {}\nenabled: false\nsummary: after\n`, + "utf-8" + ); + + const cwd = process.cwd(); + process.chdir(dir); + try { + await scheduleCommand.parse([ + "push", + "sched.schedule.yaml", + "u/admin/sched", + ]); + } finally { + process.chdir(cwd); + } +} + +describe("wmill schedule push ownership", () => { + beforeEach(() => { + updateScheduleCalls = []; + remotePermissionedAs = "u/svc"; + }); + + test("keeps the remote's permissioned_as under syncBehavior v1", async () => { + await pushIn("syncBehavior: v1\n"); + + expect(updateScheduleCalls).toHaveLength(1); + const body = updateScheduleCalls[0].requestBody; + expect(body.summary).toBe("after"); + expect(body.permissioned_as).toBe("u/svc"); + expect(body.preserve_permissioned_as).toBe(true); + }); + + // The entry to read is the one matching the workspace being pushed to, not + // the top level: a repo that varies settings per workspace puts syncBehavior + // under `overrides` and nowhere else. + test("reads syncBehavior from the target workspace's overrides", async () => { + await pushIn( + `workspaces:\n other:\n baseUrl: http://localhost/\n workspaceId: w\n overrides:\n syncBehavior: v1\n` + ); + + expect(updateScheduleCalls).toHaveLength(1); + const body = updateScheduleCalls[0].requestBody; + expect(body.permissioned_as).toBe("u/svc"); + expect(body.preserve_permissioned_as).toBe(true); + }); + + test("leaves ownership to the backend below syncBehavior v1", async () => { + await pushIn(""); + + expect(updateScheduleCalls).toHaveLength(1); + const body = updateScheduleCalls[0].requestBody; + expect(body.permissioned_as).toBeUndefined(); + expect(body.preserve_permissioned_as).toBeUndefined(); + }); +}); diff --git a/cli/test/variable_resource_push.test.ts b/cli/test/variable_resource_push.test.ts index e6bc84c326..b77ed48b50 100644 --- a/cli/test/variable_resource_push.test.ts +++ b/cli/test/variable_resource_push.test.ts @@ -361,6 +361,126 @@ describe("variable", () => { expect(content).toContain("is_secret: false"); }); }); + + test("extra_perms round-trips and pushes via /acls/* without rewriting the variable", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const varPath = `f/test/perms_var_${uniqueId}`; + + const createResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: varPath, + value: "perms_test_value", + is_secret: false, + description: "Variable for extra_perms test", + }), + } + ); + expect(createResp.status).toBeLessThan(300); + await createResp.text(); + + const aclResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/acls/add/variable/${varPath}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ owner: "g/all", write: true }), + } + ); + expect(aclResp.status).toBeLessThan(300); + await aclResp.text(); + + await writeFile( + join(tempDir, "wmill.yaml"), + `defaultTs: bun\nincludes:\n - "${varPath}**"\nexcludes: []\n`, + "utf-8" + ); + + const pullResult = await backend.runCLICommand( + ["sync", "pull", "--yes"], + tempDir + ); + expect(pullResult.code).toEqual(0); + + const localPath = join(tempDir, `${varPath}.variable.yaml`); + const pulled = await readFile(localPath, "utf-8"); + expect(pulled).toContain("extra_perms:"); + expect(pulled).toContain("g/all: true"); + + const beforeResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + const before = await beforeResp.json(); + + // Perm-only edit: downgrade the grant to read. + await writeFile( + localPath, + pulled.replace("g/all: true", "g/all: false"), + "utf-8" + ); + + const pushResult = await backend.runCLICommand( + ["sync", "push", "--yes"], + tempDir + ); + expect(pushResult.code).toEqual(0); + + const afterResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + const after = await afterResp.json(); + expect(after.extra_perms).toEqual({ "g/all": false }); + // Routed through /acls/* rather than update_variable, so the row itself + // is untouched. + expect(after.edited_at).toEqual(before.edited_at); + expect(after.value).toEqual("perms_test_value"); + + // A yaml with no extra_perms field at all is "no opinion": a checkout + // that predates ACL sync must never revoke UI-managed grants. + await writeFile( + localPath, + `description: "Variable for extra_perms test"\nvalue: perms_test_value\nis_secret: false\n`, + "utf-8" + ); + const noOpinionResult = await backend.runCLICommand( + ["sync", "push", "--yes"], + tempDir + ); + expect(noOpinionResult.code).toEqual(0); + + const noOpinionApiResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + expect((await noOpinionApiResp.json()).extra_perms).toEqual({ + "g/all": false, + }); + + // An owner present remotely but absent from a *present* map is revoked — + // the one direction that can destroy a grant. + await writeFile( + localPath, + `description: "Variable for extra_perms test"\nvalue: perms_test_value\nis_secret: false\nextra_perms: {}\n`, + "utf-8" + ); + const revokeResult = await backend.runCLICommand( + ["sync", "push", "--yes"], + tempDir + ); + expect(revokeResult.code).toEqual(0); + + const finalResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + const final = await finalResp.json(); + expect(final.extra_perms).toEqual({}); + }); + }); }); // ============================================================================= diff --git a/cli/test/workspace_key_filename_integration.test.ts b/cli/test/workspace_key_filename_integration.test.ts index ae22f191bf..a2e731eae5 100644 --- a/cli/test/workspace_key_filename_integration.test.ts +++ b/cli/test/workspace_key_filename_integration.test.ts @@ -7,8 +7,10 @@ import { stringify as yamlStringify } from "yaml"; import { resolveWsNameForGitBranch } from "../src/core/specific_items.ts"; import { findResourceFile } from "../src/commands/script/script.ts"; -import { resolveWsNameForConfigFromFlags } from "../src/commands/sync/sync.ts"; -import type { SyncOptions } from "../src/core/conf.ts"; +import { + resolveWsNameForConfigFromFlags, + type SyncOptions, +} from "../src/core/conf.ts"; // Integration tests covering the bug where workspace-specific filenames used // the raw git branch name instead of the wmill.yaml workspace config key. diff --git a/docs/dbt-runtime.md b/docs/dbt-runtime.md index 2920227eb1..a50fab2766 100644 --- a/docs/dbt-runtime.md +++ b/docs/dbt-runtime.md @@ -47,6 +47,7 @@ the dominant way dbt is orchestrated today. | 22 | Naming | Match Cosmos field names; importer deferred | | 23 | Descriptor | `wm_dbt.yaml` inside the project, OPTIONAL. See below | | 24 | Warehouse | Configured on the workspace by name, `main` by default. See below | +| 25 | Cascade direction | Into a relation, not out of a run: `// materialize manual dbt://…` declares a write from any language but dbt's own and wakes `# on dbt://…` subscribers; a finished dbt run still does not dispatch. See "No cascade *from* dbt" | ## Decision 1: engine toggle, and why the shipped default is not Fusion yet @@ -157,11 +158,15 @@ build and an enterprise build whose key did not verify. workspace warehouse's NAME, so two scripts running against the same warehouse agree on identity. -The SCHEME names the producer, because dbt is the only thing that creates one of -these: no other language derives warehouse relations, `// materialize` takes -DuckLake targets only, and a dbt run does not dispatch. Calling the kind -something generic promised a parity with native Snowflake and BigQuery scripts -that does not exist. +The SCHEME names the namespace dbt made, not an exclusive producer. dbt is what +put warehouse relations in the asset graph and is what derives them from a +project; no other language *infers* one, and calling the kind something generic +promised a parity with native Snowflake and BigQuery scripts that does not exist. +A script can nonetheless DECLARE that it writes one — `// materialize manual +dbt:////`, in any language but dbt's own, whose writes +come from its manifest — and that declaration lands on the same node the dbt model +reading the relation does, because identity is the relation rather than the tool. +See "No cascade *from* dbt" below. The PATH is the physical relation, and that is the load-bearing half. dbt-core has no cross-project `ref()`: two projects meet when one materializes a mart and @@ -648,10 +653,13 @@ Two consequences worth knowing: dropped would be filtered out of its own run's graph. The pinned version's nodes are the scope instead. -## No cascade from dbt, and no pipeline membership +## No cascade *from* dbt, and no pipeline membership A finished dbt run does not trigger anything. Its models are recorded, drawn and -tracked; they do not fan out. +tracked; they do not fan out. The opposite direction does: a script that declares +`// materialize manual dbt:////` is an ordinary producer +of that relation, and its completion wakes `# on dbt://` subscribers +through the same fan-out every other asset kind uses. A dbt script is also not a pipeline member (`in_pipeline` is forced false for `ScriptLang::Dbt` at deploy). It materializes warehouse tables, so it looks like @@ -663,22 +671,99 @@ Its models are `dbt://` assets in the shared graph regardless: that is what puts a native script reading one of them on the same node, and it is independent of pipeline membership. -dbt already orders its own DAG, so a cascade would only ever add one thing: -waking a Windmill script that reads a mart. That edge is real but narrow, and -only half of it exists — nothing outside dbt can declare a `dbt://` write -(`// materialize` accepts DuckLake targets only), so the reverse direction, an -ingestion script waking a dbt project, cannot be expressed at all. - -Against that, dispatching correctly from dbt is not cheap. A run's `select` can -build any subset of the project, so the deploy-time write set is not what ran; -using it wakes consumers of relations the run never touched, and narrowing it -needs a per-job record of what was built, which the per-relation state table -cannot supply (it keeps one row per relation, stamped with the last writer). +dbt already orders its own DAG, so a cascade out of a run would only ever add one +thing: waking a Windmill script that reads a mart. That edge is real but narrow, +and dispatching it correctly is not cheap. A run's `select` can build any subset +of the project, so the deploy-time write set is not what ran; using it wakes +consumers of relations the run never touched, and narrowing it needs a per-job +record of what was built, which the per-relation state table cannot supply (it +keeps one row per relation, stamped with the last writer). So dbt materializes and reports, and `asset_dispatch` returns early for -`ScriptLang::Dbt`. A `# on dbt://` subscription is refused outright at -deploy rather than accepted and left dormant — an edge drawn on the canvas that -can never fire is worse than an error saying so. +`ScriptLang::Dbt`. Wiring it up later means deciding what a selective run should +notify — that decision is the work, not the plumbing. + +### Declaring the write, and which subscriptions are refused + +`// materialize manual dbt:////` is how an ingestion +script says it writes a warehouse relation. `manual` is not a mode but the only +mode: nothing generates warehouse DDL, so the script issues its own write and +Windmill records the outcome — the same `materialized_partition` row a DuckLake +target lands, so the relation carries a last writer on the run page and the graph. +It is language-agnostic (the DuckLake write ENGINE is DuckDB's; this declaration +is anyone's but a dbt project's, whose writes are read from its manifest), and the +recording happens in the generic job path +(`record_declared_warehouse_write`) rather than in an executor, for the same +reason. Identity is unchanged — the physical relation — so the ingestion script +and the dbt model reading it are one node, and a `source` declared on the relation +puts the whole thing on one lineage. `// data_test` is refused beside it: those +checks are probes the DuckDB executor splices around a managed write, so on a +warehouse relation — which the script writes itself, from any language — nothing +would run them, and a declarer would deploy green with its assertions silently +skipped. Assert on the relation with a dbt test in the project that reads it. +The `` segment is resolved at +deploy for the same reason a descriptor's `profile.warehouse` is: a name no +warehouse answers to is not a namespace, it strands the write on a node nothing +else reaches. + +Known boundary, shared with every other runtime pipeline annotation: the record +is written from the normal execution path, and recording and cascading are decided +separately, so the routes off it differ. + +* A **dedicated worker** never enters that path — it bypasses the record exactly + as it bypasses `// partitioned` resolution — while its job is still a top-level + `Script`, so the fan-out (which reads the deploy-time `asset` rows) runs. It + cascades and records nothing, leaving the relation with no last writer. +* A **flow runner** bypasses the path too, and is routed by `flow_step_id`, which + `is_eligible_kind` rejects. Neither record nor cascade. +* A **flow step running a deployed script** enters the path as a `Script` job, so + it records — and carries a `flow_step_id`, so it never cascades. +* A **flow step with an inline body** is a `FlowScript` job, which the recording + guard excludes along with previews: neither. + +Fixing the recording half is one change for every runtime pipeline annotation, +not this one. + +A `# on dbt://` subscription is held to the same relation a producer +is — a whole `//` under a configured warehouse, checked +by the validator the `// materialize` target goes through, since two spellings of +that rule would refuse and accept the same string. Beyond that it is refused in +exactly one shape: when every script that writes that relation is a dbt one. Nothing +produces it yet is NOT that shape — a subscriber may be deployed before its +producer, as for every other asset kind, and refusing there would break +deploy-order-independent syncs. A dbt script may neither subscribe nor declare a +`// materialize`: its graph ingest republishes that path's trigger and asset rows +wholesale, so either annotation would deploy something the dependency job then +silently removes — while the declared write would still stamp the relation on +every run. + +The producer set is read as it stands committed, minus the deploying script's own +rows — those describe the version being replaced, so a script dropping its +`// materialize` while adding a subscription would otherwise count itself as the +producer that wakes it, which it could not be anyway (the dispatcher skips +self-loops). + +What that leaves is a subscription accepted while it was live and later orphaned. +A dbt project that claims the relation afterwards names those edges in its own log +rather than leaving them silently dormant — the same "an edge that can never fire +is worse than saying so" the refusal is for, at the other point where it is +knowable. Both points that publish ownership warn: the deploy, and a run whose +static descriptor found its profile moved. An agent run publishes none — it is +forced to per-run models, so it stores a job-pinned snapshot and leaves workspace +ownership with the deployed graph — so it cannot orphan a subscription either. + +Two orphanings are reported nowhere, and both are accepted rather than overlooked. +A native producer that drops its `// materialize` and leaves dbt alone on the +relation: the deploy that causes it does not touch the subscriber. And the +interleaving where a dbt ingest commits between a subscriber's producer check and +its own commit — the check sees no producer and accepts, the ingest's warning +query sees no trigger and says nothing. Closing the second means a per-relation +lock shared by the deploy path and the ingest, and the ingest takes +`script … FOR UPDATE` before its own advisory lock, so a deploy holding relation +locks first inverts that order into a deadlock across two subsystems — a worse +failure than the cosmetic edge it would prevent. Both are bounded the same way: +the next deploy of that project warns, and the canvas is where they show +meanwhile. A plain READ still renders the consumer beside the model, which is what makes the lineage one graph — but it is written in the script's own code, not in a @@ -687,8 +772,8 @@ comment: the body parsers resolve an asset URI from a string literal Python, TS/Bun/Deno, DuckDB or Ansible script is the read. Those four are the languages with a body-asset parser; the native warehouse ones (snowflake, bigquery, postgresql, mysql, mssql) declare no assets at all today, so a mart -they consume joins the graph only once that inference exists. Wiring the trigger up later means deciding what a -selective run should notify — that decision is the work, not the plumbing. +they consume joins the graph only once that inference exists — while a relation +one of them WRITES joins it now, through the annotation. ## Live per-model progress, and why only dbt-core 1.x has it @@ -1308,17 +1393,22 @@ Against a real dbt project (jaffle_shop shape) and the local Postgres: script reading one of the marts gets an edge to it. 6. **Shared node**: a native script that READS a mart renders as a reader of the same node the dbt model writes — one node, not two islands. Declared with a - plain read (`# dbt://`), never `# on`: a `dbt://` subscription is - refused at deploy, because nothing but dbt writes a warehouse relation and a - dbt run does not dispatch (see "no cascade from dbt"). -7. **Selection**: descriptor `select`/`exclude`, and a run-arg override, each + plain read (`# dbt://`), never `# on`: a subscription to a relation dbt + alone builds is refused at deploy, since a dbt run does not dispatch. +7. **Declared write**: a native `// materialize manual dbt://` script + and a dbt project reading that relation as a `source` render as one node; a + run of the script records its materialization and wakes a + `# on dbt://` subscriber — a subscription only that producer makes + wakeable, the dbt project reading the relation being no producer of it (see + "no cascade *from* dbt"). +8. **Selection**: descriptor `select`/`exclude`, and a run-arg override, each build only the expected subset. -8. **Dynamic descriptors**: a `{{ }}` placeholder in `vars` re-ingests the graph +9. **Dynamic descriptors**: a `{{ }}` placeholder in `vars` re-ingests the graph from the run's own manifest, so a model that placeholder enables appears in the same run that builds it. -9. **Both credential paths**: resource-rendered `profiles.yml`, and the project's +10. **Both credential paths**: resource-rendered `profiles.yml`, and the project's own `profiles.yml` with env-var injection. -10. **Caching**: a second run reuses the cached `dbt_packages/` with no network +11. **Caching**: a second run reuses the cached `dbt_packages/` with no network fetch. Keep only tests that pin behavior a future change could break. Per AGENTS.md, diff --git a/docs/ducklake-materialization.md b/docs/ducklake-materialization.md index 32fff7ca23..834b48c211 100644 --- a/docs/ducklake-materialization.md +++ b/docs/ducklake-materialization.md @@ -606,8 +606,10 @@ how-to (extract-engine choice, cursor recipes, schema-drift handling, worked examples) lives in windmilldocs `core_concepts/63_pipelines` → "Ingestion (EL)"; this section records only what future feature work must not break. -- **`// materialize` is DuckDB-only** (deploy-rejected elsewhere, managed and - `manual` alike — `windmill-api-scripts/src/scripts.rs`), and the SDK +- **A `ducklake://` `// materialize` is DuckDB-only** (deploy-rejected + elsewhere, managed and `manual` alike — `windmill-api-scripts/src/scripts.rs`; + a `dbt://` warehouse-relation target is the one any language but dbt's own may + declare, and it is track-only — see `docs/dbt-runtime.md`), and the SDK materialize helpers (`upsert_partition` / `upsertPartition`) build their SQL inside the SDK, so the asset parsers cannot see the write. A polyglot node that "writes the lake directly" therefore deploys with **no output edge** — diff --git a/docs/reusable-ai-agents.md b/docs/reusable-ai-agents.md index 8383bc2836..ede8f64b1d 100644 --- a/docs/reusable-ai-agents.md +++ b/docs/reusable-ai-agents.md @@ -22,22 +22,54 @@ every workspace via the standard cached-resource-type sync, like other built-in or flow expressions), so saving round-trips losslessly. Each host flow overrides what it needs: `tool_inputs` stores per-tool overrides (a diff from the resource tool's own transforms) that overlay onto the matching tools at runtime. Editing on a linked step edits - the flow's use of the agent; editing under the "Editing" banner edits the agent itself. + the flow's use of the agent; editing in the agent editor edits the agent itself. In the flow editor, the AI agent step's **Step Input** tab shows a single read-only card (*linked to *, with the inherited brain + tools and an explanatory tooltip) plus -*Edit* (fork into the editable step, Save changes upserts back and re-links) and *Unlink* -(fork the resolved config — including any `tool_inputs` — back into the step as a one-off). -While editing, the step is the only copy of the edits: Cancel drops them and re-links (asking -first when there is something to drop), and the unsaved-changes badge opens a diff against the -deployed agent whose Discard changes is Cancel without the question. What a fork is an edit of, -and the deployed baseline the edits are judged against, live in `agentEditStore` (in memory), so -a reload brings the step back as a standalone agent with no path to save back to. +*Edit*, which opens the agent editor over the flow, and *Unlink* (fork the resolved config — +including any `tool_inputs` — back into the step as a one-off). A linked agent's tools appear as display-only graph tool nodes (clicking one selects the agent step); below the step's inputs, each tool gets a section with the standard schema-aware input editors (prop picker included) and a read-only view of its code — edits persist into `tool_inputs`. +## Drafts + +The agent editor edits the resource through a **per-user resource draft** (`draft` table, +`item_kind = 'resource'`), autosaved by `useAgentDraft` and deployed by the editor's own Deploy +button. It is the same draft row the generic resource editor writes and the Review & Deploy page +lists, so an agent can be deployed from any of them. + +A flow does not wait for that deploy to see the draft: + +- Testing the flow, or a single linked step, runs the draft. `runFlowPreview` and `ModuleTest` + substitute each linked step for the standalone step the draft would run as + (`linkedAgentDrafts.ts`): `agent` cleared, the draft's brain as static input transforms, the + draft's tools on the step, and the step's own `user_message`/`user_attachments` kept on top — + the same overlay order `ai_executor.rs` applies to a linked step. `tool_inputs` is untouched, + since the worker overlays it in both branches. +- The step's linked card and the graph's tool nodes show the draft, with a *Draft* badge, so the + editor describes what a test would run. Read-only surfaces (the deployed flow page, the run + viewer) stay on the deployed agent: they resolve tools through `publishLinkedAgentTools` without + the draft flag. +- Deploying the flow lists every linked agent that has a draft in the confirmation dialog, beside + the draft triggers. Deploying one writes the resource and drops the draft; leaving one out keeps + its draft untouched, and the flow runs the agent as currently deployed. That is the one place + the two kinds differ: an undeployed draft trigger is deleted, because it belongs to the flow, + while an agent draft belongs to a resource other flows also use. + +Because a draft is per-user, a flow test can behave differently for two people looking at the same +flow. That is the same contract as a flow draft, and deploying the agent is what makes it shared. + +Inlining has a consequence worth knowing: a preview job's `raw_flow` then carries the agent's +config, where a linked step used to carry only the path and leave the resolution to the worker. So +an agent's prompt and tool set are readable by whoever can read that preview job, which is a wider +set than whoever can read the resource when the agent sits in a more restricted folder than the +flow. No credential travels with it — the provider stays a `$res:` reference, resolved at run time +as the runner. The agent editor's own test pane has inlined the same way since drafts existed; +closing the gap would mean the preview carrying a draft *reference* the worker resolves, rather +than the config. + Sharing works through standard resource folder permissions (save agents under `f/...`). Only the agent's brain is interpolated when the step runs. A tool's own `$res:`/`$var:` defaults are diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 7dde8c270e..a200c878e3 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.804.0", + "version": "1.805.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.804.0", + "version": "1.805.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 593ad0e404..d96e512c00 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.804.0", + "version": "1.805.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/frontend/src/lib/attachments/newTabModifier.dom.test.ts b/frontend/src/lib/attachments/newTabModifier.dom.test.ts new file mode 100644 index 0000000000..115877298c --- /dev/null +++ b/frontend/src/lib/attachments/newTabModifier.dom.test.ts @@ -0,0 +1,106 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { newTabModifier } from './newTabModifier.svelte' + +const onPlatform = (userAgent: string) => vi.stubGlobal('navigator', { userAgent }) +const LINUX = 'Mozilla/5.0 (X11; Linux x86_64)' +const MAC = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)' + +const attached: (() => void)[] = [] + +/** Attach to a fresh element and return it with its cleanup, as `{@attach}` would. */ +function pill() { + const node = document.createElement('span') + document.body.append(node) + const modifier = newTabModifier() + const cleanup = modifier.attach(node) as () => void + attached.push(cleanup) + const hover = (init: MouseEventInit = {}) => + node.dispatchEvent(new MouseEvent('mouseenter', init)) + const move = (init: MouseEventInit = {}) => node.dispatchEvent(new MouseEvent('mousemove', init)) + const unhover = () => node.dispatchEvent(new MouseEvent('mouseleave')) + return { modifier, hover, move, unhover, cleanup } +} + +const keydown = (init: KeyboardEventInit) => + window.dispatchEvent(new KeyboardEvent('keydown', init)) + +describe('newTabModifier', () => { + // The window listeners outlive the DOM, so every case has to be torn down through the + // attachment rather than by emptying the body. + afterEach(() => { + attached.splice(0).forEach((cleanup) => cleanup()) + document.body.replaceChildren() + vi.unstubAllGlobals() + }) + + // The hover event carries the live modifier state, so a modifier pressed before the pointer + // arrived (or while this window was unfocused) is picked up rather than read as false. + it('seeds from the hover event, per platform', () => { + onPlatform(LINUX) + const linux = pill() + linux.hover({ ctrlKey: true }) + expect(linux.modifier.held).toBe(true) + + onPlatform(MAC) + const mac = pill() + // macOS ctrl+click is a secondary click, so it must not read as a new-tab modifier. + mac.hover({ ctrlKey: true }) + expect(mac.modifier.held).toBe(false) + mac.hover({ metaKey: true }) + expect(mac.modifier.held).toBe(true) + }) + + // Editors and menus stop keydown propagation to keep their own shortcuts, so a bubble-phase + // listener would go blind whenever focus sits in one. + it('sees a keydown that a focused element stops from propagating', () => { + onPlatform(LINUX) + const { modifier, hover } = pill() + hover() + const input = document.createElement('input') + input.addEventListener('keydown', (e) => e.stopPropagation()) + document.body.append(input) + + input.dispatchEvent( + new KeyboardEvent('keydown', { key: 'Control', ctrlKey: true, bubbles: true }) + ) + expect(modifier.held).toBe(true) + }) + + // A modifier held across a keyboard app switch is cleared by the blur and delivers no keydown + // on the way back, while the pointer parked on the pill fires no fresh mouseenter either. + it('re-seeds from pointer movement after the window lost focus', () => { + onPlatform(LINUX) + const { modifier, hover, move } = pill() + hover({ ctrlKey: true }) + window.dispatchEvent(new Event('blur')) + expect(modifier.held).toBe(false) + + move({ ctrlKey: true }) + expect(modifier.held).toBe(true) + }) + + it('stops tracking once unhovered', () => { + onPlatform(LINUX) + const { modifier, hover, unhover } = pill() + hover({ ctrlKey: true }) + unhover() + expect(modifier.held).toBe(false) + + keydown({ key: 'Control', ctrlKey: true }) + expect(modifier.held).toBe(false) + }) + + it('stops tracking when the element is destroyed while hovered', () => { + onPlatform(LINUX) + const { modifier, hover, cleanup } = pill() + hover({ ctrlKey: true }) + // Hovering again without leaving must not strand the first hover's listeners, which nothing + // would then hold a reference to. + hover({ ctrlKey: true }) + cleanup() + expect(modifier.held).toBe(false) + + keydown({ key: 'Control', ctrlKey: true }) + expect(modifier.held).toBe(false) + }) +}) diff --git a/frontend/src/lib/attachments/newTabModifier.svelte.ts b/frontend/src/lib/attachments/newTabModifier.svelte.ts new file mode 100644 index 0000000000..06c69f36cc --- /dev/null +++ b/frontend/src/lib/attachments/newTabModifier.svelte.ts @@ -0,0 +1,66 @@ +import type { Attachment } from 'svelte/attachments' +import { isMac } from '$lib/utils' + +/** + * Tracks whether the modifier that turns a click into a new browser tab is held, but only while + * the attached element is hovered, which is the only moment the answer is used. + */ +export function newTabModifier() { + let held = $state(false) + + // Only the modifier that actually yields a tab: shift opens a window, alt can start a + // download, and on macOS ctrl+click is a secondary click. + // Taken from each event rather than accumulated across keydown/keyup pairs, so a keyup lost to + // a focus change cannot strand the flag on. + const sync = (event: KeyboardEvent | MouseEvent) => { + held = isMac() ? event.metaKey : event.ctrlKey + } + const clear = () => { + held = false + } + + const attach: Attachment = (node) => { + // One controller per hover: a mirrored remove list leaks any listener whose options drift. + let hover: AbortController | undefined + const leave = () => { + hover?.abort() + hover = undefined + clear() + } + const enter = (event: MouseEvent) => { + // Seeded from the hover itself: mouse events carry the same modifier flags as key events, + // so a modifier already held before the pointer arrived reads correctly. + sync(event) + // Re-entering without an intervening leave would strand the previous controller: nothing + // else references it, so its listeners could never be removed. + hover?.abort() + hover = new AbortController() + const { signal } = hover + // Same reason the hover seeds: a modifier held across a keyboard app switch delivers no + // keydown on the way back, so the pointer is all that is left to re-read it from. + node.addEventListener('mousemove', sync, { signal }) + // Capture: editors and menus stopPropagation the keys they handle, hiding the modifier + // from a bubble-phase listener whenever focus sits in one. + window.addEventListener('keydown', sync, { capture: true, signal }) + window.addEventListener('keyup', sync, { capture: true, signal }) + // Not capture, unlike the two above: blur does not bubble but does reach the window while + // capturing, so it would fire for every element that loses focus. + window.addEventListener('blur', clear, { signal }) + } + + const life = new AbortController() + node.addEventListener('mouseenter', enter, { signal: life.signal }) + node.addEventListener('mouseleave', leave, { signal: life.signal }) + return () => { + life.abort() + leave() + } + } + + return { + get held() { + return held + }, + attach + } +} diff --git a/frontend/src/lib/components/AppConnectInner.svelte b/frontend/src/lib/components/AppConnectInner.svelte index 320b3e36dd..7f2b646f6b 100644 --- a/frontend/src/lib/components/AppConnectInner.svelte +++ b/frontend/src/lib/components/AppConnectInner.svelte @@ -41,6 +41,13 @@ import TextInput from './text_input/TextInput.svelte' import { sameTopDomainOrigin } from '$lib/cookies' import SyncResourceTypes from './SyncResourceTypes.svelte' + import { + alphabetical, + byPopularity, + hubResourceTypePicks, + localResourceTypeCounts, + recordHubResourceTypePick + } from './pickerPopularity' import Label from './Label.svelte' import ResourcePathHint from './ResourcePathHint.svelte' @@ -332,6 +339,7 @@ export async function open(rt?: string) { if (!rt) { loadResourceTypes() + loadPopularity() } step = 1 //express && !manual ? 3 : 1 // The list is keyboard-driven from the search field, so it takes focus on open. @@ -378,12 +386,29 @@ } } + /** + * Orders the browse list: the types this workspace already has resources of lead, ranked + * among themselves by the hub's pick counts, then everything else on the same counts. + * `byPopularity` carries the full rule. Both signals are fetched, so the rows render + * alphabetically and re-sort when this lands. + */ + let popularity: (a: string, b: string) => number = $state(alphabetical) + + async function loadPopularity() { + if (!effectiveWorkspace) return + const [hub, local] = await Promise.all([ + hubResourceTypePicks(effectiveWorkspace), + localResourceTypeCounts(effectiveWorkspace) + ]) + popularity = byPopularity(hub, local) + } + async function loadConnects() { if (!connects) { try { - const list = (await OauthService.listOauthConnects()) - .filter((x) => x.name != 'supabase_wizard') - .sort((a, b) => a.name.localeCompare(b.name)) + const list = (await OauthService.listOauthConnects()).filter( + (x) => x.name != 'supabase_wizard' + ) connects = list.map((x) => x.name) connectsInfo = Object.fromEntries(list.map((x) => [x.name, x])) } catch (e) { @@ -466,19 +491,17 @@ // providers — so any of them can also be connected with the user's own // credentials or manually, not only via the shared instance setup (same as // the authorization-code behavior). - connectsManual = availableRts - .map( - (x) => - ({ - key: x, - ...(apiTokenApps[x] ?? { - instructions: '', - img: undefined, - linkedSecret: undefined - }) - }) as { key: string; img?: string; instructions: string[] } - ) - .sort((a, b) => a.key.localeCompare(b.key)) + connectsManual = availableRts.map( + (x) => + ({ + key: x, + ...(apiTokenApps[x] ?? { + instructions: '', + img: undefined, + linkedSecret: undefined + }) + }) as { key: string; img?: string; instructions: string[] } + ) const filteredNativeLanguages = filteredConnectsManual?.filter( (o) => nativeLanguagesCategory?.includes(o[0]) ?? false ) @@ -952,6 +975,10 @@ } }) } + // Saving is what "picking a type" means to the hub: reaching step 2 is still + // browsing. Both branches above count, `filling` included — an imported stub is + // a type taken into the workspace just the same. + recordHubResourceTypePick(effectiveWorkspace, resourceType) dispatch('refresh', path) dispatch('close') sendUserToast( @@ -972,6 +999,9 @@ if (step == 1) { loadConnects() loadResourceTypes() + // Opened on a specific type, `open()` skipped this; backing out to the browse + // list is the first time it is needed. + loadPopularity() } } @@ -980,24 +1010,35 @@ let filteredConnects: { key: string }[] = $state([]) let filteredConnectsManual: { key: string; img?: string; instructions: string[] }[] = $state([]) - // uFuzzy scores the name and the description as one string, so searching "google" ranks - // every type whose description mentions Google alongside the ones named after it. Re-sort - // on which field matched, keeping uFuzzy's order within a tier. + let searching = $derived(filter.trim() !== '') + + // Searching, the query owns the order: uFuzzy scores the name and the description as one + // string, so "google" ranks every type whose description mentions Google alongside the + // ones named after it — re-sort on which field matched, keeping uFuzzy's order within a + // tier. Browsing, there is no query to rank against, so popularity orders the list. const rank = (items: { key: string }[] | undefined) => items && - sortResourceTypesByMatch( - items, - filter, - (x) => x.key, - (x) => resourceTypeDescriptions[x.key] - ) + (searching + ? sortResourceTypesByMatch( + items, + filter, + (x) => x.key, + (x) => resourceTypeDescriptions[x.key] + ) + : // Both signals are keyed by resource type, and a sandbox client is a second row + // against one (`salesforce_sandbox` saves a `salesforce`), so it ranks on the + // parent's popularity. Its own key still breaks the tie the pair then have, or + // the two would order arbitrarily. + [...items].sort( + (a, b) => + popularity(stripSandboxSuffix(a.key), stripSandboxSuffix(b.key)) || + a.key.localeCompare(b.key) + )) let rankedConnects = $derived(rank(filteredConnects)) let rankedConnectsManual = $derived( rank(filteredConnectsManual) as typeof filteredConnectsManual | undefined ) - let searching = $derived(filter.trim() !== '') - // Browsing, the "Others" list leads with the native database types. Searching, that // grouping would outrank the search itself — `ms_sql_server` sorting under `mysql` on // "sql" — so the ranked order stands on its own. diff --git a/frontend/src/lib/components/DBManager.svelte b/frontend/src/lib/components/DBManager.svelte index 35bd88abec..7520cc5013 100644 --- a/frontend/src/lib/components/DBManager.svelte +++ b/frontend/src/lib/components/DBManager.svelte @@ -12,8 +12,8 @@ import { Pane, Splitpanes } from 'svelte-splitpanes' import { ClearableInput, Drawer, DrawerContent } from './common' import { sendUserToast } from '$lib/toast' - import { type ColumnDef } from './apps/components/display/dbtable/utils' - import DBTable from './DBTable.svelte' + import { renderDbEqualityFilter, type ColumnDef } from './apps/components/display/dbtable/utils' + import DBTable, { type DbForeignKeyTarget, type DbRowFilter } from './DBTable.svelte' import type { IDbSchemaOps, IDbTableOps } from './dbOps' import DropdownV2 from './DropdownV2.svelte' import ConfirmationModal from './common/confirmationModal/ConfirmationModal.svelte' @@ -46,7 +46,12 @@ dbSupportsSchemas: boolean databaseIsEmpty?: boolean colDefs: Record | undefined - dbTableOpsFactory: (params: { colDefs: ColumnDef[]; tableKey: string }) => IDbTableOps + dbTableOpsFactory: (params: { + colDefs: ColumnDef[] + tableKey: string + /** Raw SQL predicate AND-ed into the reads (already escaped). */ + whereClause?: string + }) => IDbTableOps dbSchemaOps: IDbSchemaOps refresh?: () => void initialSchemaKey?: string @@ -214,6 +219,83 @@ : selected.tableKey ) + // Set by "Go to row" on a foreign-keyed cell; pinned to the table it was + // created for so a schema change can't carry it onto an unrelated table. + let rowFilter: (DbRowFilter & { tableKey: string }) | undefined = $state() + let activeRowFilter = $derived(rowFilter?.tableKey === tableKey ? rowFilter : undefined) + let whereClause = $derived( + activeRowFilter + ? renderDbEqualityFilter(activeRowFilter.column, activeRowFilter.value, dbType) + : undefined + ) + + function selectTable(schemaKey: string | undefined, table: string) { + rowFilter = undefined + selected = { schemaKey, tableKey: table } + } + + /** Where a foreign key's `schema.table` target lives in the sidebar, or + * undefined when it cannot be opened from here. */ + function resolveForeignKeyTarget( + targetTable: string + ): { schemaKey: string; table: string } | undefined { + const parts = targetTable.split('.') + const table = parts[parts.length - 1] + const qualifier = parts.length > 1 ? parts.slice(0, -1).join('.') : undefined + // Without schema support the sidebar browses the connection's default + // schema only, and unqualified reads would hit a same-named local table. + if (!dbSupportsSchemas && qualifier && qualifier !== selected.schemaKey) return undefined + const schemaKey = dbSupportsSchemas && qualifier ? qualifier : selected.schemaKey + if (!schemaKey || !(table in (dbSchema.schema[schemaKey] ?? {}))) return undefined + return { schemaKey, table } + } + + function goToRow(target: DbForeignKeyTarget) { + const resolved = resolveForeignKeyTarget(target.table) + if (!resolved) { + sendUserToast(`Table ${target.table} cannot be opened from this schema`, true) + return + } + if (renderDbEqualityFilter(target.column, target.value, dbType) === undefined) { + sendUserToast('This value cannot be used as a filter', true) + return + } + const { schemaKey, table } = resolved + selectTable(schemaKey, table) + rowFilter = { + tableKey: dbSupportsSchemas ? `${schemaKey}.${table}` : table, + column: target.column, + value: target.value + } + } + + // The result carries the table it was fetched for: `resource` keeps the + // previous value while refetching, and a stale list would decorate the new + // table's same-named columns as foreign keys. + let foreignKeys = resource( + [() => selected.tableKey, () => selected.schemaKey, () => colDefs], + async ([table, schema], _prev, { signal }) => { + if (!table) return undefined + const forTableKey = dbSupportsSchemas && schema ? `${schema}.${table}` : table + const fks = + features?.foreignKeys === false + ? [] + : await dbSchemaOps.onFetchForeignKeys({ table, schema }) + // A newer selection started meanwhile: an AbortError keeps this result + // out of `current`, where it would shadow the newer table's keys. + if (signal.aborted) throw new DOMException('Superseded', 'AbortError') + return { tableKey: forTableKey, foreignKeys: fks } + } + ) + // Only keys whose target the sidebar can open get the "Go to row" affordance. + let currentForeignKeys = $derived.by(() => { + const fetched = foreignKeys.current + if (!fetched || fetched.tableKey !== tableKey) return undefined + return fetched.foreignKeys.filter( + (fk) => fk.targetTable && resolveForeignKeyTarget(fk.targetTable) !== undefined + ) + }) + let askingForConfirmation: | (ConfirmationModal['$$prop_def'] & { onConfirm: () => void }) | undefined = $state() @@ -395,14 +477,12 @@ role="button" tabindex="0" onclick={() => { - selected.schemaKey = schemaKey - selected.tableKey = tableKey + selectTable(schemaKey, tableKey) toggleTableSelection(schemaKey, tableKey) }} onkeydown={(e) => { if (e.key === 'Enter' || e.key === ' ') { - selected.schemaKey = schemaKey - selected.tableKey = tableKey + selectTable(schemaKey, tableKey) toggleTableSelection(schemaKey, tableKey) } }} @@ -468,7 +548,7 @@ + diff --git a/frontend/src/lib/components/FilterSearchbar.svelte b/frontend/src/lib/components/FilterSearchbar.svelte index 51026f10fd..6c89f1a039 100644 --- a/frontend/src/lib/components/FilterSearchbar.svelte +++ b/frontend/src/lib/components/FilterSearchbar.svelte @@ -112,17 +112,37 @@ } /** - * Creates a URL-synced filter instance that automatically syncs with URL search parameters + * Creates a URL-synced filter instance that automatically syncs with URL search parameters. + * + * `initial` supplies defaults for keys the URL doesn't carry (a route segment, a persisted + * toggle); the returned `seed` re-applies them after a navigation has rewritten the query. */ export function useUrlSyncedFilterInstance( - schemaRec: T - ): { val: Partial> } { + schemaRec: T, + initial?: Partial> + ): { + val: Partial> + seed: (values: Partial>) => void + } { // Build the Zod schema from the filter schema const zodSchema = filterSchemaRecToZodSchema(schemaRec) // Create URL-synced search params const urlFilter = useSearchParams(zodSchema) as Record + // A default has to arrive as a URL param: the URL→instance effect below drops whatever the + // URL lacks, so a value written to the instance is undone on the next sync. Going through + // urlFilter rather than straight to history keeps the search-param cells in step, so it + // does not matter whether a popstate follows. + function seed(values: Partial>) { + const sp = new URLSearchParams(window.location.search) + for (const [key, value] of Object.entries(values) as [string, unknown][]) { + if (value === undefined || value === null || sp.has(key)) continue + urlFilter[key] = value instanceof Date ? value.toISOString() : value + } + } + if (initial) seed(initial) + // Create the filter instance object const filterInstance: { val: Partial> } = $state({ val: {} }) @@ -173,7 +193,15 @@ }) } - return filterInstance + return { + get val() { + return filterInstance.val + }, + set val(v: Partial>) { + filterInstance.val = v + }, + seed + } } function filterToText(filter: FilterInstance, schema: F): string { diff --git a/frontend/src/lib/components/FlowBuilder.svelte b/frontend/src/lib/components/FlowBuilder.svelte index 52af60d7a7..b8a2d4a9e6 100644 --- a/frontend/src/lib/components/FlowBuilder.svelte +++ b/frontend/src/lib/components/FlowBuilder.svelte @@ -98,7 +98,18 @@ import { invalidateWorkspacePaths } from './PathNameAutocomplete.svelte' import type { Trigger } from './triggers/utils' import { deployTriggers, handleSelectTriggerFromKind } from './triggers/utils' - import DraftTriggersConfirmationModal from './common/confirmationModal/DraftTriggersConfirmationModal.svelte' + import DraftChangesConfirmationModal from './common/confirmationModal/DraftChangesConfirmationModal.svelte' + import { + agentDraftCanWrite, + linkedAgentPaths, + loadLinkedAgentDrafts, + type LinkedAgentDraft + } from './flows/linkedAgentDrafts' + import { agentDraftDeployRefusal } from './flows/agentDraft.svelte' + import { markAgentWritten } from './flows/agentEditorStore.svelte' + import { logReusableAgentUsage } from './flows/agentTelemetry' + import { deployDraft } from '$lib/utils_draft_deploy' + import { getUserExt } from '$lib/user' import { Triggers } from './triggers/triggers.svelte' import { StepsInputArgs } from './flows/stepsInputArgs.svelte' import { aiChatManager } from './copilot/chat/AIChatManager.svelte' @@ -192,8 +203,16 @@ let confirmCallback: () => void = $state(() => {}) // What happens when user clicks `override` in warning let open: boolean = $state(false) // Is confirmation modal open - // Draft triggers confirmation modal - let draftTriggersModalOpen = $state(false) + // Draft changes (triggers + linked agents) confirmation modal + let draftChangesModalOpen = $state(false) + /** The unsaved agent drafts the pending deploy found. Loaded rather than derived: it takes a + * request per linked agent, so it is resolved when the deploy asks. */ + let draftAgents = $state([]) + let agentCanWrite = $state>({}) + let agentRefusal = $state>({}) + + /** What the dialog's confirm hands back to `saveFlow`. */ + type DraftChangesToDeploy = { triggers: Trigger[]; agents: LinkedAgentDraft[] } // Top-bar responsive collapse. Measured via bind:clientWidth — we can't // rely on viewport `md:` because the editor lives inside other panes @@ -224,7 +243,7 @@ ] : [] ) - let confirmDeploymentCallback: (triggersToDeploy: Trigger[]) => void = () => {} + let confirmDeploymentCallback: (toDeploy: DraftChangesToDeploy) => void = () => {} // AI changes warning modal let aiChangesWarningOpen = $state(false) @@ -236,11 +255,57 @@ const job: Job | undefined = $derived(flowPreviewContent?.getJob()) let showJobStatus = $state(false) - async function handleDraftTriggersConfirmed(event: CustomEvent<{ selectedTriggers: Trigger[] }>) { - const { selectedTriggers } = event.detail + async function handleDraftChangesConfirmed( + event: CustomEvent<{ selectedTriggers: Trigger[]; selectedAgents: LinkedAgentDraft[] }> + ) { + const { selectedTriggers, selectedAgents } = event.detail // Continue with saving the flow - draftTriggersModalOpen = false - confirmDeploymentCallback(selectedTriggers) + draftChangesModalOpen = false + confirmDeploymentCallback({ triggers: selectedTriggers, agents: selectedAgents }) + } + + /** Deploy each selected agent's draft, the same way the Review & Deploy page deploys the same + * row: hand the path to `deployDraft` and let it promote whatever the draft holds, with no + * re-read to check it still matches what the dialog listed. Agents left out keep their draft + * untouched. */ + async function deployAgentDrafts(agents: LinkedAgentDraft[]) { + const ws = opWorkspace + if (!ws) return + for (const listed of draftAgents) { + // Only the rows the dialog gave a choice on. A `Read-only` or `Invalid config` agent can + // never be selected, so counting it as "kept" would record a decision the user was never + // offered and bias the pair towards keeping. + const selectable = agentCanWrite[listed.path] !== false && !agentRefusal[listed.path] + if (selectable && !agents.some((a) => a.path === listed.path)) { + logReusableAgentUsage('draft_kept_on_deploy') + } + } + for (const agent of agents) { + // Writes the resource, deletes the draft row, and clears the local hint and the workspace + // drafts cache. A failure aborts the flow save the way a failing trigger does, rather than + // deploying a flow against agents that were meant to change with it. + const deployed = await deployDraft('resource', agent.path, ws, { + draftOnly: agent.noDeployed + }) + if (!deployed.success) { + throw new Error(`Could not deploy agent ${agent.path}: ${deployed.error}`) + } + // Nothing was promoted: the draft had gone by the time the helper read it. Saying the agent + // deployed would be a lie about the one thing the toggle decides. + if (deployed.noop) { + throw new Error( + `The draft for ${agent.path} was deployed or discarded elsewhere while this deploy ran, so nothing was written for it.` + ) + } + // `deployDraft` deletes the server row but leaves any in-memory cell for this key, and that + // cell is what `agentDraftState` prefers — a still-mounted holder would otherwise keep + // feeding a phantom draft to the cards and to the next deploy dialog. Local only: `remove` + // would POST a second delete, debounced and past the baseline the first one cleared. + UserDraft.forgetLocal('resource', agent.path, { workspace: ws }) + // Every linked card and the graph key on this to refetch the agent they display. + markAgentWritten(ws, agent.path) + logReusableAgentUsage('draft_deployed_with_flow') + } } // Inside an AI session pane (SessionEditorTarget injects an aiChatManager via @@ -429,18 +494,53 @@ deployedBy = flow.edited_by } - async function saveFlow(deploymentMsg?: string, triggersToDeploy?: Trigger[]): Promise { - if (!triggersToDeploy) { - // Check if there are draft triggers that need confirmation + async function saveFlow(deploymentMsg?: string, toDeploy?: DraftChangesToDeploy): Promise { + if (!toDeploy) { + // Draft triggers and drafts on the agents this flow links: both are unsaved changes the + // deploy would otherwise leave behind, so they are confirmed together. const draftTriggers = triggersState.triggers.filter((trigger) => trigger.draftConfig) - if (draftTriggers.length > 0) { - draftTriggersModalOpen = true - confirmDeploymentCallback = async (triggersToDeploy: Trigger[]) => { - await saveFlow(deploymentMsg, triggersToDeploy) + try { + draftAgents = [ + ...( + await loadLinkedAgentDrafts(linkedAgentPaths(flowStore.val.value), opWorkspace) + ).values() + ] + } catch (err: any) { + // This runs before the try below, and `withAIChangesWarning` invokes its callback without + // awaiting, so a rejection here would be unhandled: the button would do nothing at all, + // with no toast and no `onDeployError`. Report it the way the rest of the save does. + // Deploying anyway is not the fallback — this throws only when an agent's unsaved changes + // cannot be read, which is exactly when the dialog must not claim there are none. + onDeployError?.({ error: err }) + sendUserToast(`The flow could not be saved: ${err?.body ?? err}`, true) + return + } + agentCanWrite = {} + agentRefusal = {} + if (draftAgents.length > 0) { + // One lookup for the whole list: an agent lives in a folder, and the groups and admin + // flag that answer for it are per workspace, so the nav user would answer for the wrong + // membership when a session editor operates on another workspace. + const user = await getUserExt(opWorkspace ?? '').catch(() => undefined) + agentCanWrite = Object.fromEntries( + draftAgents.map((a) => [a.path, agentDraftCanWrite(a, user ?? $userStore ?? undefined)]) + ) + // The path is passed, so a draft that renames the agent is refused here too: a rename is + // the resource editor's to deploy, and this dialog lists the agent under the path the + // flow links. + agentRefusal = Object.fromEntries( + draftAgents.map((a) => [a.path, agentDraftDeployRefusal(a.state, a.path)]) + ) + } + if (draftTriggers.length > 0 || draftAgents.length > 0) { + draftChangesModalOpen = true + confirmDeploymentCallback = async (confirmed: DraftChangesToDeploy) => { + await saveFlow(deploymentMsg, confirmed) } return } } + const triggersToDeploy = toDeploy?.triggers loadingSave = true try { @@ -471,6 +571,11 @@ // loadingSave = false // del // return + // Ahead of the flow itself, as the update branch deploys its triggers: an agent is a + // resource of its own, so the flow should land on top of the agent set it was tested + // against rather than the other way round. + await deployAgentDrafts(toDeploy?.agents ?? []) + // `newFlow` comes from the embedder, and updating a path that has no // deployed flow 404s. Confirm with the server before taking the update // branch so a first deploy still lands. @@ -648,7 +753,7 @@ const resolved = linkedAgentToolsForScope(to) for (const [moduleId, agentPath] of linkedAgentEntries(linkedAgentRefs)) { if (resolved[moduleId] === undefined) { - publishLinkedAgentTools(agentPath, ws, to, moduleId) + publishLinkedAgentTools(agentPath, ws, to, moduleId, true) } } } @@ -716,7 +821,7 @@ // writing overrides against — the tool ids of the agent that was just replaced. claimLinkedToolsFetch(scope, moduleId) clearLinkedAgentTools(scope, moduleId) - publishLinkedAgentTools(agentPath, ws, scope, moduleId) + publishLinkedAgentTools(agentPath, ws, scope, moduleId, true) } publishedAgentByModule = next }) @@ -1343,14 +1448,17 @@ currentValue={flowStore.val} /> - t.draftConfig)} + {draftAgents} + {agentCanWrite} + {agentRefusal} isFlow={true} on:canceled={() => { - draftTriggersModalOpen = false + draftChangesModalOpen = false }} - on:confirmed={handleDraftTriggersConfirmed} + on:confirmed={handleDraftChangesConfirmed} /> diff --git a/frontend/src/lib/components/FlowGraphViewer.svelte b/frontend/src/lib/components/FlowGraphViewer.svelte index 47da29bb57..945533bcc8 100644 --- a/frontend/src/lib/components/FlowGraphViewer.svelte +++ b/frontend/src/lib/components/FlowGraphViewer.svelte @@ -67,6 +67,14 @@ const dispatch = createEventDispatcher() + // A bucket of this viewer's own, never the editor's. Both are keyed by (workspace, flow path), so + // a viewer mounted over the flow being edited — the version-history drawer, which renders the + // same path — would otherwise publish its deployed tools into the editor's bucket and replace the + // drafted tool nodes there. The editor's graph has to keep showing the tools a preview would + // actually run, and nothing republishes when the drawer closes. `FlowStatusViewerInner` scopes + // itself the same way, with `job:`. + let linkedToolsPath = $derived(`view:${flow?.path ?? ''}`) + // This read-only viewer doesn't run initFlowState, so linked agents' tools would otherwise never // resolve. Resolve them for display, keyed by module id. Best-effort: publishLinkedAgentTools // swallows access errors and publishes [], so an inaccessible agent simply shows no tool nodes @@ -80,7 +88,15 @@ for (const m of modules) { const value = m?.value as { type?: string; agent?: string } | undefined if (value?.type === 'aiagent' && value.agent) { - publishLinkedAgentTools(value.agent, ws, linkedToolsScope(ws, flow?.path), m.id) + // Without the draft: this viewer shows a deployed flow or a past run, both of which + // used the deployed agent. + publishLinkedAgentTools( + value.agent, + ws, + linkedToolsScope(ws, linkedToolsPath), + m.id, + false + ) } } }) @@ -101,6 +117,7 @@ earlyStop={flow?.value?.skip_expr !== undefined} cache={flow?.value?.cache_ttl !== undefined} path={flow?.path} + {linkedToolsPath} {download} minHeight={fillAvailableHeight ? Math.max(minHeight, availableHeight) : minHeight} {workspace} diff --git a/frontend/src/lib/components/FlowLoopIterationPreview.svelte b/frontend/src/lib/components/FlowLoopIterationPreview.svelte index 1378e20f04..0a4f5c60ee 100644 --- a/frontend/src/lib/components/FlowLoopIterationPreview.svelte +++ b/frontend/src/lib/components/FlowLoopIterationPreview.svelte @@ -5,6 +5,7 @@ import { createEventDispatcher, getContext } from 'svelte' import type { FlowEditorContext } from './flows/types' import { runFlowPreview } from './flows/utils.svelte' + import { sendUserToast } from '$lib/toast' import SchemaForm from './SchemaForm.svelte' import FlowStatusViewer from '../components/FlowStatusViewer.svelte' import FlowProgressBar from './flows/FlowProgressBar.svelte' @@ -101,15 +102,22 @@ // The preview flow holds only the loop body, so it inherits none of the flow's settings: // carry the tag over so the iteration lands on the worker group the flow runs on. const newFlow = { value: { modules }, summary: '', tag: flowStore.val.tag } - jobId = await runFlowPreview( - whileLoop ? withWhileLoopIter(args) : args, - newFlow, - $pathStore, - restartedFrom, - undefined, - undefined, - opWorkspace?.() - ) + try { + jobId = await runFlowPreview( + whileLoop ? withWhileLoopIter(args) : args, + newFlow, + $pathStore, + restartedFrom, + undefined, + undefined, + opWorkspace?.() + ) + } catch (err: any) { + // `runFlowPreview` resolves a linked agent's draft first and refuses when it cannot be read. + // Without this the rejection is unhandled and the button just does nothing. + sendUserToast(`Could not run preview: ${err?.body ?? err}`, true) + return + } isRunning = true } diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index 96b6eed293..3b860dc3a1 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -294,7 +294,9 @@ untrack(() => { for (const entry of refs ? refs.split('\u0001') : []) { const [moduleId, agentPath] = entry.split('\u0000') - publishLinkedAgentTools(agentPath, ws, scope, moduleId) + // Without the draft: this viewer describes a run that has already happened, and it ran + // the deployed agent. + publishLinkedAgentTools(agentPath, ws, scope, moduleId, false) } }) }) diff --git a/frontend/src/lib/components/ModuleTest.svelte b/frontend/src/lib/components/ModuleTest.svelte index b4e82ced2f..64a318acc7 100644 --- a/frontend/src/lib/components/ModuleTest.svelte +++ b/frontend/src/lib/components/ModuleTest.svelte @@ -14,6 +14,14 @@ import JobLoader, { type Callbacks } from './JobLoader.svelte' import { getStepHistoryLoaderContext } from './stepHistoryLoader.svelte' import { loadSchemaFromModule } from './flows/flowInfers' + import { + inlineAgentDraft, + loadLinkedAgentDrafts, + normalizeAgentRef, + type LinkedAgentDraft + } from './flows/linkedAgentDrafts' + import { AGENT_FLOW_LOCAL_KEYS } from './flows/agentResourceUtils' + import { sendUserToast } from '$lib/toast' interface Props { mod: FlowModule @@ -138,7 +146,35 @@ } else if (val.type == 'aiagent') { const { schema } = await loadSchemaFromModule(mod, opWs) - const agentVal = val + // A linked step whose agent has an unsaved draft is tested as the draft, the same way the + // whole-flow preview and the agent editor's own test pane run it. `inlineAgentDraft` + // clears `agent` and moves the draft's brain and tools onto the step, so the branches + // below then treat it as a standalone agent. + let draft: LinkedAgentDraft | undefined + if (val.agent) { + const linked = normalizeAgentRef(val.agent) + try { + draft = (await loadLinkedAgentDrafts([linked], opWs)).get(linked) + } catch (err: any) { + // The load refuses when the agent's unsaved changes cannot be read, and this function's + // caller neither awaits nor catches: without this the rejection is unhandled and the + // button appears to do nothing, with the test already marked as started. + sendUserToast(`Could not run test: ${err?.body ?? err}`, true) + // Guarded like every other access to it here: the entry is only created for steps the + // panel is tracking, and this runs on a path where it may never have been. + if (modulesTestStates.states[mod.id]) { + modulesTestStates.states[mod.id].loading = false + } + return + } + } + const agentVal = draft ? inlineAgentDraft(val, draft.args) : val + + // `args` is built from the whole AI agent schema whatever the step is, so on a linked step + // it carries every brain key as undefined even though the form renders only the flow-local + // ones (`flowLocalAgentSchema`). Overlaying those would shadow the brain the draft just + // supplied with nothing, so an inlined step takes only the inputs its form actually offers. + const formKeys = draft ? (AGENT_FLOW_LOCAL_KEYS as readonly string[]) : Object.keys(args) // The test form only covers the schema it was given, and for a standalone agent that may be // the flow-local one (the agent editor shows the brain in its own form, not here). Take the @@ -150,7 +186,7 @@ ? {} : ((agentVal.input_transforms ?? {}) as Record)), ...Object.fromEntries( - Object.keys(args).map((key) => [ + formKeys.map((key) => [ key, { expr: `flow_input.${key}`, diff --git a/frontend/src/lib/components/ResourceEditor.svelte b/frontend/src/lib/components/ResourceEditor.svelte index 89acac79de..3c9eda9467 100644 --- a/frontend/src/lib/components/ResourceEditor.svelte +++ b/frontend/src/lib/components/ResourceEditor.svelte @@ -2,7 +2,7 @@ import type { Schema } from '$lib/common' import { ResourceService, WorkspaceService, type Resource, type ResourceType } from '$lib/gen' import { canWrite } from '$lib/utils' - import { createEventDispatcher, untrack } from 'svelte' + import { createEventDispatcher, onDestroy, untrack } from 'svelte' import { userStore, workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' import { clearJsonSchemaResourceCache } from './schema/jsonSchemaResource.svelte' @@ -13,7 +13,9 @@ import { getUserExt } from '$lib/user' import type { UserExt } from '$lib/stores' import { UserDraft, draftValuesEqual, type UserDraftHandle } from '$lib/userDraft.svelte' + import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' import { setLocalDraftHint } from '$lib/localDraftHints.svelte' + import { onUserInput } from '$lib/userDraftEditGate' interface Props { canSave?: boolean @@ -108,6 +110,55 @@ workspaceSpecs.push({ ws, defaultValue }) } + // Gated per workspace until the user puts something into that workspace's + // form (see `onUserInput`): the autosave stays suspended and the deployed + // baseline absorbs whatever the form settles on. A workspace opened ON a + // saved draft keeps its baseline — that divergence is the user's own. + let userEdited: Record = $state({}) + let openedOnDraft: Record = $state({}) + const suspendedWorkspaces = new Set() + + function setGated(ws: string, gated: boolean): void { + if (!initialPath) return + if (gated === suspendedWorkspaces.has(ws)) return + if (gated) { + UserDraft.stopSync('resource', initialPath, { workspace: ws }) + suspendedWorkspaces.add(ws) + } else { + UserDraft.restartSync('resource', initialPath, { workspace: ws }) + suspendedWorkspaces.delete(ws) + } + } + + // Nothing counts until this workspace's form is on screen, and while the + // schema is still arriving a precursor alone does not: it would open the gate + // just in time for the schema's materialized values to POST. `Path`, the + // labels and the description render above that skeleton and stay editable + // throughout, so a real value event still counts and keeps the edit. + onUserInput((kind) => { + if (!selected || !(selected in states)) return + if (kind === 'precursor' && loadingSchema) return + userEdited[selected] = true + }) + + $effect(() => { + const wss = Object.keys(states) + const edited = { ...userEdited } + const onDraft = { ...openedOnDraft } + untrack(() => { + // A workspace opened on a saved draft is never suspended — there is no + // phantom to prevent, and a write made while suspended is dropped for + // good. Without `onDraft` here this effect re-suspends it the moment its + // handle appears, undoing the decision made when it was opened. + for (const ws of wss) setGated(ws, !edited[ws] && !onDraft[ws]) + }) + }) + + // `stopSync` must be paired or the key stays unsynced for the session. + onDestroy(() => { + for (const ws of [...suspendedWorkspaces]) setGated(ws, false) + }) + let isValid = $state(true) let jsonError = $state('') let perWsValid: Record = $state({}) @@ -245,6 +296,13 @@ } // Open with the saved draft if present, else the deployed. const s: ResourceState = savedDraftState ?? deployedState + openedOnDraft[ws] = !!savedDraftState + // Gate BEFORE the handle is acquired: `stopSync` queues on a + // not-yet-live entry, and the form can settle before the effect + // above gets a chance to run. Only worth doing when no draft exists + // yet — where one does, there is no phantom to prevent and + // suspending could only drop a write. + if (!savedDraftState) setGated(ws, true) ensureHandle(ws, s) initialStates[ws] = structuredClone(deployedState) // Draft-only paths (`no_deployed`) have no row — saving must @@ -259,6 +317,47 @@ }) }) + /** The schema can only ever write `args`. `path`, `labels`, `description` and + * `wsSpecific` are beyond its reach, so a difference in one of those is the + * user's — whatever event did or didn't reach the gate. Removing a label runs + * a click handler and emits nothing native, and would otherwise be absorbed. */ + function differsOutsideArgs(a: ResourceState, b: ResourceState | undefined): boolean { + return !!b && !draftValuesEqual({ ...a, args: null }, { ...b, args: null }) + } + + // Absorb the form's settling writes into the deployed baseline while the + // selected workspace is gated, so they show up neither as the "unsaved + // changes" banner nor, once `discardIf` reads the baseline, as a draft. + // Only the selected workspace has a form rendered against it. + $effect(() => { + const ws = selected + if (!ws || !initialPath) return + if (userEdited[ws] || openedOnDraft[ws]) return + // `$state.snapshot` deep-reads, so nested `args` mutations re-run this. + const settled = states[ws]?.draft + ? ($state.snapshot(states[ws].draft) as ResourceState) + : undefined + untrack(() => { + if (!settled) return + if (differsOutsideArgs(settled, initialStates[ws])) { + // An edit, not settling. This runs AFTER the write landed, and a write + // made while suspended is swallowed for good (the mirror advances its + // baseline either way), so un-suspend and push the value here rather + // than leaving it to whichever effect happens to run next. + userEdited[ws] = true + setGated(ws, false) + void UserDraftDbSyncer.save({ + workspace: ws, + itemKind: 'resource', + path: initialPath, + value: settled + }) + return + } + if (!draftValuesEqual(settled, initialStates[ws])) initialStates[ws] = settled + }) + }) + // Keep current.path bound to the outer `path` prop for consumers $effect(() => { if (current) path = current.path @@ -292,6 +391,13 @@ } export function discardLocalDraft(): void { if (!selected) return + // Back to the deployed value with nothing of the user's left in it, so + // the gate closes again — otherwise the form settles on the schema's + // values a second time and the discarded draft comes straight back. + // `discard` POSTs the delete itself, so suspending first is safe. + openedOnDraft[selected] = false + userEdited[selected] = false + setGated(selected, true) UserDraft.discard('resource', initialPath ?? '', initialStates[selected], { workspace: selected }) diff --git a/frontend/src/lib/components/RunsPage.svelte b/frontend/src/lib/components/RunsPage.svelte index 03b410d232..cac8952706 100644 --- a/frontend/src/lib/components/RunsPage.svelte +++ b/frontend/src/lib/components/RunsPage.svelte @@ -84,6 +84,8 @@ initialPath?: string } + let { initialPath }: Props = $props() + let paths: string[] = $state([]) let usernames: string[] = $state([]) let folders: string[] = $state([]) @@ -100,25 +102,30 @@ let perPage = useLocalStorageValue('runs_per_page', 1000, 'number') let showSchedulesStorage = useLocalStorageValue('runs_show_schedules', true, 'boolean') let showFutureJobsStorage = useLocalStorageValue('runs_show_future_jobs', true, 'boolean') - let filters = useUrlSyncedFilterInstance(untrack(() => runsFilterSearchbarSchema)) + function filterSeeds() { + return { + path: initialPath || undefined, + job_trigger_kind: showSchedulesStorage.val === false ? ('!schedule' as const) : undefined, + show_future_jobs: showFutureJobsStorage.val === false ? false : undefined + } + } - let { initialPath }: Props = $props() + let filters = useUrlSyncedFilterInstance( + untrack(() => runsFilterSearchbarSchema), + untrack(filterSeeds) + ) + + // `runs/[...path]` is a single route, so a navigation between its URLs — the sidebar's own + // "Runs" entry, `/runs/` → `/runs`, Back — rewrites the query without remounting, and + // what was seeded at mount is gone. Re-apply it. Editing a filter writes with `replaceState`, + // which never reaches `page.url`, so a filter the user clears stays cleared. + $effect(() => { + page.url.href + untrack(() => filters.seed(filterSeeds())) + }) let batchRerunOptionsIsOpen = $state(false) - // Initialize path filter from route param if provided and not already set via query params - if (untrack(() => initialPath) && !filters.val.path) { - filters.val.path = untrack(() => initialPath) - } - - // Apply persistent toggle values from local storage if URL doesn't specify them - if (!page.url.searchParams.has('job_trigger_kind') && showSchedulesStorage.val === false) { - filters.val.job_trigger_kind = '!schedule' - } - if (!page.url.searchParams.has('show_future_jobs') && showFutureJobsStorage.val === false) { - filters.val.show_future_jobs = false - } - // Sync toggle state back to local storage when filters change $effect(() => { if (!filters.val.job_trigger_kind || filters.val.job_trigger_kind === '!schedule') { diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index e069fe50b8..393744dbca 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -102,7 +102,7 @@ import type { SavedAndModifiedValue } from './common/confirmationModal/unsavedTypes' import DeployButton from './DeployButton.svelte' import { type Trigger, deployTriggers, handleSelectTriggerFromKind } from './triggers/utils' - import DraftTriggersConfirmationModal from './common/confirmationModal/DraftTriggersConfirmationModal.svelte' + import DraftChangesConfirmationModal from './common/confirmationModal/DraftChangesConfirmationModal.svelte' import { Triggers } from './triggers/triggers.svelte' import type { ScriptBuilderProps } from './script_builder' import WorkerTagSelect from './WorkerTagSelect.svelte' @@ -1149,7 +1149,7 @@ currentValue={script} /> - t.draftConfig)} on:canceled={() => { diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index fdab141d03..f8028aeb1f 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -15,7 +15,7 @@ import { copyToClipboard, emptySchema, sendUserToast } from '$lib/utils' import Editor from './Editor.svelte' import { inferArgs, inferAssets, inferAnsibleExecutionMode } from '$lib/infer' - import { parsePipelineAnnotations } from '$lib/components/assets/AssetGraph/parsePipelineAnnotations' + import { injectPartitionArg } from '$lib/scriptEditorSchema' import { isWorkflowAsCode } from '$lib/components/graph/wacToFlow' import WacDiagram from '$lib/components/graph/WacDiagram.svelte' import { Pane, Splitpanes } from 'svelte-splitpanes' @@ -1074,55 +1074,6 @@ } } - // A `// partitioned` pipeline script is materialized one slice at a time and - // receives the slice as a runtime `partition` arg (the cascade injects it in - // production). It isn't a code parameter, so schema inference doesn't see it — - // surface it in the test form so a partitioned script can be run manually. - function injectPartitionArg( - s: any, - a: Record | undefined, - l: string | undefined, - c: string - ) { - try { - if (l !== 'duckdb' || !s?.properties) return - const part = parsePipelineAnnotations(c).partition - if (!part) return - // Date-based partition kinds render a date / datetime picker; a dynamic - // key is a free-form string. - const format = - part.kind === 'hourly' - ? 'date-time' - : part.kind === 'daily' || part.kind === 'weekly' || part.kind === 'monthly' - ? 'date' - : undefined - if (!s.properties['partition']) { - s.properties['partition'] = { - type: 'string', - ...(format ? { format } : {}), - // ISO output so partition keys sort lexicographically (the date - // picker defaults to dd-MM-yyyy otherwise). - ...(format === 'date' ? { dateFormat: 'yyyy-MM-dd' } : {}), - description: - part.kind === 'dynamic' - ? 'Partition key value to materialize.' - : `Partition (${part.kind}) to materialize.` - } - if (Array.isArray(s.order) && !s.order.includes('partition')) { - s.order = ['partition', ...s.order] - } - } - // Pre-fill the *test* arg with the current slice for date kinds — a - // convenience default, kept on the args (not baked into the schema, - // where it would persist to the deployed script and go stale). - if (format && a && (a['partition'] == null || a['partition'] === '')) { - const now = new Date() - a['partition'] = - format === 'date' ? now.toISOString().slice(0, 10) : now.toISOString().slice(0, 16) - } - } catch (e) {} - } - async function inferModuleSchema() { if (activeModuleTab === null) return try { diff --git a/frontend/src/lib/components/apps/components/display/dbtable/renderDbLiteral.test.ts b/frontend/src/lib/components/apps/components/display/dbtable/renderDbLiteral.test.ts new file mode 100644 index 0000000000..1dec5e9da8 --- /dev/null +++ b/frontend/src/lib/components/apps/components/display/dbtable/renderDbLiteral.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest' +import { renderDbEqualityFilter, renderDbLiteral } from './utils' + +describe('renderDbLiteral', () => { + it('doubles single quotes on every dialect', () => { + expect(renderDbLiteral("O'Brien", 'postgresql')).toBe("'O''Brien'") + expect(renderDbLiteral("O'Brien", 'mysql')).toBe("'O''Brien'") + }) + + it('doubles backslashes only where the dialect treats them as escapes', () => { + expect(renderDbLiteral('C:\\dir\\', 'postgresql')).toBe("'C:\\dir\\'") + expect(renderDbLiteral('C:\\dir\\', 'mysql')).toBe("'C:\\\\dir\\\\'") + expect(renderDbLiteral('C:\\dir\\', 'snowflake')).toBe("'C:\\\\dir\\\\'") + }) + + it('marks SQL Server strings as Unicode constants', () => { + expect(renderDbLiteral("Zoë's", 'ms_sql_server')).toBe("N'Zoë''s'") + }) + + it('renders numbers and booleans without quotes', () => { + expect(renderDbLiteral(42, 'postgresql')).toBe('42') + expect(renderDbLiteral(true, 'postgresql')).toBe('TRUE') + expect(renderDbLiteral(true, 'ms_sql_server')).toBe('1') + }) + + it('has no literal for values that cannot be compared safely', () => { + expect(renderDbLiteral(null, 'postgresql')).toBeUndefined() + expect(renderDbLiteral({ a: 1 }, 'postgresql')).toBeUndefined() + expect(renderDbLiteral(NaN, 'postgresql')).toBeUndefined() + }) +}) + +describe('renderDbEqualityFilter', () => { + it('quotes the identifier per dialect', () => { + expect(renderDbEqualityFilter('user id', 'x', 'postgresql')).toBe(`"user id" = 'x'`) + expect(renderDbEqualityFilter('user id', 'x', 'ms_sql_server')).toBe(`[user id] = N'x'`) + expect(renderDbEqualityFilter('user id', 'x', 'mysql')).toBe("`user id` = 'x'") + expect(renderDbEqualityFilter('user id', null, 'postgresql')).toBeUndefined() + }) + + it('doubles a delimiter embedded in the identifier', () => { + expect(renderDbEqualityFilter('a"b', 1, 'postgresql')).toBe(`"a""b" = 1`) + expect(renderDbEqualityFilter('a"b', 1, 'snowflake')).toBe(`"a""b" = 1`) + expect(renderDbEqualityFilter('a"b', 1, 'duckdb')).toBe(`"a""b" = 1`) + expect(renderDbEqualityFilter('a]b', 1, 'ms_sql_server')).toBe(`[a]]b] = 1`) + expect(renderDbEqualityFilter('a`b', 1, 'mysql')).toBe('`a``b` = 1') + }) +}) diff --git a/frontend/src/lib/components/apps/components/display/dbtable/utils.ts b/frontend/src/lib/components/apps/components/display/dbtable/utils.ts index bf81ec07bb..c0477e8408 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/utils.ts +++ b/frontend/src/lib/components/apps/components/display/dbtable/utils.ts @@ -333,25 +333,58 @@ export function duckdbQuicksearchColumns(columnDefs: ColumnDef[]): string { .join(', ') } +/** Mirrors the backend's `render_db_quoted_identifier`, including doubling an + * embedded delimiter. */ export function renderDbQuotedIdentifier(identifier: string, dbType: DbType): string { switch (dbType) { case 'postgresql': - return `"${identifier}"` // PostgreSQL uses double quotes for identifiers - case 'ms_sql_server': - return `[${identifier}]` // MSSQL uses square brackets for identifiers - case 'mysql': - return `\`${identifier}\`` // MySQL uses backticks case 'snowflake': - return `"${identifier}"` // Snowflake uses double quotes for identifiers - case 'bigquery': - return `\`${identifier}\`` // BigQuery uses backticks case 'duckdb': - return `"${identifier}"` // DuckDB uses double quotes for identifiers + return `"${identifier.replace(/"/g, '""')}"` + case 'ms_sql_server': + return `[${identifier.replace(/]/g, ']]')}]` + case 'mysql': + case 'bigquery': + return `\`${identifier.replace(/`/g, '``')}\`` default: throw new Error('Unsupported database type: ' + dbType) } } +/** Renders a cell value as a SQL literal. Returns undefined for values that + * have no safe literal form (null, objects, non-finite numbers). */ +export function renderDbLiteral(value: unknown, dbType: DbType): string | undefined { + if (value === null || value === undefined) return undefined + if (typeof value === 'number') return Number.isFinite(value) ? String(value) : undefined + if (typeof value === 'bigint') return value.toString() + if (typeof value === 'boolean') { + if (dbType === 'ms_sql_server') return value ? '1' : '0' + return value ? 'TRUE' : 'FALSE' + } + if (typeof value !== 'string') return undefined + let escaped = value.replace(/'/g, "''") + // MySQL, Snowflake and BigQuery treat a backslash inside a string literal as + // an escape character. + if (dbType === 'mysql' || dbType === 'snowflake' || dbType === 'bigquery') { + escaped = escaped.replace(/\\/g, '\\\\') + } + // A plain constant is varchar on SQL Server and goes through the database + // code page; the N prefix keeps it Unicode against nvarchar columns. + return dbType === 'ms_sql_server' ? `N'${escaped}'` : `'${escaped}'` +} + +/** `"column" = ` predicate, or undefined when the value can't be + * rendered as a literal. */ +export function renderDbEqualityFilter( + column: string, + value: unknown, + dbType: DbType +): string | undefined { + const literal = renderDbLiteral(value, dbType) + if (literal === undefined) return undefined + return `${renderDbQuotedIdentifier(column, dbType)} = ${literal}` +} + export function getLanguageByResourceType(name: string): ScriptLang { const language = { postgresql: 'postgresql', diff --git a/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte b/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte index d5839c6766..99ea07c8fd 100644 --- a/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte +++ b/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte @@ -22,19 +22,22 @@ let { workspace, - path, - editHref + path }: { workspace: string path: string - /** Where the Edit button points (low-code vs raw editor). */ - editHref: string } = $props() let app: any = $state(undefined) let notExists = $state(false) let noPermission = $state(false) let canWriteApp = $state(false) + /** Raw vs low-code, read from the app itself rather than from the route: + * both kinds render here and either route serves either kind (links to a raw + * app point at /apps/get all over the app), so only the app can say which + * editor the Edit button must open. */ + let isRawApp = $state(false) + let editHref = $derived(`${base}/${isRawApp ? 'apps_raw' : 'apps'}/edit/${path}?nodraft=true`) let refresh: (() => void) | undefined // The opaque iframe loads the dedicated cookieless, chrome-less viewer route. @@ -103,11 +106,14 @@ } } - // Edit button: determine write access on this real-origin page (cookie). + // Edit button: determine write access and which editor to open on this + // real-origin page (cookie). The sandboxed low-code app never loads on this + // page (it loads inside the opaque iframe), so `app` can't be the source. async function loadPerms() { try { const lite: any = await AppService.getAppLiteByPath({ workspace, path }) canWriteApp = canWrite(lite?.path, lite?.extra_perms ?? {}, $userStore) + isRawApp = !!lite?.raw_app } catch (_) { canWriteApp = false } diff --git a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts index de00aef2be..c57687d78b 100644 --- a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts @@ -319,6 +319,18 @@ describe('resolveGraph', () => { expect(assetTrigKeys(r, 'f/x/open')).toEqual(['ducklake:main.orders']) }) + // A relation a native `// materialize manual dbt://…` script writes wakes its + // subscribers, so hiding the edge would leave the author's own annotation off + // the canvas. The deploy refuses the ones that cannot fire. + it('draws an explicit dbt:// subscription as an unsaved trigger overlay', () => { + const liveAnnotations = { + scriptPath: 'f/x/open', + annotations: ann({ triggerAssets: [{ kind: 'dbt', path: 'main/analytics/orders' }] }) + } + const r = resolveGraph(input({ liveAnnotations })) + expect(assetTrigKeys(r, 'f/x/open')).toEqual(['dbt:main/analytics/orders']) + }) + it('open-script live annotations add unsaved triggers, deduped vs persisted', () => { const base = baseGraph({ triggers: [ diff --git a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts index 867fa6f036..5c8aa697a4 100644 --- a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts +++ b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts @@ -77,23 +77,15 @@ function persistedNativeKinds(base: AssetGraphResponse, path: string): Set = new Set(['ducklake', 's3object']) -/** Whether a subscription on this kind can ever fire once deployed. - * - * A `dbt://` one cannot: dbt is the only producer of a warehouse relation and - * a dbt run does not dispatch, so the deploy refuses `// on dbt://…` outright - * (`scripts.rs`). The editor must not draw an arrow the deploy will reject — - * applied to the EXPLICIT overlays; auto-derivation is already scoped by - * `AUTO_TRIGGER_KINDS`. Parsing is left alone so the Rust-parity test still - * compares like for like. */ -function canTrigger(kind: AssetKind): boolean { - return kind !== 'dbt' -} - /** `kind:path` refs of a script's `// materialize` write target(s) (base + * the scd2 `_current` companion), which the body `SELECT` doesn't express. */ function materializeWriteRefs(parsed: PipelineAnnotations): string[] { @@ -381,7 +373,7 @@ function makeContext(input: ResolveGraphInput): ResolveContext { const liveRefKeys = new Set() if (openIsSavedEdit) { if (liveAnnotations.scriptPath === openPath) { - for (const a of liveAnnotations.annotations.triggerAssets.filter((a) => canTrigger(a.kind))) + for (const a of liveAnnotations.annotations.triggerAssets) liveRefKeys.add(`${a.kind}:${a.path}`) // The `// materialize ` target is a declared *output*, but it // lives in an annotation (not the SQL body), so neither triggerAssets @@ -625,7 +617,7 @@ function seedDraftOverlays(acc: Accumulator, input: ResolveGraphInput) { // stable when the user clicks off this draft. Live annotations // (below) take over for the currently-open draft so keystroke // edits still update in real time. - for (const a of parsed.triggerAssets.filter((a) => canTrigger(a.kind))) { + for (const a of parsed.triggerAssets) { extraTriggers.push({ trigger_kind: 'asset', asset_kind: a.kind, @@ -704,7 +696,7 @@ function applyLiveBufferOverlay(acc: Accumulator, input: ResolveGraphInput, ctx: for (let i = extraTriggers.length - 1; i >= 0; i--) { if (extraTriggers[i].runnable_path === livePath) extraTriggers.splice(i, 1) } - for (const a of liveAnnotations.annotations.triggerAssets.filter((a) => canTrigger(a.kind))) { + for (const a of liveAnnotations.annotations.triggerAssets) { const key = `${a.kind}:${a.path}` if (assetKeys.has(key)) continue extraTriggers.push({ diff --git a/frontend/src/lib/components/assets/lib.ts b/frontend/src/lib/components/assets/lib.ts index 23531d71f2..93367a725e 100644 --- a/frontend/src/lib/components/assets/lib.ts +++ b/frontend/src/lib/components/assets/lib.ts @@ -96,10 +96,11 @@ export function formatAssetKind(asset: { case 'volume': return 'Volume' case 'dbt': - // The SCHEME says dbt because dbt is the only thing that creates one; - // the PATH stays the relation, so a mart one project builds and the - // `source` the next reads land on one node — their dbt `unique_id`s - // differ where the relation does not (docs/dbt-runtime.md, decision 11). + // The SCHEME says dbt because dbt is what derives these relations; the + // PATH stays the relation, so a mart one project builds, the `source` + // the next reads, and a native `// materialize manual dbt://…` writer + // land on one node — their dbt `unique_id`s differ where the relation + // does not (docs/dbt-runtime.md, decision 11). return 'dbt table' } } diff --git a/frontend/src/lib/components/common/confirmationModal/DraftChangesConfirmationModal.svelte b/frontend/src/lib/components/common/confirmationModal/DraftChangesConfirmationModal.svelte new file mode 100644 index 0000000000..dadf814a80 --- /dev/null +++ b/frontend/src/lib/components/common/confirmationModal/DraftChangesConfirmationModal.svelte @@ -0,0 +1,323 @@ + + + dispatch('canceled')} + on:confirmed={() => dispatch('confirmed', { selectedTriggers, selectedAgents })} +> +
+ {#if draftTriggers.length > 0} +
+
+ {`Your ${runnable} has draft triggers. Select which draft triggers to deploy with the ${runnable}. Undeployed draft triggers will be permanently deleted.`} +
+ +
5 ? 'h-[300px]' : ''}> + +
+ + + + + + + {#each draftTriggers as trigger} + {@const SvelteComponent = triggerIconMap[trigger.type]} + {@const permission = checkSavePermissions(trigger)} + {@const isSelectedTrigger = isSelected(selectedTriggers, trigger)} + + + + + + {/each} + + + + + {/if} + + {#if draftAgents.length > 0} +
+
+ Saved agents this flow uses have unsaved changes. Select which ones to deploy with the + flow. An agent kept as a draft stays editable, and the flow runs the agent as currently + deployed. +
+ +
5 ? 'h-[300px]' : ''}> + +
+ + + + + + + + {#each draftAgents as agent (agent.path)} + {@const permission = checkAgentPermissions(agent)} + {@const isSelectedAgent = selectedAgents.some((a) => a.path === agent.path)} + + + + + + + + {/each} + + + + + {/if} + + diff --git a/frontend/src/lib/components/common/confirmationModal/DraftTriggersConfirmationModal.svelte b/frontend/src/lib/components/common/confirmationModal/DraftTriggersConfirmationModal.svelte deleted file mode 100644 index cfebae0e9f..0000000000 --- a/frontend/src/lib/components/common/confirmationModal/DraftTriggersConfirmationModal.svelte +++ /dev/null @@ -1,168 +0,0 @@ - - - dispatch('canceled')} - on:confirmed={() => dispatch('confirmed', { selectedTriggers })} -> -
-
- {`${isFlow ? 'Your flow' : 'Your script'} has draft triggers. Select which draft triggers to deploy with the ${isFlow ? 'flow' : 'script'}. Undeployed - draft triggers will be permanently deleted.`} -
- -
5 ? 'h-[300px]' : ''}> - -
- - - - - - - {#each draftTriggers as trigger} - {@const SvelteComponent = triggerIconMap[trigger.type]} - {@const permission = checkSavePermissions(trigger)} - {@const isSelectedTrigger = isSelected(selectedTriggers, trigger)} - - - - - - {/each} - - {#if draftTriggers.length === 0} - - - - {/if} - - - - - diff --git a/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte b/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte index 1e885fa304..e93017b095 100644 --- a/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte +++ b/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte @@ -3,6 +3,7 @@ import { ExternalLink, PanelRight } from 'lucide-svelte' import { Button } from '$lib/components/common' import RowIcon from '$lib/components/common/table/RowIcon.svelte' + import { newTabModifier } from '$lib/attachments/newTabModifier.svelte' import { hasToolDisplayActionHandler, runToolDisplayAction @@ -44,6 +45,8 @@ const previewAction = $derived(available?.type === 'open_item_preview' ? available : undefined) const drawerAction = $derived(available?.type === 'open_created_resource' ? available : undefined) + const modifier = newTabModifier() + const hint = $derived( previewAction ? `Open ${wmPath} in the preview panel` : `Open ${wmPath} in a new tab` ) @@ -67,7 +70,11 @@ {#if href} {#if wmKind} - + + - {#if previewAction} + + {#if previewAction && !modifier.held} {:else} diff --git a/frontend/src/lib/components/copilot/chat/pipeline/core.ts b/frontend/src/lib/components/copilot/chat/pipeline/core.ts index 7583c1b356..f696b1c869 100644 --- a/frontend/src/lib/components/copilot/chat/pipeline/core.ts +++ b/frontend/src/lib/components/copilot/chat/pipeline/core.ts @@ -338,8 +338,8 @@ export function getPipelinePromptSection(ctx: PipelineContext): string { Data Pipeline editor (ACTIVE): - The user has the /pipeline/${ctx.folder} editor open. A pipeline is a DAG of scripts (nodes) connected by storage assets (DuckLake tables, data tables, S3 objects, volumes, resources) and execution triggers. - Annotations are top-of-file comments in the NODE'S OWN comment syntax: \`--\` for SQL (duckdb/postgresql), \`#\` for python3/bash, \`//\` for bun/TS. The \`//\` shown below is the TS form — translate it (a \`// pipeline\` line in a SQL node is a syntax error that won't deploy). -- A script becomes a pipeline node when its source starts with the \`// pipeline\` annotation. Declare execution-DAG inputs with \`// on \` (e.g. \`// on ducklake://main/orders\`). Outputs are inferred from what the body writes (wmill SDK calls / SQL CREATE TABLE / writeS3File); declare a managed output with \`// materialize \`. Optional badges: \`// partitioned \`, \`// freshness \`, \`// tag \`, \`// retry [delay]\`, \`// data_test ...\`, \`// measure = [where ]\`, \`// dimension = \`. -- \`materialize\` (the managed output): \`// materialize \` means the runtime writes the node's output table FOR you — write the body as a single SELECT and the runtime wraps it in the create/replace, so do NOT also write your own CREATE TABLE / INSERT. IMPORTANT: \`// materialize\` is **DuckDB-only** and its target MUST be a DuckLake table (\`ducklake:///
Triggers to deploy
+
+
+ + {#if trigger.isPrimary} + + {/if} +
+ +
+ +
+
+ 		+ {#if permission === 'deploy'} +
+ toggleTrigger(trigger, e.detail)} + > + {#snippet children({ item })} + + + {/snippet} + +
+ {:else if permission === 'admin-only'} + Admin only + {:else if permission === 'invalid-config'} + Invalid config + {/if} +
Agents to deploy
+
+ +
+
+ + + {agent.path} + + {#if agent.noDeployed} + Never deployed + {/if} +
+ {#if agent.noDeployed && !isSelectedAgent} + + + Never deployed, so the flow will not run until this agent is deployed. + + {/if} +
+
+ 		+ {#if permission.state === 'deploy'} +
+ toggleAgent(agent, e.detail)} + > + {#snippet children({ item })} + + + {/snippet} + +
+ {:else if permission.state === 'read-only'} + + Read-only + + {:else} + Invalid config + {/if} +
Triggers to deploy
-
-
- - {#if trigger.isPrimary} - - {/if} -
-
- -
-
- 		- {#if permission === 'deploy'} -
- toggleTrigger(trigger, e.detail)} - > - {#snippet children({ item })} - - - {/snippet} - -
- {:else if permission === 'admin-only'} - Admin only - {:else if permission === 'invalid-config'} - Invalid config - {/if} -
- No draft triggers found -
\`) — deploy rejects it on any other language or target. For a \`python3\`/\`bun\`/\`postgresql\` node, do NOT use \`// materialize\`; write the output via the SDK instead (e.g. \`wmill.writeS3File(...)\`, a \`CREATE TABLE\` in postgresql, or \`wmill.databaseUrlFromResource\`/ducklake helpers) and let the output be inferred. Reach for \`duckdb\` when a node should materialize a DuckLake table. Write strategy: with no option it REPLACES the whole table each run (full refresh; the only mode whose output columns may change); \`// materialize append\` INSERT-appends rows (incremental); \`// materialize key=\` merges/upserts on \`\`. \`// materialize manual \` opts OUT of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. \`materialize\` is paired with partitioning for incremental pipelines: a \`// partitioned \` node runs once per partition (append/merge into a fixed-schema table), and the \`{partition}\` token — usable in any asset URI AND in the body SQL — is substituted with the current partition's IDENTITY string at run time. To filter the source to the active slice on a time grain, use the runtime-injected macro: \`WHERE wm_partition() = {partition}\`. \`wm_partition(ts)\` buckets a timestamp with the exact identity format the runtime used (daily/hourly/weekly/monthly), so it always matches and you never hand-write a \`strftime\` format. Do NOT write \`= TIMESTAMP {partition}\`: the identity string is not a valid timestamp literal for hourly/weekly/monthly and errors at runtime. For \`dynamic\` partitioning the identity is your caller-supplied key (not a timestamp, no macro), so filter on it directly: \`WHERE = {partition}\`. \`materialize\` is an output DECLARATION on the node — it is not a command; there is no "materialize run". +- A script becomes a pipeline node when its source starts with the \`// pipeline\` annotation. Declare execution-DAG inputs with \`// on \` (e.g. \`// on ducklake://main/orders\`). Outputs are inferred from what the body writes (wmill SDK calls / SQL CREATE TABLE / writeS3File); declare a managed output with \`// materialize \`. Optional badges: \`// partitioned \`, \`// freshness \`, \`// tag \`, \`// retry [delay]\`, \`// data_test ...\` (managed DuckLake targets only — deploy rejects it beside a \`dbt://\` target), \`// measure = [where ]\`, \`// dimension = \`. +- \`materialize\` (the managed output): a managed \`// materialize ducklake:///
\` means the runtime writes the node's output table FOR you — write the body as a single SELECT and the runtime wraps it in the create/replace, so do NOT also write your own CREATE TABLE / INSERT. The \`dbt://\` target below is the opposite: the node writes its own DDL and none of the write strategies apply to it. IMPORTANT: a MANAGED \`// materialize\` is **DuckDB-only** and its target MUST be a DuckLake table (\`ducklake:///
\`) — deploy rejects a \`ducklake://\` target on any other language. For a \`python3\`/\`bun\`/\`postgresql\` node writing the lake, do NOT use \`// materialize\`; write the output via the SDK instead (e.g. \`wmill.writeS3File(...)\`, a \`CREATE TABLE\` in postgresql, or \`wmill.databaseUrlFromResource\`/ducklake helpers) and let the output be inferred. Reach for \`duckdb\` when a node should materialize a DuckLake table. The one target any language BUT DBT'S OWN may declare (a dbt project's writes come from its manifest, so \`// materialize\` on a dbt script is rejected at deploy) is a WAREHOUSE RELATION: \`// materialize manual dbt:////\`, with \`\` a warehouse the workspace configures under Settings → dbt. \`manual\` is its only mode — nothing generates warehouse DDL, so the node issues its own write and the annotation records the outcome. Use it on an ingestion node a dbt project reads as a \`source\`: the declared relation and the dbt model become ONE graph node, and a downstream \`// on dbt:////\` fires when that node completes. Write strategy: with no option it REPLACES the whole table each run (full refresh; the only mode whose output columns may change); \`// materialize append\` INSERT-appends rows (incremental); \`// materialize key=\` merges/upserts on \`\`. \`// materialize manual \` opts OUT of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. \`materialize\` is paired with partitioning for incremental pipelines: a \`// partitioned \` node runs once per partition (append/merge into a fixed-schema table), and the \`{partition}\` token — usable in any asset URI AND in the body SQL — is substituted with the current partition's IDENTITY string at run time. To filter the source to the active slice on a time grain, use the runtime-injected macro: \`WHERE wm_partition() = {partition}\`. \`wm_partition(ts)\` buckets a timestamp with the exact identity format the runtime used (daily/hourly/weekly/monthly), so it always matches and you never hand-write a \`strftime\` format. Do NOT write \`= TIMESTAMP {partition}\`: the identity string is not a valid timestamp literal for hourly/weekly/monthly and errors at runtime. For \`dynamic\` partitioning the identity is your caller-supplied key (not a timestamp, no macro), so filter on it directly: \`WHERE = {partition}\`. \`materialize\` is an output DECLARATION on the node — it is not a command; there is no "materialize run". - \`measure\` / \`dimension\` (declared metrics): on a node that materializes a DuckLake table, \`// measure = [where ]\` names the canonical way to aggregate that table (e.g. \`// measure revenue = sum(amount) where not is_refund\`), and \`// dimension = \` names a way to slice it (e.g. \`// dimension region = region\`, \`// dimension month = date_trunc('month', ordered_at)\`). They execute nothing: they are catalogued at deploy so the editor and other agents can reuse the definition instead of re-deriving it and silently disagreeing. Keep the predicate in the \`where\` clause rather than folding it into the aggregate: it is rendered as \` FILTER (WHERE )\`, which is what lets two measures with different predicates sit under one GROUP BY. DuckLake-only, and only meaningful next to \`// materialize\`. Declare one when a number carries a judgement call someone else would get wrong (refunds excluded, test rows dropped, which column is the amount); do NOT blanket every table with measures, an obvious \`count(*)\` earns nothing. To USE a metric another node declares, read that node with read_pipeline_node and reuse its exact expression rather than guessing it. - Use get_pipeline_graph to see the current nodes/assets/triggers, and read_pipeline_node before editing one. - Every node of this pipeline lives at \`f/${ctx.folder}/\` — \`${ctx.folder}\` is the folder name and \`f/\` is the owner prefix every workspace path carries, so write it exactly once (never \`f/f/…\`, and never a bare \`\`). diff --git a/frontend/src/lib/components/dbOps.ts b/frontend/src/lib/components/dbOps.ts index a89abf4374..5005cc931c 100644 --- a/frontend/src/lib/components/dbOps.ts +++ b/frontend/src/lib/components/dbOps.ts @@ -14,6 +14,7 @@ import { WorkspaceService } from '$lib/gen' import { pendingMigrations } from './workspaceSettings/datatableMigrationUtils' import { buildTableEditorValues, + type TableEditorForeignKey, type TableEditorValues } from './apps/components/display/dbtable/tableEditor' import { type AlterTableValues } from './apps/components/display/dbtable/queries/alterTable' @@ -250,6 +251,10 @@ export type IDbSchemaOps = { schema?: string colDefs: TableMetadata }) => Promise + onFetchForeignKeys: (params: { + table: string + schema?: string + }) => Promise } /** Thrown by a schema op when the user declines the out-of-order run warning. @@ -396,6 +401,48 @@ export function dbSchemaOpsWithPreviewScripts({ } } + /** Resolves to [] when the database has no foreign key introspection + * (BigQuery) or the query fails: callers treat foreign keys as optional. */ + async function fetchForeignKeys({ + table, + schema + }: { + table: string + schema?: string + }): Promise { + if (dbType === 'bigquery') return [] + try { + const fkContent = makeMarker('FOREIGN_KEYS', { table, schema }) + const fkResult = await runScriptAndPollResult({ + workspace, + requestBody: { args: dbArg, content: fkContent, language, tag } + }) + + let rawForeignKeys: RawForeignKey[] + if (dbType === 'snowflake') { + rawForeignKeys = transformSnowflakeForeignKeys(fkResult as any[]) + } else { + rawForeignKeys = fkResult as RawForeignKey[] + if (rawForeignKeys && Array.isArray(rawForeignKeys)) { + rawForeignKeys = rawForeignKeys.map((fk) => { + const lowerFk: any = {} + Object.keys(fk).forEach((key) => { + lowerFk[key.toLowerCase()] = fk[key] + }) + return lowerFk + }) + } + } + + if (rawForeignKeys && Array.isArray(rawForeignKeys)) { + return transformForeignKeys(rawForeignKeys) + } + } catch (e) { + console.warn('Failed to fetch foreign keys:', e) + } + return [] + } + return { onDelete: async ({ tableKey, schema }) => { const content = makeMarker('DROP_TABLE', { table: tableKey, schema }) @@ -454,44 +501,11 @@ export function dbSchemaOpsWithPreviewScripts({ const downContent = makeMarker('CREATE_SCHEMA', { schema }) await applyDdl(migrationName('drop_schema', schema), content, downContent) }, + onFetchForeignKeys: fetchForeignKeys, onFetchTableEditorDefinition: async ({ table, schema, colDefs }) => { - let foreignKeys: import('./apps/components/display/dbtable/tableEditor').TableEditorForeignKey[] = - [] + const foreignKeys = await fetchForeignKeys({ table, schema }) let pk_constraint_name: string | undefined - // Fetch foreign keys (not supported for BigQuery) - if (dbType !== 'bigquery') { - try { - const fkContent = makeMarker('FOREIGN_KEYS', { table, schema }) - const fkResult = await runScriptAndPollResult({ - workspace, - requestBody: { args: dbArg, content: fkContent, language, tag } - }) - - let rawForeignKeys: RawForeignKey[] - if (dbType === 'snowflake') { - rawForeignKeys = transformSnowflakeForeignKeys(fkResult as any[]) - } else { - rawForeignKeys = fkResult as RawForeignKey[] - if (rawForeignKeys && Array.isArray(rawForeignKeys)) { - rawForeignKeys = rawForeignKeys.map((fk) => { - const lowerFk: any = {} - Object.keys(fk).forEach((key) => { - lowerFk[key.toLowerCase()] = fk[key] - }) - return lowerFk - }) - } - } - - if (rawForeignKeys && Array.isArray(rawForeignKeys)) { - foreignKeys = transformForeignKeys(rawForeignKeys) - } - } catch (e) { - console.warn('Failed to fetch foreign keys:', e) - } - } - // Fetch primary key constraint name (not supported for BigQuery/MySQL) if (dbType !== 'bigquery' && dbType !== 'mysql') { try { diff --git a/frontend/src/lib/components/flows/agentDraft.svelte.ts b/frontend/src/lib/components/flows/agentDraft.svelte.ts index 5a6a67ccb8..d4f8fa35de 100644 --- a/frontend/src/lib/components/flows/agentDraft.svelte.ts +++ b/frontend/src/lib/components/flows/agentDraft.svelte.ts @@ -6,6 +6,7 @@ import { sendUserToast } from '$lib/toast' import { canWrite } from '$lib/utils' import { userStore } from '$lib/stores' import { getUserExt } from '$lib/user' +import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' import { useTriggerDraftSync, type TriggerDraftSync } from '../triggers/useTriggerDraftSync.svelte' import { logReusableAgentUsage } from './agentTelemetry' import { @@ -84,6 +85,95 @@ export interface AgentResourceState { wsSpecific: boolean } +/** + * Why writing this draft to its resource would be refused, if it would. Shared with the flow's + * deploy dialog, which lists the drafts of the agents a flow links: an agent the editor's own + * Deploy button rejects must not be deployable from a flow either. + * + * `currentPath` is the path the draft is being deployed to; pass undefined where the caller has + * none of its own to compare against. + */ +export function agentDraftDeployRefusal( + state: AgentResourceState, + currentPath: string | undefined +): string | undefined { + // The editor offers only values, but a transform can arrive from a step that was forked + // before this existed, or from the generic resource editor: say so rather than writing it. + const transformValued = transformValuedBrainKeys(state.args) + if (transformValued.length > 0) { + const fields = transformValued.map((key) => AGENT_BRAIN_LABELS[key] ?? key) + const many = fields.length > 1 + return `${fields.join(', ')} ${many ? 'are' : 'is'} set to an expression or an AI-filled value, which a saved agent cannot store. Replace ${many ? 'them' : 'it'} with a plain value before deploying.` + } + // The resource endpoint takes any JSON, so nothing downstream stops an agent that cannot run: + // the worker needs a provider to call and rejects a tool whose name it cannot pass to the + // model. Deploying one would break every flow linking it, so it is refused here. + const blocked = agentConfigRunError(state.args) + if (blocked) { + return blocked + } + // Renaming is not the agent editor's to do: moving the resource leaves every step that links to + // it naming a path that no longer exists, and reconciling those is a feature of its own. A + // renamed path can still reach here, the generic editor writing the same draft row and offering + // a path field, so refuse it rather than performing half of a rename. + if (currentPath && state.path !== currentPath) { + return `This draft renames the agent to ${state.path}. Deploy it from the resource editor instead.` + } + // Only a draft naming another type: the load refuses a resource that is not an agent, while a + // draft the generic resource editor wrote names no type at all and inherits the loaded one. + if (state.resource_type && state.resource_type !== 'ai_agent') { + return `This draft is a ${state.resource_type} resource, not an agent.` + } + return undefined +} + +/** + * Write an agent to its resource from the state the editor holds, which can be ahead of the + * persisted draft row: the form stays editable while a deploy is in flight. Surfaces that deploy + * the row itself go through `deployDraft` instead. + * + * `notAnAgent` separates the one failure that invalidates the caller's whole view of the path, its + * holding something else now, from a write that merely failed. + */ +type AgentWriteResult = { ok: true } | { ok: false; error: string; notAnAgent?: true } + +async function writeAgentResource( + workspace: string, + state: AgentResourceState, + noDeployed: boolean +): Promise { + const body = { + path: state.path, + value: state.args, + description: state.description, + labels: state.labels, + ws_specific: state.wsSpecific + } + try { + if (noDeployed) { + // A create needs a type, and every caller proved this path is an agent before offering it. + await ResourceService.createResource({ + workspace, + requestBody: { ...body, resource_type: state.resource_type ?? 'ai_agent' } + }) + } else { + // The type the caller proved is as old as its own load, and an update carries no type of + // its own: were the path deleted and recreated as something else meanwhile, this write + // would put an agent config inside that resource. Reading it again narrows the window to + // the request rather than to however long the editor or the dialog stayed open. + const current = await ResourceService.getResource({ workspace, path: state.path }) + const refused = agentEditorRefusal(state.path, current.resource_type) + if (refused) { + return { ok: false, error: refused, notAnAgent: true } + } + await ResourceService.updateResource({ workspace, path: state.path, requestBody: body }) + } + } catch (err) { + return { ok: false, error: `Could not save agent: ${err}` } + } + return { ok: true } +} + export interface AgentDraftOptions { /** The `ai_agent` resource being edited. */ path: () => string | undefined @@ -219,6 +309,16 @@ export function useAgentDraft(opts: AgentDraftOptions): AgentDraftHandle { // config before the autosave lands. state = ((r as any).draft as AgentResourceState | undefined) ?? structuredClone(deployedState) + // Adopt the row's timestamp as this tab's baseline. Without it the first save from + // each tab goes out with no `last_sync`, which the backend treats as unconditional + // and so silently overwrites another tab's newer draft. It also clears any parked + // conflict or failure for the key: a conflict is deliberately sticky (the retry + // keeps the same baseline), and nothing else mounts a resolver for `resource` + // drafts, so re-opening the agent is the only place it can be resolved. + UserDraftDbSyncer.recordRemoteSync( + { workspace: ws, itemKind: 'resource', path }, + (r as { draft_saved_at?: string }).draft_saved_at + ) loading = false await sync.maybeRestore() }, @@ -237,42 +337,9 @@ export function useAgentDraft(opts: AgentDraftOptions): AgentDraftHandle { const ws = opts.workspace() const s = state if (!ws || !s) return false - // The editor offers only values, but a transform can arrive from a step that was forked - // before this existed, or from the generic resource editor: say so rather than writing it. - const transformValued = transformValuedBrainKeys(s.args) - if (transformValued.length > 0) { - const fields = transformValued.map((key) => AGENT_BRAIN_LABELS[key] ?? key) - const many = fields.length > 1 - sendUserToast( - `${fields.join(', ')} ${many ? 'are' : 'is'} set to an expression or an AI-filled value, which a saved agent cannot store. Replace ${many ? 'them' : 'it'} with a plain value before deploying.`, - true - ) - return false - } - // The resource endpoint takes any JSON, so nothing downstream stops an agent that cannot run: - // the worker needs a provider to call and rejects a tool whose name it cannot pass to the - // model. Deploying one would break every flow linking it, so it is refused here. - const blocked = agentConfigRunError(s.args) - if (blocked) { - sendUserToast(blocked, true) - return false - } - // Renaming is not this editor's to do: moving the resource leaves every step that links to it - // naming a path that no longer exists, and reconciling those is a feature of its own. A - // renamed path can still reach here, the generic editor writing the same draft row and - // offering a path field, so refuse it rather than performing half of a rename. - const currentPath = opts.path() - if (currentPath && s.path !== currentPath) { - sendUserToast( - `This draft renames the agent to ${s.path}. Deploy it from the resource editor instead.`, - true - ) - return false - } - // Only a draft naming another type: the load refuses a resource that is not an agent, while a - // draft the generic resource editor wrote names no type at all and inherits the loaded one. - if (s.resource_type && s.resource_type !== 'ai_agent') { - sendUserToast(`This draft is a ${s.resource_type} resource, not an agent.`, true) + const refused = agentDraftDeployRefusal(s, opts.path()) + if (refused) { + sendUserToast(refused, true) return false } // The form stays editable while the request is in flight, so everything below works from a @@ -280,39 +347,15 @@ export function useAgentDraft(opts: AgentDraftOptions): AgentDraftHandle { // made during the request as saved, and the banner would clear on a value the server never // received; against the snapshot it stays a draft, which is what it is. const submitted = structuredClone($state.snapshot(s)) as AgentResourceState - const body = { - path: submitted.path, - value: submitted.args, - description: submitted.description, - labels: submitted.labels, - ws_specific: submitted.wsSpecific - } - try { - if (noDeployed) { - await ResourceService.createResource({ - workspace: ws, - // A create needs a type, and the load proved this path is an agent before opening. - requestBody: { ...body, resource_type: submitted.resource_type ?? 'ai_agent' } - }) + const written = await writeAgentResource(ws, submitted, noDeployed) + if (!written.ok) { + // A path that is no longer an agent tears this editor down; anything else is a plain error + // the user can retry from the form as it stands. + if (written.notAnAgent) { + refuse(written.error) } else { - // The type this editor proved is as old as the load, and an update carries no type of - // its own: were the path deleted and recreated as something else meanwhile, this write - // would put an agent config inside that resource. Reading it again narrows the window - // to the request rather than to however long the editor stayed open. - const current = await ResourceService.getResource({ workspace: ws, path: submitted.path }) - const refused = agentEditorRefusal(submitted.path, current.resource_type) - if (refused) { - refuse(refused) - return false - } - await ResourceService.updateResource({ - workspace: ws, - path: submitted.path, - requestBody: body - }) + sendUserToast(written.error, true) } - } catch (err) { - sendUserToast(`Could not save agent: ${err}`, true) return false } // The counter the step card's write-back used to report, from the surface that now owns the diff --git a/frontend/src/lib/components/flows/agentEditorStore.svelte.ts b/frontend/src/lib/components/flows/agentEditorStore.svelte.ts index 83660212b3..025a426d28 100644 --- a/frontend/src/lib/components/flows/agentEditorStore.svelte.ts +++ b/frontend/src/lib/components/flows/agentEditorStore.svelte.ts @@ -6,6 +6,8 @@ * resources page. Module-level rather than a context value because what opens it — a step's card, * a list row — unmounts the moment the selection moves. */ +import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' + export interface AgentEditorTarget { path: string /** The workspace the opener operates on; the nav workspace when absent. */ @@ -61,3 +63,25 @@ export function markAgentWritten(workspace: string | undefined, path: string) { export function agentWriteCount(workspace: string | undefined, path: string | undefined): number { return agentWrites[writeKey(workspace, path)] ?? 0 } + +/** How many times each agent's DRAFT has been saved, for the surfaces that display an agent by + * fetching it. A draft write moves no deployed version, so `agentWriteCount` never sees it, and a + * card keyed on that alone would keep describing the config a test no longer runs. */ +let agentDraftSaves = $state>({}) + +/** Every writer in this document goes through the draft syncer — this editor, the generic resource + * editor — so one subscription answers for them all, firing when the write lands rather than on + * each keystroke. `resource` is the item kind agent draft rows use. In-memory: a save in another + * tab never arrives here, so a card lags it until reload, while what a test runs is read live. */ +UserDraftDbSyncer.onAnySaved(({ workspace, itemKind, path }) => { + if (itemKind !== 'resource') return + const key = writeKey(workspace, path) + agentDraftSaves[key] = (agentDraftSaves[key] ?? 0) + 1 +}) + +export function agentDraftSaveCount( + workspace: string | undefined, + path: string | undefined +): number { + return agentDraftSaves[writeKey(workspace, path)] ?? 0 +} diff --git a/frontend/src/lib/components/flows/agentTelemetry.ts b/frontend/src/lib/components/flows/agentTelemetry.ts index fcfad440ff..87fcb0dad4 100644 --- a/frontend/src/lib/components/flows/agentTelemetry.ts +++ b/frontend/src/lib/components/flows/agentTelemetry.ts @@ -13,6 +13,10 @@ export type ReusableAgentEvent = | 'linked' /** A linked step was forked back into a standalone agent. */ | 'unlinked' + /** A linked agent's unsaved draft was deployed alongside the flow that uses it. */ + | 'draft_deployed_with_flow' + /** A linked agent's unsaved draft was left as a draft when its flow was deployed. */ + | 'draft_kept_on_deploy' export function logReusableAgentUsage(event: ReusableAgentEvent): void { logFeatureUsage('ai_agent', 'reusable', { key: event }) diff --git a/frontend/src/lib/components/flows/content/AgentEditorHost.svelte b/frontend/src/lib/components/flows/content/AgentEditorHost.svelte index 79db0cedb3..a7c867a6f4 100644 --- a/frontend/src/lib/components/flows/content/AgentEditorHost.svelte +++ b/frontend/src/lib/components/flows/content/AgentEditorHost.svelte @@ -29,6 +29,7 @@ inputTransformsToAgentConfig, type AIAgentConfig } from '../agentResourceUtils' + import { agentArgsToTransforms } from '../linkedAgentDrafts' import { AGENT_TOOLS_ROW } from '../agentFormFields' import { toolDisplayName, type AgentTool } from '../agentToolUtils' import { useAgentDraft } from '../agentDraft.svelte' @@ -223,19 +224,6 @@ }) }) - /** Every argument the resource carries, as a static transform. `tools` is the roster rather than - * a field, so it rides on the module's own key instead. Not only the keys the form renders: a - * run reads them all, and an agent holding its own `user_message` answers with it when nothing - * overrides it, so a test here has to run the configuration a linked step would. */ - function agentArgsToTransforms(args: AIAgentConfig): Record { - const it: Record = {} - for (const [key, value] of Object.entries(args ?? {})) { - if (key === 'tools' || value === undefined) continue - it[key] = { type: 'static', value } as InputTransform - } - return it - } - /** Everything the form does not model. `inputTransformsToAgentConfig` rebuilds the value from * `AGENT_BRAIN_KEYS` alone, so a key this editor never renders — one a newer backend added, or * the `user_message` default a resource may carry, which the runtime does read when the step diff --git a/frontend/src/lib/components/flows/content/AgentEditorModal.svelte b/frontend/src/lib/components/flows/content/AgentEditorModal.svelte index 3ded2deb9f..22ac424144 100644 --- a/frontend/src/lib/components/flows/content/AgentEditorModal.svelte +++ b/frontend/src/lib/components/flows/content/AgentEditorModal.svelte @@ -183,7 +183,9 @@ const moduleIds = new Set(linkedModulesForAgent(scope, path)) moduleIds.add(at.host.moduleId) return Promise.all( - [...moduleIds].map((moduleId) => publishLinkedAgentTools(path, at.ws, scope, moduleId)) + // With the draft: a deploy leaves none, but a version restore leaves the draft standing and + // it is still what a test of the host step would run. + [...moduleIds].map((moduleId) => publishLinkedAgentTools(path, at.ws, scope, moduleId, true)) ) } diff --git a/frontend/src/lib/components/flows/content/AgentResourceBar.svelte b/frontend/src/lib/components/flows/content/AgentResourceBar.svelte index 3a870f911d..96548b10af 100644 --- a/frontend/src/lib/components/flows/content/AgentResourceBar.svelte +++ b/frontend/src/lib/components/flows/content/AgentResourceBar.svelte @@ -4,7 +4,7 @@ import Badge from '$lib/components/common/badge/Badge.svelte' import Path from '$lib/components/Path.svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' - import { ResourceService, type InputTransform } from '$lib/gen' + import { ResourceService, type InputTransform, type Resource } from '$lib/gen' import { workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' import { Bot, ChevronDown, ChevronUp, Save, Unlink, Pencil } from 'lucide-svelte' @@ -19,14 +19,24 @@ type AIAgentConfig, type AgentTool } from '../agentResourceUtils' - import { agentWriteCount, markAgentWritten, openAgentEditor } from '../agentEditorStore.svelte' + import { + agentDraftSaveCount, + agentWriteCount, + markAgentWritten, + openAgentEditor + } from '../agentEditorStore.svelte' import { setLinkedAgentTools, clearLinkedAgentTools, + linkedModulesForAgent, linkedToolsScope } from '../linkedAgentToolsStore.svelte' import { logReusableAgentUsage } from '../agentTelemetry' import { claimLinkedToolsFetch } from '../flowState' + import { AgentDraftUnavailable, fetchAgentWithDraft } from '../linkedAgentDrafts' + import type { AgentResourceState } from '../agentDraft.svelte' + import { getLocalDraftHint } from '$lib/localDraftHints.svelte' + import Tooltip from '$lib/components/meltComponents/Tooltip.svelte' import type { AgentTool as AgentToolStrict } from '../agentToolUtils' import { resource } from 'runed' import { untrack } from 'svelte' @@ -65,6 +75,10 @@ // deploy from the agent editor mounted alongside it. Both reads below key on it, so neither // keeps naming the config and version a write has just replaced. let writes = $derived(agentWriteCount(ws, agent)) + // Draft saves as well, for the link fetch: the card shows what a test of this step would run, + // and that is the draft. Only the deploy moves `writes`, so without this the card would keep + // describing the config the agent held before it was edited. + let draftSaves = $derived(agentDraftSaveCount(ws, agent)) let saveDrawer: Drawer | undefined = $state() let newPath = $state('') @@ -75,14 +89,18 @@ type LinkedInfo = { // What this result was fetched for. runed's resource neither aborts nor tags a superseded // request, so a slow fetch can land after a newer one: every consumer gates on these matching - // the current (ws, agent, writes). `writes` is what covers a refetch of the *same* link after - // a deploy — without it a pre-deploy response is indistinguishable from the current one, and - // accepting it republishes the tools the deploy just replaced. + // the current (ws, agent, writes, draftSaves). `writes` is what covers a refetch of the *same* + // link after a deploy — without it a pre-deploy response is indistinguishable from the current + // one, and accepting it republishes the tools the deploy just replaced. `draftSaves` does the + // same for a draft save, which the card follows just as closely. ws?: string path?: string writes: number + draftSaves: number config: AIAgentConfig tools: AgentTool[] + /** The config shown came from the agent's unsaved draft rather than the deployed resource. */ + fromDraft: boolean providerPath?: string providerOk: boolean } @@ -90,14 +108,37 @@ // A linked agent is rigid and read-only: its brain and tools come from the resource. We // load them here for display, and probe the provider resource so we can warn when it isn't // accessible in this workspace (the user then needs to unlink/fork or gain access). + // The draft when there is one, since that is what a test of this step runs. let linkedResource = resource( - () => ({ ws, path: agent, writes }), - async ({ ws, path, writes }): Promise => { + () => ({ ws, path: agent, writes, draftSaves }), + async ({ ws, path, writes, draftSaves }): Promise => { if (!ws || !path) { - return { ws, path, writes, config: {}, tools: [], providerOk: true } + return { + ws, + path, + writes, + draftSaves, + config: {}, + tools: [], + fromDraft: false, + providerOk: true + } + } + let response: Resource + let draft: AgentResourceState | undefined + try { + ;({ response, draft } = await fetchAgentWithDraft(path, ws)) + } catch (err) { + // Only the DRAFT was unreadable. This card is a display, so fall back to the deployed + // agent rather than rendering one with no brain and no tools, which reads as "the agent + // is empty" while the Draft badge still says it has unsaved changes. Same fallback the + // graph's tool nodes take; the paths that run or deploy the draft still refuse. + if (!(err instanceof AgentDraftUnavailable)) throw err + response = await ResourceService.getResource({ workspace: ws, path }) + } + const cfg = (draft?.args ?? response.value ?? {}) as AIAgentConfig & { + provider?: { resource?: string } } - const res = await ResourceService.getResource({ workspace: ws, path }) - const cfg = (res.value ?? {}) as AIAgentConfig & { provider?: { resource?: string } } const tools = (cfg.tools ?? []) as AgentTool[] const providerRef = cfg.provider?.resource const providerPath = @@ -116,8 +157,10 @@ ws, path, writes, + draftSaves, config: cfg, tools, + fromDraft: draft != undefined, providerPath, providerOk } @@ -129,7 +172,13 @@ let loadedInfo = $state(undefined) $effect(() => { const current = linkedResource.current - if (current && current.ws === ws && current.path === agent && current.writes === writes) { + if ( + current && + current.ws === ws && + current.path === agent && + current.writes === writes && + current.draftSaves === draftSaves + ) { loadedInfo = current } }) @@ -140,6 +189,12 @@ let brainParams = $derived(summarizeAgentBrain(linkedInfo?.config)) let providerPath = $derived(linkedInfo?.providerPath) let providerOk = $derived(linkedInfo?.providerOk ?? true) + // The hint flips on the first keystroke in the agent editor, so the badge does not wait for the + // debounced autosave and the refetch behind it; the fetched answer covers a draft written + // elsewhere, which no editor here has published an opinion about. + let hasDraft = $derived( + getLocalDraftHint(ws, 'resource', agent ?? '') ?? linkedInfo?.fromDraft ?? false + ) /** The agent the card is about: the one this step links to, or the one being edited. */ let cardPath = $derived(agent) // The version eval runs are recorded against. The resource does not hold it; its newest history @@ -190,9 +245,18 @@ } const loaded = linkedInfo if (loaded) { - claimLinkedToolsFetch(toolScope, moduleId) - // linkedResource types tools loosely; they are the same resource tools the store holds. - setLinkedAgentTools(toolScope, moduleId, loaded.tools as AgentToolStrict[], agent) + // Every step of this flow linking this agent, not just this one. Tools belong to the agent, + // so the sibling steps show the same set, and only the selected step mounts this card: + // without them a draft saved from here leaves their nodes on what the flow load resolved, + // while a test of those steps runs the draft. Claimed like this card's own publish, so a + // sibling's in-flight fetch cannot land afterwards and put the old tools back. + const modules = new Set(linkedModulesForAgent(toolScope, agent)) + modules.add(moduleId) + for (const id of modules) { + claimLinkedToolsFetch(toolScope, id) + // linkedResource types tools loosely; they are the same resource tools the store holds. + setLinkedAgentTools(toolScope, id, loaded.tools as AgentToolStrict[], agent) + } publishedFor = agent } else if (publishedFor !== undefined && publishedFor !== agent) { // The link moved and the new agent hasn't resolved, so the stored tools are the old one's. @@ -358,13 +422,15 @@ // `tools` is one array per module value, so it identifies the step itself — the path alone // would not, since a replacement can carry the same link. const stepMarker = tools - const res = await ResourceService.getResource({ workspace: ws, path }) + // The draft, like the card above and like a test of this step: forking the deployed value + // while the card displays a drafted prompt would hand back something the user never saw. + const { response, draft } = await fetchAgentWithDraft(path, ws) // The module may have been replaced while the fetch was in flight (undo, session drafts); // applying a stale fork would overwrite the restored state. if (agent !== path || tools !== stepMarker) { return false } - const cfg = (res.value ?? {}) as AIAgentConfig + const cfg = (draft?.args ?? response.value ?? {}) as AIAgentConfig // Preserve the flow-local inputs already wired in the step. const local: Record = {} for (const key of AGENT_FLOW_LOCAL_KEYS) { @@ -448,6 +514,15 @@ v{version} {/if} + {#if hasDraft} + + Draft + {#snippet text()} + This agent has unsaved changes. Testing this flow runs the draft, and deploying the + flow offers to deploy it. + {/snippet} + + {/if}
{#if brainParams.length > 0 || inheritedTools.length > 0} diff --git a/frontend/src/lib/components/flows/flowState.ts b/frontend/src/lib/components/flows/flowState.ts index 3123396a9d..595227f57d 100644 --- a/frontend/src/lib/components/flows/flowState.ts +++ b/frontend/src/lib/components/flows/flowState.ts @@ -5,6 +5,7 @@ import { get } from 'svelte/store' import { workspaceStore } from '$lib/stores' import { isFlowModuleTool, agentToolToFlowModule, type AgentTool } from './agentToolUtils' import { linkedToolsScope, setLinkedAgentTools } from './linkedAgentToolsStore.svelte' +import { fetchAgentWithDraft, normalizeAgentRef } from './linkedAgentDrafts' import { loadFlowModuleState } from './flowStateUtils.svelte' import { emptyFlowModuleState } from './utils.svelte' import type { StateStore } from '$lib/utils' @@ -90,7 +91,9 @@ async function mapFlowModule( // the graph can render its tool nodes. They are display-only (their inputs are edited in // the step panel, which infers schemas itself), so no per-tool module state is loaded — // resource tool ids are not flow-unique and must not key into the flow state. - await publishLinkedAgentTools(agentRef, workspace, scope, flowModule.id) + // Drafts included: every caller of `initFlowState` is a flow editor, where the graph has + // to show the tools a test would run. Read-only viewers publish for themselves. + await publishLinkedAgentTools(agentRef, workspace, scope, flowModule.id, true) } else { // Shape-checked because `tools` is JSON-authored: throwing here would skip the agent's // own state below, leaving it with no schema rather than with no tool schemas. @@ -119,11 +122,17 @@ export async function publishLinkedAgentTools( agentRef: string, workspace: string | undefined, scope: string, - moduleId: string + moduleId: string, + /** Resolve from the agent's unsaved draft when there is one. Editors pass true so the graph + * shows the tool set a test would run; read-only viewers pass false, since a run they are + * displaying used the deployed agent. Required rather than defaulted: an editor call site that + * forgets it republishes the deployed tools over the drafted ones, which reads as the graph + * spontaneously reverting. */ + withDraft: boolean ) { const genKey = `${scope}:${moduleId}` const gen = claimLinkedToolsFetch(scope, moduleId) - const tools = await resolveLinkedAgentTools(agentRef, workspace) + const tools = await resolveLinkedAgentTools(agentRef, workspace, withDraft) if (linkedToolFetchGen.get(genKey) === gen) { setLinkedAgentTools(scope, moduleId, tools, agentRef) } @@ -155,12 +164,27 @@ export function claimLinkedToolsFetch(scope: string, moduleId: string): number { // resource is missing or inaccessible so a broken link never stalls the flow load. export async function resolveLinkedAgentTools( agentRef: string, - workspace?: string + workspace: string | undefined, + withDraft: boolean ): Promise { const ws = workspace ?? get(workspaceStore) if (!ws) return [] - const path = agentRef.replace(/^\$res:/, '').replace(/^res:\/\//, '') + const path = normalizeAgentRef(agentRef) try { + if (withDraft) { + try { + const { response, draft } = await fetchAgentWithDraft(path, ws) + const value = (draft?.args ?? response.value) as { tools?: AgentTool[] } | undefined + return (value?.tools ?? []) as AgentTool[] + } catch { + // The draft read failed for any reason. This is a display, not a run, so fall through to + // the deployed tools rather than showing an agent with none: an empty node list reads as + // "the agent lost its tools" instead of "we could not reach the server". The paths that + // act on a draft — the previews and the deploy dialog — surface the failure instead. + // Not rethrowing anything here: the outer catch turns every throw into `[]`, so a + // rethrow would skip the very fallback this exists for. + } + } const res = await ResourceService.getResource({ workspace: ws, path }) return ((res.value as { tools?: AgentTool[] } | undefined)?.tools ?? []) as AgentTool[] } catch { diff --git a/frontend/src/lib/components/flows/linkedAgentDrafts.test.ts b/frontend/src/lib/components/flows/linkedAgentDrafts.test.ts new file mode 100644 index 0000000000..563007615f --- /dev/null +++ b/frontend/src/lib/components/flows/linkedAgentDrafts.test.ts @@ -0,0 +1,148 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { + inlineAgentDraft, + inlineAgentDrafts, + loadLinkedAgentDrafts, + type LinkedAgentDraft +} from './linkedAgentDrafts' +import { ResourceService, type FlowModule, type FlowValue } from '$lib/gen' + +type AiAgentValue = Extract + +function linkedStep(input_transforms: Record): AiAgentValue { + return { + type: 'aiagent', + agent: 'f/team/support', + tool_inputs: { t1: { query: { type: 'javascript', expr: 'flow_input.q' } } }, + input_transforms + } as unknown as AiAgentValue +} + +describe('inlineAgentDraft', () => { + // The overlay order is the worker's: the resource brain first, the step's flow-local inputs on + // top. Reversing it would run the agent author's own `user_message` instead of the flow's. + it('keeps the step wired to the flow while the brain comes from the draft', () => { + const inlined = inlineAgentDraft( + linkedStep({ + user_message: { type: 'javascript', expr: 'flow_input.question' }, + user_attachments: { type: 'static', value: [] } + }), + { + provider: { kind: 'openai', model: 'gpt-4o', resource: '$res:f/team/openai' }, + system_prompt: 'answer in french', + user_message: 'the default the agent carries', + tools: [{ id: 't1', summary: 'search' }] + } as any + ) + + expect(inlined.agent).toBeUndefined() + expect(inlined.tools).toEqual([{ id: 't1', summary: 'search' }]) + expect(inlined.input_transforms).toEqual({ + provider: { + type: 'static', + value: { kind: 'openai', model: 'gpt-4o', resource: '$res:f/team/openai' } + }, + system_prompt: { type: 'static', value: 'answer in french' }, + user_message: { type: 'javascript', expr: 'flow_input.question' }, + user_attachments: { type: 'static', value: [] } + }) + // Host bindings are the step's, not the agent's, and the worker overlays them either way. + expect(inlined.tool_inputs).toEqual({ + t1: { query: { type: 'javascript', expr: 'flow_input.q' } } + }) + }) + + // A linked step carries only the flow-local inputs, but one persisted before linking existed can + // still hold stale brain transforms. They must not shadow the draft the test is meant to run. + it('drops brain transforms the step still carries', () => { + const inlined = inlineAgentDraft( + linkedStep({ + user_message: { type: 'static', value: 'hi' }, + system_prompt: { type: 'static', value: 'stale' } + }), + { system_prompt: 'from the draft' } as any + ) + + expect(inlined.input_transforms).toEqual({ + system_prompt: { type: 'static', value: 'from the draft' }, + user_message: { type: 'static', value: 'hi' } + }) + }) +}) + +describe('inlineAgentDrafts', () => { + // The index is keyed on the bare path while a step may name its agent `$res:`-prefixed, and the + // walk has to reach inside branches and loops. Miss either and every preview silently runs the + // deployed agent — the failure this whole path exists to prevent, and a silent one. + it('reaches a $res:-prefixed link nested in a branch', () => { + const value = { + modules: [ + { + id: 'b', + value: { + type: 'branchone', + default: [], + branches: [ + { + modules: [ + { + id: 'inner', + value: { + type: 'aiagent', + agent: '$res:f/team/support', + tools: [], + input_transforms: { user_message: { type: 'static', value: 'hi' } } + } + } + ] + } + ] + } + } + ] + } as unknown as FlowValue + + const drafts = new Map([ + ['f/team/support', { args: { system_prompt: 'drafted' } } as unknown as LinkedAgentDraft] + ]) + + const inner = (inlineAgentDrafts(value, drafts).modules[0].value as any).branches[0].modules[0] + expect(inner.value.agent).toBeUndefined() + expect(inner.value.input_transforms.system_prompt).toEqual({ + type: 'static', + value: 'drafted' + }) + // The input the flow supplies survives the rewrite. + expect(inner.value.input_transforms.user_message).toEqual({ type: 'static', value: 'hi' }) + }) +}) + +// A link the user cannot resolve is an ordinary state and must not block the flow; anything else is +// an outage, and answering "no draft" to one would silently test or deploy against the deployed +// agent while the editor shows the draft. +describe('loadLinkedAgentDrafts error handling', () => { + function failWith(status: number | undefined) { + return async () => { + const err: Error & { status?: number } = new Error('boom') + err.status = status + throw err + } + } + + beforeEach(() => { + vi.restoreAllMocks() + }) + + it.each([401, 403, 404])('treats %i as no draft', async (status) => { + vi.spyOn(ResourceService, 'getResource').mockImplementation(failWith(status) as any) + await expect(loadLinkedAgentDrafts(['f/team/support'], 'ws')).resolves.toEqual(new Map()) + }) + + it.each([500, undefined])('propagates %s rather than reporting no draft', async (status) => { + vi.spyOn(ResourceService, 'getResource').mockImplementation(failWith(status) as any) + await expect(loadLinkedAgentDrafts(['f/team/support'], 'ws')).rejects.toThrow( + 'Could not load the agent f/team/support' + ) + }) +}) diff --git a/frontend/src/lib/components/flows/linkedAgentDrafts.ts b/frontend/src/lib/components/flows/linkedAgentDrafts.ts new file mode 100644 index 0000000000..8da732e7ff --- /dev/null +++ b/frontend/src/lib/components/flows/linkedAgentDrafts.ts @@ -0,0 +1,229 @@ +import { + ResourceService, + type FlowModule, + type FlowValue, + type InputTransform, + type Resource +} from '$lib/gen' +import { UserDraft } from '$lib/userDraft.svelte' +import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' +import { canWrite } from '$lib/utils' +import type { UserExt } from '$lib/stores' +import { dfs } from './dfs' +import { flowLocalInputs, type AIAgentConfig } from './agentResourceUtils' +import type { AgentResourceState } from './agentDraft.svelte' +import type { AgentTool } from './agentToolUtils' + +/** A step names its agent bare or as `$res:`/`res://`; all three are the same agent, + * and a draft index has to answer for a lookup written any of those ways. Same normalization as + * `linkedAgentToolsStore`, and as the `trim_start_matches` the worker applies. */ +export function normalizeAgentRef(agentRef: string): string { + return agentRef.replace(/^\$res:/, '').replace(/^res:\/\//, '') +} + +/** Every `ai_agent` resource this flow links to, deduped. `dfs` walks agent tool nodes as well as + * branches and loops, so a nested linked agent tool is included. */ +export function linkedAgentPaths(value: FlowValue | undefined): string[] { + if (!value?.modules) return [] + const paths = new Set() + for (const module of dfs(value.modules, (m) => m)) { + const v = module?.value as { type?: string; agent?: string } | undefined + if (v?.type === 'aiagent' && v.agent) { + paths.add(normalizeAgentRef(v.agent)) + } + } + return [...paths] +} + +/** + * The unsaved draft for an agent, freshest first: the cell an open agent editor is writing, then + * what a `get_draft` response carried. + * + * Only the live cell is reliably current, and only while an editor holds it: `releaseEntry` drops + * the cached write at refcount 0 on purpose, so once the agent editor closes the persisted row is + * the sole answer. Read through `fetchAgentWithDraft`, which settles that row first. + */ +export function agentDraftState( + response: { draft?: unknown }, + path: string, + workspace: string | undefined +): AgentResourceState | undefined { + const live = UserDraft.get('resource', path, { workspace }) + return live ?? (response.draft as AgentResourceState | undefined) +} + +/** A refusal that already names the agent and says what is wrong with it, so a caller wrapping it + * would only repeat itself. */ +export class AgentDraftUnavailable extends Error {} + +/** + * An agent's resource together with the draft a run of it would use. + * + * The flush is what makes the answer current. Autosave is debounced by 1.5s (10s ceiling), and + * closing the agent editor releases the in-memory cell without cancelling that pending POST — so + * testing or deploying right after closing would otherwise read a row the last edits have not + * reached yet. `flush` replays the parked save and is a no-op when there is none. + */ +export async function fetchAgentWithDraft( + path: string, + workspace: string +): Promise<{ response: Resource; draft: AgentResourceState | undefined }> { + const query = { workspace, itemKind: 'resource' as const, path } + await UserDraftDbSyncer.flush(query) + // `flush` resolves whether or not the save actually landed: `postSave` catches network and + // server errors into its failure map, and answers a conflicting write by parking a snapshot, + // returning normally in both cases. The row about to be read is then older than the edit still + // held in the browser, and nothing downstream could tell. Running that row is a test of the + // wrong agent; deploying it is worse, because the deploy deletes the draft and takes the newer + // edit with it. Neither is recoverable from here, so refuse the read. + const failure = UserDraftDbSyncer.getState(query).failureMessage + if (failure) { + throw new AgentDraftUnavailable(`The unsaved changes to ${path} could not be saved: ${failure}`) + } + if (UserDraftDbSyncer.getConflict(query).conflict) { + throw new AgentDraftUnavailable( + `The unsaved changes to ${path} could not be saved because it was edited elsewhere. Open the agent to resolve it.` + ) + } + const response = await ResourceService.getResource({ workspace, path, getDraft: true }) + return { response, draft: agentDraftState(response, path, workspace) } +} + +/** One linked agent whose resource the user has an unsaved draft for. */ +export interface LinkedAgentDraft { + path: string + /** The draft's resource value: what a run of this agent would use. */ + args: AIAgentConfig + /** The whole draft row, as the resource editors write it — the deploy payload. */ + state: AgentResourceState + /** No deployed row at this path, so deploying has to create rather than update. */ + noDeployed: boolean + /** Of the deployed resource, for `agentDraftCanWrite`. */ + extraPerms: Record +} + +/** Whether `user` may write this agent's resource. Split from the load so that resolving the + * drafts of a whole flow costs no `whoami` — only the deploy dialog needs the answer, and it + * looks the user up once for every agent it lists. */ +export function agentDraftCanWrite(draft: LinkedAgentDraft, user: UserExt | undefined): boolean { + return canWrite(draft.path, draft.extraPerms, user) +} + +/** A link that cannot resolve for the user rather than because something went wrong: the agent was + * deleted, or sits in a folder they cannot read. Both are ordinary states of a rigid link, and + * neither should stop the caller — the flow still tests and deploys, against the deployed agent. + * Every other failure is an outage, and answering "no draft" to one would quietly run or deploy + * the wrong configuration, which is the whole thing this module exists to prevent. */ +function isExpectedLinkFailure(err: unknown): boolean { + const status = (err as { status?: number } | null | undefined)?.status + return status === 401 || status === 403 || status === 404 +} + +/** + * The unsaved draft of every given `ai_agent` path, for the paths that have one. + * + * Throws when a path fails to load for any reason other than being missing or unreadable, so a + * caller cannot mistake an outage for an agent with nothing unsaved. + */ +export async function loadLinkedAgentDrafts( + paths: string[], + workspace: string | undefined +): Promise> { + const out = new Map() + if (!workspace || paths.length === 0) return out + await Promise.all( + paths.map(async (path) => { + let response: Resource + let draft: AgentResourceState | undefined + try { + ;({ response, draft } = await fetchAgentWithDraft(path, workspace)) + } catch (err) { + if (isExpectedLinkFailure(err)) return + if (err instanceof AgentDraftUnavailable) throw err + throw new Error(`Could not load the agent ${path}: ${err}`) + } + if (!draft) return + out.set(path, { + path, + args: (draft.args ?? {}) as AIAgentConfig, + state: draft, + noDeployed: Boolean((response as { no_deployed?: boolean }).no_deployed), + extraPerms: response.extra_perms ?? {} + }) + }) + ) + return out +} + +/** + * Every argument a saved agent carries, as a static input transform. Not only the keys the agent + * form renders: a run reads them all, and an agent holding its own `user_message` answers with it + * when nothing overrides it. `tools` is the step's own roster rather than an input, so it rides on + * the module's `tools` key instead. + */ +export function agentArgsToTransforms(args: AIAgentConfig): Record { + const it: Record = {} + for (const [key, value] of Object.entries(args ?? {})) { + if (key === 'tools' || value === undefined) continue + it[key] = { type: 'static', value } as InputTransform + } + return it +} + +type AiAgentValue = Extract + +/** + * The standalone step a linked step's draft would run as: the draft's brain and tools inlined, with + * the step's own flow-local inputs kept on top. + * + * The overlay order is the worker's (`ai_executor.rs`): its linked branch interpolates the whole + * resource brain and only then writes `user_message`/`user_attachments` back from the step's own + * args. `tool_inputs` stays untouched — the worker overlays it onto the tools in both branches, so + * an inlined step keeps the host flow's tool bindings. + */ +export function inlineAgentDraft(value: AiAgentValue, args: AIAgentConfig): AiAgentValue { + const { agent: _agent, ...rest } = value + return { + ...rest, + tools: (args.tools ?? []) as AgentTool[], + input_transforms: { + ...agentArgsToTransforms(args), + ...flowLocalInputs(value.input_transforms as Record) + } + } as AiAgentValue +} + +/** + * Replace every linked agent step that has a draft with the draft's own configuration, so a preview + * runs what the agent editor is showing rather than the deployed resource. Returns a new value: the + * flow editor hands its live store object to previews. + */ +export function inlineAgentDrafts( + value: FlowValue, + drafts: Map +): FlowValue { + if (drafts.size === 0) return value + // JSON rather than `structuredClone`: the flow editor's value is a Svelte `$state` proxy, which + // `structuredClone` refuses outright. A flow value is JSON by definition — it is about to be + // posted as one — so the round trip loses nothing this preview would have carried. + const next = JSON.parse(JSON.stringify(value)) as FlowValue + for (const module of dfs(next.modules ?? [], (m) => m)) { + const v = module?.value as AiAgentValue | undefined + if (v?.type !== 'aiagent' || !v.agent) continue + const draft = drafts.get(normalizeAgentRef(v.agent)) + if (!draft) continue + module.value = inlineAgentDraft(v, draft.args) + } + return next +} + +/** Load the drafts this flow's linked agents have and inline them. The whole substitution, for a + * caller holding nothing but the value it is about to preview. */ +export async function withAgentDrafts( + value: FlowValue, + workspace: string | undefined +): Promise { + const paths = linkedAgentPaths(value) + if (paths.length === 0) return value + return inlineAgentDrafts(value, await loadLinkedAgentDrafts(paths, workspace)) +} diff --git a/frontend/src/lib/components/flows/linkedToolsFetchGuard.test.ts b/frontend/src/lib/components/flows/linkedToolsFetchGuard.test.ts index ea1674e0b6..29207f58f7 100644 --- a/frontend/src/lib/components/flows/linkedToolsFetchGuard.test.ts +++ b/frontend/src/lib/components/flows/linkedToolsFetchGuard.test.ts @@ -8,7 +8,9 @@ vi.mock('./agentToolUtils', () => ({ isFlowModuleTool: () => false, agentToolToFlowModule: (t: unknown) => t })) -vi.mock('$lib/stores', () => ({ workspaceStore: { subscribe: (f: (v: string) => void) => (f('ws'), () => {}) } })) +vi.mock('$lib/stores', () => ({ + workspaceStore: { subscribe: (f: (v: string) => void) => (f('ws'), () => {}) } +})) import { claimLinkedToolsFetch, @@ -38,8 +40,8 @@ describe('linked tools fetch guard', () => { ) .mockResolvedValueOnce({ value: { tools: [tool('new')] } } as never) - const stale = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step') - await publishLinkedAgentTools('f/a/new', 'ws', scope, 'step') + const stale = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step', false) + await publishLinkedAgentTools('f/a/new', 'ws', scope, 'step', false) release?.({ value: { tools: [tool('old')] } }) await stale @@ -56,7 +58,7 @@ describe('linked tools fetch guard', () => { () => new Promise((r) => (release = r)) as ReturnType ) - const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step') + const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step', false) setLinkedAgentTools(scope, 'step', [tool('kept')], 'u/admin/a') invalidateLinkedToolsFetches(scope) @@ -73,7 +75,7 @@ describe('linked tools fetch guard', () => { () => new Promise((r) => (release = r)) as ReturnType ) - const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step') + const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step', false) claimLinkedToolsFetch(scope, 'step') setLinkedAgentTools(scope, 'step', [tool('direct')], 'u/admin/a') release?.({ value: { tools: [tool('stale')] } }) diff --git a/frontend/src/lib/components/flows/pickers/PickHubScript.svelte b/frontend/src/lib/components/flows/pickers/PickHubScript.svelte index 207e8612e9..d84228dd35 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubScript.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubScript.svelte @@ -8,8 +8,13 @@ import { IntegrationService, ScriptService, type HubScriptKind } from '$lib/gen' import { Loader2 } from 'lucide-svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' - import { disableHubStore } from '$lib/stores' + import { disableHubStore, workspaceStore } from '$lib/stores' import { logHubScriptPick } from '$lib/utils/featureUsage' + import { + alphabetical, + byPopularity, + localCountsByIntegration + } from '$lib/components/pickerPopularity' interface Props { kind?: HubScriptKind & string @@ -46,19 +51,25 @@ }[] = $state([]) let allApps: string[] = $state([]) + let popularity: (a: string, b: string) => number = $state(alphabetical) let apps: string[] = $derived.by(() => - filter.length > 0 ? Array.from(new Set(items?.map((x) => x.app) ?? [])).sort() : allApps + filter.length > 0 + ? Array.from(new Set(items?.map((x) => x.app) ?? [])).sort(popularity) + : allApps ) async function getAllApps(filterKind: typeof kind) { if ($disableHubStore) return try { hubNotAvailable = false - allApps = ( - await IntegrationService.listHubIntegrations({ - kind: filterKind - }) - ).map((x) => x.name) + // Independent reads, so they share one round trip before first paint. + const [integrations, local] = await Promise.all([ + IntegrationService.listHubIntegrations({ kind: filterKind }), + $workspaceStore ? localCountsByIntegration($workspaceStore) : {} + ]) + const hubPicks = Object.fromEntries(integrations.map((x) => [x.name, x.picks ?? 0])) + popularity = byPopularity(hubPicks, local) + allApps = integrations.map((x) => x.name).sort(popularity) } catch (err) { console.error('Hub is not available') allApps = [] @@ -154,72 +165,73 @@ {#if $disableHubStore} {:else} -
- {@render children?.()} -
- - {#if loading} - - {/if} -
-
- -{#if hubNotAvailable} - - Could not connect to the Windmill Hub. If you are in a closed environment, you can disable the Hub in the instance settings. - -{:else if (items.length > 0 && apps.length > 0) || !loading} - - {#if items.length == 0} - - {:else} -
    - {#each items as item (item.path)} -
  • - -
    • - {/each} -
- {/if} - {#if items.length == 20} -
- There are more items than being displayed. Refine your search. +
+ {@render children?.()} +
+ + {#if loading} + + {/if}
+
+ + {#if hubNotAvailable} + + Could not connect to the Windmill Hub. If you are in a closed environment, you can disable the + Hub in the instance settings. + + {:else if (items.length > 0 && apps.length > 0) || !loading} + + {#if items.length == 0} + + {:else} +
    + {#each items as item (item.path)} +
  • + +
    • + {/each} +
+ {/if} + {#if items.length == 20} +
+ There are more items than being displayed. Refine your search. +
+ {/if} + {:else} + {#each Array(10).fill(0) as _} + + {/each} {/if} -{:else} - {#each Array(10).fill(0) as _} - - {/each} -{/if} {/if} diff --git a/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte b/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte index 5204892051..d32e82c702 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte @@ -44,12 +44,17 @@ import { Circle, ExternalLink } from 'lucide-svelte' import Popover from '$lib/components/Popover.svelte' import { usePromise } from '$lib/svelte5Utils.svelte' - import { disableHubStore, hubBaseUrlStore, userStore } from '$lib/stores' + import { disableHubStore, hubBaseUrlStore, userStore, workspaceStore } from '$lib/stores' import { get } from 'svelte/store' import Button from '$lib/components/common/button/Button.svelte' import { Alert } from '$lib/components/common' import type { FlowBuilderWhitelabelCustomUi } from '$lib/components/custom_ui' import { logHubScriptPick } from '$lib/utils/featureUsage' + import { + alphabetical, + byPopularity, + localCountsByIntegration + } from '$lib/components/pickerPopularity' let customUi: undefined | FlowBuilderWhitelabelCustomUi = getContext('customUi') @@ -94,9 +99,10 @@ }: Props = $props() let allApps: string[] = $state([]) + let popularity: (a: string, b: string) => number = $state(alphabetical) $effect(() => { if (filter.length > 0) { - apps = Array.from(new Set(items?.map((x) => x.app) ?? [])).sort() + apps = Array.from(new Set(items?.map((x) => x.app) ?? [])).sort(popularity) } else { apps = allApps } @@ -106,9 +112,14 @@ if ($disableHubStore) return try { hubNotAvailable = false - allApps = (await listHubIntegrationsCached({ kind: filterKind, refreshCount })).map( - (x) => x.name - ) + // Independent reads, so they share one round trip before first paint. + const [integrations, local] = await Promise.all([ + listHubIntegrationsCached({ kind: filterKind, refreshCount }), + $workspaceStore ? localCountsByIntegration($workspaceStore) : {} + ]) + const hubPicks = Object.fromEntries(integrations.map((x) => [x.name, x.picks ?? 0])) + popularity = byPopularity(hubPicks, local) + allApps = integrations.map((x) => x.name).sort(popularity) } catch (err) { console.error('Failed to fetch hub integrations:', err) allApps = [] diff --git a/frontend/src/lib/components/flows/utils.svelte.ts b/frontend/src/lib/components/flows/utils.svelte.ts index 8d8e4366e5..df27960dc7 100644 --- a/frontend/src/lib/components/flows/utils.svelte.ts +++ b/frontend/src/lib/components/flows/utils.svelte.ts @@ -17,6 +17,7 @@ import { get } from 'svelte/store' import type { FlowModuleState } from './flowState' import { type PickableProperties, dfs } from './previousResults' import { forEachFlowModule } from './dfs' +import { withAgentDrafts } from './linkedAgentDrafts' import { NEVER_TESTED_THIS_FAR } from './models' import { sendUserToast } from '$lib/toast' import type { ExtendedOpenFlow } from './types' @@ -187,6 +188,12 @@ export function jobsToResults(jobs: Job[]) { }) } +/** + * Run the flow the editor currently holds. A step linked to a saved agent runs that agent's + * unsaved draft when there is one (`withAgentDrafts`), so testing exercises what the agent editor + * is showing rather than the deployed resource — the same rule the agent editor's own test pane + * follows. The value passed in is left alone; only what goes to the server is substituted. + */ export async function runFlowPreview( args: Record, flow: OpenFlow & { tag?: string }, @@ -198,14 +205,15 @@ export async function runFlowPreview( // editor; falls back to the navigation workspace for full-page previews. workspace?: string ) { - const newFlow = flow + const ws = workspace ?? get(workspaceStore) ?? '' + const value = await withAgentDrafts(flow.value, ws) return await JobService.runFlowPreview({ - workspace: workspace ?? get(workspaceStore) ?? '', + workspace: ws, requestBody: { args, - value: newFlow.value, + value, path: path, - tag: newFlow.tag, + tag: flow.tag, restarted_from: restartedFrom, temp_script_refs: tempScriptRefs }, diff --git a/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte b/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte index 8183be16d6..db5abfde20 100644 --- a/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte +++ b/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte @@ -7,7 +7,7 @@ import Popover from '$lib/components/meltComponents/Popover.svelte' import AssetNode from './AssetNode.svelte' import type { FlowGraphAssetContext } from '$lib/components/flows/types' - import { getContext } from 'svelte' + import { getContext, untrack } from 'svelte' import { assetEq } from '$lib/components/assets/lib' import { getNodeColorClasses } from '../../util' @@ -24,15 +24,17 @@ data.overflowedAssets.some((asset) => assetEq(flowGraphAssetsCtx?.val.selectedAsset, asset)) ) - let wasOpenedBecauseOfExternalSelected = false + // Open while a sibling asset node is hovered and one of the hidden assets is the same asset. + let openedByHover = $state(false) $effect(() => { - if (includesSelected && !isOpen) { - isOpen = true - wasOpenedBecauseOfExternalSelected = true - } - if (wasOpenedBecauseOfExternalSelected && !includesSelected) { + if (includesSelected) { + if (!untrack(() => isOpen)) { + isOpen = true + openedByHover = true + } + } else if (untrack(() => openedByHover)) { isOpen = false - wasOpenedBecauseOfExternalSelected = false + openedByHover = false } }) const colors = $derived(getNodeColorClasses(undefined, includesSelected)) @@ -41,9 +43,17 @@ {#snippet children({ darkMode })} + {#snippet trigger()} - - +{data.overflowedAssets.length} - - {/snippet} + +{data.overflowedAssets.length} + {/snippet} {#snippet content()} - -
    - {#each data.overflowedAssets as asset} -
  • - -
    • - {/each} -
- - {/snippet} +
    + {#each data.overflowedAssets as asset} +
  • + +
    • + {/each} +
+ {/snippet} {/snippet} diff --git a/frontend/src/lib/components/meltComponents/Popover.svelte b/frontend/src/lib/components/meltComponents/Popover.svelte index f850e53f6c..6a6b7d681b 100644 --- a/frontend/src/lib/components/meltComponents/Popover.svelte +++ b/frontend/src/lib/components/meltComponents/Popover.svelte @@ -60,6 +60,8 @@ documentationLink?: string | undefined disableFocusTrap?: boolean openFocus?: string | HTMLElement | (() => HTMLElement | null) | null | undefined + /** Element to focus when the popover closes; defaults to the trigger, `null` leaves focus alone. */ + closeFocus?: string | HTMLElement | (() => HTMLElement | null) | null | undefined escapeBehavior?: EscapeBehaviorType enableFlyTransition?: boolean onKeyDown?: (e: KeyboardEvent) => void @@ -99,6 +101,7 @@ documentationLink = undefined, disableFocusTrap = false, openFocus = undefined, + closeFocus = undefined, escapeBehavior = 'close', enableFlyTransition = false, onKeyDown = () => {}, @@ -133,6 +136,7 @@ disableFocusTrap: untrack(() => disableFocusTrap), escapeBehavior: untrack(() => escapeBehavior), openFocus: untrack(() => openFocus), + closeFocus: untrack(() => closeFocus), onOpenChange: ({ curr, next }) => { if (curr != next) { dispatch('openChange', next) diff --git a/frontend/src/lib/components/pickerPopularity.test.ts b/frontend/src/lib/components/pickerPopularity.test.ts new file mode 100644 index 0000000000..f9440981ad --- /dev/null +++ b/frontend/src/lib/components/pickerPopularity.test.ts @@ -0,0 +1,80 @@ +import { describe, it, expect } from 'vitest' +import { alphabetical, byPopularity, totalLocalCountsByApp } from './pickerPopularity' + +const order = (names: string[], hub: Record, local: Record = {}) => + [...names].sort(byPopularity(hub, local)) + +describe('byPopularity', () => { + // The tier that stops a filling-up hub from squeezing the workspace's own stack out of + // the ordering: a global pick count grows without bound, a local one does not. + it('leads with what the workspace uses, whatever the hub says', () => { + expect(order(['slack', 'stripe'], { slack: 900 }, { stripe: 1 })).toEqual(['stripe', 'slack']) + }) + + it('ranks the used types among themselves by hub picks', () => { + expect(order(['slack', 'stripe'], { slack: 900 }, { slack: 1, stripe: 1 })).toEqual([ + 'slack', + 'stripe' + ]) + }) + + it('breaks a hub tie on how much the workspace uses it', () => { + expect(order(['slack', 'stripe'], { slack: 5, stripe: 5 }, { slack: 1, stripe: 2 })).toEqual([ + 'stripe', + 'slack' + ]) + }) + + it('ranks the unused types by hub picks, below every used one', () => { + expect(order(['ably', 'github', 'stripe'], { ably: 900, github: 5 }, { stripe: 1 })).toEqual([ + 'stripe', + 'ably', + 'github' + ]) + }) + + it('falls back to alphabetical for everything neither signal ranks', () => { + expect(order(['stripe', 'ably', 'github'], { github: 3 })).toEqual(['github', 'ably', 'stripe']) + }) + + it('orders on local usage alone when the hub ranks nothing', () => { + expect(order(['stripe', 'ably', 'github'], {}, { stripe: 1 })).toEqual([ + 'stripe', + 'ably', + 'github' + ]) + }) + + // The lists render before either signal lands, and one of them arrives in a server-side + // HashMap's iteration order, so the resting comparator has to sort rather than no-op. + it('leaves an alphabetical order with no signal at all', () => { + expect(['stripe', 'ably', 'github'].sort(alphabetical)).toEqual(['ably', 'github', 'stripe']) + }) +}) + +describe('totalLocalCountsByApp', () => { + const HUB = [ + { name: 'discord_webhook', app: 'discord', picks: 0 }, + { name: 'discord_bot_configuration', app: 'discord', picks: 0 }, + { name: 'ms_teams_webhook', app: 'msteams', picks: 0 }, + { name: 'slack', app: 'slack', picks: 0 } + ] + + // The integration pickers list app names, the counts arrive keyed by resource type, and + // the two only usually agree. Without the mapping a workspace whose Discord credential is + // a `discord_webhook` never reaches the used-here tier at all. + it('totals a resource type under the integration it belongs to', () => { + expect(totalLocalCountsByApp({ discord_webhook: 2 }, HUB)).toEqual({ discord: 2 }) + }) + + it('sums the several types one integration can have', () => { + expect( + totalLocalCountsByApp({ discord_webhook: 2, discord_bot_configuration: 1 }, HUB) + ).toEqual({ discord: 3 }) + }) + + // What a workspace-made type, and an unreachable hub, both leave every entry with. + it('keeps a type the hub has no mapping for under its own name', () => { + expect(totalLocalCountsByApp({ c_acme: 1, slack: 2 }, HUB)).toEqual({ c_acme: 1, slack: 2 }) + }) +}) diff --git a/frontend/src/lib/components/pickerPopularity.ts b/frontend/src/lib/components/pickerPopularity.ts new file mode 100644 index 0000000000..2d585ff858 --- /dev/null +++ b/frontend/src/lib/components/pickerPopularity.ts @@ -0,0 +1,149 @@ +import { get } from 'svelte/store' +import { ResourceService } from '$lib/gen' +import { disableHubStore } from '$lib/stores' +import { createCache } from '$lib/utils' +import { isCustomResourceTypeName } from './resourceTypeDisplay' + +/** + * How often something has been picked or used, keyed by integration or resource type name. + * A name the caller lists but this map does not mention counts as zero, which is what an + * unpicked entry and an absent signal both mean. + */ +export type PopularityCounts = Record + +/** + * The signals are read on every picker open, so they are cached briefly; both resolve to an + * empty map rather than rejecting, since an ordering hint is never worth a broken list. + */ +const CACHE_MS = 60_000 + +type HubResourceTypeInfo = { name: string; app: string; picks: number } + +const hubInfoCached = createCache( + async ({ workspace }: { workspace: string }): Promise => { + try { + return await ResourceService.listHubResourceTypeInfo({ workspace }) + } catch { + return [] + } + }, + { invalidateMs: CACHE_MS } +) + +const localCountsCached = createCache( + async ({ workspace }: { workspace: string }): Promise => { + try { + const counts = await ResourceService.listResourceCountsByType({ workspace }) + return Object.fromEntries(counts.map((c) => [c.resource_type, c.count])) + } catch { + return {} + } + }, + { invalidateMs: CACHE_MS } +) + +/** + * What the hub sees people pick, per resource type. Empty on a hub that counts nothing, + * and on an instance that has switched the hub off — a closed environment must not spend a + * request on hub.windmill.dev just to order a list. + */ +export async function hubResourceTypePicks(workspace: string): Promise { + if (get(disableHubStore)) return {} + const info = await hubInfoCached({ workspace }) + return Object.fromEntries(info.map((rt) => [rt.name, rt.picks])) +} + +/** + * How many resources of each type this workspace holds — the only evidence about this + * particular team. Keyed by resource type, which is what the add-resource drawer lists. + */ +export function localResourceTypeCounts(workspace: string): Promise { + return localCountsCached({ workspace }) +} + +/** + * The same counts totalled per integration, which is what the flow step picker lists. + * + * A type usually shares its integration's name, but often enough it does not: + * `discord_webhook` and `discord_bot_configuration` are both Discord, `ms_teams_webhook` and + * `azure_bot` are both MS Teams. Only the hub knows that, so a workspace whose Discord + * credential is a `discord_webhook` would otherwise read as one that has never touched + * Discord — and since local usage is the leading tier, that decides which half of the list + * the integration lands in, not merely its position within one. + * + * A type the hub has no mapping for counts under its own name, which is the right guess and + * also what an unreachable hub leaves every type with. + */ +export async function localCountsByIntegration(workspace: string): Promise { + const [counts, info] = await Promise.all([ + localCountsCached({ workspace }), + get(disableHubStore) ? Promise.resolve([]) : hubInfoCached({ workspace }) + ]) + return totalLocalCountsByApp(counts, info) +} + +/** The mapping half of {@link localCountsByIntegration}, separated so it can be tested alone. */ +export function totalLocalCountsByApp( + counts: PopularityCounts, + hub: { name: string; app: string }[] +): PopularityCounts { + const appOf = new Map(hub.map((rt) => [rt.name, rt.app])) + const byApp: PopularityCounts = {} + for (const [name, count] of Object.entries(counts)) { + const app = appOf.get(name) ?? name + byApp[app] = (byApp[app] ?? 0) + count + } + return byApp +} + +/** + * Tell the hub a resource type was taken into a workspace, which is what its ranking counts. + * Fire-and-forget: a hub that does not count picks must not be felt by the user who just + * saved a resource. Workspace-made types exist on no hub, so they are not reported. + */ +export function recordHubResourceTypePick(workspace: string, resourceType: string): void { + if (get(disableHubStore)) return + if (!resourceType || isCustomResourceTypeName(resourceType)) return + ResourceService.pickHubResourceType({ workspace, name: resourceType }).catch(() => {}) +} + +/** + * Orders the lists that offer hub content: integrations in the flow step picker, resource + * types in the add-resource drawer. + * + * Four tiers. **Whether this workspace already holds a resource of the type leads**, then the + * hub's pick count, then how many local resources there are, then the name. + * + * Used-here leads rather than merely breaking hub ties because the two counts are on + * incomparable scales: a hub pick count is global and grows without bound, a local count is + * usually single digits. Ranked the other way round, local usage only ever sorts the slice + * where hub counts are equal — which, since they are distinct integers, is just the tail + * that nobody has picked. That reads fine on a hub with few picks and silently stops + * mattering as one fills up, so the ordering would drift away from the workspace's own + * stack with no change to this code. + * + * Within each half the hub decides, so "yours" and "everyone's" are both honoured rather + * than blended with a weighting constant that would need tuning. Alphabetical is the floor, + * and it is where an entry neither signal knows about lands. + */ +export function byPopularity( + hub: PopularityCounts, + local: PopularityCounts +): (a: string, b: string) => number { + const usedHere = (name: string) => ((local[name] ?? 0) > 0 ? 1 : 0) + return (a, b) => + usedHere(b) - usedHere(a) || + (hub[b] ?? 0) - (hub[a] ?? 0) || + (local[b] ?? 0) - (local[a] ?? 0) || + a.localeCompare(b) +} + +/** + * The ordering to hold before either signal has landed: the alphabetical floor, which is + * what `byPopularity` degrades to anyway. + * + * A list has to be sorted by *something* from its first paint — one source of these names + * is a `HashMap` on the server, so leaving them unsorted means hash order, which differs + * between processes. + */ +export const alphabetical = byPopularity({}, {}) diff --git a/frontend/src/lib/components/search/GlobalSearchModal.svelte b/frontend/src/lib/components/search/GlobalSearchModal.svelte index a117d2769c..084ad9d492 100644 --- a/frontend/src/lib/components/search/GlobalSearchModal.svelte +++ b/frontend/src/lib/components/search/GlobalSearchModal.svelte @@ -427,7 +427,7 @@ path = `/apps/get/${e.path}` break case 'raw_app': - path = `/raw_apps/get/${e.path}` + path = `/apps_raw/get/${e.path}` break default: path = '/' diff --git a/frontend/src/lib/components/triggers/useTriggerDraftSync.svelte.ts b/frontend/src/lib/components/triggers/useTriggerDraftSync.svelte.ts index ba09167d6d..98d38ac25f 100644 --- a/frontend/src/lib/components/triggers/useTriggerDraftSync.svelte.ts +++ b/frontend/src/lib/components/triggers/useTriggerDraftSync.svelte.ts @@ -2,9 +2,21 @@ import { untrack } from 'svelte' import { deepEqual } from 'fast-equals' import { UserDraft, normalizeDraftForCompare, type UserDraftItemKind } from '$lib/userDraft.svelte' import { setLocalDraftHint } from '$lib/localDraftHints.svelte' +import { onUserInput } from '$lib/userDraftEditGate' type Cfg = Record +/** + * Detach a config from whatever holds it. The draft cell is deeply reactive, + * so handing its object straight to `applyCfg` would make the form's own + * `$state` (a schedule's `args`, say) the very object the cell holds — every + * later keystroke would then mutate the draft in place behind the autosave's + * back. + */ +function snapshotCfg(cfg: V): V { + return structuredClone($state.snapshot(cfg)) as V +} + /** * Whether `a` differs from `b` after `normalizeDraftForCompare` (JSON * round-trip to drop `undefined`-valued keys, plus ignored deploy-directive @@ -84,8 +96,10 @@ export interface TriggerDraftSync { * …)` (another tab, a programmatic write) propagate into the open editor. * * - **apply-effect**: reflects external `handle.draft` changes into the form. + * - **absorb-effect**: folds the form's own settling into the baseline until + * the user's first input, so a schema that moved on is not a draft. * - **persist-effect**: writes form edits back through the handle, dropping - * the draft when the form is back at the deployed baseline. + * the draft when the form is back at the baseline. * * Both effect bodies are `untrack`ed and gated by `cfgDiffers` * idempotence so they can't feed back into each other. Must be called once @@ -99,14 +113,56 @@ export function useTriggerDraftSync(opts: TriggerDraftSyncOptions): TriggerDraft }) const handle = $derived(handles[0]) + // Gated until the user puts something in (see `onUserInput`): the baseline + // absorbs whatever the form settles on and nothing persists, so an untouched + // trigger never reports unsaved changes. A drawer opened ON a restored draft + // absorbs nothing — that divergence is the user's own. + let settledBaseline: Cfg | undefined = $state(undefined) + let userEdited = $state(false) + let openedOnDraft = $state(false) + // These editors are mounted by the list page, not by the drawer, so input + // arriving while the drawer is still loading is the click that opened it + // (or anything else on the page behind) — never an edit to this form. + onUserInput(() => { + if (!opts.drawerLoading()) userEdited = true + }) + + /** The deployed config, plus whatever the form settled on by itself. */ + const baseline = $derived(settledBaseline ?? opts.deployed()) + + $effect(() => { + // A reload re-opens the gate's window: the drawer is being pointed at a + // different trigger, or the same one re-read from the backend. The click + // that opened it landed before this, hence the reset of `userEdited`. + if (!opts.drawerLoading()) return + untrack(() => { + settledBaseline = undefined + userEdited = false + openedOnDraft = false + }) + }) + + // absorb-effect: pre-edit form drift joins the baseline. + $effect(() => { + if (opts.drawerLoading() || userEdited || openedOnDraft) return + const cfg = opts.getCfg() + const deployed = opts.deployed() + if (cfg == null || deployed == null) return + // Snapshot before untracking: `getCfg` hands back the form's `$state` + // objects by reference, so only a deep read subscribes to the nested + // writes the form makes as it settles. + const settled = snapshotCfg(cfg) + untrack(() => { + if (cfgDiffers(settled, settledBaseline ?? deployed)) settledBaseline = settled + }) + }) + // Live "is there a local draft?" — the form diverges from the deployed // baseline. Gated on `!drawerLoading` (the baseline isn't settled yet // mid-load) and on a non-null baseline (a brand-new trigger has none, so // "unsaved changes" / discard-to-deployed is meaningless there). const hasDraft = $derived( - !opts.drawerLoading() && - opts.deployed() != null && - cfgDiffers(opts.getCfg() as Cfg, opts.deployed() as Cfg) + !opts.drawerLoading() && baseline != null && cfgDiffers(opts.getCfg() as Cfg, baseline as Cfg) ) // Reactive "banner is possible" — depends on `drawerLoading()` so it @@ -131,7 +187,7 @@ export function useTriggerDraftSync(opts: TriggerDraftSyncOptions): TriggerDraft if (opts.drawerLoading() || d == null) return untrack(() => { if (cfgDiffers(d, opts.getCfg() as Cfg)) { - void opts.applyCfg(d) + void opts.applyCfg(snapshotCfg(d)) } }) }) @@ -148,30 +204,32 @@ export function useTriggerDraftSync(opts: TriggerDraftSyncOptions): TriggerDraft discardTimer = undefined if (opts.drawerLoading()) return const cfg = opts.getCfg() - const deployed = opts.deployed() const h = handle if (!h || cfg == null) return - if (!cfgDiffers(cfg, deployed) && cfgDiffers(h.draft, deployed)) { - discard(opts.path(), deployed, true) + if (!cfgDiffers(cfg, baseline) && cfgDiffers(h.draft, baseline)) { + discard(opts.path(), baseline, true) } }, 600) } // persist-effect: form edits → handle; drop the draft when back at the - // deployed baseline. + // baseline. $effect(() => { if (opts.drawerLoading() || !opts.path()) return + // Nothing persists before the user's first input — the form's own + // settling is not an edit, and gating here rather than relying on the + // absorb-effect having run first keeps the two effects order-independent. + if (!userEdited && !openedOnDraft) return const cfg = opts.getCfg() if (cfg == null) return untrack(() => { const h = handle if (!h) return - const deployed = opts.deployed() - if (cfgDiffers(cfg, deployed)) { + if (cfgDiffers(cfg, baseline)) { if (cfgDiffers(cfg, h.draft)) h.draft = cfg - } else if (cfgDiffers(h.draft, deployed)) { - // Only when a draft actually exists to drop: `h.draft` equals - // `deployed` right after a discard or the post-load seed. + } else if (cfgDiffers(h.draft, baseline)) { + // Only when a draft actually exists to drop: `h.draft` equals the + // baseline right after a discard or the post-load seed. scheduleAutoDiscard() } }) @@ -202,7 +260,7 @@ export function useTriggerDraftSync(opts: TriggerDraftSyncOptions): TriggerDraft return hasBaseline }, get deployed() { - return opts.deployed() + return baseline }, get current() { return opts.getCfg() @@ -211,8 +269,14 @@ export function useTriggerDraftSync(opts: TriggerDraftSyncOptions): TriggerDraft const d = handle?.draft if (cfgDiffers(d, opts.getCfg() as Cfg)) { // Overlay the local autosave on the just-loaded backend config. - await opts.applyCfg(d) + await opts.applyCfg(snapshotCfg(d)) } + // The form is not rendered while the drawer loads, so anything that + // diverges from the deployed config right now is a draft restored onto + // it — by the overlay above, or by the editor from the backend before + // calling this — never the form settling. Absorbing that into the + // baseline would hide the user's own work behind a clean drawer. + openedOnDraft = cfgDiffers(opts.getCfg() as Cfg, opts.deployed()) // Adopt the post-load form state as the cell's baseline without // POSTing, consuming the entry's one-shot first-write seed guard. // Trigger drawers never write the cell programmatically on open, so @@ -221,14 +285,20 @@ export function useTriggerDraftSync(opts: TriggerDraftSyncOptions): TriggerDraft const p = opts.path() const cfg = opts.getCfg() if (ws && p && cfg != null) { - UserDraft.seed(opts.itemKind, p, structuredClone($state.snapshot(cfg)) as Cfg, { + UserDraft.seed(opts.itemKind, p, snapshotCfg(cfg) as Cfg, { workspace: ws }) } }, async resetToDeployed(path: string) { - const deployedCfg = structuredClone($state.snapshot(opts.deployed())) as Cfg + const deployedCfg = snapshotCfg(opts.deployed()) as Cfg discard(path, deployedCfg) + // Nothing of the user's is left in the form, so the gate closes again + // — otherwise the form settles on the schema's values a second time + // and the discarded draft comes straight back. + settledBaseline = undefined + userEdited = false + openedOnDraft = false await opts.applyCfg(deployedCfg) }, discard diff --git a/frontend/src/lib/scriptEditorSchema.ts b/frontend/src/lib/scriptEditorSchema.ts new file mode 100644 index 0000000000..7bd13b32a7 --- /dev/null +++ b/frontend/src/lib/scriptEditorSchema.ts @@ -0,0 +1,90 @@ +import type { Schema, SupportedLanguage } from '$lib/common' +import type { Script } from '$lib/gen' +import { inferArgs } from '$lib/infer' +import { emptySchema } from '$lib/utils' +import { parsePipelineAnnotations } from '$lib/components/assets/AssetGraph/parsePipelineAnnotations' + +/** + * A `// partitioned` pipeline script is materialized one slice at a time and + * receives the slice as a runtime `partition` arg (the cascade injects it in + * production). It isn't a code parameter, so schema inference doesn't see it — + * surface it in the test form so a partitioned script can be run manually. + */ +export function injectPartitionArg( + s: any, + a: Record | undefined, + l: string | undefined, + c: string +) { + try { + if (l !== 'duckdb' || !s?.properties) return + const part = parsePipelineAnnotations(c).partition + if (!part) return + // Date-based partition kinds render a date / datetime picker; a dynamic + // key is a free-form string. + const format = + part.kind === 'hourly' + ? 'date-time' + : part.kind === 'daily' || part.kind === 'weekly' || part.kind === 'monthly' + ? 'date' + : undefined + if (!s.properties['partition']) { + s.properties['partition'] = { + type: 'string', + ...(format ? { format } : {}), + // ISO output so partition keys sort lexicographically (the date + // picker defaults to dd-MM-yyyy otherwise). + ...(format === 'date' ? { dateFormat: 'yyyy-MM-dd' } : {}), + description: + part.kind === 'dynamic' + ? 'Partition key value to materialize.' + : `Partition (${part.kind}) to materialize.` + } + if (Array.isArray(s.order) && !s.order.includes('partition')) { + s.order = ['partition', ...s.order] + } + } + // Pre-fill the *test* arg with the current slice for date kinds — a + // convenience default, kept on the args (not baked into the schema, + // where it would persist to the deployed script and go stale). + if (format && a && (a['partition'] == null || a['partition'] === '')) { + const now = new Date() + a['partition'] = + format === 'date' ? now.toISOString().slice(0, 10) : now.toISOString().slice(0, 16) + } + } catch (e) {} +} + +/** + * The schema the script editor holds right after mounting on `content`, without + * any user input: `ScriptEditor.inferSchema` re-infers the main signature into + * the stored schema and injects the duckdb partition arg. A schema stored by + * another producer (a CLI push, an older parser) can lack keys the current + * parser emits, so the editor rewrites it on mount; a draft baseline taken from + * the raw stored schema would then never compare equal to the untouched editor. + * Returns a copy; a parse failure leaves it as far as inference got, which is + * also what the editor keeps. + */ +export async function schemaAsEditorMounts( + language: SupportedLanguage | undefined, + content: string, + schema: unknown, + kind: Script['kind'] | undefined +): Promise { + const copy: Schema = schema ? structuredClone(schema as Schema) : emptySchema() + const infer = async () => { + await inferArgs(language, content, copy, kind === 'preprocessor' ? 'preprocessor' : undefined) + injectPartitionArg(copy, undefined, language, content) + } + try { + await infer() + } catch { + // The editors retry a failed mount inference once (transient wasm init); + // the baseline has to land where that retry leaves the editor. + if (!content || !language) return copy + try { + await infer() + } catch {} + } + return copy +} diff --git a/frontend/src/lib/userDraft.svelte.ts b/frontend/src/lib/userDraft.svelte.ts index 24572ee1ab..57e911dd3c 100644 --- a/frontend/src/lib/userDraft.svelte.ts +++ b/frontend/src/lib/userDraft.svelte.ts @@ -219,7 +219,10 @@ const DRAFT_COMPARE_IGNORED_FIELDS = [ 'version_id', 'parent_version', 'is_draft', - 'assets' + 'assets', + // Fixed at creation and absent from the resource editor's draft shape, so + // it only ever shows up on the deployed side of a comparison. + 'resource_type' ] as const /** @@ -349,6 +352,32 @@ export const UserDraft = { void UserDraftDbSyncer.save({ workspace: ws, itemKind, path, value: null }) }, + /** + * Drop this key's in-memory cell and cached write WITHOUT touching the + * server. For callers that have already deleted the row by another route + * (a deploy) and only need the local mirror to stop answering `get`/`has` + * with a value that is gone. + * + * MUST be used instead of `remove` there. `remove` POSTs its own + * `value: null`, and that POST is debounced and carries whatever + * `last_sync` is left — which a preceding successful delete has already + * cleared. The backend treats a delete with no `last_sync` as + * unconditional, so the second POST lands ~1.5s later with nothing to + * compare against and removes a draft saved in the meantime. + */ + forgetLocal(itemKind: UserDraftItemKind, path: string, opts?: UserDraftOptions): void { + const ws = resolveWorkspace(opts) + const mk = mapKey(ws, itemKind, path) + const entry = entries.get(mk) + if (entry) { + // Same as `remove`: clear the cell so live observers see the delete, and arm + // `skipNextSync` so the mirror does not turn that write into a POST of its own. + entry.skipNextSync = true + entry.state.val = undefined + } + writtenCache.delete(mk) + }, + clear(itemKind: UserDraftItemKind, path: string, opts?: UserDraftOptions): void { UserDraft.discard(itemKind, path, undefined, opts) }, diff --git a/frontend/src/lib/userDraftEditGate.ts b/frontend/src/lib/userDraftEditGate.ts new file mode 100644 index 0000000000..e3057a73ee --- /dev/null +++ b/frontend/src/lib/userDraftEditGate.ts @@ -0,0 +1,66 @@ +import { onDestroy } from 'svelte' + +/** + * What kind of event opened the gate. + * + * - `value`: the user changed something — the event *is* the edit. + * - `precursor`: a gesture that usually precedes an edit. Needed because a + * custom component (a picker, a toggle built out of divs) writes its value + * through Svelte state and fires no native value event at all, so waiting for + * one would drop those edits. The cost is that a bare click counts too. + */ +export type UserInputKind = 'value' | 'precursor' + +/** Events that ARE an edit. `drop` and `paste` matter on their own: text + * dragged in from another application produces no pointer or key event in this + * document at all. */ +const VALUE_EVENTS = ['input', 'change', 'drop', 'paste'] as const +/** `click` is here for the controls that mutate state from a click handler and + * fire no native value event — ArgInput's "Add item", say. A mouse always sends + * `pointerdown` first, but an assistive technology can activate one with a + * trusted `click` alone, and that is a real edit with nothing else to catch it. */ +const PRECURSOR_EVENTS = ['pointerdown', 'keydown', 'click'] as const + +/** + * A draft is supposed to record what the USER changed, but an editor built + * from a schema writes into the value on its own: the form materializes a + * property the stored item never carried (an empty string, `false`, the first + * option of a required enum, a schema `default`) and deletes one a `showExpr` + * hides. So merely opening an item whose schema has moved on makes it diverge + * from the deployed value with nobody having touched it — a draft nobody asked + * for, cluttering the workspace. + * + * An editor guards against that by gating its draft on this: nothing the form + * settles on counts until the user has actually put something in. Callers + * decide what a gate covers (the resource editor keys it by workspace, since + * switching workspaces re-renders the form against a fresh value) and what + * gating means for them — suspending the autosave, absorbing the settled value + * into the deployed baseline, or both. + * + * Capture phase puts this ahead of the handler that writes the value, so a gate + * opened here is already open by the time the edit lands. Listening on the + * document rather than the editor's own subtree is deliberate: pickers and + * modals render in portals outside it, and missing a real edit would silently + * drop the user's work, while opening the gate too eagerly only costs the + * phantom draft that existed before. + * + * Registers for the lifetime of the calling component — call it during init. + */ +export function onUserInput(handle: (kind: UserInputKind) => void): void { + if (typeof document === 'undefined') return + const listeners: Array<[string, (e: Event) => void]> = [] + const register = (type: string, kind: UserInputKind) => { + const onEvent = (e: Event) => { + // A programmatic `dispatchEvent` is untrusted, which is what keeps the + // form's own settling from opening the gate it is gated by. + if (e.isTrusted) handle(kind) + } + document.addEventListener(type, onEvent, true) + listeners.push([type, onEvent]) + } + for (const type of VALUE_EVENTS) register(type, 'value') + for (const type of PRECURSOR_EVENTS) register(type, 'precursor') + onDestroy(() => { + for (const [type, onEvent] of listeners) document.removeEventListener(type, onEvent, true) + }) +} diff --git a/frontend/src/lib/userDraftPrune.test.ts b/frontend/src/lib/userDraftPrune.test.ts new file mode 100644 index 0000000000..0276598d45 --- /dev/null +++ b/frontend/src/lib/userDraftPrune.test.ts @@ -0,0 +1,243 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest' + +// The sweep DELETES drafts, so the guard that matters is which rows it picks. +// Stub the two collaborators it decides from — the draft listing and the +// per-kind diff — and assert on what it discards. +const listDrafts = vi.fn() +const getDraftDiffValues = vi.fn() +const updateDraft = vi.fn(async () => ({ status: 'saved', current_timestamp: 'x' })) + +vi.mock('./gen', () => ({ + DraftService: { + listDrafts: (...a: unknown[]) => listDrafts(...(a as [])), + updateDraft: (...a: unknown[]) => updateDraft(...(a as [])) + } +})) +// Only `getDraftDiffValues` is stubbed; `canDiffDraftKind` is the real one, so +// the kind filter is pinned against the actual overlay table. +vi.mock('./utils_draft_deploy', async (orig) => ({ + ...(await orig>()), + getDraftDiffValues: (...a: unknown[]) => getDraftDiffValues(...(a as [])) +})) +vi.mock('./localDraftHints.svelte', () => ({ setLocalDraftHint: vi.fn() })) +vi.mock('./workspaceDrafts.svelte', () => ({ invalidateWorkspaceDrafts: vi.fn() })) +const sendUserToast = vi.fn() +vi.mock('./toast', () => ({ sendUserToast: (...a: unknown[]) => sendUserToast(...(a as [])) })) + +// The sweep reads exactly one thing from the syncer — whether this tab is +// mid-write on the key — and writes nothing back to it. +let syncState = 'none' +vi.mock('./userDraftDbSyncer.svelte', () => ({ + UserDraftDbSyncer: { + getState: () => ({ + get state() { + return syncState + } + }) + } +})) + +let liveDraft = false +vi.mock('./userDraft.svelte', async (orig) => ({ + ...(await orig>()), + UserDraft: { has: () => liveDraft } +})) + +import { pruneMeaninglessDrafts } from './userDraftPrune' + +const row = (over: Record = {}) => ({ + kind: 'resource', + path: 'u/me/r', + draft_only: false, + legacy_draft: false, + mine: true, + can_write: true, + created_at: '2026-01-01T00:00:00Z', + ...over +}) +const diff = (over: Record = {}) => ({ + deployed: { value: { host: 'h' } }, + draft: { value: { host: 'h' } }, + hasDraft: true, + noDeployed: false, + ...over +}) +const discardedPaths = () => updateDraft.mock.calls.map((c: any[]) => c[0].path as string) + +beforeEach(() => { + localStorage.clear() + vi.clearAllMocks() + updateDraft.mockResolvedValue({ status: 'saved', current_timestamp: 'x' }) + syncState = 'none' + liveDraft = false +}) + +describe('pruneMeaninglessDrafts', () => { + it('discards a draft whose diff against the deployed value is empty', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(discardedPaths()).toEqual(['u/me/r']) + }) + + it('keeps a draft that carries a real change', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff({ draft: { value: { host: 'other' } } })) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('never touches a draft-only item — the draft is the whole item', async () => { + listDrafts.mockResolvedValue([row({ draft_only: true })]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('never touches another user’s row, or one it cannot write', async () => { + listDrafts.mockResolvedValue([row({ mine: false }), row({ path: 'u/me/b', can_write: false })]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('leaves a draft alone when its diff cannot be fetched, and retries it later', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockRejectedValue(new Error('boom')) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + // A row that could not be judged is not a row that carries changes, so + // the pass must stay open rather than strand it. + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(discardedPaths()).toEqual(['u/me/r']) + }) + + it('conditions the delete on the timestamp it judged, so a row that moved is spared', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + updateDraft.mockResolvedValue({ status: 'conflict', current_timestamp: 'newer' }) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).toHaveBeenCalledWith({ + workspace: 'main', + kind: 'resource', + path: 'u/me/r', + requestBody: { value: null, last_sync: '2026-01-01T00:00:00Z', force: false } + }) + // Refused, so nothing is reported as cleared — and the sweep leaves no + // state behind for the editor's own autosave to trip over. + expect(sendUserToast).not.toHaveBeenCalled() + }) + + it('does not keep retrying a row the server will never judge', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockRejectedValue({ status: 404 }) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + // Sealed: a 4xx is final, unlike the transient case above. + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('gives up after a bounded number of unresolved passes', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockRejectedValue(new Error('network')) + for (let i = 0; i < 3; i++) await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(listDrafts).toHaveBeenCalledTimes(3) + // Sealed on the third: an unresolvable row cannot re-list forever. + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(listDrafts).toHaveBeenCalledTimes(3) + }) + + it('skips a kind no diff can be computed for, and still seals', async () => { + listDrafts.mockResolvedValue([row({ kind: 'trigger_webhook', path: 'u/me/hook' })]) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(getDraftDiffValues).not.toHaveBeenCalled() + // Unjudgeable is permanent, not transient: leaving the pass open would + // re-run the sweep on every page load forever. + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('leaves alone a draft this tab is editing', async () => { + liveDraft = true + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('leaves alone a draft with a write queued or in flight', async () => { + syncState = 'pending' + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('does not count, or seal the pass on, a delete that failed to send', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + updateDraft.mockRejectedValueOnce(new Error('network')) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(sendUserToast).not.toHaveBeenCalled() + // The pass stayed open, so the draft left behind is retried. + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(discardedPaths()).toEqual(['u/me/r', 'u/me/r']) + }) + + it('never touches a legacy workspace-level row', async () => { + listDrafts.mockResolvedValue([row({ legacy_draft: true })]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('only sweeps the kinds whose editors are gated', async () => { + listDrafts.mockResolvedValue([ + row({ kind: 'script', path: 'u/me/s' }), + row({ kind: 'flow', path: 'u/me/f' }), + row({ kind: 'app', path: 'u/me/a' }), + row({ kind: 'trigger_schedule', path: 'u/me/sched' }), + row() + ]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(discardedPaths().sort()).toEqual(['u/me/r', 'u/me/sched']) + // The expensive payload fetches are never made for the ungated kinds. + expect(getDraftDiffValues).toHaveBeenCalledTimes(2) + }) + + it('leaves the pass open when a row was skipped as busy', async () => { + liveDraft = true + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + liveDraft = false + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(discardedPaths()).toEqual(['u/me/r']) + }) + + it('runs once per workspace and user', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).toHaveBeenCalledTimes(1) + await pruneMeaninglessDrafts('other', 'me@x.dev') + expect(updateDraft).toHaveBeenCalledTimes(2) + }) + + it('retries next mount when the listing failed', async () => { + listDrafts.mockRejectedValueOnce(new Error('offline')) + await pruneMeaninglessDrafts('main', 'me@x.dev') + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(discardedPaths()).toEqual(['u/me/r']) + }) +}) diff --git a/frontend/src/lib/userDraftPrune.ts b/frontend/src/lib/userDraftPrune.ts new file mode 100644 index 0000000000..465048d09d --- /dev/null +++ b/frontend/src/lib/userDraftPrune.ts @@ -0,0 +1,230 @@ +/** + * One-off sweep that drops drafts carrying no changes. + * + * `onUserInput` stops new ones from being written; the ones already stored need + * this pass to clear. Runs once per (workspace, user) per browser, after the + * localStorage→DB migration so anything it just uploaded is swept too. + * + * Scoped to the kinds whose editors that gate covers. A script, flow or app + * draft only ever came from an explicit edit, so there is no phantom to clear + * there — and `getDraftDiffValues` would fetch each one's full deployed payload + * at login to prove it. + * + * A draft is dropped only when the diff the user would be shown is empty: both + * sides come from `getDraftDiffValues`, the same canonicalization the diff + * drawer renders, compared with the same `draftValuesEqual` the editors use. + * Anything that can't be established is left alone — a `draft_only` item (no + * deployed counterpart, so discarding would destroy the item itself), a kind + * with no diff support, a failed fetch, and the legacy workspace-level rows, + * which belong to nobody and are admin-gated to migrate. + * + * Deleting is the dangerous half, and the equality behind it is always stale: + * it was read one round trip ago, and every candidate is read before any is + * deleted. So the delete is a compare-and-delete — `last_sync` is the + * timestamp the row was judged on, and the backend drops it only if nothing has + * written it since, whoever wrote it. + * + * It is sent straight to `DraftService`, NOT through `UserDraftDbSyncer`. That + * syncer exists to autosave an editor's own live value, and everything it does + * for that — parking the payload for the `pagehide` flush, debouncing, holding + * a per-tab `last_sync` baseline and conflict state — is a way for a one-shot + * delete to reach back into whatever the user is doing in the same tab. This + * sweep wants exactly one conditional request and no state afterwards. + */ + +import { DraftService } from './gen' +import type { UserDraftItemKind } from './gen' +import { sendUserToast } from './toast' +import { setLocalDraftHint } from './localDraftHints.svelte' +import { UserDraft, draftValuesEqual } from './userDraft.svelte' +import { UserDraftDbSyncer } from './userDraftDbSyncer.svelte' +import { canDiffDraftKind, getDraftDiffValues } from './utils_draft_deploy' +import { invalidateWorkspaceDrafts } from './workspaceDrafts.svelte' + +const SENTINEL_PREFIX = 'userdraft/pruned/v1/' + +/** A pass that leaves anything unresolved runs again next mount, which costs a + * listing plus an overlay GET per row. Bounded so no permanently-unresolvable + * row can make that repeat forever — whatever the reason it can't be judged. */ +const MAX_PASSES = 3 + +/** The editors whose forms are built from a schema, and so the only kinds that + * could have banked a draft nobody wrote — minus the ones no diff can be + * computed for, which would throw and keep the pass unsealed forever. */ +function isSweepableKind(kind: UserDraftItemKind): boolean { + return (kind === 'resource' || kind.startsWith('trigger_')) && canDiffDraftKind(kind) +} + +/** Overlay GETs are one round trip each and a cluttered workspace has dozens; + * a small window keeps the sweep off the critical path of a fresh login. */ +const CONCURRENCY = 4 + +/** Guards against the layout effect firing again before the sentinel lands. */ +const inFlight = new Set() + +type Candidate = { + kind: UserDraftItemKind + path: string + /** The row's `created_at` as listed — the baseline the delete is conditioned on. */ + createdAt: string +} + +const attemptsKey = (sentinel: string) => `${sentinel}:attempts` + +function readAttempts(sentinel: string): number { + try { + const n = Number(localStorage.getItem(attemptsKey(sentinel))) + return Number.isFinite(n) && n > 0 ? n : 0 + } catch { + return 0 + } +} + +/** Is this tab holding or writing this draft right now? */ +function busyLocally(workspace: string, kind: UserDraftItemKind, path: string): boolean { + if (UserDraft.has(kind, path, { workspace })) return true + return UserDraftDbSyncer.getState({ workspace, itemKind: kind, path }).state !== 'none' +} + +/** A 4xx is the server's final answer for this row — the item is gone, or the + * kind's overlay endpoint isn't served by this build (a feature-gated trigger + * on CE). Retrying it on every page load would never succeed. Anything else + * (network, 5xx) is worth another pass. 429 asks for exactly that. */ +function isPermanentlyUnjudgeable(e: unknown): boolean { + const status = (e as { status?: unknown })?.status + return typeof status === 'number' && status >= 400 && status < 500 && status !== 429 +} + +/** `undefined` when the diff could not be fetched and might be next time — + * distinct from `false`, so the caller can leave the pass open rather than + * strand a row it never judged. */ +async function carriesNoChanges( + workspace: string, + { kind, path }: Candidate +): Promise { + try { + const { deployed, draft, hasDraft, noDeployed } = await getDraftDiffValues( + kind, + path, + workspace + ) + // `hasDraft` false means the overlay had no draft row and the item's own + // value stood in for the draft side — there is nothing to discard, and the + // two sides would compare equal by construction. + if (!hasDraft || noDeployed) return false + return draftValuesEqual(draft, deployed) + } catch (e) { + return isPermanentlyUnjudgeable(e) ? false : undefined + } +} + +async function mapWithLimit( + items: T[], + limit: number, + fn: (item: T) => Promise +): Promise { + const out = new Array(items.length) + let next = 0 + await Promise.all( + Array.from({ length: Math.min(limit, items.length) }, async () => { + while (next < items.length) { + const i = next++ + out[i] = await fn(items[i]) + } + }) + ) + return out +} + +export async function pruneMeaninglessDrafts(workspace: string, userKey: string): Promise { + if (typeof localStorage === 'undefined') return + const sentinel = `${SENTINEL_PREFIX}${workspace}/${userKey}` + if (inFlight.has(sentinel)) return + try { + if (localStorage.getItem(sentinel)) return + } catch { + // Storage unavailable (private mode): the sweep can't record that it ran, + // and re-running it on every mount would cost an overlay GET per draft. + return + } + inFlight.add(sentinel) + try { + const rows = await DraftService.listDrafts({ workspace }) + // Anything left unresolved keeps the pass open: a row skipped as busy was + // never judged, and one whose delete failed is still there. Sealing on + // either would strand it. + let unresolved = 0 + const candidates: Candidate[] = rows + // `draft_only` rows ARE the item; `mine` / `can_write` are the same + // gate the discard endpoint enforces, so anything else would 403. + .filter((r) => !r.draft_only && r.mine && r.can_write) + .filter((r) => isSweepableKind(r.kind) && !r.legacy_draft) + .filter((r) => { + if (!busyLocally(workspace, r.kind, r.path)) return true + unresolved++ + return false + }) + .map((r) => ({ + kind: r.kind, + path: r.path, + createdAt: r.created_at + })) + + const empty: Candidate[] = [] + await mapWithLimit(candidates, CONCURRENCY, async (c) => { + const verdict = await carriesNoChanges(workspace, c) + if (verdict === undefined) unresolved++ + else if (verdict) empty.push(c) + }) + + let discarded = 0 + for (const c of empty) { + // Re-check: the reads above took a while, and the user may have opened + // this item in the meantime. + if (busyLocally(workspace, c.kind, c.path)) { + unresolved++ + continue + } + try { + const resp = await DraftService.updateDraft({ + workspace, + kind: c.kind, + path: c.path, + requestBody: { value: null, last_sync: c.createdAt, force: false } + }) + // `conflict` means the row moved past the timestamp we judged it on, + // so it is no longer the empty draft we decided to drop. + if (resp.status === 'saved') { + setLocalDraftHint(workspace, c.kind, c.path, false) + discarded++ + } + } catch { + unresolved++ + } + } + if (discarded > 0) { + invalidateWorkspaceDrafts(workspace) + sendUserToast(`Cleared ${discarded} draft${discarded > 1 ? 's' : ''} that carried no changes`) + } + // Seal once nothing is left hanging, or once we have tried enough times + // that whatever is hanging is not going to resolve. + const attempts = readAttempts(sentinel) + 1 + if (unresolved === 0 || attempts >= MAX_PASSES) { + try { + localStorage.setItem(sentinel, new Date().toISOString()) + localStorage.removeItem(attemptsKey(sentinel)) + } catch { + // Nothing to do — the pass is idempotent, it just runs again. + } + } else { + try { + localStorage.setItem(attemptsKey(sentinel), String(attempts)) + } catch {} + } + } catch { + // Fire-and-forget from the layout: a workspace whose draft list can't be + // read is left exactly as it was. + } finally { + inFlight.delete(sentinel) + } +} diff --git a/frontend/src/lib/utils_draft_deploy.test.ts b/frontend/src/lib/utils_draft_deploy.test.ts index 9bed95917b..4495155f24 100644 --- a/frontend/src/lib/utils_draft_deploy.test.ts +++ b/frontend/src/lib/utils_draft_deploy.test.ts @@ -7,7 +7,7 @@ vi.mock('$lib/gen', () => ({ DraftService: { deleteDraft: vi.fn() }, AppService: {}, VariableService: {}, - ResourceService: {}, + ResourceService: { getResource: vi.fn(), updateResource: vi.fn(), createResource: vi.fn() }, ScheduleService: {}, HttpTriggerService: {}, WebsocketTriggerService: {}, @@ -21,7 +21,9 @@ vi.mock('$lib/gen', () => ({ AzureTriggerService: {}, EmailTriggerService: {} })) -vi.mock('$lib/userDraftDbSyncer.svelte', () => ({ UserDraftDbSyncer: { save: vi.fn() } })) +vi.mock('$lib/userDraftDbSyncer.svelte', () => ({ + UserDraftDbSyncer: { save: vi.fn(), recordRemoteSync: vi.fn() } +})) vi.mock('$lib/workspaceDrafts.svelte', () => ({ invalidateWorkspaceDrafts: vi.fn() })) vi.mock('$lib/workspaceComparison', () => ({ invalidateWorkspaceComparison: vi.fn() })) vi.mock('$lib/localDraftHints.svelte', () => ({ setLocalDraftHint: vi.fn() })) @@ -30,7 +32,8 @@ vi.mock('$lib/components/raw_apps/utils', () => ({ canonicalRawAppDiffValue: vi. vi.mock('$lib/appDiffSides', () => ({ classicAppDraftParts: vi.fn() })) vi.mock('$lib/utils_deployable', () => ({ TRIGGER_RUNTIME_IGNORE: [] })) -import { ScriptService, FlowService } from '$lib/gen' +import { ScriptService, FlowService, ResourceService } from '$lib/gen' +import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' // draftBaseIsStale compares a draft's base pointer against the deployed head // of the item it was fetched with (`get_draft=true`). Shared by CompareDrafts @@ -124,3 +127,65 @@ describe('deployDraft preserves on_behalf_of', () => { ) }) }) + +// The resource branch reads the item again when the deploy lands, and falls back to the deployed +// row when the draft has gone. That row keeps its value under `value` and carries no `args` at +// all, so reading it as a draft (`value: d.args ?? {}`) would replace a live resource with `{}`. +describe('deployDraft: resource with no draft', () => { + beforeEach(() => vi.clearAllMocks()) + + it('writes nothing rather than `{}` over the deployed value', async () => { + vi.mocked(ResourceService.getResource).mockResolvedValueOnce({ + path: 'f/support/triage_agent', + resource_type: 'ai_agent', + value: { system_prompt: 'deployed' } + } as any) + + // `noop` is what lets a caller deploying one specific draft tell "nothing to promote" apart + // from "deployed", instead of reporting an agent as deployed that was never written. + expect(await deployDraft('resource', 'f/support/triage_agent', 'ws')).toEqual({ + success: true, + noop: true + }) + expect(ResourceService.updateResource).not.toHaveBeenCalled() + expect(ResourceService.createResource).not.toHaveBeenCalled() + // Nor does it touch the draft row. There was none of this user's to delete, so the only row + // the cleanup could reach is one written after the read: an edit destroyed without ever + // having been deployed. Clearing the baseline would be the same bug by another route, since + // a delete with no baseline is the unconditional one. + expect(UserDraftDbSyncer.save).not.toHaveBeenCalled() + expect(UserDraftDbSyncer.recordRemoteSync).not.toHaveBeenCalled() + }) + + it('still deploys normally when the draft is there, and keys the cleanup to the row it read', async () => { + vi.mocked(ResourceService.getResource).mockResolvedValueOnce({ + path: 'f/support/triage_agent', + resource_type: 'ai_agent', + value: { system_prompt: 'deployed' }, + draft_saved_at: '2026-01-01T00:00:00Z', + draft: { path: 'f/support/triage_agent', args: { system_prompt: 'drafted' } } + } as any) + + expect(await deployDraft('resource', 'f/support/triage_agent', 'ws')).toEqual({ success: true }) + expect(ResourceService.updateResource).toHaveBeenCalledWith( + expect.objectContaining({ + requestBody: expect.objectContaining({ value: { system_prompt: 'drafted' } }) + }) + ) + // The draft delete that follows is conditional on this baseline. With no baseline the backend + // deletes unconditionally, destroying a draft saved between the read and the delete without + // ever having deployed it, so the timestamp has to be the one from the row just promoted. + expect(UserDraftDbSyncer.recordRemoteSync).toHaveBeenCalledWith( + { workspace: 'ws', itemKind: 'resource', path: 'f/support/triage_agent' }, + '2026-01-01T00:00:00Z' + ) + expect(UserDraftDbSyncer.save).toHaveBeenCalledWith( + expect.objectContaining({ path: 'f/support/triage_agent', value: null, immediate: true }) + ) + // Order is the whole point: a delete issued before the seed carries whatever baseline the tab + // happened to hold, which for a caller that only read through a listing is none at all. + expect(vi.mocked(UserDraftDbSyncer.recordRemoteSync).mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(UserDraftDbSyncer.save).mock.invocationCallOrder[0] + ) + }) +}) diff --git a/frontend/src/lib/utils_draft_deploy.ts b/frontend/src/lib/utils_draft_deploy.ts index b322d6adc4..0d121d0627 100644 --- a/frontend/src/lib/utils_draft_deploy.ts +++ b/frontend/src/lib/utils_draft_deploy.ts @@ -95,6 +95,19 @@ const OVERLAY_GETTERS: Partial< EmailTriggerService.getEmailTrigger({ workspace, path, getDraft: true }) } +/** Whether `getDraftDiffValues` can produce a diff for this kind at all. The + * script/flow/app family is handled inline; every other kind needs an overlay + * getter and throws without one — several trigger kinds have none. */ +export function canDiffDraftKind(kind: DraftKind): boolean { + return ( + kind === 'script' || + kind === 'flow' || + kind === 'app' || + kind === 'raw_app' || + OVERLAY_GETTERS[kind] !== undefined + ) +} + /** Strip the per-user draft-overlay metadata, returning `{deployed, draft}`. */ function splitOverlay(r: any): { deployed: any @@ -435,8 +448,13 @@ export async function deployDraft( path: string, workspace: string, opts: { draftOnly?: boolean; rawApp?: boolean; deploymentMessage?: string } = {} -): Promise { +): Promise { const { draftOnly = false, rawApp = false, deploymentMessage } = opts + // Set when the branch found nothing to promote and wrote nothing. Success, because the item is + // already at the value a deploy would have left it at and its stale draft state still wants + // clearing — but a caller deploying one specific draft it showed the user has to be able to tell + // that apart from having deployed it. + let noop = false try { if (kind === 'raw_app' || (kind === 'app' && rawApp)) { // Raw apps bundle their source files and deploy via the raw-app @@ -579,10 +597,34 @@ export async function deployDraft( } void deployed } else if (kind === 'resource') { - const { deployed, draft: d } = splitOverlay(await OVERLAY_GETTERS.resource!(workspace, path)) + const overlay = await OVERLAY_GETTERS.resource!(workspace, path) + // Adopt the row this promote is based on as the baseline for the delete below. Without one + // the backend deletes unconditionally, so a draft saved between that read and the delete is + // destroyed having never been deployed — a caller that only ever read through a listing has + // no baseline of its own to supply. With it the delete is refused instead and the newer + // draft survives, which is the recoverable outcome of the two. Only ever seeded, never + // cleared: passing no timestamp drops whatever baseline the tab already held, which would + // turn that same delete back into an unconditional one. + if (overlay?.draft_saved_at) { + UserDraftDbSyncer.recordRemoteSync( + { workspace, itemKind: kind, path }, + overlay.draft_saved_at + ) + } + const { deployed, draft: d, hasDraft } = splitOverlay(overlay) // ResourceEditor's `ResourceState` draft shape: // { path, description, args, resource_type?, labels?, wsSpecific } - if (draftOnly) { + // The deployed row is a different shape (`value`, `ws_specific`, no `args` at all), and + // `splitOverlay` hands it back as the draft side when the draft row has gone — deployed or + // discarded from another tab between the listing and this click. Reading it as a draft is + // what made `value: d.args ?? {}` replace a live resource with `{}`. Nothing to promote + // then, so write nothing and fall through to the cleanup below, which clears the stale + // local draft hint and the drafts listing. The item is already at the value a successful + // deploy would have left it at, so this reports success rather than an error, matching + // what the other kinds end up doing when their own draft is gone. + if (!hasDraft) { + noop = true + } else if (draftOnly) { await ResourceService.createResource({ workspace, requestBody: { @@ -633,8 +675,8 @@ export async function deployDraft( return { success: false, error: `Deploy not supported for draft kind ${kind}` } } // Delete the draft at its STORAGE path (the row key, = the `path` arg). - // Two reasons it must happen here for every kind, mirroring the editors' - // post-deploy `discardDraftAfterDeploy(draftPath)`: + // Two reasons it must happen here for every kind that promoted something, + // mirroring the editors' post-deploy `discardDraftAfterDeploy(draftPath)`: // - Drawer kinds (variable / resource / triggers) aren't deleted by // their create/update endpoints at all. // - script/flow/app/raw_app DO delete server-side, but only the draft at @@ -642,13 +684,18 @@ export async function deployDraft( // synthetic `u/{user}/draft_{uuid}` storage path ≠ `d.path`, so its // draft row survives the deploy and keeps listing. Deleting the // storage-path draft removes it (a no-op when the server already did). - await UserDraftDbSyncer.save({ - workspace, - itemKind: kind, - path, - value: null, - immediate: true - }) + // Skipped when nothing was promoted: the read that set `noop` found no draft of this user's to + // delete, so the only row this could reach is one written after it — destroying an edit that + // was never deployed, and never even listed. + if (!noop) { + await UserDraftDbSyncer.save({ + workspace, + itemKind: kind, + path, + value: null, + immediate: true + }) + } // Mutated the workspace's Server Drafts — refresh every mounted reader. invalidateWorkspaceDrafts(workspace) // The DEPLOYED state moved: cached fork comparisons involving this @@ -659,7 +706,7 @@ export async function deployDraft( // so the syncer-owned hint won't auto-clear — clear it explicitly. // (Idempotent: the drawer-kind delete above already cleared it.) setLocalDraftHint(workspace, kind, path, false) - return { success: true } + return noop ? { success: true, noop: true } : { success: true } } catch (e: any) { return { success: false, error: e?.body ?? e?.message ?? String(e) } } diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index a20c61f2ff..e82d328c76 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -74,6 +74,7 @@ import { createUsageResources, registerUsageResources } from '$lib/usage.svelte' import { purgeLegacyUserDrafts } from '$lib/userDraftLegacyMigration' import { migrateUserDraftsToDb } from '$lib/userDraftDbMigration' + import { pruneMeaninglessDrafts } from '$lib/userDraftPrune' import DraftMigrationErrorModal from '$lib/components/DraftMigrationErrorModal.svelte' import { onDestroy, setContext, untrack } from 'svelte' import { base } from '$app/paths' @@ -792,12 +793,16 @@ // drafts). `migrateUserDraftsToDb` then pushes the workspace-scoped // `userdraft/w/{ws}/{kind}/{path}` keys — written by the editor with the // correct workspace — onto the server-side draft table, clearing LS on - // success. + // success. `pruneMeaninglessDrafts` then clears the drafts an older, stricter + // comparison saved for changes nobody made; it runs after the upload so the + // entries that just landed are swept in the same pass. $effect(() => { - if ($workspaceStore && $userStore) { + const ws = $workspaceStore + const email = $userStore?.email + if (ws && email) { untrack(() => { purgeLegacyUserDrafts() - void migrateUserDraftsToDb() + void migrateUserDraftsToDb().then(() => pruneMeaninglessDrafts(ws, email)) }) } }) diff --git a/frontend/src/routes/(root)/(logged)/apps/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/apps/get/[...path]/+page.svelte index 0af3cdeded..4ff8670cda 100644 --- a/frontend/src/routes/(root)/(logged)/apps/get/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/apps/get/[...path]/+page.svelte @@ -1,11 +1,11 @@