starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-08 08:04:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into dbt-column-lineage-index

Browse Source
This commit is contained in:
Ruben Fiszel
2026-09-07 22:12:06 +02:00
parent 484fdb3de3 a9d42b489f
commit 7dc0cd521c
127 changed files with 6230 additions and 1120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.release-please-manifest.json
+1 -1
View File
@@ -1,3 +1,3 @@
{
".": "1.804.0"
".": "1.805.0"
}
CHANGELOG.md
+25
View File
@@ -1,5 +1,30 @@
# Changelog
## [1.805.0](https://github.com/windmill-labs/windmill/compare/v1.804.0...v1.805.0) (2026-09-07)
### Features
* **git-sync:** sync extra_perms for variables ([#11004](https://github.com/windmill-labs/windmill/issues/11004)) ([ee9e550](https://github.com/windmill-labs/windmill/commit/ee9e550a484fda286eeab43b7db5f314b8b2d0d9))
* go to referenced row from foreign-keyed cells in the database manager ([#10998](https://github.com/windmill-labs/windmill/issues/10998)) ([e2b63d1](https://github.com/windmill-labs/windmill/commit/e2b63d177ae4e5c980cb5da34154540c90771b63))
* let `// materialize` declare a `dbt://` warehouse-relation write ([#10978](https://github.com/windmill-labs/windmill/issues/10978)) ([c6e0302](https://github.com/windmill-labs/windmill/commit/c6e0302d7c1c60147f19d55a3923be8b1aa99c9c))
* report resource type picks to the hub and rank pickers by popularity ([#10982](https://github.com/windmill-labs/windmill/issues/10982)) ([48a5615](https://github.com/windmill-labs/windmill/commit/48a56158c135c3b13a02f73b7b8438bc691f85b4))
* run a linked AI agent's draft when testing a flow, and offer to deploy it ([#10993](https://github.com/windmill-labs/windmill/issues/10993)) ([7feaf61](https://github.com/windmill-labs/windmill/commit/7feaf619cf0ec021d66be14ef535cf2149bec58a))
* show the new-tab icon on a chat path pill while the modifier is held ([#10976](https://github.com/windmill-labs/windmill/issues/10976)) ([5da4ea4](https://github.com/windmill-labs/windmill/commit/5da4ea43fbd01e43aa14e75dc597d7ce5d8797ab))
### Bug Fixes
* **cli:** keep permissioned_as on single-item push, as sync push does ([#11000](https://github.com/windmill-labs/windmill/issues/11000)) ([5f3f99b](https://github.com/windmill-labs/windmill/commit/5f3f99ba6915b7c5df663a30b35f4cd02050e728))
* **cli:** say which workspace id is targeted, and when wmill.yaml is bypassed ([#11006](https://github.com/windmill-labs/windmill/issues/11006)) ([7643e9b](https://github.com/windmill-labs/windmill/commit/7643e9bd77c56f72596b8dca50801baf58984198))
* **frontend:** no phantom draft when opening a CLI-pushed script ([#10997](https://github.com/windmill-labs/windmill/issues/10997)) ([1be390a](https://github.com/windmill-labs/windmill/commit/1be390aa878e15a58f530f3a878e8f9caeb89c43))
* **frontend:** stop hover flicker on asset nodes shared with an overflow popover ([#10996](https://github.com/windmill-labs/windmill/issues/10996)) ([519a5c8](https://github.com/windmill-labs/windmill/commit/519a5c8bc70b44a7417e83c26c7b9c58b2c4fb9c))
* let a draft-only schedule, trigger or resource be deleted ([#11010](https://github.com/windmill-labs/windmill/issues/11010)) ([8d0f475](https://github.com/windmill-labs/windmill/commit/8d0f4754e4e0c78696ee0c97ff2de3016ece3bac))
* point the app viewer's edit button at the editor for the app's kind ([#11009](https://github.com/windmill-labs/windmill/issues/11009)) ([8f553ea](https://github.com/windmill-labs/windmill/commit/8f553eab353103fd8a28a00532e1766f133590de))
* seed runs page filter defaults through the url so they survive sync ([#11005](https://github.com/windmill-labs/windmill/issues/11005)) ([f381acd](https://github.com/windmill-labs/windmill/commit/f381acdb37f66f5e272bc37938e69f734987d53f))
* stop an untouched item's form from saving a draft nobody wrote ([#10964](https://github.com/windmill-labs/windmill/issues/10964)) ([c3f7f8a](https://github.com/windmill-labs/windmill/commit/c3f7f8a45830fb548aa628ebf6e2b6c95c6de67f))
* write and read python job files as utf-8, not the platform locale ([#10994](https://github.com/windmill-labs/windmill/issues/10994)) ([670404f](https://github.com/windmill-labs/windmill/commit/670404ffe27fedc3858b46b0c6b3312fbe175e13))
## [1.804.0](https://github.com/windmill-labs/windmill/compare/v1.803.0...v1.804.0) (2026-09-05)
backend/.sqlx/query-1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6.json
Generated
+12
View File
@@ -0,0 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace_settings\n SET dbt_warehouses = '{\"main\": {\"resource_path\": \"u/test-user/wh\"}}'::jsonb\n WHERE workspace_id = 'test-workspace'",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6"
}
backend/.sqlx/query-1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704.json
Generated
+12
View File
@@ -0,0 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind)\n VALUES ('test-workspace', 'main/analytics/marts', 'dbt', 'w', 'u/test-user/project',\n 'script')",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704"
}
backend/.sqlx/query-33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db.json
Generated
+20
View File
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "SELECT path FROM asset WHERE workspace_id = 'test-workspace' AND kind = 'dbt' AND usage_path = 'u/test-user/ingest' AND usage_access_type = 'w'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
}
],
"parameters": {
"Left": []
},
"nullable": [
false
]
},
"hash": "33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db"
}
backend/.sqlx/query-49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "WITH producer AS (\n SELECT 'dbt://' || a.path AS trigger_ref, a.usage_path, s.language\n FROM asset a\n JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path\n AND s.archived = false AND s.deleted = false\n WHERE a.workspace_id = $1 AND a.kind = 'dbt'\n AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw')\n AND 'dbt://' || a.path = ANY($2)\n )\n SELECT DISTINCT st.trigger_ref || ' → ' || st.runnable_path AS \"edge!\"\n FROM script_trigger st\n WHERE st.workspace_id = $1 AND st.trigger_kind = 'asset'\n AND st.trigger_ref = ANY($2)\n AND EXISTS (SELECT 1 FROM producer p\n WHERE p.trigger_ref = st.trigger_ref\n AND p.usage_path <> st.runnable_path\n AND p.language = 'dbt')\n AND NOT EXISTS (SELECT 1 FROM producer p\n WHERE p.trigger_ref = st.trigger_ref\n AND p.usage_path <> st.runnable_path\n AND p.language <> 'dbt')\n ORDER BY 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "edge!",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text",
"TextArray"
]
},
"nullable": [
null
]
},
"hash": "49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e"
}
backend/.sqlx/query-6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7.json
Generated
+12
View File
@@ -0,0 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by,\n language)\n VALUES ('test-workspace', 1, 'u/test-user/project', '', '', '', 'test-user', 'dbt')",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7"
}
backend/.sqlx/query-69c9ac0e5761c00a9e9a7ff96431e5ebfce96786448fb5273d49f13978df65c0.json
Generated
+28
View File
@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "SELECT resource_type, count(*) as \"count!\" FROM resource WHERE workspace_id = $1 GROUP BY resource_type",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "resource_type",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "count!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
null
]
},
"hash": "69c9ac0e5761c00a9e9a7ff96431e5ebfce96786448fb5273d49f13978df65c0"
}
backend/.sqlx/query-ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf.json
Generated
+20
View File
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "SELECT trigger_ref FROM script_trigger WHERE workspace_id = 'test-workspace' AND runnable_path = 'u/test-user/consumer' AND trigger_kind = 'asset'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "trigger_ref",
"type_info": "Text"
}
],
"parameters": {
"Left": []
},
"nullable": [
false
]
},
"hash": "ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf"
}
backend/.sqlx/query-b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946.json
Generated
+63
View File
@@ -0,0 +1,63 @@
{
"db_name": "PostgreSQL",
"query": "SELECT s.path AS \"path!\", s.language AS \"language!: ScriptLang\"\n FROM asset a\n JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path\n AND s.archived = false AND s.deleted = false\n WHERE a.workspace_id = $1 AND a.kind = 'dbt' AND a.path = $2\n AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw')\n AND a.usage_path <> ALL($3)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path!",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "language!: ScriptLang",
"type_info": {
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb",
"ruby",
"rlang",
"dbt"
]
}
}
}
}
],
"parameters": {
"Left": [
"Text",
"Text",
"TextArray"
]
},
"nullable": [
false,
false
]
},
"hash": "b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946"
}
backend/.sqlx/query-e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570.json
Generated
+12
View File
@@ -0,0 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind)\n VALUES ('test-workspace', 'main/analytics/orders', 'dbt', 'w', 'u/test-user/project',\n 'script')",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570"
}
backend/Cargo.lock
Generated
+101 -101
View File
@@ -2763,18 +2763,18 @@ dependencies = [
[[package]]
name = "crossbeam-channel"
version = "0.5.16"
version = "0.5.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e"
checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.7"
version = "0.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb"
checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
@@ -2782,27 +2782,27 @@ dependencies = [
[[package]]
name = "crossbeam-epoch"
version = "0.9.20"
version = "0.9.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-queue"
version = "0.3.13"
version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26"
checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.22"
version = "0.8.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"
checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
[[package]]
name = "crossterm_winapi"
@@ -6640,9 +6640,9 @@ dependencies = [
[[package]]
name = "io-uring"
version = "0.7.14"
version = "0.7.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d64d8ca234d152948ceaede1f419b6a83983a5ecccaac05fb337a809c96d3aa6"
checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb"
dependencies = [
"bitflags 2.13.1",
"cfg-if",
@@ -6664,9 +6664,9 @@ dependencies = [
[[package]]
name = "ipnet"
version = "2.12.1"
version = "2.12.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78"
checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0"
[[package]]
name = "ipnetwork"
@@ -9077,9 +9077,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
[[package]]
name = "pest"
version = "2.9.0"
version = "2.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a07a60cc7a4d00c91f95c685609d1d2f79050e6804b70ebedd7650f0b839bcf"
checksum = "6d45aeb61b4bf818e12d4205f2466f8c4748f85f4fce0146d1c03d69d753f0ad"
dependencies = [
"memchr",
"ucd-trie",
@@ -9087,9 +9087,9 @@ dependencies = [
[[package]]
name = "pest_derive"
version = "2.9.0"
version = "2.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3a83744a5c8455b8b3e0dc5031362780a347c878bdd11584d1a8984228cc88d"
checksum = "89cc5a242e25ed4e7704d0be240f2cfbe20a8c27e7e252d94835be93d92dc39f"
dependencies = [
"pest",
"pest_generator",
@@ -9097,9 +9097,9 @@ dependencies = [
[[package]]
name = "pest_generator"
version = "2.9.0"
version = "2.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0cd3451aa3de60d4b9a1e736885e4dea6b31617598026f12256ad566d63304a"
checksum = "7abf21475cc3820fe4b2ca2dc2142902f67a02189f3b5b3a229f4febc01a43e5"
dependencies = [
"pest",
"pest_meta",
@@ -9110,9 +9110,9 @@ dependencies = [
[[package]]
name = "pest_meta"
version = "2.9.0"
version = "2.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e04d3a0849e241d7dfce834c83b1c5edc8622009e8dd51a12ba1927c32f05496"
checksum = "adba4db388f687393c18c51348d44a41d870ca9df71a2c98172ea3035dc6936e"
dependencies = [
"pest",
]
@@ -14747,7 +14747,7 @@ dependencies = [
[[package]]
name = "windmill"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"async-nats",
@@ -14835,7 +14835,7 @@ dependencies = [
[[package]]
name = "windmill-ai"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"async-stream",
"async-trait",
@@ -14868,7 +14868,7 @@ dependencies = [
[[package]]
name = "windmill-alerting"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14881,7 +14881,7 @@ dependencies = [
[[package]]
name = "windmill-api"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"argon2",
@@ -15021,7 +15021,7 @@ dependencies = [
[[package]]
name = "windmill-api-agent-workers"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15044,7 +15044,7 @@ dependencies = [
[[package]]
name = "windmill-api-assets"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15061,7 +15061,7 @@ dependencies = [
[[package]]
name = "windmill-api-auth"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15087,7 +15087,7 @@ dependencies = [
[[package]]
name = "windmill-api-client"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"reqwest 0.12.28",
"serde",
@@ -15097,7 +15097,7 @@ dependencies = [
[[package]]
name = "windmill-api-configs"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15114,7 +15114,7 @@ dependencies = [
[[package]]
name = "windmill-api-debug"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"axum 0.8.9",
"base64 0.22.1",
@@ -15136,7 +15136,7 @@ dependencies = [
[[package]]
name = "windmill-api-embeddings"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15159,7 +15159,7 @@ dependencies = [
[[package]]
name = "windmill-api-flow-conversations"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15175,7 +15175,7 @@ dependencies = [
[[package]]
name = "windmill-api-flows"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15197,7 +15197,7 @@ dependencies = [
[[package]]
name = "windmill-api-groups"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15218,7 +15218,7 @@ dependencies = [
[[package]]
name = "windmill-api-inputs"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15232,7 +15232,7 @@ dependencies = [
[[package]]
name = "windmill-api-integration-tests"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"async-nats",
@@ -15267,7 +15267,7 @@ dependencies = [
[[package]]
name = "windmill-api-jobs"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15292,7 +15292,7 @@ dependencies = [
[[package]]
name = "windmill-api-npm-proxy"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15320,7 +15320,7 @@ dependencies = [
[[package]]
name = "windmill-api-openapi"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15342,7 +15342,7 @@ dependencies = [
[[package]]
name = "windmill-api-schedule"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15362,7 +15362,7 @@ dependencies = [
[[package]]
name = "windmill-api-scripts"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15400,7 +15400,7 @@ dependencies = [
[[package]]
name = "windmill-api-settings"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15428,7 +15428,7 @@ dependencies = [
[[package]]
name = "windmill-api-sse"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"lazy_static",
"serde",
@@ -15440,7 +15440,7 @@ dependencies = [
[[package]]
name = "windmill-api-users"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"argon2",
"axum 0.8.9",
@@ -15464,7 +15464,7 @@ dependencies = [
[[package]]
name = "windmill-api-workers"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15478,7 +15478,7 @@ dependencies = [
[[package]]
name = "windmill-api-workspaces"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15513,7 +15513,7 @@ dependencies = [
[[package]]
name = "windmill-audit"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"chrono",
"lazy_static",
@@ -15527,7 +15527,7 @@ dependencies = [
[[package]]
name = "windmill-autoscaling"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15546,7 +15546,7 @@ dependencies = [
[[package]]
name = "windmill-common"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"aes-gcm",
"aho-corasick",
@@ -15652,7 +15652,7 @@ dependencies = [
[[package]]
name = "windmill-dep-map"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"chrono",
"futures",
@@ -15672,7 +15672,7 @@ dependencies = [
[[package]]
name = "windmill-git-sync"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"regex",
"serde",
@@ -15687,7 +15687,7 @@ dependencies = [
[[package]]
name = "windmill-indexer"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"astral-tokio-tar",
@@ -15714,7 +15714,7 @@ dependencies = [
[[package]]
name = "windmill-jseval"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"futures",
@@ -15731,7 +15731,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"itertools 0.14.0",
"lazy_static",
@@ -15747,7 +15747,7 @@ dependencies = [
[[package]]
name = "windmill-mcp"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15768,7 +15768,7 @@ dependencies = [
[[package]]
name = "windmill-native-triggers"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15799,7 +15799,7 @@ dependencies = [
[[package]]
name = "windmill-oauth"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"arc-swap",
@@ -15824,7 +15824,7 @@ dependencies = [
[[package]]
name = "windmill-object-store"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"async-stream",
@@ -15858,7 +15858,7 @@ dependencies = [
[[package]]
name = "windmill-operator"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"futures",
@@ -15876,7 +15876,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"convert_case 0.6.0",
"serde",
@@ -15885,7 +15885,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -15897,7 +15897,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"serde_json",
@@ -15909,7 +15909,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"gosyn",
@@ -15921,7 +15921,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -15933,7 +15933,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"serde_json",
@@ -15945,7 +15945,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"nu-parser",
@@ -15956,7 +15956,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -15967,7 +15967,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -15979,7 +15979,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-asset"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -15990,7 +15990,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -16012,7 +16012,7 @@ dependencies = [
[[package]]
name = "windmill-parser-r"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"serde_json",
@@ -16024,7 +16024,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ruby"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16038,7 +16038,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"convert_case 0.6.0",
@@ -16055,7 +16055,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16068,7 +16068,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql-asset"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"serde",
@@ -16080,7 +16080,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16098,7 +16098,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts-asset"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"serde-wasm-bindgen",
@@ -16114,7 +16114,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wac"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -16130,7 +16130,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16144,7 +16144,7 @@ dependencies = [
[[package]]
name = "windmill-queue"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -16183,7 +16183,7 @@ dependencies = [
[[package]]
name = "windmill-runtime-nativets"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"const_format",
@@ -16223,7 +16223,7 @@ dependencies = [
[[package]]
name = "windmill-sql-datatype-parser-wasm"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"getrandom 0.3.4",
"wasm-bindgen",
@@ -16234,7 +16234,7 @@ dependencies = [
[[package]]
name = "windmill-store"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -16269,7 +16269,7 @@ dependencies = [
[[package]]
name = "windmill-test-utils"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16293,7 +16293,7 @@ dependencies = [
[[package]]
name = "windmill-trigger"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16326,7 +16326,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-amqp"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16353,7 +16353,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-azure"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16386,7 +16386,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-email"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16406,7 +16406,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-gcp"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16440,7 +16440,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-http"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16476,7 +16476,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-kafka"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16499,7 +16499,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-mqtt"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16523,7 +16523,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-nats"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"async-nats",
@@ -16547,7 +16547,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-postgres"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16582,7 +16582,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-sqs"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16610,7 +16610,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-websocket"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16635,7 +16635,7 @@ dependencies = [
[[package]]
name = "windmill-types"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"bitflags 2.13.1",
@@ -16654,7 +16654,7 @@ dependencies = [
[[package]]
name = "windmill-worker"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"async-once-cell",
@@ -16772,7 +16772,7 @@ dependencies = [
[[package]]
name = "windmill-worker-volumes"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"bytes",
"futures",
backend/Cargo.toml
+2 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.804.0"
version = "1.805.0"
authors.workspace = true
edition.workspace = true
@@ -88,7 +88,7 @@ members = [
exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"]
[workspace.package]
version = "1.804.0"
version = "1.805.0"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
backend/parsers/windmill-parser-wasm/Cargo.lock
Generated
+24 -24
View File
@@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windmill-common"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"aho-corasick",
"anyhow",
@@ -6274,7 +6274,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"proc-macro2",
"quote",
@@ -6286,7 +6286,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"convert_case",
"serde",
@@ -6295,7 +6295,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6307,7 +6307,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"serde_json",
@@ -6319,7 +6319,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"gosyn",
@@ -6331,7 +6331,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6343,7 +6343,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"serde_json",
@@ -6355,7 +6355,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"nu-parser",
@@ -6366,7 +6366,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -6377,7 +6377,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -6389,7 +6389,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-asset"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -6400,7 +6400,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -6422,7 +6422,7 @@ dependencies = [
[[package]]
name = "windmill-parser-r"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"serde_json",
@@ -6434,7 +6434,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ruby"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6448,7 +6448,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"convert_case",
@@ -6465,7 +6465,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6478,7 +6478,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql-asset"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"serde",
@@ -6490,7 +6490,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6508,7 +6508,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts-asset"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"serde-wasm-bindgen",
@@ -6524,7 +6524,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wac"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -6540,7 +6540,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wasm"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"getrandom 0.2.17",
@@ -6572,7 +6572,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6586,7 +6586,7 @@ dependencies = [
[[package]]
name = "windmill-types"
version = "1.804.0"
version = "1.805.0"
dependencies = [
"anyhow",
"bitflags",
backend/parsers/windmill-parser-wasm/Cargo.toml
+1 -1
View File
@@ -12,7 +12,7 @@ resolver = "2"
members = ["."]
[workspace.package]
version = "1.804.0"
version = "1.805.0"
edition = "2021"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
backend/parsers/windmill-parser/src/asset_parser.rs
+44 -25
View File
@@ -29,10 +29,10 @@ pub enum AssetKind {
Ducklake,
DataTable,
Volume,
/// A warehouse relation a dbt project builds or reads,
/// `dbt://<warehouse>/<schema>/<name>`, the warehouse named as the
/// workspace configures it. The scheme names the producer, the path stays
/// the relation — see `windmill_types::AssetKind::Dbt`.
/// A warehouse relation, `dbt://<warehouse>/<schema>/<name>`, the warehouse
/// named as the workspace configures it. The scheme names the namespace dbt
/// made — a script in any language but dbt's own can declare a write to one —
/// and the path stays the relation. See `windmill_types::AssetKind::Dbt`.
Dbt,
}
@@ -288,11 +288,13 @@ pub struct RetrySpec {
}
// `// materialize [manual] <asset> [append] [key=<col>] [history] [track=<c1,c2>]`
// — declares that this script produces a *managed* materialization of `<asset>`
// (a `ducklake://` table). By default the runtime generates the write DDL around
// — declares that this script produces `<asset>`. A `ducklake://` table is
// materialized *managed* by default: the runtime generates the write DDL around
// the script's single trailing `SELECT` and owns idempotency, partition-state
// and snapshot capture. `manual` is the escape hatch: the script writes its own
// DDL and the runtime only records state (track-only). The reconciliation
// DDL and the runtime only records state (track-only) — and it is the only mode a
// `dbt://` warehouse relation has, since nothing generates warehouse DDL (deploy
// enforces that; see docs/dbt-runtime.md). The reconciliation
// strategy options apply to managed mode: none → DELETE-by-partition + INSERT
// (replace); `key=<col>` → MERGE (dedup within slice, SCD type 1); `append` →
// INSERT-only. `append` wins if both are given (deploy-time warning).
@@ -804,6 +806,18 @@ pub fn canonicalize_table_asset_path(path: &str) -> String {
)
}
/// Whether a `dbt://` path names a whole relation, `<warehouse>/<schema>/<name>`.
///
/// Every producer spells one that way — the manifest ingest derives it from
/// `relation_name`, a `// materialize` target is checked against it — so anything
/// else can be produced by nothing and read by nothing. Both sides of the deploy
/// ask here rather than counting segments themselves: a subscription and a write
/// that disagreed on the shape would refuse and accept the same string.
pub fn is_full_relation_path(path: &str) -> bool {
let mut segments = path.split('/');
segments.clone().count() == 3 && !segments.any(str::is_empty)
}
/// A doubled delimiter inside a quoted identifier is that delimiter, literally —
/// the same rule the worker's `split_relation` applies to `relation_name`. Both
/// have to decode it or one spelling of a table becomes two graph nodes: the dbt
@@ -1740,10 +1754,7 @@ mod pipeline_annotation_tests {
// just stop being the same node and the cross-boundary cascade never fires.
#[test]
fn table_paths_from_every_spelling_canonicalize_to_one_key() {
let canonical = Some((
AssetKind::Dbt,
Cow::Owned("main/analytics/orders".into()),
));
let canonical = Some((AssetKind::Dbt, Cow::Owned("main/analytics/orders".into())));
for spelling in [
// Hand-written annotation.
"dbt://main/analytics/orders",
@@ -1765,6 +1776,17 @@ mod pipeline_annotation_tests {
}
}
/// The shape both halves of the deploy check against: a subscription and a
/// write that disagreed on it would refuse and accept the same string.
#[test]
fn a_whole_relation_is_three_non_empty_segments() {
assert!(is_full_relation_path("main/analytics/orders"));
assert!(is_full_relation_path("main/archive.sales/orders"));
for partial in ["main", "main/analytics", "main/analytics/orders/x", "", "main//orders"] {
assert!(!is_full_relation_path(partial), "{partial} is not a relation");
}
}
// A relation that overrode its database carries `<database>.<schema>` in
// one segment, and each half can be quoted independently. Stripping only
// the outer pair leaves a key the manifest ingest never produces, so the
@@ -1791,10 +1813,7 @@ mod pipeline_annotation_tests {
// database qualifier.
assert_eq!(
parse_asset_syntax("dbt://main/\"sales.v2\"/orders", false),
Some((
AssetKind::Dbt,
Cow::Owned("main/sales.v2/orders".into())
))
Some((AssetKind::Dbt, Cow::Owned("main/sales.v2/orders".into())))
);
}
@@ -1813,14 +1832,8 @@ mod pipeline_annotation_tests {
"dbt://main/analytics/\"order\"\"s\"",
"main/analytics/order\"s",
),
(
"dbt://main/`da``ta`/`orders`",
"main/da`ta/orders",
),
(
"dbt://main/[my]]schema]/[orders]",
"main/my]schema/orders",
),
("dbt://main/`da``ta`/`orders`", "main/da`ta/orders"),
("dbt://main/[my]]schema]/[orders]", "main/my]schema/orders"),
// And in one half of a database-qualified segment.
(
"dbt://main/\"arch\"\"ive\".\"sales\"/orders",
@@ -1839,8 +1852,14 @@ mod pipeline_annotation_tests {
// apart. A lone delimiter treated as opening a quote would be dropped —
// `sa"les` filed as `sales` — and the two derivations would split.
for (decoded, spelled) in [
("dbt://main/sa\"les/orders", "dbt://main/\"sa\"\"les\"/orders"),
("dbt://main/analytics/order\"s", "dbt://main/analytics/\"order\"\"s\""),
(
"dbt://main/sa\"les/orders",
"dbt://main/\"sa\"\"les\"/orders",
),
(
"dbt://main/analytics/order\"s",
"dbt://main/analytics/\"order\"\"s\"",
),
(
"dbt://main/arch\"ive.sales/orders",
"dbt://main/\"arch\"\"ive\".\"sales\"/orders",
backend/windmill-api-groups/src/granular_acls.rs
+26
View File
@@ -318,6 +318,19 @@ async fn add_granular_acl(
)
.await?
}
"variable" => {
handle_deployment_metadata(
&authed.email,
&authed.username,
&db,
&w_id,
DeployedObject::Variable { path: path.to_string(), parent_path: None },
Some(format!("Variable '{}' changed permissions", path)),
true,
None,
)
.await?
}
_ => (),
}
@@ -528,6 +541,19 @@ async fn remove_granular_acl(
)
.await?
}
"variable" => {
handle_deployment_metadata(
&authed.email,
&authed.username,
&db,
&w_id,
DeployedObject::Variable { path: path.to_string(), parent_path: None },
Some(format!("Variable '{}' changed permissions", path)),
true,
None,
)
.await?
}
_ => (),
}
}
backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs
+250
View File
@@ -0,0 +1,250 @@
use serde_json::json;
use sqlx::{Pool, Postgres};
use windmill_test_utils::*;
fn client() -> reqwest::Client {
reqwest::Client::new()
}
fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
builder.header("Authorization", "Bearer SECRET_TOKEN")
}
async fn deploy(port: u16, path: &str, content: &str) -> reqwest::Response {
authed(client().post(format!(
"http://localhost:{port}/api/w/test-workspace/scripts/create"
)))
.json(&json!({
"path": path,
"summary": "",
"description": "",
"content": content,
"language": "deno",
"schema": { "type": "object", "properties": {}, "required": [] }
}))
.send()
.await
.unwrap()
}
/// A `dbt://` relation is one graph node only while every side spells it the same
/// way, and three sides derive that spelling independently: the `// materialize`
/// target becomes an `asset.path`, a `// on` ref becomes a `script_trigger`, and
/// the deploy-time refusal joins the two. The unit tests on `sole_dbt_producer`
/// prove the predicate; only a deploy proves the handler feeds it the key the
/// table actually holds — so a canonicalization that drifted on one side would
/// pass those and split the node here.
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_dbt_materialize_target_deploy_contract(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
sqlx::query!(
r#"UPDATE workspace_settings
SET dbt_warehouses = '{"main": {"resource_path": "u/test-user/wh"}}'::jsonb
WHERE workspace_id = 'test-workspace'"#
)
.execute(&db)
.await?;
// Nothing generates warehouse DDL, so a managed target is refused rather than
// degraded into the track-only mode it would silently become.
let resp = deploy(
port,
"u/test-user/managed",
"// materialize dbt://main/analytics/orders\nexport async function main() {}",
)
.await;
assert_eq!(resp.status(), 400);
assert!(resp.text().await?.contains("must be `manual`"));
// The warehouse segment is the identity a dbt model keys on; a name the
// workspace does not configure strands the write on an unreachable node.
let resp = deploy(
port,
"u/test-user/unknown_wh",
"// materialize manual dbt://nope/analytics/orders\nexport async function main() {}",
)
.await;
assert_eq!(resp.status(), 400);
assert!(resp.text().await?.contains("does not configure"));
// Only the DuckDB executor runs `// data_test` probes, and it runs them around
// a managed write — so a declarer in another language would deploy green with
// its assertions silently never executed.
let resp = deploy(
port,
"u/test-user/tested",
"// materialize manual dbt://main/analytics/orders\n// data_test not_null id\nexport async function main() {}",
)
.await;
assert_eq!(resp.status(), 400);
assert!(resp
.text()
.await?
.contains("`// data_test` is not supported"));
// Both halves are held to the same relation: every producer is a whole
// `<warehouse>/<schema>/<name>` under a configured warehouse, so a
// subscription to anything else names something nothing can ever write.
for (path, ref_, expected) in [
(
"u/test-user/partial_sub",
"dbt://main/analytics",
"not a whole warehouse relation",
),
(
"u/test-user/unknown_wh_sub",
"dbt://nope/analytics/orders",
"does not configure",
),
] {
let resp = deploy(
port,
path,
&format!("// on {ref_}\nexport async function main() {{}}"),
)
.await;
assert_eq!(resp.status(), 400);
assert!(resp.text().await?.contains(expected));
}
// Past `asset.path`'s column, where the manifest ingest drops the relation and
// no producer row can exist on either side — computed from the bound so it
// cannot drift under it.
let overlong = format!(
"main/analytics/{}",
"o".repeat(windmill_common::dbt_manifest::MAX_ASSET_PATH_LEN)
);
let resp = deploy(
port,
"u/test-user/overlong_sub",
&format!("// on dbt://{overlong}\nexport async function main() {{}}"),
)
.await;
assert_eq!(resp.status(), 400);
assert!(resp.text().await?.contains("characters an asset path holds"));
// Any language may declare the write — the DuckLake write engine is DuckDB's,
// this declaration is not — and the target is canonicalized on the way into
// `asset`, so a hand-written mixed-case spelling lands on the model's key.
let resp = deploy(
port,
"u/test-user/ingest",
"// materialize manual dbt://main/ANALYTICS/Orders\nexport async function main() {}",
)
.await;
assert_eq!(resp.status(), 201);
// The create response, not `{:x}` over the stored i64: `ScriptHash` decodes
// hex and demands 8 bytes, while `LowerHex` drops leading zeros, so a hash
// under 2^60 would 422 the rename below instead of reaching the refusal.
let ingest_hash = resp.text().await?;
let write = sqlx::query_scalar!(
"SELECT path FROM asset WHERE workspace_id = 'test-workspace' AND kind = 'dbt' \
AND usage_path = 'u/test-user/ingest' AND usage_access_type = 'w'"
)
.fetch_one(&db)
.await?;
assert_eq!(write, "main/analytics/orders");
// That producer is native, so subscribing to what it writes is accepted — and
// the `// on` ref has to canonicalize identically, or the row it stores names
// a relation nothing produces.
let resp = deploy(
port,
"u/test-user/consumer",
"// on dbt://main/\"Analytics\"/\"Orders\"\nexport async function main() {}",
)
.await;
assert_eq!(resp.status(), 201);
let trigger_ref = sqlx::query_scalar!(
"SELECT trigger_ref FROM script_trigger WHERE workspace_id = 'test-workspace' \
AND runnable_path = 'u/test-user/consumer' AND trigger_kind = 'asset'"
)
.fetch_one(&db)
.await?;
assert_eq!(trigger_ref, "dbt://main/analytics/orders");
// With dbt as the only producer the same subscription can never be woken — a
// dbt run does not dispatch — so the deploy refuses it and names the project.
sqlx::query!(
"INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by,
language)
VALUES ('test-workspace', 1, 'u/test-user/project', '', '', '', 'test-user', 'dbt')"
)
.execute(&db)
.await?;
sqlx::query!(
"INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind)
VALUES ('test-workspace', 'main/analytics/marts', 'dbt', 'w', 'u/test-user/project',
'script')"
)
.execute(&db)
.await?;
let resp = deploy(
port,
"u/test-user/mart_consumer",
"// on dbt://main/analytics/MARTS\nexport async function main() {}",
)
.await;
assert_eq!(resp.status(), 400);
assert!(resp.text().await?.contains("u/test-user/project"));
// A rename is the other half of that: the producer's write still sits at the
// OLD path in the committed snapshot this deploy reads, while the same
// transaction removes it — so it must not count as the producer that would
// wake the subscription the rename adds.
sqlx::query!(
"INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind)
VALUES ('test-workspace', 'main/analytics/orders', 'dbt', 'w', 'u/test-user/project',
'script')"
)
.execute(&db)
.await?;
let resp = authed(client().post(format!(
"http://localhost:{port}/api/w/test-workspace/scripts/create"
)))
.json(&json!({
"path": "u/test-user/ingest_renamed",
"parent_hash": ingest_hash,
"summary": "",
"description": "",
"content": "// on dbt://main/analytics/orders\nexport async function main() {}",
"language": "deno",
"schema": { "type": "object", "properties": {}, "required": [] }
}))
.send()
.await
.unwrap();
assert_eq!(resp.status(), 400);
assert!(resp.text().await?.contains("u/test-user/project"));
// Neither annotation is accepted on a dbt script: the graph ingest
// republishes that path's asset and trigger rows wholesale, so either would
// deploy something the dependency job then silently removes.
for content in [
"# materialize manual dbt://main/analytics/orders\nprofile:\n warehouse: main\n",
"# on dbt://main/analytics/orders\nprofile:\n warehouse: main\n",
] {
let resp = authed(client().post(format!(
"http://localhost:{port}/api/w/test-workspace/scripts/create"
)))
.json(&json!({
"path": "u/test-user/dbt_project",
"summary": "",
"description": "",
"content": content,
"language": "dbt",
"modules": { "dbt_project.yml": { "content": "name: p\n", "language": "dbt" } },
"schema": { "type": "object", "properties": {}, "required": [] }
}))
.send()
.await
.unwrap();
assert_eq!(resp.status(), 400);
assert!(resp.text().await?.contains("a dbt script cannot"));
}
Ok(())
}
backend/windmill-api-integration-tests/tests/protection_rules.rs
+66
View File
@@ -123,6 +123,72 @@ async fn test_protection_rules(db: Pool<Postgres>) -> anyhow::Result<()> {
.await?;
assert!(!resp.status().is_success(), "Non-admin should be blocked from flows: {}", resp.status());
// ========================================
// 4b. ...but a draft-only resource stays deletable: nothing is deployed at
// its path, so its DELETE is a draft discard rather than a deployment.
// ========================================
let draft_only_path = "u/test-user-2/draft_only_resource";
let resp = authed(
client().post(format!("{base}/drafts/update/resource/{draft_only_path}")),
"SECRET_TOKEN_2",
)
.json(&json!({ "value": {
"path": draft_only_path,
"value": { "a": 1 },
"resource_type": "c_test",
"description": ""
}}))
.send()
.await?;
assert_eq!(
resp.status(),
200,
"Non-admin should save a draft: {}",
resp.text().await?
);
let resp = authed(
client().delete(format!("{base}/resources/delete/{draft_only_path}")),
"SECRET_TOKEN_2",
)
.send()
.await?;
assert_eq!(
resp.status(),
200,
"Draft-only delete should not be gated by the deploy rules: {}",
resp.text().await?
);
let remaining: i64 = sqlx::query_scalar(
"SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND path = $1 \
AND typ = 'resource'::DRAFT_KIND",
)
.bind(draft_only_path)
.fetch_one(&db)
.await?;
assert_eq!(remaining, 0, "the draft should be gone");
// The gate itself is still there for a DEPLOYED resource at the same path.
let resp = authed(
client().post(format!("{base}/resources/create")),
"SECRET_TOKEN_2",
)
.json(&json!({
"path": draft_only_path,
"value": { "a": 1 },
"resource_type": "c_test",
"description": ""
}))
.send()
.await?;
assert!(
!resp.status().is_success(),
"Non-admin should still be blocked from creating a resource: {}",
resp.status()
);
// ========================================
// 5. Admin bypasses protection rule
// ========================================
backend/windmill-api-integration-tests/tests/schedules.rs
+117 -31
View File
@@ -113,7 +113,9 @@ async fn test_schedule_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
"expected at least 2 schedules, got {}",
list.len()
);
assert!(list.iter().any(|s| s["path"] == "u/test-user/test_schedule"));
assert!(list
.iter()
.any(|s| s["path"] == "u/test-user/test_schedule"));
// --- list_with_jobs ---
let resp = authed(client().get(format!("{base}/list_with_jobs")))
@@ -125,18 +127,14 @@ async fn test_schedule_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
assert!(!list.is_empty());
// --- update ---
let resp = authed(client().post(schedule_url(
port,
"update",
"u/test-user/test_schedule",
)))
.json(&json!({
"schedule": "0 0 */12 * * *",
"timezone": "Europe/Paris"
}))
.send()
.await
.unwrap();
let resp = authed(client().post(schedule_url(port, "update", "u/test-user/test_schedule")))
.json(&json!({
"schedule": "0 0 */12 * * *",
"timezone": "Europe/Paris"
}))
.send()
.await
.unwrap();
assert_eq!(resp.status(), 200, "update: {}", resp.text().await?);
// verify update
@@ -204,14 +202,11 @@ async fn test_schedule_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
assert_eq!(resp.status(), 200);
// --- delete ---
let resp = authed(client().delete(schedule_url(
port,
"delete",
"u/test-user/another_schedule",
)))
.send()
.await
.unwrap();
let resp =
authed(client().delete(schedule_url(port, "delete", "u/test-user/another_schedule")))
.send()
.await
.unwrap();
assert_eq!(resp.status(), 200);
let resp = authed_get(port, "exists", "u/test-user/another_schedule").await;
@@ -220,17 +215,108 @@ async fn test_schedule_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
// ===== Global endpoints =====
// --- preview ---
let resp = authed(client().post(format!(
"http://localhost:{port}/api/schedules/preview"
)))
.json(&json!({
"schedule": "0 0 */6 * * *",
"timezone": "UTC"
}))
.send()
.await
.unwrap();
let resp = authed(client().post(format!("http://localhost:{port}/api/schedules/preview")))
.json(&json!({
"schedule": "0 0 */6 * * *",
"timezone": "UTC"
}))
.send()
.await
.unwrap();
assert_eq!(resp.status(), 200, "preview: {}", resp.text().await?);
Ok(())
}
/// A schedule with no `schedule` row is listed from the `draft` table, so its
/// DELETE drops that draft, then 404s once nothing is left at the path. A legacy
/// (`email IS NULL`) draft is owned by nobody and stays put.
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_delete_draft_only_schedule(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let path = "u/test-user/draft_only_schedule";
let resp = authed(client().post(format!(
"http://localhost:{port}/api/w/test-workspace/drafts/update/trigger_schedule/{path}"
)))
.json(&json!({ "value": {
"path": path,
"schedule": "0 0 */6 * * *",
"timezone": "UTC",
"script_path": "u/test-user/never_deployed",
"is_flow": false,
}}))
.send()
.await
.unwrap();
assert_eq!(resp.status(), 200, "save draft: {}", resp.text().await?);
let resp = authed(client().get(format!(
"http://localhost:{port}/api/w/test-workspace/schedules/list?include_draft_only=true"
)))
.send()
.await
.unwrap();
let listed: Vec<serde_json::Value> = resp.json().await?;
assert!(
listed
.iter()
.any(|s| s["path"] == path && s["draft_only"] == json!(true)),
"draft-only schedule should be listed: {listed:?}"
);
let resp = authed(client().delete(schedule_url(port, "delete", path)))
.send()
.await
.unwrap();
assert_eq!(resp.status(), 200, "delete: {}", resp.text().await?);
let remaining: i64 = sqlx::query_scalar(
"SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND path = $1 \
AND typ = 'trigger_schedule'::DRAFT_KIND",
)
.bind(path)
.fetch_one(&db)
.await?;
assert_eq!(remaining, 0, "the draft should be gone");
let resp = authed(client().delete(schedule_url(port, "delete", path)))
.send()
.await
.unwrap();
assert_eq!(resp.status(), 404, "nothing left at the path");
// A legacy (email IS NULL) draft is owned by nobody and keeps the write gate
// on the drafts routes, so this one must not become a second door to it.
let legacy_path = "u/test-user/legacy_draft_only_schedule";
sqlx::query(
"INSERT INTO draft (workspace_id, email, path, typ, value) \
VALUES ('test-workspace', NULL, $1, 'trigger_schedule'::DRAFT_KIND, '{}'::json)",
)
.bind(legacy_path)
.execute(&db)
.await?;
let resp = authed(client().delete(schedule_url(port, "delete", legacy_path)))
.send()
.await
.unwrap();
assert_eq!(
resp.status(),
404,
"legacy draft is not this route's to delete"
);
let legacy_remaining: i64 = sqlx::query_scalar(
"SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND path = $1 \
AND typ = 'trigger_schedule'::DRAFT_KIND",
)
.bind(legacy_path)
.fetch_one(&db)
.await?;
assert_eq!(legacy_remaining, 1, "the legacy draft should survive");
Ok(())
}
backend/windmill-api-schedule/src/lib.rs
+14 -2
View File
@@ -31,8 +31,8 @@ use windmill_common::{
self, TriggerHistoryEvent, TriggerOperation, TriggerSource, SCHEDULE_TRIGGER_KIND,
},
user_drafts::{
delete_all_drafts_for_path, fetch_draft_only_list_rows, overlay_or_draft_only,
UserDraftItemKind, WithDraftOverlay, WithDraftQuery,
delete_all_drafts_for_path, delete_draft_only_for_path, fetch_draft_only_list_rows,
overlay_or_draft_only, UserDraftItemKind, WithDraftOverlay, WithDraftQuery,
},
utils::{
escape_ilike_pattern, not_found_if_none, paginate, Pagination, ScheduleType, StripPath,
@@ -1331,6 +1331,18 @@ async fn delete_schedule(
.flatten();
if exists.is_none() {
drop(tx);
if delete_draft_only_for_path(
&db,
&w_id,
UserDraftItemKind::TriggerSchedule,
path,
&authed.email,
)
.await?
{
return Ok(format!("Draft-only schedule {} deleted", path));
}
return Err(windmill_common::error::Error::NotFound(format!(
"Schedule {} not found",
path
backend/windmill-api-scripts/src/scripts.rs
+168 -34
View File
@@ -39,8 +39,8 @@ use sqlx::{FromRow, Postgres, Transaction};
use std::{collections::HashMap, sync::Arc};
use windmill_audit::audit_oss::{audit_log, AuditAuthorable};
use windmill_audit::ActionKind;
use windmill_dep_map::{lock_hash::record_lock_hashes, process_relative_imports};
use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap;
use windmill_dep_map::{lock_hash::record_lock_hashes, process_relative_imports};
use windmill_common::{
assets::{
@@ -1081,6 +1081,57 @@ fn lock_hash_entry(path: &str, lock: &str) -> [(String, i64); 1] {
[(path.to_string(), hash_script(lock))]
}
/// The `dbt://` relation both halves of a deploy have to agree on: a whole
/// `<warehouse>/<schema>/<name>`, under a warehouse this workspace configures.
///
/// Every producer is held to exactly this — a `// materialize` target here, a
/// descriptor's `profile.warehouse` in the worker — so a subscription to anything
/// else names a relation nothing can ever write. No later deploy fixes that and
/// no dormant-edge warning reports it, since the warning fires on a dbt project's
/// ingest and no project can claim a relation under a warehouse that isn't there.
/// Asking here rather than at each site is what keeps the two from drifting into
/// refusing and accepting the same string.
async fn validate_dbt_relation(
db: &sqlx::Pool<Postgres>,
w_id: &str,
relation: &str,
what: &str,
) -> Result<()> {
if !windmill_parser::asset_parser::is_full_relation_path(relation) {
return Err(Error::BadRequest(format!(
"{what} `dbt://{relation}` is not a whole warehouse relation \
(`dbt://<warehouse>/<schema>/<name>`)."
)));
}
// `asset.path` is VARCHAR(255) and the manifest ingest drops a relation that
// outgrows it rather than failing the whole graph, so past the column no
// producer row can exist on either side — a write would be rejected by
// Postgres mid-deploy, and `script_trigger.trigger_ref` is unbounded text
// that would take the subscription and keep it dormant for good.
let max = windmill_common::dbt_manifest::MAX_ASSET_PATH_LEN;
if relation.chars().count() > max {
return Err(Error::BadRequest(format!(
"{what} `dbt://{relation}` is longer than the {max} characters an asset path \
holds, so it cannot be recorded."
)));
}
let warehouse = relation.split('/').next().unwrap_or_default();
// Only the resolver's own "no such warehouse" is the annotation's fault. Its
// other failures — the query, and a setting entry with no `resource_path` —
// keep their own error: blaming the warehouse name for those misdescribes
// them, and flattening the malformed-setting one to a 400 hides a server
// fault behind a client one.
windmill_common::workspaces::dbt_warehouse_exists(db, w_id, warehouse)
.await
.map_err(|e| match e {
Error::NotFound(_) => Error::BadRequest(format!(
"{what} `dbt://{relation}` names a warehouse this workspace does not \
configure: {e}"
)),
other => other,
})
}
async fn create_script_internal<'c>(
mut ns: NewScript,
w_id: String,
@@ -1565,34 +1616,90 @@ async fn create_script_internal<'c>(
// membership; parsed writes tell us what is produced (we don't record
// them in auto_kind itself).
let pipeline_annotations = parse_pipeline_annotations(&ns.content);
// `// materialize` materializes a `ducklake://<name>/<table>` target from a
// DuckDB script. These two constraints hold for *both* modes: a non-DuckLake
// target would otherwise deploy, register a producer in the asset graph, then
// silently no-op at run time (`build_materialized_query` returns `Ok(None)`),
// and a non-DuckDB script never reaches the executor that records state. The
// managed-only checks (single trailing SELECT, no SQL args) come after — a
// `manual` script owns its DDL and skips them.
// `// materialize` names what this script produces. Two target kinds, and the
// runtime behind each is what constrains the annotation:
// • `ducklake://<name>/<table>` — the DuckDB executor generates the write
// (or, in `manual` mode, records the state the script wrote itself), so
// the script has to be a DuckDB one and the target has to name a table.
// A non-DuckDB script never reaches that executor.
// • `dbt://<warehouse>/<schema>/<name>` — a warehouse relation. Nothing
// generates warehouse DDL, so the declaration is track-only (`manual`)
// and any language but dbt's own may make it: the script writes the
// relation, the worker records the materialization, and the relation's
// asset node is shared with whatever dbt model reads it. A dbt project's
// own writes are read from its manifest, so it may not declare one.
// Any other kind would deploy, register a producer in the asset graph, then
// silently no-op at run time (`build_materialized_query` returns `Ok(None)`).
// The managed-only checks (single trailing SELECT, no SQL args) come after —
// a `manual` script owns its DDL and skips them.
if let Some(m) = pipeline_annotations.materialize.as_ref() {
if ns.language != ScriptLang::DuckDb {
return Err(Error::BadRequest(format!(
"`// materialize` is only supported for DuckDB scripts, not {}. Use the \
wmll.ducklake helpers to materialize from other languages.",
ns.language.as_str()
)));
}
if m.target_kind != windmill_parser::asset_parser::AssetKind::Ducklake {
use windmill_parser::asset_parser::AssetKind as PAssetKind;
// The producer half of the rule the trigger loop below applies to `// on`:
// a dbt project's writes come from its manifest, and the graph ingest
// republishes this path's asset rows wholesale, so a declared one would be
// wiped by the very deploy that accepted it while its runs kept stamping
// the relation.
if ns.language == ScriptLang::Dbt {
return Err(Error::BadRequest(
"`// materialize` only supports a DuckLake target \
(`ducklake://<name>/<table>`); other asset kinds aren't materializable."
"a dbt script cannot declare `// materialize`: what a project builds is read \
from its manifest and published by the graph ingest, not annotated."
.to_string(),
));
}
if !m.target_path.contains('/') {
return Err(Error::BadRequest(format!(
"`// materialize` needs a table in the target: \
`ducklake://{0}/<table>` (got `ducklake://{0}`).",
m.target_path
)));
match m.target_kind {
PAssetKind::Ducklake => {
if ns.language != ScriptLang::DuckDb {
return Err(Error::BadRequest(format!(
"`// materialize` is only supported for DuckDB scripts, not {}. Use the \
wmll.ducklake helpers to materialize from other languages, or declare a \
warehouse relation with `// materialize manual dbt://…`.",
ns.language.as_str()
)));
}
if !m.target_path.contains('/') {
return Err(Error::BadRequest(format!(
"`// materialize` needs a table in the target: \
`ducklake://{0}/<table>` (got `ducklake://{0}`).",
m.target_path
)));
}
}
PAssetKind::Dbt => {
// `// data_test` runs as verifier probes the DuckDB executor
// splices around a MANAGED write. Nothing generates a warehouse
// write, so nothing would run them — and unlike the DuckLake
// `manual` case, which at least fails loudly in that executor, a
// declarer in another language would deploy green with its
// data-quality assertions silently never executed.
if !pipeline_annotations.data_tests.is_empty() {
return Err(Error::BadRequest(
"`// data_test` is not supported with a `dbt://` target: the checks run \
against a managed materialization, and a warehouse relation is \
written by the script itself. Assert on the relation with a dbt \
test in the project that reads it."
.to_string(),
));
}
if !m.manual {
return Err(Error::BadRequest(
"`// materialize dbt://…` must be `manual`: Windmill generates no \
warehouse DDL, so the script issues its own write and only the outcome \
is recorded. Write \
`// materialize manual dbt://<warehouse>/<schema>/<name>`."
.to_string(),
));
}
validate_dbt_relation(&db, &w_id, &m.target_path, "`// materialize` target")
.await?;
}
_ => {
return Err(Error::BadRequest(
"`// materialize` only supports a DuckLake (`ducklake://<name>/<table>`) or \
warehouse-relation (`dbt://<warehouse>/<schema>/<name>`) target; other asset \
kinds aren't materializable."
.to_string(),
));
}
}
if !m.manual {
if let Err(e) = windmill_parser::sql_materialize::classify_wrap(&ns.content) {
@@ -2375,16 +2482,43 @@ async fn create_script_internal<'c>(
let Some((trigger_kind, trigger_ref)) = trigger_spec_to_row(spec) else {
continue;
};
// A `dbt://` subscription can never fire: dbt is the only producer of a
// warehouse relation (`// materialize` takes DuckLake targets only) and a
// dbt run does not dispatch. Refusing beats persisting a row that draws a
// cascade arrow on the canvas and then never wakes anything.
if trigger_ref.starts_with("dbt://") {
return Err(Error::BadRequest(format!(
"`{trigger_ref}` cannot be subscribed to: a dbt run does not trigger downstream \
runs, and nothing else writes a warehouse relation. Declare the read without \
`on` to keep the lineage edge, or schedule this script."
)));
// A `dbt://` subscription fires only when a NON-dbt job materialized the
// relation: `// materialize manual dbt://…` declares such a write, while a
// dbt run records its models and does not dispatch. So refuse exactly the
// edge that cannot fire — one whose relation is already claimed by dbt and
// by nothing else — rather than every `dbt://` edge (`sole_dbt_producer`,
// which takes the workspace pool: under RLS an unreadable native producer
// would refuse a live subscription).
if let Some(relation) = trigger_ref.strip_prefix("dbt://") {
// The subscriber side of the same rule: a dbt project is not woken by
// the asset cascade. Its graph ingest clears these rows for its own
// path, so accepting one here would deploy an edge the dependency job
// then silently removes.
if ns.language == ScriptLang::Dbt {
return Err(Error::BadRequest(format!(
"a dbt script cannot subscribe to `{trigger_ref}`: dbt orders its own DAG \
and a project is run on its schedule, not woken by an asset cascade."
)));
}
validate_dbt_relation(&db, &w_id, relation, "subscription target").await?;
// Both paths under a rename: the old one's committed write row is
// still there and this transaction is about to remove it.
let deploying_paths = match p_path_opt.as_deref().filter(|old| *old != ns.path) {
Some(old) => vec![ns.path.clone(), old.to_string()],
None => vec![ns.path.clone()],
};
if let Some(dbt_owner) =
windmill_common::assets::sole_dbt_producer(&db, &w_id, relation, &deploying_paths)
.await?
{
return Err(Error::BadRequest(format!(
"`{trigger_ref}` cannot be subscribed to: it is built by the dbt project at \
`{dbt_owner}`, and a dbt run does not trigger downstream runs. Declare the \
read without `on` to keep the lineage edge, or schedule this script. A \
relation written by a `// materialize manual {trigger_ref}` script can be \
subscribed to."
)));
}
}
// Effective debounce for this edge: per-`// on debounce=` wins,
// else the script-level `// debounce` default. Debounce only
backend/windmill-api/openapi.yaml
+82 -1
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.804.0
version: 1.805.0
title: Windmill API
contact:
@@ -8528,6 +8528,84 @@ paths:
items:
type: string
/w/{workspace}/resources/type/resource_counts:
get:
summary: count the workspace's resources per resource type
operationId: listResourceCountsByType
tags:
- resource
parameters:
- $ref: "#/components/parameters/WorkspaceId"
responses:
"200":
description: resource count per resource_type
content:
application/json:
schema:
type: array
items:
type: object
properties:
resource_type:
type: string
count:
type: integer
required:
- resource_type
- count
/w/{workspace}/resources/type/hub/info:
get:
summary: list what the hub knows about its resource types
operationId: listHubResourceTypeInfo
tags:
- resource
parameters:
- $ref: "#/components/parameters/WorkspaceId"
responses:
"200":
description: each hub resource type with its integration and pick count, empty if the hub answers neither read
content:
application/json:
schema:
type: array
items:
type: object
properties:
name:
type: string
app:
description: the integration the resource type belongs to, which is not always its own name
type: string
picks:
type: integer
required:
- name
- app
- picks
/w/{workspace}/resources/type/hub/pick/{name}:
post:
summary: record a hub resource type pick
operationId: pickHubResourceType
tags:
- resource
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Name"
responses:
"200":
description: whether the hub recorded the pick
content:
application/json:
schema:
type: object
properties:
success:
type: boolean
required:
- success
/w/{workspace}/npm_proxy/config:
get:
summary: get npm proxy configuration
@@ -8777,6 +8855,9 @@ paths:
properties:
name:
type: string
picks:
description: how often the integration has been picked, absent on a hub that does not count picks
type: integer
required:
- name
backend/windmill-api/src/apps.rs
+3
View File
@@ -5825,6 +5825,9 @@ mod embed_token_tests {
"GET",
),
("/api/w/test/resources/list_search", "GET"),
// The metadata allowlist matches `resources/type/` by prefix, so a route
// added under it that is not a read must be denied by its method.
("/api/w/test/resources/type/hub/pick/slack", "POST"),
// Workspace-wide job enumeration/export must NOT be reachable — an app
// reads only jobs it launched, by id (blocked via the app_embed sentinel).
("/api/w/test/jobs/list", "GET"),
backend/windmill-api/src/workspaces_export.rs
+5 -6
View File
@@ -346,7 +346,7 @@ pub(crate) struct ArchiveQueryParams {
default_ts: Option<String>,
/// Settings format version: "v1" (default) returns legacy flat format, "v2" returns grouped format
settings_version: Option<String>,
/// Opt-in: include `extra_perms` on flow / script / app rows. Default `false`
/// Opt-in: include `extra_perms` on script / flow / app / variable rows. Default `false`
/// so cross-workspace tarball imports do not carry over ACLs referring to
/// identities that may not exist in the target workspace. `wmill sync pull`
/// passes `true` to surface ACLs in the git-tracked yaml.
@@ -365,8 +365,8 @@ pub(crate) struct ArchiveQueryParams {
/// pre-existing serialization for folders and groups so
/// no customer sees a one-time noisy diff on upgrade.
/// * `KeepIfNonEmpty` — keep when there is at least one entry, drop when `{}`
/// or null. New surface for flow / script / app, which
/// never carried ACLs in source before this change.
/// or null. New surface for script / flow / app / variable,
/// which never carried ACLs in source before this change.
#[derive(Clone, Copy)]
pub enum ExtraPermsBehavior {
Drop,
@@ -665,7 +665,7 @@ pub(crate) async fn tarball_workspace(
check_scopes(&authed, || "variables:read".to_string())?;
}
// Opt-in behavior for surfacing per-resource ACLs on flow/app rows.
// Opt-in behavior for surfacing per-resource ACLs on script/flow/app/variable rows.
// Folder and group rows have always carried `extra_perms` in source and
// continue to do so unconditionally (`KeepEvenEmpty`) so existing
// customer git repos see no one-time noisy diff.
@@ -1002,8 +1002,7 @@ pub(crate) async fn tarball_workspace(
Error::internal_err(format!("Error decrypting variable {}: {}", var.path, e))
})?);
}
let var_str =
&to_string_without_metadata(&var, ExtraPermsBehavior::Drop, None).unwrap();
let var_str = &to_string_without_metadata(&var, new_kinds_extra_perms, None).unwrap();
archive
.write_to_archive(&var_str, &format!("{}.variable.json", var.path))
.await?;
backend/windmill-common/src/assets.rs
+134 -4
View File
@@ -175,10 +175,12 @@ fn is_write_access(access: Option<AssetUsageAccessType>) -> bool {
/// producers). Resource / datatable / volume reads stay explicit-`// on`:
/// a config/lookup read cascading is more often surprising than wanted.
fn is_auto_trigger_kind(kind: AssetKind) -> bool {
// `Dbt` is deliberately NOT here. dbt is the only thing that can produce a
// warehouse relation (`// materialize` takes DuckLake targets only) and a dbt
// run does not dispatch, so a derived `dbt://` edge could never fire — it
// would draw a cascade arrow into a script nothing can wake.
// `Dbt` is deliberately NOT here. A warehouse relation is usually built by
// the dbt project that reads it, and a dbt run does not dispatch, so deriving
// an edge from every `dbt://` read would draw cascade arrows that mostly never
// fire. The relations a native `// materialize manual dbt://…` script writes
// do wake subscribers, but only through an explicit `// on`, which is where
// the author states that this particular relation has such a producer.
matches!(kind, AssetKind::Ducklake | AssetKind::S3Object)
}
@@ -235,6 +237,134 @@ pub fn derive_pipeline_asset_trigger_refs(
out
}
/// A dbt script that builds the `dbt://` relation at `asset_path`, when dbt is
/// its ONLY producer.
///
/// That is the one shape in which subscribing to a warehouse relation can never
/// be woken: a dbt run records the models it built and does not dispatch
/// (`asset_dispatch` returns early for `ScriptLang::Dbt`), while a script that
/// declares `// materialize manual dbt://…` fans out on the ordinary path.
///
/// `None` covers both "some non-dbt script materializes it" and "nothing
/// produces it yet" — the second is the ordinary deploy-order case, identical to
/// every other asset kind, not a dormant edge.
///
/// **Give it the workspace pool, not an RLS-scoped transaction.** `script`
/// carries RLS while `asset` does not, so a scoped executor hides producers, and
/// the hidden ones fail in the harmful direction: a native producer the deployer
/// cannot read leaves a dbt-only set behind and refuses a subscription that would
/// have fired. What it discloses in exchange is the path of a dbt script building
/// a relation the caller already named, which the workspace asset graph hands out
/// for every `dbt://` node anyway (the source that script wrote stays gated).
/// Callers must therefore already be scoped to `workspace_id`.
///
/// `deploying_paths` is excluded from the producer set, and has to be: reading
/// committed rows means the deploying script's own are the version being
/// replaced, so one that just dropped its `// materialize` would still count as a
/// producer and let a now-dormant subscription through. Pass every path this
/// deploy is rewriting — under a rename that is the old path as well as the new
/// one, whose committed write row the transaction is about to remove. Excluding
/// them is free of the opposite error, because a script never wakes its own
/// subscription — the dispatcher skips that as a self-loop.
///
/// A producer another deploy is committing concurrently is invisible either way,
/// and the outcome depends on which side it is. An uncommitted NATIVE producer
/// leaves a dbt-only set and refuses, with a message the user can retry past. An
/// uncommitted DBT one leaves an empty set and accepts — and if that ingest then
/// commits and runs [`dormant_dbt_subscriptions`] before this deploy's trigger
/// row lands, neither side reports the edge it left dormant. Serializing the two
/// is not worth it: they would have to share a per-relation lock, and the ingest
/// takes `script … FOR UPDATE` before its own advisory lock, so a deploy holding
/// relation locks first inverts that order into a deadlock across the two
/// subsystems. The next deploy of that project warns (docs/dbt-runtime.md).
pub async fn sole_dbt_producer<'e>(
executor: impl PgExecutor<'e>,
workspace_id: &str,
asset_path: &str,
deploying_paths: &[String],
) -> error::Result<Option<String>> {
use crate::scripts::ScriptLang;
let producers = sqlx::query!(
r#"SELECT s.path AS "path!", s.language AS "language!: ScriptLang"
FROM asset a
JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path
AND s.archived = false AND s.deleted = false
WHERE a.workspace_id = $1 AND a.kind = 'dbt' AND a.path = $2
AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw')
AND a.usage_path <> ALL($3)"#,
workspace_id,
asset_path,
deploying_paths
)
.fetch_all(executor)
.await?;
if producers
.iter()
.any(|p| !matches!(p.language, ScriptLang::Dbt))
{
return Ok(None);
}
Ok(producers.into_iter().next().map(|p| p.path))
}
/// The set form of [`sole_dbt_producer`], for asking about many relations at
/// once: every `// on dbt://<relation>` edge among `relations` whose producers
/// are all dbt scripts, rendered as `dbt://<relation> → <subscriber path>`.
///
/// A dbt deploy asks this about the relations it just ingested, because that
/// ingest is what can retroactively leave a subscription accepted earlier — when
/// nothing produced the relation — with dbt as its only producer.
///
/// Spells the predicate the same way its singular sibling does, per subscriber:
/// the producer set excludes the subscriber's own path (a script never wakes
/// itself) and has to be non-empty (nothing produces it yet is deploy order, not
/// a dormant edge). Two "is dbt the sole producer" rules that drifted apart would
/// silence this warning with nothing failing.
///
/// Same disclosure and executor contract as [`sole_dbt_producer`]: workspace
/// pool, caller already scoped to `workspace_id`.
pub async fn dormant_dbt_subscriptions<'e>(
executor: impl PgExecutor<'e>,
workspace_id: &str,
relations: &[String],
) -> error::Result<Vec<String>> {
if relations.is_empty() {
return Ok(vec![]);
}
let refs = relations
.iter()
.map(|r| format!("dbt://{r}"))
.collect::<Vec<_>>();
Ok(sqlx::query_scalar!(
r#"WITH producer AS (
SELECT 'dbt://' || a.path AS trigger_ref, a.usage_path, s.language
FROM asset a
JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path
AND s.archived = false AND s.deleted = false
WHERE a.workspace_id = $1 AND a.kind = 'dbt'
AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw')
AND 'dbt://' || a.path = ANY($2)
)
SELECT DISTINCT st.trigger_ref || ' → ' || st.runnable_path AS "edge!"
FROM script_trigger st
WHERE st.workspace_id = $1 AND st.trigger_kind = 'asset'
AND st.trigger_ref = ANY($2)
AND EXISTS (SELECT 1 FROM producer p
WHERE p.trigger_ref = st.trigger_ref
AND p.usage_path <> st.runnable_path
AND p.language = 'dbt')
AND NOT EXISTS (SELECT 1 FROM producer p
WHERE p.trigger_ref = st.trigger_ref
AND p.usage_path <> st.runnable_path
AND p.language <> 'dbt')
ORDER BY 1"#,
workspace_id,
&refs
)
.fetch_all(executor)
.await?)
}
/// Clear and reinsert the full static-asset usage set of a script in one tx,
/// invalidating the producer-writes cache at most once and only on a real
/// change. The cache (asset_dispatch::ASSET_PRODUCER_WRITES_CACHE) keys a
backend/windmill-common/src/dbt_manifest.rs
+5 -3
View File
@@ -8,9 +8,11 @@
//! Two things this module is deliberate about:
//!
//! * **Asset identity is the physical relation.** A model becomes
//! `dbt://<warehouse>/<schema>/<name>`: the scheme names dbt, which is the
//! only thing that creates one, but the PATH is the relation and never dbt's
//! own `unique_id`. Two projects meet at a handoff — one materializes a mart,
//! `dbt://<warehouse>/<schema>/<name>`: the scheme names the namespace dbt
//! made — it is the only thing that DERIVES one, while any other language can
//! declare a write to one (decision 25) — but the PATH is the relation and
//! never dbt's own `unique_id`. Two projects meet at a handoff — one
//! materializes a mart,
//! the next declares it a `source` — where `model.a.orders` and
//! `source.b.analytics.orders` differ but the relation does not, so keying on
//! the node id would leave each project an island; a native script reading the
backend/windmill-common/src/user_drafts.rs
+39
View File
@@ -401,6 +401,45 @@ pub async fn fetch_draft_only_list_rows(
Ok(rows)
}
/// Delete the caller's OWN draft at a path with no deployed row, for the DELETE
/// route of a kind whose list synthesizes such rows via
/// `fetch_draft_only_list_rows`. The `NOT EXISTS` leaves a deployed row's draft
/// alone, so a route may call this on its not-found branch whatever the reason
/// for the miss. `Ok(false)` means nothing matched: the caller reports its own error.
///
/// Takes no permission check and callers must not add one: an email-scoped row
/// belongs to the caller, who can always discard it, as `update_draft`'s
/// own-discard does. Legacy (`email IS NULL`) rows are owned by nobody and keep
/// their write gate, so discarding one stays on the `update_draft` route.
pub async fn delete_draft_only_for_path(
db: &DB,
w_id: &str,
kind: UserDraftItemKind,
path: &str,
email: &str,
) -> Result<bool> {
let Some(table) = kind.deployed_table() else {
return Ok(false);
};
// `table` is from the closed `deployed_table()` enum, never user input.
let sql = format!(
"DELETE FROM draft \
WHERE workspace_id = $1 AND typ = $2::text::DRAFT_KIND AND path = $3 \
AND email = $4 \
AND NOT EXISTS (SELECT 1 FROM {table} t \
WHERE t.workspace_id = draft.workspace_id AND t.path = draft.path)"
);
let deleted = sqlx::query(&sql)
.bind(w_id)
.bind(kind.as_str())
.bind(path)
.bind(email)
.execute(db)
.await?
.rows_affected();
Ok(deleted > 0)
}
/// The get-by-path draft choreography, shared by every entity's "get by path"
/// route. Given the deployed entity as an `Option` (caller maps its own "not
/// found" to `None`):
backend/windmill-common/tests/dbt_producer_rules.rs
+196
View File
@@ -0,0 +1,196 @@
/*!
* One rule, two spellings: "is dbt the sole producer of this warehouse
* relation". `sole_dbt_producer` decides whether a `// on dbt://…` subscription
* is refused at deploy; `dormant_dbt_subscriptions` names the edges a dbt deploy
* retroactively leaves unwakeable. Both must answer "yes, dormant" only when
* every script writing the relation is a dbt one — a dbt run does not dispatch —
* and "no" both when a native `// materialize manual dbt://…` producer exists and
* when nothing produces the relation yet, which is the ordinary deploy-order
* case. Every way of getting this wrong is silent: a dormant edge on the canvas,
* a refused deploy of a valid pipeline, or a warning that stops appearing.
*/
use sqlx::{Pool, Postgres};
use windmill_common::assets::{dormant_dbt_subscriptions, sole_dbt_producer};
const WS: &str = "test-workspace";
const RELATION: &str = "main/analytics/orders";
const SUBSCRIBER: &str = "u/test-user/consumer";
async fn plant_producer(db: &Pool<Postgres>, path: &str, language: &str, hash: i64) {
sqlx::query(
"INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by,
language)
VALUES ($1, $2, $3, '', '', '', 'test-user', $4::text::script_lang)",
)
.bind(WS)
.bind(hash)
.bind(path)
.bind(language)
.execute(db)
.await
.expect("insert script");
sqlx::query(
"INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind)
VALUES ($1, $2, 'dbt', 'w', $3, 'script') ON CONFLICT DO NOTHING",
)
.bind(WS)
.bind(RELATION)
.bind(path)
.execute(db)
.await
.expect("insert asset");
}
async fn plant_subscriber(db: &Pool<Postgres>, path: &str) {
sqlx::query(
"INSERT INTO script_trigger (workspace_id, runnable_kind, runnable_path, trigger_kind,
trigger_ref)
VALUES ($1, 'script', $2, 'asset', 'dbt://' || $3)",
)
.bind(WS)
.bind(path)
.bind(RELATION)
.execute(db)
.await
.expect("insert script_trigger");
}
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn no_producer_is_not_dormant(db: Pool<Postgres>) {
assert_eq!(
sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()])
.await
.unwrap(),
None,
"a relation nothing produces yet must not refuse the subscription"
);
}
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn dbt_only_producer_is_dormant(db: Pool<Postgres>) {
plant_producer(&db, "u/test-user/project", "dbt", 1).await;
assert_eq!(
sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()])
.await
.unwrap(),
Some("u/test-user/project".to_string())
);
}
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn a_native_producer_beside_dbt_is_not_dormant(db: Pool<Postgres>) {
plant_producer(&db, "u/test-user/project", "dbt", 1).await;
plant_producer(&db, "u/test-user/ingest", "postgresql", 2).await;
assert_eq!(
sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()])
.await
.unwrap(),
None
);
}
/// `asset` is keyed by path while `script` holds every version of it, so the
/// language has to be read off the live one: a path converted to dbt still has
/// its old native versions sitting in `script`.
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn a_superseded_native_version_does_not_count(db: Pool<Postgres>) {
plant_producer(&db, "u/test-user/project", "postgresql", 1).await;
sqlx::query("UPDATE script SET archived = true WHERE hash = 1")
.execute(&db)
.await
.expect("archive the old version");
plant_producer(&db, "u/test-user/project", "dbt", 2).await;
assert_eq!(
sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()])
.await
.unwrap(),
Some("u/test-user/project".to_string())
);
}
/// The rows of the script being deployed describe the version it replaces, so a
/// script dropping its `// materialize` while adding a subscription would
/// otherwise count itself as the producer that wakes it — and commit a dormant
/// edge. It can never be that producer anyway: the dispatcher skips self-loops.
/// Under a rename that write sits at the OLD path, which the deploy is removing
/// in the same uncommitted transaction, so both paths have to be excluded.
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn the_subscriber_is_never_its_own_producer(db: Pool<Postgres>) {
plant_producer(&db, "u/test-user/project", "dbt", 1).await;
plant_producer(&db, SUBSCRIBER, "postgresql", 2).await;
assert_eq!(
sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()])
.await
.unwrap(),
Some("u/test-user/project".to_string())
);
}
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn a_renamed_producer_is_excluded_too(db: Pool<Postgres>) {
plant_producer(&db, "u/test-user/project", "dbt", 1).await;
plant_producer(&db, "u/test-user/old_ingest", "postgresql", 2).await;
assert_eq!(
sole_dbt_producer(
&db,
WS,
RELATION,
&[SUBSCRIBER.to_string(), "u/test-user/old_ingest".to_string()]
)
.await
.unwrap(),
Some("u/test-user/project".to_string()),
"the write this deploy is moving off the old path cannot wake the subscription"
);
}
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn the_set_form_agrees_with_the_singular_one(db: Pool<Postgres>) {
let relations = vec![RELATION.to_string()];
plant_subscriber(&db, "u/test-user/consumer").await;
plant_producer(&db, "u/test-user/project", "dbt", 1).await;
assert_eq!(
dormant_dbt_subscriptions(&db, WS, &relations)
.await
.unwrap(),
vec![format!("dbt://{RELATION} → u/test-user/consumer")],
"dbt alone builds it, so the subscription can never be woken"
);
plant_producer(&db, "u/test-user/ingest", "postgresql", 2).await;
assert!(
dormant_dbt_subscriptions(&db, WS, &relations)
.await
.unwrap()
.is_empty(),
"a native producer wakes it, so the edge is live"
);
}
/// The two ways the set form could stop meaning what the singular one means: a
/// relation nothing produces is deploy order rather than a dormant edge, and a
/// subscriber's own write is not a producer that can wake it — the dispatcher
/// skips self-loops, so that edge is dormant and has to be named.
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn the_set_form_matches_on_the_edge_cases_too(db: Pool<Postgres>) {
let relations = vec![RELATION.to_string()];
plant_subscriber(&db, SUBSCRIBER).await;
assert!(
dormant_dbt_subscriptions(&db, WS, &relations)
.await
.unwrap()
.is_empty(),
"nothing produces it yet, so nothing is dormant"
);
plant_producer(&db, "u/test-user/project", "dbt", 1).await;
plant_producer(&db, SUBSCRIBER, "postgresql", 2).await;
assert_eq!(
dormant_dbt_subscriptions(&db, WS, &relations)
.await
.unwrap(),
vec![format!("dbt://{RELATION} → {SUBSCRIBER}")],
"the subscriber's own write cannot wake it, so dbt is still the sole producer"
);
}
backend/windmill-queue/src/asset_dispatch.rs
+7 -5
View File
@@ -248,11 +248,13 @@ async fn try_dispatch(db: &DB, job: &MiniCompletedJob) -> Result<DispatchResult>
// A dbt run records the relations it builds, so it looks like a producer
// here — but dbt does not trigger downstream runs. Its own DAG is dbt's to
// order; the only thing a cascade would add is waking Windmill scripts that
// read a mart, and nothing outside dbt can declare a `dbt://` write, so
// that edge exists in one direction only. Cascading from a project whose
// per-run selection can build any subset of itself needs a per-run write set
// to be correct, which is a design worth doing deliberately rather than
// inferring. Until then dbt materializes and reports; it does not dispatch.
// read a mart. Cascading from a project whose per-run selection can build any
// subset of itself needs a per-run write set to be correct, which is a design
// worth doing deliberately rather than inferring: the deploy-time write set is
// not what ran, and the per-relation state table keeps one row per relation.
// Until then dbt materializes and reports; it does not dispatch. The opposite
// direction does: a native `// materialize manual dbt://…` script reaches the
// fan-out below on the ordinary path, on the strength of its own asset rows.
if job.script_lang == Some(ScriptLang::Dbt) {
return Ok(DispatchResult::default());
}
backend/windmill-store/src/resources.rs
+368 -4
View File
@@ -50,9 +50,9 @@ use windmill_common::{
error::{self, Error, JsonResult, Result},
get_database_url,
user_drafts::{
delete_all_drafts_for_path, delete_own_draft_for_path, fetch_draft_only,
fetch_draft_only_list_rows, maybe_overlay_draft, UserDraftItemKind, WithDraftOverlay,
WithDraftQuery,
delete_all_drafts_for_path, delete_draft_only_for_path, delete_own_draft_for_path,
fetch_draft_only, fetch_draft_only_list_rows, maybe_overlay_draft, UserDraftItemKind,
WithDraftOverlay, WithDraftQuery,
},
utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath},
variables,
@@ -89,6 +89,9 @@ pub fn workspaced_service() -> Router {
.route("/git_commit_hash/{*path}", get(get_git_commit_hash))
.route("/type/list", get(list_resource_types))
.route("/type/listnames", get(list_resource_types_names))
.route("/type/resource_counts", get(list_resource_counts_by_type))
.route("/type/hub/info", get(list_hub_resource_type_info))
.route("/type/hub/pick/{name}", post(pick_hub_resource_type))
.route("/type/get/{name}", get(get_resource_type))
.route("/type/exists/{name}", get(exists_resource_type))
.route("/type/update/{name}", post(update_resource_type))
@@ -134,7 +137,10 @@ pub struct EditResourceType {
/// `Option` conflates: an absent field leaves the extension alone, while an
/// explicit `null` clears it. A hub pull relies on both — a type that stops
/// being a file type has to stop being one locally too.
#[serde(default, deserialize_with = "windmill_common::more_serde::double_option")]
#[serde(
default,
deserialize_with = "windmill_common::more_serde::double_option"
)]
pub format_extension: Option<Option<String>>,
}
@@ -1309,6 +1315,17 @@ async fn delete_resource(
let path = path.to_path();
check_scopes(&authed, || format!("resources:write:{}", path))?;
// Ahead of the deploy rules: nothing is deployed at a draft-only path, so
// gating this discard on them would strand the row in a protected workspace.
// Ahead of the transaction too — the not-found branch other kinds hang this
// off is the `not_found_if_none` below, past the linked-variable cascade.
if delete_draft_only_for_path(&db, &w_id, UserDraftItemKind::Resource, path, &authed.email)
.await?
{
return Ok(format!("draft-only resource {} deleted", path));
}
if let RuleCheckResult::Blocked(msg) = check_deploy_rules(
&w_id,
AuditAuthorable::username(&authed),
@@ -1320,6 +1337,7 @@ async fn delete_resource(
{
return Err(Error::PermissionDenied(msg));
}
let mut tx = user_db.begin(&authed).await?;
// Capture resource data for trashbin before deleting
@@ -2566,6 +2584,352 @@ async fn list_resource_types_names(
Ok(Json(rows))
}
#[derive(Serialize)]
struct ResourceTypeCount {
resource_type: String,
count: i64,
}
/// How many resources of each type this workspace holds — how popular a type is *here*,
/// which is what the pickers rank on below the hub's own pick counts.
async fn list_resource_counts_by_type(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
) -> JsonResult<Vec<ResourceTypeCount>> {
// A count per type is aggregate, so there is no path to narrow it by: a token scoped
// to individual resources gets nothing rather than a total spanning paths it cannot
// read. Callers treat the refusal as "no local signal".
check_scopes(&authed, || "resources:read".to_string())?;
let mut tx = user_db.begin(&authed).await?;
let rows = sqlx::query!(
"SELECT resource_type, count(*) as \"count!\" FROM resource WHERE workspace_id = $1 GROUP BY resource_type",
&w_id
)
.fetch_all(&mut *tx)
.await?;
tx.commit().await?;
Ok(Json(
rows.into_iter()
.map(|r| ResourceTypeCount { resource_type: r.resource_type, count: r.count })
.collect(),
))
}
/// A hub read, remembered with the hub it came from: `hub_base_url` is a live instance
/// setting, so a cache ignoring it would keep serving the previous hub's answers.
struct HubCached<T> {
hub_base_url: String,
fetched_at: std::time::Instant,
value: T,
}
/// The index changes only when a resource type is published to the hub; picks move slowly
/// and only reorder a list. Both are read on every drawer open, hence caching at all.
const HUB_RT_INDEX_TTL: std::time::Duration = std::time::Duration::from_secs(60 * 60);
const HUB_RT_PICKS_TTL: std::time::Duration = std::time::Duration::from_secs(5 * 60);
/// How long a *failed* index read is remembered. Short, and deliberately not the hour a
/// success is good for: this read is on the path of every picker open, so an unreachable
/// hub must not cost an outbound timeout each time, while one blip must not silence pick
/// reporting for an hour.
const HUB_RT_INDEX_FAILURE_TTL: std::time::Duration = std::time::Duration::from_secs(60);
static HUB_RT_INDEX: LazyLock<
std::sync::RwLock<Option<HubCached<Option<HashMap<String, HubResourceType>>>>>,
> = LazyLock::new(|| std::sync::RwLock::new(None));
static HUB_RT_PICKS: LazyLock<std::sync::RwLock<Option<HubCached<Vec<HubResourceTypePicks>>>>> =
LazyLock::new(|| std::sync::RwLock::new(None));
fn hub_cache_get<T: Clone>(
cache: &std::sync::RwLock<Option<HubCached<T>>>,
hub_base_url: &str,
ttl: std::time::Duration,
) -> Option<T> {
let guard = cache.read().ok()?;
let entry = guard.as_ref()?;
(entry.hub_base_url == hub_base_url && entry.fetched_at.elapsed() < ttl)
.then(|| entry.value.clone())
}
fn hub_cache_put<T>(cache: &std::sync::RwLock<Option<HubCached<T>>>, hub_base_url: &str, value: T) {
if let Ok(mut guard) = cache.write() {
*guard = Some(HubCached {
hub_base_url: hub_base_url.to_string(),
fetched_at: std::time::Instant::now(),
value,
});
}
}
#[derive(Deserialize)]
struct HubResourceTypeEntry {
id: i64,
name: String,
/// Optional so a hub that does not send it costs only the mapping. Required, it would
/// fail the whole parse and take pick reporting — which needs just the id — with it.
#[serde(default)]
app: Option<String>,
}
#[derive(Clone)]
struct HubResourceType {
id: i64,
/// The integration the type belongs to. Usually the type's own name, but not always:
/// `discord_webhook` and `discord_bot_configuration` are both `discord`, and only the
/// hub knows that. Without it a workspace holding a `discord_webhook` resource looks
/// like one that has never touched Discord.
app: String,
}
/// Reads the index cache, choosing the TTL by what is stored: a failure expires far sooner
/// than a success. `None` is a miss, `Some(None)` a remembered failure.
fn hub_index_cached(hub_base_url: &str) -> Option<Option<HashMap<String, HubResourceType>>> {
let guard = HUB_RT_INDEX.read().ok()?;
let entry = guard.as_ref()?;
if entry.hub_base_url != hub_base_url {
return None;
}
let ttl = if entry.value.is_some() {
HUB_RT_INDEX_TTL
} else {
HUB_RT_INDEX_FAILURE_TTL
};
(entry.fetched_at.elapsed() < ttl).then(|| entry.value.clone())
}
/// What the hub knows about every published resource type, keyed by the name Windmill
/// addresses it by. `None` when the hub cannot be reached or does not answer with a list.
async fn hub_resource_types(
db: &DB,
hub_base_url: &str,
) -> Option<HashMap<String, HubResourceType>> {
if let Some(cached) = hub_index_cached(hub_base_url) {
return cached;
}
let index = async {
let response = windmill_common::utils::http_get_from_hub(
&windmill_common::utils::HTTP_CLIENT,
&format!("{hub_base_url}/resource_types/list"),
false,
None,
Some(db),
)
.await
.ok()?;
if !response.status().is_success() {
return None;
}
// Only the id and the app are kept. That listing carries every type's schema —
// around a megabyte — and neither reporting a pick nor grouping types by
// integration needs it.
Some(
response
.json::<Vec<HubResourceTypeEntry>>()
.await
.ok()?
.into_iter()
.map(|rt| {
let app = rt.app.unwrap_or_else(|| rt.name.clone());
(rt.name, HubResourceType { id: rt.id, app })
})
.collect::<HashMap<String, HubResourceType>>(),
)
}
.await;
hub_cache_put(&HUB_RT_INDEX, hub_base_url, index.clone());
index
}
#[derive(Serialize)]
struct PickHubResourceTypeResult {
success: bool,
}
/// Tells the hub a resource type was taken into a workspace, which is the counter its
/// `/resource_types/picked` ranking reads.
///
/// Never fails the caller: a hub predating the route, an unreachable one, and a type that
/// is local-only all mean the same thing — not counted — and the request that reaches here
/// has already saved the user's resource.
///
/// POST, and scoped as a write, because it changes state on the hub under the instance's
/// own credentials. The sibling `/type/*` routes are metadata reads that a `resources:run`
/// app-embed token may make, and both the method and this check keep such a token — which
/// is untrusted app JavaScript — from driving hub counters through us.
async fn pick_hub_resource_type(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path((_w_id, name)): Path<(String, String)>,
) -> JsonResult<PickHubResourceTypeResult> {
check_scopes(&authed, || "resources:write".to_string())?;
let hub_base_url = (**windmill_common::HUB_BASE_URL.load()).clone();
let success = async {
let id = hub_resource_types(&db, &hub_base_url).await?.get(&name)?.id;
let response = windmill_common::utils::http_get_from_hub(
&windmill_common::utils::HTTP_CLIENT,
&format!("{hub_base_url}/resource_types/{id}/pick"),
false,
None,
Some(&db),
)
.await
.ok()?;
Some(response.status().is_success())
}
.await
.unwrap_or(false);
if !success {
tracing::debug!("hub did not record a pick for resource type {name}");
}
Ok(Json(PickHubResourceTypeResult { success }))
}
#[derive(Deserialize, Clone)]
struct HubResourceTypePicks {
name: String,
/// The hub counts picks in a bigint, which its driver serialises as a string.
#[serde(deserialize_with = "windmill_common::more_serde::maybe_number")]
picks: i64,
}
#[derive(Deserialize)]
struct HubPickedResourceTypes {
resource_types: Vec<HubResourceTypePicks>,
}
#[cfg(test)]
mod hub_picks_tests {
use super::*;
/// Two properties of the index cache that a later edit could quietly drop: it is keyed on
/// the hub it came from, and a failure is forgotten long before a success is.
#[test]
fn the_index_cache_is_keyed_on_the_hub_and_forgets_failures_sooner() {
let index = || {
Some(HashMap::from([(
"slack".to_string(),
HubResourceType { id: 1, app: "slack".to_string() },
)]))
};
hub_cache_put(&HUB_RT_INDEX, "https://hub.example", index());
assert!(hub_index_cached("https://hub.example").is_some_and(|v| v.is_some()));
// Switching hubs must miss rather than serve the previous hub's mapping.
assert!(hub_index_cached("https://other.example").is_none());
// A remembered failure reads as a hit (so the hub is not re-attempted) carrying
// nothing, and only until the shorter of the two TTLs.
hub_cache_put(&HUB_RT_INDEX, "https://hub.example", None);
assert!(hub_index_cached("https://hub.example").is_some_and(|v| v.is_none()));
assert!(HUB_RT_INDEX_FAILURE_TTL < HUB_RT_INDEX_TTL);
if let Ok(mut guard) = HUB_RT_INDEX.write() {
*guard = None;
}
}
/// The hub counts picks in a bigint, which postgres.js serialises as a string. Typing
/// the field as a plain i64 fails the whole response, and the ranking silently empties.
#[test]
fn picks_decode_from_a_string_or_a_number() {
let parsed: HubPickedResourceTypes = serde_json::from_str(
r#"{"resource_types":[{"name":"slack","picks":"42"},{"name":"github","picks":7}]}"#,
)
.unwrap();
assert_eq!(
parsed
.resource_types
.iter()
.map(|rt| (rt.name.as_str(), rt.picks))
.collect::<Vec<_>>(),
vec![("slack", 42), ("github", 7)]
);
}
}
/// One hub resource type as the pickers need it.
#[derive(Serialize)]
struct HubResourceTypeInfo {
name: String,
/// The integration it belongs to, so a caller can total a workspace's resources per
/// integration rather than per type.
app: String,
picks: i64,
}
/// What the hub knows about its resource types: which integration each belongs to, and how
/// often each has been picked.
///
/// Empty rather than an error when the hub answers neither read, so the pickers treat an
/// older or private hub as "no hub signal" and fall back to what the workspace itself uses.
/// The two reads degrade independently: a hub that lists types but has no `picked` route
/// still supplies the type-to-integration mapping, which is what decides whether a
/// workspace's resources are recognised as belonging to an integration at all.
async fn list_hub_resource_type_info(
Extension(db): Extension<DB>,
) -> JsonResult<Vec<HubResourceTypeInfo>> {
let hub_base_url = (**windmill_common::HUB_BASE_URL.load()).clone();
let picks = match hub_cache_get(&HUB_RT_PICKS, &hub_base_url, HUB_RT_PICKS_TTL) {
Some(picks) => picks,
None => {
let fetched = async {
let response = windmill_common::utils::http_get_from_hub(
&windmill_common::utils::HTTP_CLIENT,
&format!("{hub_base_url}/resource_types/picked"),
false,
Some(vec![("limit", "200".to_string())]),
Some(&db),
)
.await
.ok()?;
if !response.status().is_success() {
return None;
}
Some(
response
.json::<HubPickedResourceTypes>()
.await
.ok()?
.resource_types,
)
}
.await
.unwrap_or_default();
hub_cache_put(&HUB_RT_PICKS, &hub_base_url, fetched.clone());
fetched
}
};
let mut picks_by_name: HashMap<String, i64> =
picks.into_iter().map(|rt| (rt.name, rt.picks)).collect();
let index = hub_resource_types(&db, &hub_base_url)
.await
.unwrap_or_default();
let mut info: Vec<HubResourceTypeInfo> = index
.into_iter()
.map(|(name, rt)| HubResourceTypeInfo {
picks: picks_by_name.remove(&name).unwrap_or(0),
name,
app: rt.app,
})
.collect();
// What the index did not account for is a type the picks read knows and the listing does
// not — which is what a hub answering only the second read looks like. Its own name is
// the same guess a caller makes for any type the mapping misses.
info.extend(
picks_by_name
.into_iter()
.map(|(name, picks)| HubResourceTypeInfo { app: name.clone(), name, picks }),
);
Ok(Json(info))
}
async fn get_resource_type(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
backend/windmill-trigger/src/handler.rs
+15 -2
View File
@@ -18,8 +18,9 @@ use windmill_common::{
error::{Error, JsonResult, Result},
trigger_history::{self, TriggerHistoryEvent, TriggerOperation, TriggerSource},
user_drafts::{
delete_all_drafts_for_path, delete_own_draft_for_path, fetch_draft_only_list_rows,
overlay_or_draft_only, UserDraftItemKind, WithDraftOverlay, WithDraftQuery,
delete_all_drafts_for_path, delete_draft_only_for_path, delete_own_draft_for_path,
fetch_draft_only_list_rows, overlay_or_draft_only, UserDraftItemKind, WithDraftOverlay,
WithDraftQuery,
},
utils::{paginate, Pagination, StripPath},
worker::CLOUD_HOSTED,
@@ -990,6 +991,18 @@ async fn delete_trigger<T: TriggerCrud>(
.await?;
if !deleted {
drop(tx);
if delete_draft_only_for_path(
&db,
&workspace_id,
T::user_draft_item_kind(),
path,
&authed.email,
)
.await?
{
return Ok(format!("Draft-only trigger '{}' deleted", path));
}
return Err(Error::NotFound(format!(
"Trigger not found at path: {}",
path
backend/windmill-types/src/assets.rs
+13 -10
View File
@@ -14,17 +14,20 @@ pub enum AssetKind {
Ducklake,
DataTable,
Volume,
/// A warehouse relation a dbt project builds or reads,
/// `dbt://<warehouse>/<schema>/<name>`, where `<warehouse>` is the name the
/// workspace configures it under.
/// A warehouse relation, `dbt://<warehouse>/<schema>/<name>`, where
/// `<warehouse>` is the name the workspace configures it under.
///
/// The SCHEME names the producer — dbt is the only thing that creates one —
/// while the PATH stays the physical relation, because that is what two
/// projects agree on: a mart one builds is a `source` the next reads, and
/// their dbt `unique_id`s differ (`model.a.orders` vs
/// `source.b.analytics.orders`) where the relation does not
/// (docs/dbt-runtime.md, decision 11). A dbt run does not trigger that
/// reader — the shared node is lineage, not a cascade edge.
/// The SCHEME names the namespace dbt made rather than an exclusive
/// producer: dbt is what derives these relations from a project, and a script
/// in any language but dbt's own can DECLARE one it writes
/// (`// materialize manual dbt://…`) — a project's writes are read from its
/// manifest, never annotated. The PATH stays the physical relation,
/// because that is what two producers agree on: a mart one builds is a
/// `source` the next reads, and their dbt `unique_id`s differ
/// (`model.a.orders` vs `source.b.analytics.orders`) where the relation does
/// not (docs/dbt-runtime.md, decision 11). A dbt run does not trigger the
/// readers of what it built — that shared node is lineage, not a cascade
/// edge — while a declared write does (decision 25).
Dbt,
}
backend/windmill-worker/src/dbt_executor.rs
+64 -5
View File
@@ -693,6 +693,7 @@ pub(crate) async fn dbt_dep(
&conn,
)
.await;
warn_dormant_subscribers(db, w_id, job_id, &ingested, &conn).await;
}
!published
} else {
@@ -3268,7 +3269,7 @@ async fn ingest_from_run(
let snapshot_job = p.graph_refresh.snapshot_job(job.id);
match conn {
Connection::Sql(db) => {
persist_ingest(
let published = persist_ingest(
db,
&job.workspace_id,
script_path,
@@ -3281,6 +3282,13 @@ async fn ingest_from_run(
p.graph_refresh.publishes_ownership(),
)
.await?;
// Publishing ownership from a RUN makes this project the owner of
// those relations exactly as a deploy does, so it can be what leaves
// a subscription accepted while the relation had no producer with dbt
// as its only one. Same warning the deploy emits.
if published && p.graph_refresh.publishes_ownership() {
warn_dormant_subscribers(db, &job.workspace_id, &job.id, &ingested, conn).await;
}
}
// An agent worker reaches these tables only through the API. Publishing
// is the whole of what it needs: a worker that can replace the graph
@@ -3333,7 +3341,7 @@ enum GraphPublisher {
/// Replace this script's graph, unless a newer version of it has been deployed.
///
/// Write one ingest: the sidecar rows and the `asset` usages the manifest
/// implies. No subscriptions — a `dbt://` one could never fire.
/// implies. No subscriptions — a dbt project is not woken by the cascade.
///
/// Returns whether this job was still the one entitled to the path-keyed half —
/// false once a newer version has superseded it, or once the version is gone.
@@ -3424,9 +3432,10 @@ async fn persist_ingest(
&ingested.assets,
)
.await?;
// A `dbt://` subscription can never fire, so none are derived from the
// manifest. The delete stays to clear what earlier versions wrote, which would
// otherwise keep drawing cascade arrows that wake nothing.
// A dbt project is not woken by the asset cascade (refused at deploy), so
// none are derived from the manifest either. The delete stays to clear what
// earlier versions wrote, which would otherwise keep drawing cascade arrows
// that wake nothing.
sqlx::query!(
"DELETE FROM script_trigger
WHERE workspace_id = $1 AND runnable_kind = 'script' AND runnable_path = $2
@@ -3440,6 +3449,56 @@ async fn persist_ingest(
Ok(true)
}
/// Log the `// on dbt://…` subscriptions this project's relations leave dormant.
///
/// Subscribing to a relation dbt already owns is refused at the subscriber's
/// deploy, but one deployed while nothing produced that relation is accepted —
/// as it is for every other asset kind — and an ingest is what can afterwards
/// make dbt its only producer. A dbt run does not dispatch, so such an edge is
/// drawn on the canvas and never fires; a job's own log is where that ordering is
/// visible.
///
/// Called from both points that publish ownership, the deploy and a run whose
/// static descriptor found its profile moved — either can be the one that claims
/// the relation.
async fn warn_dormant_subscribers(
db: &sqlx::Pool<sqlx::Postgres>,
w_id: &str,
job_id: &Uuid,
ingested: &windmill_common::dbt_manifest::IngestedManifest,
conn: &Connection,
) {
use windmill_common::assets::AssetUsageAccessType;
let relations: Vec<String> = ingested
.assets
.iter()
.filter(|a| {
matches!(
a.access_type.or(a.alt_access_type),
Some(AssetUsageAccessType::W) | Some(AssetUsageAccessType::RW)
)
})
.map(|a| a.path.clone())
.collect();
match windmill_common::assets::dormant_dbt_subscriptions(db, w_id, &relations).await {
Ok(edges) if !edges.is_empty() => {
append_logs(
job_id,
w_id,
format!(
"\nThese subscriptions will not fire — a dbt run does not trigger downstream \
runs, and nothing else writes their relation:\n {}\n",
edges.join("\n ")
),
conn,
)
.await;
}
Ok(_) => {}
Err(e) => tracing::warn!("listing dormant `dbt://` subscribers failed: {e:#}"),
}
}
/// Serialize publishers for one script path and confirm this job's version is
/// still the newest. Both happen inside the caller's transaction, so a newer
/// publisher either commits before this check sees it, or waits behind it and
backend/windmill-worker/src/duckdb_executor.rs
+3 -1
View File
@@ -112,7 +112,9 @@ async fn fetch_custom_test_body(conn: &Connection, w_id: &str, path: &str) -> Re
// the user's own ATTACH. `// data_test` lines append verifier probes that run
// against the freshly-materialized target and raise (failing the run) on
// violation. Returns `None` when there is no materialize annotation or the
// target isn't a ducklake (only ducklake is materialized in v1).
// target isn't a ducklake: only ducklake has a write engine, and a `dbt://`
// target is recorded by the generic job path instead (worker.rs,
// `record_declared_warehouse_write`).
fn build_materialized_query(
query: &str,
partition_value: Option<&str>,
backend/windmill-worker/src/worker.rs
+105 -2
View File
@@ -5527,7 +5527,7 @@ async fn handle_code_execution_job(
.await?;
let language = language.clone();
run_language_executor(
let result = run_language_executor(
job,
conn,
client,
@@ -5553,7 +5553,110 @@ async fn handle_code_execution_job(
false,
in_pipeline,
)
.await
.await;
record_declared_warehouse_write(job, conn, code, &result).await;
result
}
/// Record the outcome of a `// materialize manual dbt://<warehouse>/<schema>/<name>`
/// declaration, the way the DuckDB executor records a DuckLake target.
///
/// Nothing generates warehouse DDL, so the script issues its own write and this
/// is the only thing that turns it into a `materialized_partition` row — the
/// relation's last writer on the run page and the graph. Language-agnostic on
/// purpose — the DuckLake write engine is DuckDB's, this declaration is anyone's
/// — except dbt's own, which is refused at deploy.
///
/// Best-effort, and it can be: the cascade fans out from the deploy-time `asset`
/// rows, not from this one, so a lost row costs the relation its last writer and
/// nothing else. It must not fail a job whose write already landed.
///
/// Shares the reach of every other runtime pipeline annotation, which is this
/// function's caller: a job handed to a dedicated worker or a flow runner never
/// passes through it, so — exactly as `// partitioned` is not resolved there —
/// such a run performs its write and records no row.
async fn record_declared_warehouse_write(
job: &MiniPulledJob,
conn: &Connection,
code: &str,
result: &error::Result<Box<RawValue>>,
) {
use windmill_common::materialization::{
MaterializationStatus, RecordMaterializationRequest, UNPARTITIONED,
};
// A DEPLOYED script only. The annotation is a deploy-time contract — `manual`,
// a three-segment relation, a configured warehouse — checked in
// `create_script_internal`, which also required write access to the path. A
// preview, hub or inline-flow body reaches this function without any of that,
// so honouring it there would let `jobs:run` alone restamp any relation's last
// writer from a script that never touched it.
if job.kind != JobKind::Script {
return;
}
// Cheap guard: the annotation scan is skipped for the overwhelming majority
// of jobs, which carry no `materialize` line at all.
if !code.contains("materialize") {
return;
}
let Some(m) = windmill_parser::asset_parser::parse_pipeline_annotations(code)
.materialize
.filter(|m| m.target_kind == windmill_parser::asset_parser::AssetKind::Dbt)
else {
return;
};
// The slice this run wrote, resolved once upstream (`resolve_partition_for_job`)
// and carried in the args the cascade reads too, so a partitioned producer
// records the same identity everything else propagates.
let partition = job
.args
.as_ref()
.and_then(|a| a.0.get(windmill_common::partition::PARTITION_ARG))
.and_then(|v| serde_json::from_str::<String>(v.get()).ok())
.unwrap_or_else(|| UNPARTITIONED.to_string());
let (status, error) = match result {
Ok(_) => (MaterializationStatus::Materialized, None),
Err(e) => (MaterializationStatus::Failed, Some(e.to_string())),
};
let recorded = match conn {
Connection::Sql(db) => windmill_common::materialization::record_materialization(
db,
&job.workspace_id,
windmill_common::assets::AssetKind::Dbt,
&m.target_path,
&partition,
status,
None,
None,
Some(job.id),
error.as_deref(),
)
.await
.map_err(|e| anyhow::anyhow!("{e:#}")),
Connection::Http(client) => {
crate::agent_workers::record_materialization_from_agent_http(
client,
&job.workspace_id,
&RecordMaterializationRequest {
asset_kind: windmill_common::assets::AssetKind::Dbt,
asset_path: m.target_path.clone(),
partition,
status,
snapshot_id: None,
row_count: None,
job_id: Some(job.id),
error,
schema: None,
},
)
.await
}
};
if let Err(e) = recorded {
tracing::warn!(
"recording the materialization of dbt://{} failed: {e:#}",
m.target_path
);
}
}
/// True when `path` contains only `Normal`/`CurDir` components, i.e. it cannot
benchmarks/lib.ts
+1 -1
View File
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
export const VERSION = "v1.804.0";
export const VERSION = "v1.805.0";
export async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({
cli/src/commands/app/app.ts
+18 -2
View File
@@ -12,7 +12,11 @@ import * as wmill from "../../../gen/services.gen.ts";
import { ListableApp, Policy } from "../../../gen/types.gen.ts";
import { GlobalOptions, isSuperset } from "../../types.ts";
import { getWmillYamlPath, mergeConfigWithConfigFile } from "../../core/conf.ts";
import {
getWmillYamlPath,
mergeConfigWithConfigFile,
readEffectiveSyncBehavior,
} from "../../core/conf.ts";
import { readInlinePathSync } from "../../utils/utils.ts";
import devCommand from "./dev.ts";
import lintCommand from "./lint.ts";
@@ -21,6 +25,7 @@ import newCommand from "./new.ts";
import generateAgentsCommand from "./generate_agents.ts";
import { isVersionsGeq1585 } from "../sync/global.ts";
import type { PermissionedAsContext } from "../../core/permissioned_as.ts";
import { buildPermissionedAsContext } from "../../core/permissioned_as.ts";
import { applyExtraPermsDiff } from "../../core/extra_perms.ts";
export interface AppFile {
@@ -420,6 +425,8 @@ async function push(
if (isRawAppByName || hasRawAppYaml) {
const { pushRawApp } = await import("./raw_apps.ts");
const merged = await mergeConfigWithConfigFile(opts);
// Raw-app ownership preservation is not implemented on either push
// path: sync push hands pushRawApp no context either.
await pushRawApp(
workspace.workspaceId,
remotePath,
@@ -429,7 +436,16 @@ async function push(
);
log.info(colors.bold.underline.green("Raw app pushed"));
} else {
await pushApp(workspace.workspaceId, remotePath, absoluteFilePath);
await pushApp(
workspace.workspaceId,
remotePath,
absoluteFilePath,
undefined,
await buildPermissionedAsContext(
workspace.workspaceId,
await readEffectiveSyncBehavior(opts, workspace),
),
);
log.info(colors.bold.underline.green("App pushed"));
}
}
cli/src/commands/flow/flow.ts
+18 -3
View File
@@ -4,7 +4,7 @@ import { Command } from "@cliffy/command";
import { Confirm } from "@cliffy/prompt/confirm";
import { Table } from "@cliffy/table";
import * as log from "../../core/log.ts";
import { dirname, sep as SEP } from "node:path";
import { dirname, sep as SEP, resolve as pathResolve } from "node:path";
import { stringify as yamlStringify } from "yaml";
import { yamlParseFile } from "../../utils/yaml.ts";
import { readTextFile, validateRequiredArgs } from "../../utils/utils.ts";
@@ -21,11 +21,16 @@ import {
} from "../../core/context.ts";
import { resolve, track_job, pollForJobResult } from "../script/script.ts";
import { defaultFlowDefinition } from "../../../bootstrap/flow_bootstrap.ts";
import { SyncOptions, mergeConfigWithConfigFile } from "../../core/conf.ts";
import {
SyncOptions,
mergeConfigWithConfigFile,
readEffectiveSyncBehavior,
} from "../../core/conf.ts";
import { FSFSElement, elementsToMap, ignoreF } from "../sync/sync.ts";
import { Flow } from "../../../gen/types.gen.ts";
import { applyExtraPermsDiff } from "../../core/extra_perms.ts";
import type { PermissionedAsContext } from "../../core/permissioned_as.ts";
import { buildPermissionedAsContext } from "../../core/permissioned_as.ts";
import {
collectPathScriptPaths,
replaceInlineScripts,
@@ -327,10 +332,20 @@ async function push(opts: Options & { message?: string }, filePath: string, remo
if (!validatePath(remotePath)) {
return;
}
// Reading the config moves the cwd to the wmill.yaml root when it sits in a
// parent directory, so pin the file against the invocation cwd first.
filePath = pathResolve(filePath);
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
const syncBehavior = await readEffectiveSyncBehavior(opts, workspace);
await pushFlow(workspace.workspaceId, remotePath, filePath, opts.message);
await pushFlow(
workspace.workspaceId,
remotePath,
filePath,
opts.message,
await buildPermissionedAsContext(workspace.workspaceId, syncBehavior)
);
log.info(colors.bold.underline.green("Flow pushed"));
}
cli/src/commands/schedule/schedule.ts
+15 -4
View File
@@ -6,13 +6,19 @@ import { Command } from "@cliffy/command";
import { Table } from "@cliffy/table";
import { colors } from "@cliffy/ansi/colors";
import * as log from "../../core/log.ts";
import { sep as SEP } from "node:path";
import { sep as SEP, resolve as pathResolve } from "node:path";
import { requireLogin } from "../../core/auth.ts";
import { resolveWorkspace, validatePath } from "../../core/context.ts";
import { mergeConfigWithConfigFile } from "../../core/conf.ts";
import {
mergeConfigWithConfigFile,
readEffectiveSyncBehavior,
} from "../../core/conf.ts";
import * as wmill from "../../../gen/services.gen.ts";
import type { PermissionedAsContext } from "../../core/permissioned_as.ts";
import { lookupUsernameByEmail } from "../../core/permissioned_as.ts";
import {
buildPermissionedAsContext,
lookupUsernameByEmail,
} from "../../core/permissioned_as.ts";
import {
GlobalOptions,
@@ -299,8 +305,12 @@ async function disable(opts: GlobalOptions, path: string) {
}
async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
// Reading the config moves the cwd to the wmill.yaml root when it sits in a
// parent directory, so pin the file against the invocation cwd first.
filePath = pathResolve(filePath);
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
const syncBehavior = await readEffectiveSyncBehavior(opts, workspace);
if (!validatePath(remotePath)) {
return;
@@ -317,7 +327,8 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
workspace.workspaceId,
remotePath,
undefined,
parseFromFile(filePath)
parseFromFile(filePath),
await buildPermissionedAsContext(workspace.workspaceId, syncBehavior)
);
console.log(colors.bold.underline.green("Schedule pushed"));
}
cli/src/commands/script/script.ts
+7 -1
View File
@@ -7,6 +7,7 @@ import {
validatePath,
} from "../../core/context.ts";
import type { PermissionedAsContext } from "../../core/permissioned_as.ts";
import { buildPermissionedAsContext } from "../../core/permissioned_as.ts";
import { applyExtraPermsDiff } from "../../core/extra_perms.ts";
import { writeFile, stat, mkdir } from "node:fs/promises";
import { Buffer } from "node:buffer";
@@ -58,6 +59,7 @@ import {
SyncOptions,
mergeConfigWithConfigFile,
readConfigFile,
readEffectiveSyncBehavior,
} from "../../core/conf.ts";
import { SyncCodebase, listSyncCodebases } from "../../utils/codebase.ts";
import { pollJobWithQueueLogging } from "../../utils/job_polling.ts";
@@ -231,7 +233,11 @@ async function push(opts: PushOptions, filePath: string) {
opts.message,
opts,
await getRawWorkspaceDependencies(true),
codebases
codebases,
await buildPermissionedAsContext(
workspace.workspaceId,
await readEffectiveSyncBehavior(opts, workspace)
)
);
log.info(colors.bold.underline.green(`Script ${filePath} pushed`));
}
cli/src/commands/sync/pull.ts
+13 -2
View File
@@ -104,7 +104,7 @@ export async function downloadZip(
// from v1 the on-behalf-of address is stripped below, so the tarball sends the
// `has_on_behalf_of` marker instead and never resolves an address.
// `preserve_extra_perms=true` opts the tarball into surfacing granular ACLs
// on flow / script / app rows. Default-off on the server protects cross-
// on script / flow / app / variable rows. Default-off on the server protects cross-
// workspace tarball imports from carrying ACLs that reference identities
// missing in the target workspace; the CLI sync flow explicitly wants them.
const baseParams = `&plain_secret=${plainSecrets ?? false
@@ -150,7 +150,18 @@ export async function downloadZip(
}
if (zipResponse.status === 404 || body.includes("no rows returned")) {
log.info(colors.red(`Workspace '${workspace.workspaceId}' not found on ${workspace.remote}. Please check your --workspace and try again.`));
log.info(
colors.red(
`Workspace id '${workspace.workspaceId}' not found on ${workspace.remote}` +
(workspace.name !== workspace.workspaceId
? ` (resolved from profile '${workspace.name}')`
: "") +
`.\n` +
`Note this is the workspace *id* sent to the API, which is not necessarily what you passed to --workspace:\n` +
` - check 'wmill workspace list' (the 'workspace id' column)\n` +
` - check the 'workspaces' block of wmill.yaml ('workspaceId' overrides the workspace name)`
)
);
} else {
log.info(colors.red(`Failed to request tarball from API: ${zipResponse.status} ${zipResponse.statusText}`));
if (body) log.info(colors.red(body));
cli/src/commands/sync/sync.ts
+13 -75
View File
@@ -76,7 +76,8 @@ import {
} from "../../utils/utils.ts";
import {
getEffectiveSettings,
getWorkspaceNames,
inferWsNameFromProfile,
resolveWsNameForConfigFromFlags,
mergeConfigWithConfigFile,
parseSyncBehavior,
SyncOptions,
@@ -85,7 +86,10 @@ import {
WorkspaceEntryConfig,
} from "../../core/conf.ts";
import type { PermissionedAsContext } from "../../core/permissioned_as.ts";
import { preCheckPermissionedAs } from "../../core/permissioned_as.ts";
import {
buildPermissionedAsContext,
preCheckPermissionedAs,
} from "../../core/permissioned_as.ts";
import {
fromWorkspaceSpecificPath,
toWorkspaceSpecificPath,
@@ -429,37 +433,6 @@ export function computeWsSpecificFlagOnlyPushes(
return out;
}
// Resolve workspace name from a --branch override (git branch → workspace name).
// Falls back to using the branch value as-is (backward compat: old key = branch name).
function resolveWsNameFromBranch(
opts: SyncOptions,
branchName: string,
): string {
const match = findWorkspaceByGitBranch(opts.workspaces, branchName);
return match ? match[0] : branchName;
}
// Resolve wsNameForConfig from CLI flags. Prefers --branch → matching config key,
// then --workspace → matching config key (incl. when --base-url is set). Returns
// undefined when no flag-based resolution applies; callers then fall back to
// inferWsNameFromProfile on the resolved workspace profile.
export function resolveWsNameForConfigFromFlags(
opts: SyncOptions & { branch?: string; workspace?: string },
): string | undefined {
if (opts.branch) {
return resolveWsNameFromBranch(opts, opts.branch);
}
if (opts.workspace) {
// Use getWorkspaceNames so reserved keys (e.g. commonSpecificItems) are filtered out,
// matching the behavior of findWorkspaceByGitBranch / inferWsNameFromProfile.
const validKeys = getWorkspaceNames(opts.workspaces);
if (validKeys.includes(opts.workspace)) {
return opts.workspace;
}
}
return undefined;
}
// Warn if --workspace overrides auto-detected branch or if workspace not in config.
function warnWorkspaceOverride(
opts: SyncOptions,
@@ -507,33 +480,6 @@ function resolveWsNameForFiles(_opts: SyncOptions, wsName: string): string {
return wsName;
}
// After resolveWorkspace, infer the workspace config name from the resolved profile
// by matching baseUrl + workspaceId against the workspaces config entries.
function inferWsNameFromProfile(
opts: SyncOptions,
profile: { remote: string; workspaceId: string },
): string | undefined {
if (!opts.workspaces) return undefined;
const wsNames = Object.keys(opts.workspaces).filter(
(k) => k !== "commonSpecificItems",
);
for (const name of wsNames) {
const entry = (opts.workspaces as any)[name] as WorkspaceEntryConfig;
if (!entry?.baseUrl) continue;
try {
const entryUrl = new URL(entry.baseUrl).toString();
const profileUrl = new URL(profile.remote).toString();
const entryWsId = entry.workspaceId ?? name;
if (entryUrl === profileUrl && entryWsId === profile.workspaceId) {
return name;
}
} catch {
continue;
}
}
return undefined;
}
// Merge CLI options with effective settings, preserving CLI flags as overrides
function mergeCliWithEffectiveOptions<
T extends GlobalOptions & SyncOptions & { repository?: string },
@@ -5540,27 +5486,19 @@ export async function push(
return;
}
let permissionedAsContext: PermissionedAsContext | undefined = undefined;
if (parseSyncBehavior(opts.syncBehavior) >= 1) {
const user = await wmill.whoami({ workspace: workspace.workspaceId });
const userIsAdminOrDeployer =
user.is_admin || (user.groups ?? []).includes("wm_deployers");
log.debug(
`permissioned_as: user=${user.email}, is_admin=${user.is_admin}, groups=${JSON.stringify(user.groups)}, isAdminOrDeployer=${userIsAdminOrDeployer}`,
const permissionedAsContext: PermissionedAsContext | undefined =
await buildPermissionedAsContext(
workspace.workspaceId,
opts.syncBehavior,
);
permissionedAsContext = {
userCache: new Map(),
userIsAdminOrDeployer,
userEmail: user.email,
};
if (permissionedAsContext) {
// ws_specific_flag changes have no content payload, so they don't
// affect permissioned_as resolution — filter them out before the
// pre-check (which expects only added/edited/deleted).
await preCheckPermissionedAs(
changes.filter((c) => c.name !== "ws_specific_flag"),
user.email,
userIsAdminOrDeployer,
permissionedAsContext.userEmail,
permissionedAsContext.userIsAdminOrDeployer,
opts.acceptOverridingPermissionedAsWithSelf ?? false,
!!process.stdin.isTTY,
);
cli/src/commands/trigger/trigger.ts
+9 -2
View File
@@ -23,7 +23,7 @@ import { Command } from "@cliffy/command";
import { Table } from "@cliffy/table";
import { colors } from "@cliffy/ansi/colors";
import * as log from "../../core/log.ts";
import { sep as SEP } from "node:path";
import { sep as SEP, resolve as pathResolve } from "node:path";
import {
GlobalOptions,
isSuperset,
@@ -41,6 +41,8 @@ import { getCurrentGitBranch } from "../../utils/git.ts";
import { requireLogin } from "../../core/auth.ts";
import { validatePath, resolveWorkspace } from "../../core/context.ts";
import type { PermissionedAsContext } from "../../core/permissioned_as.ts";
import { buildPermissionedAsContext } from "../../core/permissioned_as.ts";
import { readEffectiveSyncBehavior } from "../../core/conf.ts";
type Trigger = {
http: HttpTrigger;
@@ -620,8 +622,12 @@ async function extractTriggerKindFromPath(filePath: string): Promise<string | un
}
async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
// Reading the config moves the cwd to the wmill.yaml root when it sits in a
// parent directory, so pin the file against the invocation cwd first.
filePath = pathResolve(filePath);
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
const syncBehavior = await readEffectiveSyncBehavior(opts, workspace);
if (!validatePath(remotePath)) {
return;
@@ -643,7 +649,8 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
workspace.workspaceId,
remotePath,
undefined,
parseFromFile(filePath)
parseFromFile(filePath),
await buildPermissionedAsContext(workspace.workspaceId, syncBehavior)
);
console.log(colors.bold.underline.green("Trigger pushed"));
}
cli/src/commands/variable/variable.ts
+45 -27
View File
@@ -19,6 +19,7 @@ import { sep as SEP } from "node:path";
import * as wmill from "../../../gen/services.gen.ts";
import { ListableVariable } from "../../../gen/types.gen.ts";
import { applyExtraPermsDiff } from "../../core/extra_perms.ts";
async function list(opts: GlobalOptions & { json?: boolean }) {
if (opts.json) log.setSilent(true);
@@ -97,6 +98,7 @@ export interface VariableFile {
description: string;
account?: number;
is_oauth?: boolean;
extra_perms?: Record<string, boolean>;
}
/**
@@ -152,37 +154,42 @@ export async function pushVariable(
log.debug(`Variable ${remotePath} does not exist on remote`);
}
// extra_perms is synced independently via /acls/* (see applyExtraPermsDiff)
// so a perm-only edit never rewrites the variable value. Strip the field from
// the body that goes to update_variable / create_variable and treat it as a
// separate step both for the up-to-date short-circuit and after the write.
const { extra_perms: localPerms, ...localVariableBody } = localVariable;
if (variable) {
if (isSuperset(localVariable, variable)) {
if (isSuperset(localVariableBody, variable)) {
log.debug(`Variable ${remotePath} is up-to-date`);
return;
}
} else {
log.debug(`Variable ${remotePath} is not up-to-date, updating`);
log.debug(`Variable ${remotePath} is not up-to-date, updating`);
// Apply is_secret only when it differs from the remote (the value is always
// sent, so the server allows the flag change). Upgrades (non-secret->secret)
// always apply; downgrades only when explicitly allowed (single-file push) —
// see allowSecretDowngrade. `undefined` leaves the flag untouched.
let nextIsSecret: boolean | undefined = undefined;
if (localVariable.is_secret !== variable.is_secret) {
if (localVariable.is_secret) {
nextIsSecret = true;
} else if (allowSecretDowngrade) {
nextIsSecret = false;
// Apply is_secret only when it differs from the remote (the value is always
// sent, so the server allows the flag change). Upgrades (non-secret->secret)
// always apply; downgrades only when explicitly allowed (single-file push) —
// see allowSecretDowngrade. `undefined` leaves the flag untouched.
let nextIsSecret: boolean | undefined = undefined;
if (localVariableBody.is_secret !== variable.is_secret) {
if (localVariableBody.is_secret) {
nextIsSecret = true;
} else if (allowSecretDowngrade) {
nextIsSecret = false;
}
}
}
await wmill.updateVariable({
workspace,
path: remotePath.replaceAll(SEP, "/"),
alreadyEncrypted: !plainSecrets,
requestBody: {
...localVariable,
is_secret: nextIsSecret,
...(wsSpecific !== undefined ? { ws_specific: wsSpecific } : {}),
},
});
await wmill.updateVariable({
workspace,
path: remotePath.replaceAll(SEP, "/"),
alreadyEncrypted: !plainSecrets,
requestBody: {
...localVariableBody,
is_secret: nextIsSecret,
...(wsSpecific !== undefined ? { ws_specific: wsSpecific } : {}),
},
});
}
} else {
log.info(colors.yellow.bold(`Creating new variable ${remotePath}...`));
await wmill.createVariable({
@@ -190,11 +197,22 @@ export async function pushVariable(
alreadyEncrypted: !plainSecrets,
requestBody: {
path: remotePath.replaceAll(SEP, "/"),
...localVariable,
...localVariableBody,
...(wsSpecific !== undefined ? { ws_specific: wsSpecific } : {}),
},
});
}
// Synced whether or not the body changed. No refetch: folder perms are never
// merged onto item.extra_perms, and the update/create body carries no
// extra_perms, so the value getVariable read above is still the remote one.
await applyExtraPermsDiff(
workspace,
"variable",
remotePath.replaceAll(SEP, "/"),
localPerms,
(variable as any)?.extra_perms,
);
}
async function push(
cli/src/core/conf.ts
+118 -1
View File
@@ -145,7 +145,9 @@ function getGitRepoRoot(): string | null {
}
export const GLOBAL_CONFIG_OPT = { noCdToRoot: false };
function findWmillYaml(): string | null {
// Pure upward search: no chdir, no logging. findWmillYaml() adds the chdir.
function locateWmillYaml(): string | null {
const startDir = resolve(process.cwd());
const isInGitRepo = isGitRepository();
const gitRoot = isInGitRepo ? getGitRepoRoot() : null;
@@ -176,6 +178,13 @@ function findWmillYaml(): string | null {
currentDir = parentDir;
}
return foundPath;
}
function findWmillYaml(): string | null {
const startDir = resolve(process.cwd());
const foundPath = locateWmillYaml();
// If wmill.yaml was found in a parent directory, warn the user and change working directory
if (
!GLOBAL_CONFIG_OPT.noCdToRoot &&
@@ -198,6 +207,37 @@ export function getWmillYamlPath(): string | null {
return findWmillYaml();
}
/**
* Look up one `workspaces` entry, for diagnostics only. readConfigFile() must
* not be used for that: it chdirs to the config's directory, exits on an
* unsupported syncBehavior and throws on a malformed file. A diagnostic may
* never fail or relocate the command it is diagnosing.
*/
export async function peekWorkspaceEntry(
workspaceName: string
): Promise<WorkspaceEntryConfig | undefined> {
if (RESERVED_WORKSPACE_KEYS.has(workspaceName)) {
return undefined;
}
const wmillYamlPath = locateWmillYaml();
if (!wmillYamlPath) {
return undefined;
}
try {
const conf = (await yamlParseFile(wmillYamlPath)) as SyncOptions;
const workspaces =
conf?.workspaces ??
conf?.gitBranches ??
conf?.environments ??
conf?.git_branches;
const entry = (workspaces as any)?.[workspaceName];
return typeof entry === "object" && entry !== null ? entry : undefined;
} catch (e) {
log.debug(`Failed to parse ${wmillYamlPath} for workspace lookup: ${e}`);
return undefined;
}
}
let legacyConfigWarned = false;
export async function readConfigFile(opts?: { warnIfMissing?: boolean }): Promise<SyncOptions> {
@@ -630,6 +670,83 @@ export async function getEffectiveSettings(
return effective;
}
// Resolve workspace name from a --branch override (git branch → workspace name).
// Falls back to using the branch value as-is (backward compat: old key = branch name).
function resolveWsNameFromBranch(opts: SyncOptions, branchName: string): string {
const match = findWorkspaceByGitBranch(opts.workspaces, branchName);
return match ? match[0] : branchName;
}
// Resolve wsNameForConfig from CLI flags. Prefers --branch → matching config key,
// then --workspace → matching config key (incl. when --base-url is set). Returns
// undefined when no flag-based resolution applies; callers then fall back to
// inferWsNameFromProfile on the resolved workspace profile.
export function resolveWsNameForConfigFromFlags(
opts: SyncOptions & { branch?: string; workspace?: string }
): string | undefined {
if (opts.branch) {
return resolveWsNameFromBranch(opts, opts.branch);
}
if (opts.workspace) {
// Use getWorkspaceNames so reserved keys (e.g. commonSpecificItems) are filtered out,
// matching the behavior of findWorkspaceByGitBranch / inferWsNameFromProfile.
const validKeys = getWorkspaceNames(opts.workspaces);
if (validKeys.includes(opts.workspace)) {
return opts.workspace;
}
}
return undefined;
}
/**
* Match a workspace config entry to a resolved workspace profile by remote +
* workspace id. The fallback for when no flag names the entry outright.
*/
export function inferWsNameFromProfile(
opts: SyncOptions,
profile: { remote: string; workspaceId: string }
): string | undefined {
if (!opts.workspaces) return undefined;
for (const name of getWorkspaceNames(opts.workspaces)) {
const entry = (opts.workspaces as any)[name] as WorkspaceEntryConfig;
if (!entry?.baseUrl) continue;
try {
const entryUrl = new URL(entry.baseUrl).toString();
const profileUrl = new URL(profile.remote).toString();
const entryWsId = entry.workspaceId ?? name;
if (entryUrl === profileUrl && entryWsId === profile.workspaceId) {
return name;
}
} catch {
continue;
}
}
return undefined;
}
/**
* `syncBehavior` as the workspace being pushed to sees it. The top level alone
* misses a `workspaces.<name>.overrides.syncBehavior`, which is where a repo
* that varies settings per workspace puts it, and the entry to read is the one
* `--workspace` names — falling back to the profile, then to the git branch —
* the same order `sync push` resolves it in.
*/
export async function readEffectiveSyncBehavior(
opts: { workspace?: string },
profile?: { remote: string; workspaceId: string }
): Promise<string | undefined> {
const config = await readConfigFile({ warnIfMissing: false });
const named = resolveWsNameForConfigFromFlags({ ...config, ...opts });
const effective = await getEffectiveSettings(
config,
undefined,
false,
true,
named ?? (profile ? inferWsNameFromProfile(config, profile) : undefined)
);
return effective.syncBehavior;
}
const RESERVED_WORKSPACE_KEYS = new Set(["commonSpecificItems"]);
/**
cli/src/core/constants.ts
+1 -1
View File
@@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork";
// (e.g. utils.ts) can read it without importing main.ts and creating a circular
// dependency (main → workspace → utils → main) that triggers a TDZ.
// Re-exported from main.ts for backwards compatibility.
export const VERSION = "1.804.0";
export const VERSION = "1.805.0";
cli/src/core/context.ts
+52 -2
View File
@@ -20,6 +20,7 @@ import {
import { getLastUsedProfile, setLastUsedProfile } from "./branch-profiles.ts";
import {
readConfigFile,
peekWorkspaceEntry,
findWorkspaceByGitBranch,
getEffectiveWorkspaceId,
getWmillYamlPath,
@@ -219,6 +220,9 @@ async function tryResolveWorkspace(
// First try: look up workspace by name in wmill.yaml workspaces config
const config = await readConfigFile({ warnIfMissing: false });
const wsEntry = config.workspaces?.[opts.workspace] as WorkspaceEntryConfig | undefined;
// What wmill.yaml said to target, kept for the fallback below: a profile
// found by name can silently point somewhere else entirely.
let configuredTarget: { workspaceId: string; baseUrl: string } | undefined;
if (wsEntry?.baseUrl) {
const workspaceId = getEffectiveWorkspaceId(opts.workspace, wsEntry);
let normalizedBaseUrl: string;
@@ -231,6 +235,8 @@ async function tryResolveWorkspace(
};
}
configuredTarget = { workspaceId, baseUrl: normalizedBaseUrl };
// Find matching profile by baseUrl + workspaceId
const allProfs = await allWorkspaces(opts.configDir);
const matching = allProfs.filter(
@@ -283,6 +289,22 @@ async function tryResolveWorkspace(
),
};
}
if (
configuredTarget &&
(e.workspaceId !== configuredTarget.workspaceId ||
e.remote !== configuredTarget.baseUrl)
) {
log.warnStderr(
colors.yellow(
`⚠️ Falling back to the local profile named '${opts.workspace}' (${e.workspaceId} on ${e.remote}), which does NOT match wmill.yaml:\n` +
` wmill.yaml maps workspace '${opts.workspace}' to ${configuredTarget.workspaceId} on ${configuredTarget.baseUrl}, but no profile targets it.\n` +
` Run: wmill workspace add <profile-name> ${configuredTarget.workspaceId} ${configuredTarget.baseUrl}`
)
);
}
log.infoStderr(
`Using local profile '${e.name}' → ${e.workspaceId} on ${e.remote}`
);
(opts as any).__secret_workspace = e;
return { isError: false, value: e };
}
@@ -486,6 +508,8 @@ export async function resolveWorkspace(
return process.exit(-1);
}
let resolved: Workspace | undefined;
// Try to find existing workspace profile by name, then by workspaceId + remote
if (opts.workspace) {
let existingWorkspace = await getWorkspaceByName(
@@ -523,19 +547,45 @@ export async function resolveWorkspace(
);
return process.exit(-1);
}
return {
resolved = {
...existingWorkspace,
token: opts.token,
};
}
}
return {
resolved ??= {
remote: normalizedBaseUrl,
workspaceId: opts.workspace,
name: opts.workspace,
token: opts.token,
};
// --base-url pins the target, so wmill.yaml's `workspaces` block is never
// consulted and `--workspace` reaches the API as a workspace id. Name the
// id being sent, and the mapping being skipped, before the request 404s
// on an id the user never typed.
// Only an explicit `workspaceId:` is worth reporting: an entry without one
// maps the name to itself, leaving nothing to correct.
const yamlEntry = await peekWorkspaceEntry(opts.workspace);
const yamlWorkspaceId = yamlEntry?.workspaceId;
if (yamlWorkspaceId && yamlWorkspaceId !== resolved.workspaceId) {
log.warnStderr(
colors.yellow(
`⚠️ --base-url is set, so wmill.yaml is not consulted: workspace id '${resolved.workspaceId}' is sent to the API.\n` +
` wmill.yaml maps workspace '${opts.workspace}' to workspace id '${yamlWorkspaceId}'${yamlEntry!.baseUrl ? ` on ${yamlEntry!.baseUrl}` : ""}.\n` +
` Use '--workspace ${yamlWorkspaceId}', or drop --base-url/--token to resolve through wmill.yaml.`
)
);
}
log.infoStderr(
`Using workspace id '${resolved.workspaceId}' on ${normalizedBaseUrl} (--base-url given` +
(resolved.name !== resolved.workspaceId
? `, profile '${resolved.name}')`
: ")")
);
(opts as any).__secret_workspace = resolved;
return resolved;
} else {
log.infoStderr(
colors.red(
cli/src/core/permissioned_as.ts
+30
View File
@@ -3,6 +3,7 @@ import * as log from "./log.ts";
import { colors } from "@cliffy/ansi/colors";
import { Confirm } from "@cliffy/prompt/confirm";
import { getTypeStrFromPath } from "../types.ts";
import { parseSyncBehavior } from "./conf.ts";
export interface PermissionedAsContext {
userCache: Map<string, { username: string; email: string }>;
@@ -10,6 +11,35 @@ export interface PermissionedAsContext {
userEmail: string;
}
/**
* The whole-tree `sync push` and the single-item `push` commands must resolve
* ownership the same way, so both build the context here: a push that leaves it
* undefined reassigns `permissioned_as` / `on_behalf_of` to whoever ran it.
* Undefined below syncBehavior v1, where that reassignment is the contract, and
* for a caller who is neither admin nor in `wm_deployers` the backend enforces
* it anyway — the flag on the context is what keeps the CLI from claiming
* otherwise.
*/
export async function buildPermissionedAsContext(
workspace: string,
syncBehavior: string | number | undefined
): Promise<PermissionedAsContext | undefined> {
if (parseSyncBehavior(syncBehavior) < 1) {
return undefined;
}
const user = await wmill.whoami({ workspace });
const userIsAdminOrDeployer =
user.is_admin || (user.groups ?? []).includes("wm_deployers");
log.debug(
`permissioned_as: user=${user.email}, is_admin=${user.is_admin}, groups=${JSON.stringify(user.groups)}, isAdminOrDeployer=${userIsAdminOrDeployer}`
);
return {
userCache: new Map(),
userIsAdminOrDeployer,
userEmail: user.email,
};
}
async function ensureUserCache(
workspace: string,
cache: Map<string, { username: string; email: string }>
cli/test/base_url_workspace_resolution_unit.test.ts
+68
View File
@@ -0,0 +1,68 @@
import { describe, expect, test } from "bun:test";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { resolveWorkspace } from "../src/core/context.ts";
import { getWorkspaceConfigFilePath } from "../windmill-utils-internal/src/config/config.ts";
import type { GlobalOptions } from "../src/types.ts";
const BASE_URL = "http://localhost:9999/";
// --base-url pins the target: --workspace reaches the API as a workspace id and
// wmill.yaml is not consulted. The warning that says so may only peek at the
// file — readConfigFile() exits on an unsupported syncBehavior and throws on a
// malformed one, so resolving through it lets an unrelated config fail a
// command that never needed it.
async function withWmillYaml(
wmillYaml: string,
fn: (opts: GlobalOptions) => Promise<void>
): Promise<void> {
const repoDir = await mkdtemp(path.join(os.tmpdir(), "wmill_baseurl_repo_"));
const configDir = await mkdtemp(path.join(os.tmpdir(), "wmill_baseurl_conf_"));
const originalCwd = process.cwd();
try {
await writeFile(path.join(repoDir, "wmill.yaml"), wmillYaml);
await writeFile(await getWorkspaceConfigFilePath(configDir), "");
process.chdir(repoDir);
await fn({
configDir,
baseUrl: BASE_URL,
token: "sometoken",
workspace: "staging",
} as GlobalOptions);
} finally {
process.chdir(originalCwd);
await rm(repoDir, { recursive: true, force: true });
await rm(configDir, { recursive: true, force: true });
}
}
describe("--base-url workspace resolution", () => {
const rejectedConfigs: [string, string][] = [
["an unsupported syncBehavior", "syncBehavior: v2\n"],
["a malformed file", 'workspaces:\n staging:\n baseUrl: "unterminated\n'],
];
for (const [label, wmillYaml] of rejectedConfigs) {
test(`resolves despite ${label}`, async () => {
await withWmillYaml(wmillYaml, async (opts) => {
const workspace = await resolveWorkspace(opts);
expect(workspace.workspaceId).toBe("staging");
expect(workspace.remote).toBe(BASE_URL);
});
});
}
test("a workspaces mapping never overrides the explicit workspace id", async () => {
await withWmillYaml(
"workspaces:\n staging:\n baseUrl: http://elsewhere.example/\n workspaceId: admins\n",
async (opts) => {
const workspace = await resolveWorkspace(opts);
expect(workspace.workspaceId).toBe("staging");
expect(workspace.remote).toBe(BASE_URL);
}
);
});
});
cli/test/schedule_push_permissioned_as_unit.test.ts
+125
View File
@@ -0,0 +1,125 @@
/**
* Regression guard: the standalone `wmill schedule push` must resolve ownership
* the same way `wmill sync push` does. It only preserves the remote's
* `permissioned_as` when the command hands `pushSchedule` a context, so a push
* that builds none silently reassigns the schedule to whoever ran it.
*/
import { expect, test, describe, beforeEach, mock } from "bun:test";
import { mkdtemp, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
let updateScheduleCalls: any[] = [];
let remotePermissionedAs: string | undefined = "u/svc";
const REMOTE_SCHEDULE = () => ({
path: "u/admin/sched",
schedule: "0 0 */6 * * *",
timezone: "Etc/UTC",
script_path: "u/admin/script",
is_flow: false,
args: {},
enabled: false,
summary: "before",
permissioned_as: remotePermissionedAs,
});
mock.module("../gen/services.gen.ts", () => ({
getSchedule: async () => REMOTE_SCHEDULE(),
updateSchedule: async (a: unknown) => {
updateScheduleCalls.push(a);
},
whoami: async () => ({
email: "deployer@windmill.dev",
username: "deployer",
is_admin: true,
groups: [],
}),
}));
const realContext = await import("../src/core/context.ts");
mock.module("../src/core/context.ts", () => ({
...realContext,
resolveWorkspace: async () => ({
workspaceId: "w",
name: "w",
remote: "http://localhost/",
token: "t",
}),
}));
const realAuth = await import("../src/core/auth.ts");
mock.module("../src/core/auth.ts", () => ({
...realAuth,
requireLogin: async () => ({}),
}));
const scheduleCommand = (await import("../src/commands/schedule/schedule.ts"))
.default;
async function pushIn(wmillYamlTail: string): Promise<void> {
const dir = await mkdtemp(join(tmpdir(), "windmill_sched_push_"));
await writeFile(
join(dir, "wmill.yaml"),
`defaultTs: bun\nincludeSchedules: true\n${wmillYamlTail}`,
"utf-8"
);
await writeFile(
join(dir, "sched.schedule.yaml"),
`schedule: "0 0 */6 * * *"\ntimezone: Etc/UTC\nscript_path: u/admin/script\nis_flow: false\nargs: {}\nenabled: false\nsummary: after\n`,
"utf-8"
);
const cwd = process.cwd();
process.chdir(dir);
try {
await scheduleCommand.parse([
"push",
"sched.schedule.yaml",
"u/admin/sched",
]);
} finally {
process.chdir(cwd);
}
}
describe("wmill schedule push ownership", () => {
beforeEach(() => {
updateScheduleCalls = [];
remotePermissionedAs = "u/svc";
});
test("keeps the remote's permissioned_as under syncBehavior v1", async () => {
await pushIn("syncBehavior: v1\n");
expect(updateScheduleCalls).toHaveLength(1);
const body = updateScheduleCalls[0].requestBody;
expect(body.summary).toBe("after");
expect(body.permissioned_as).toBe("u/svc");
expect(body.preserve_permissioned_as).toBe(true);
});
// The entry to read is the one matching the workspace being pushed to, not
// the top level: a repo that varies settings per workspace puts syncBehavior
// under `overrides` and nowhere else.
test("reads syncBehavior from the target workspace's overrides", async () => {
await pushIn(
`workspaces:\n other:\n baseUrl: http://localhost/\n workspaceId: w\n overrides:\n syncBehavior: v1\n`
);
expect(updateScheduleCalls).toHaveLength(1);
const body = updateScheduleCalls[0].requestBody;
expect(body.permissioned_as).toBe("u/svc");
expect(body.preserve_permissioned_as).toBe(true);
});
test("leaves ownership to the backend below syncBehavior v1", async () => {
await pushIn("");
expect(updateScheduleCalls).toHaveLength(1);
const body = updateScheduleCalls[0].requestBody;
expect(body.permissioned_as).toBeUndefined();
expect(body.preserve_permissioned_as).toBeUndefined();
});
});
cli/test/variable_resource_push.test.ts
+120
View File
@@ -361,6 +361,126 @@ describe("variable", () => {
expect(content).toContain("is_secret: false");
});
});
test("extra_perms round-trips and pushes via /acls/* without rewriting the variable", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspaceProfile(backend);
const uniqueId = Date.now();
const varPath = `f/test/perms_var_${uniqueId}`;
const createResp = await backend.apiRequest!(
`/api/w/${backend.workspace}/variables/create`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
path: varPath,
value: "perms_test_value",
is_secret: false,
description: "Variable for extra_perms test",
}),
}
);
expect(createResp.status).toBeLessThan(300);
await createResp.text();
const aclResp = await backend.apiRequest!(
`/api/w/${backend.workspace}/acls/add/variable/${varPath}`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ owner: "g/all", write: true }),
}
);
expect(aclResp.status).toBeLessThan(300);
await aclResp.text();
await writeFile(
join(tempDir, "wmill.yaml"),
`defaultTs: bun\nincludes:\n - "${varPath}**"\nexcludes: []\n`,
"utf-8"
);
const pullResult = await backend.runCLICommand(
["sync", "pull", "--yes"],
tempDir
);
expect(pullResult.code).toEqual(0);
const localPath = join(tempDir, `${varPath}.variable.yaml`);
const pulled = await readFile(localPath, "utf-8");
expect(pulled).toContain("extra_perms:");
expect(pulled).toContain("g/all: true");
const beforeResp = await backend.apiRequest!(
`/api/w/${backend.workspace}/variables/get/${varPath}`
);
const before = await beforeResp.json();
// Perm-only edit: downgrade the grant to read.
await writeFile(
localPath,
pulled.replace("g/all: true", "g/all: false"),
"utf-8"
);
const pushResult = await backend.runCLICommand(
["sync", "push", "--yes"],
tempDir
);
expect(pushResult.code).toEqual(0);
const afterResp = await backend.apiRequest!(
`/api/w/${backend.workspace}/variables/get/${varPath}`
);
const after = await afterResp.json();
expect(after.extra_perms).toEqual({ "g/all": false });
// Routed through /acls/* rather than update_variable, so the row itself
// is untouched.
expect(after.edited_at).toEqual(before.edited_at);
expect(after.value).toEqual("perms_test_value");
// A yaml with no extra_perms field at all is "no opinion": a checkout
// that predates ACL sync must never revoke UI-managed grants.
await writeFile(
localPath,
`description: "Variable for extra_perms test"\nvalue: perms_test_value\nis_secret: false\n`,
"utf-8"
);
const noOpinionResult = await backend.runCLICommand(
["sync", "push", "--yes"],
tempDir
);
expect(noOpinionResult.code).toEqual(0);
const noOpinionApiResp = await backend.apiRequest!(
`/api/w/${backend.workspace}/variables/get/${varPath}`
);
expect((await noOpinionApiResp.json()).extra_perms).toEqual({
"g/all": false,
});
// An owner present remotely but absent from a *present* map is revoked —
// the one direction that can destroy a grant.
await writeFile(
localPath,
`description: "Variable for extra_perms test"\nvalue: perms_test_value\nis_secret: false\nextra_perms: {}\n`,
"utf-8"
);
const revokeResult = await backend.runCLICommand(
["sync", "push", "--yes"],
tempDir
);
expect(revokeResult.code).toEqual(0);
const finalResp = await backend.apiRequest!(
`/api/w/${backend.workspace}/variables/get/${varPath}`
);
const final = await finalResp.json();
expect(final.extra_perms).toEqual({});
});
});
});
// =============================================================================
cli/test/workspace_key_filename_integration.test.ts
+4 -2
View File
@@ -7,8 +7,10 @@ import { stringify as yamlStringify } from "yaml";
import { resolveWsNameForGitBranch } from "../src/core/specific_items.ts";
import { findResourceFile } from "../src/commands/script/script.ts";
import { resolveWsNameForConfigFromFlags } from "../src/commands/sync/sync.ts";
import type { SyncOptions } from "../src/core/conf.ts";
import {
resolveWsNameForConfigFromFlags,
type SyncOptions,
} from "../src/core/conf.ts";
// Integration tests covering the bug where workspace-specific filenames used
// the raw git branch name instead of the wmill.yaml workspace config key.
docs/dbt-runtime.md
+120 -30
View File
@@ -47,6 +47,7 @@ the dominant way dbt is orchestrated today.
| 22 | Naming | Match Cosmos field names; importer deferred |
| 23 | Descriptor | `wm_dbt.yaml` inside the project, OPTIONAL. See below |
| 24 | Warehouse | Configured on the workspace by name, `main` by default. See below |
| 25 | Cascade direction | Into a relation, not out of a run: `// materialize manual dbt://…` declares a write from any language but dbt's own and wakes `# on dbt://…` subscribers; a finished dbt run still does not dispatch. See "No cascade *from* dbt" |
## Decision 1: engine toggle, and why the shipped default is not Fusion yet
@@ -157,11 +158,15 @@ build and an enterprise build whose key did not verify.
workspace warehouse's NAME, so two scripts running against the same warehouse
agree on identity.
The SCHEME names the producer, because dbt is the only thing that creates one of
these: no other language derives warehouse relations, `// materialize` takes
DuckLake targets only, and a dbt run does not dispatch. Calling the kind
something generic promised a parity with native Snowflake and BigQuery scripts
that does not exist.
The SCHEME names the namespace dbt made, not an exclusive producer. dbt is what
put warehouse relations in the asset graph and is what derives them from a
project; no other language *infers* one, and calling the kind something generic
promised a parity with native Snowflake and BigQuery scripts that does not exist.
A script can nonetheless DECLARE that it writes one — `// materialize manual
dbt://<warehouse>/<schema>/<name>`, in any language but dbt's own, whose writes
come from its manifest — and that declaration lands on the same node the dbt model
reading the relation does, because identity is the relation rather than the tool.
See "No cascade *from* dbt" below.
The PATH is the physical relation, and that is the load-bearing half. dbt-core
has no cross-project `ref()`: two projects meet when one materializes a mart and
@@ -648,10 +653,13 @@ Two consequences worth knowing:
dropped would be filtered out of its own run's graph. The pinned version's
nodes are the scope instead.
## No cascade from dbt, and no pipeline membership
## No cascade *from* dbt, and no pipeline membership
A finished dbt run does not trigger anything. Its models are recorded, drawn and
tracked; they do not fan out.
tracked; they do not fan out. The opposite direction does: a script that declares
`// materialize manual dbt://<warehouse>/<schema>/<name>` is an ordinary producer
of that relation, and its completion wakes `# on dbt://<relation>` subscribers
through the same fan-out every other asset kind uses.
A dbt script is also not a pipeline member (`in_pipeline` is forced false for
`ScriptLang::Dbt` at deploy). It materializes warehouse tables, so it looks like
@@ -663,22 +671,99 @@ Its models are `dbt://` assets in the shared graph regardless: that is what
puts a native script reading one of them on the same node, and it is independent
of pipeline membership.
dbt already orders its own DAG, so a cascade would only ever add one thing:
waking a Windmill script that reads a mart. That edge is real but narrow, and
only half of it exists — nothing outside dbt can declare a `dbt://` write
(`// materialize` accepts DuckLake targets only), so the reverse direction, an
ingestion script waking a dbt project, cannot be expressed at all.
Against that, dispatching correctly from dbt is not cheap. A run's `select` can
build any subset of the project, so the deploy-time write set is not what ran;
using it wakes consumers of relations the run never touched, and narrowing it
needs a per-job record of what was built, which the per-relation state table
cannot supply (it keeps one row per relation, stamped with the last writer).
dbt already orders its own DAG, so a cascade out of a run would only ever add one
thing: waking a Windmill script that reads a mart. That edge is real but narrow,
and dispatching it correctly is not cheap. A run's `select` can build any subset
of the project, so the deploy-time write set is not what ran; using it wakes
consumers of relations the run never touched, and narrowing it needs a per-job
record of what was built, which the per-relation state table cannot supply (it
keeps one row per relation, stamped with the last writer).
So dbt materializes and reports, and `asset_dispatch` returns early for
`ScriptLang::Dbt`. A `# on dbt://<mart>` subscription is refused outright at
deploy rather than accepted and left dormant — an edge drawn on the canvas that
can never fire is worse than an error saying so.
`ScriptLang::Dbt`. Wiring it up later means deciding what a selective run should
notify — that decision is the work, not the plumbing.
### Declaring the write, and which subscriptions are refused
`// materialize manual dbt://<warehouse>/<schema>/<name>` is how an ingestion
script says it writes a warehouse relation. `manual` is not a mode but the only
mode: nothing generates warehouse DDL, so the script issues its own write and
Windmill records the outcome — the same `materialized_partition` row a DuckLake
target lands, so the relation carries a last writer on the run page and the graph.
It is language-agnostic (the DuckLake write ENGINE is DuckDB's; this declaration
is anyone's but a dbt project's, whose writes are read from its manifest), and the
recording happens in the generic job path
(`record_declared_warehouse_write`) rather than in an executor, for the same
reason. Identity is unchanged — the physical relation — so the ingestion script
and the dbt model reading it are one node, and a `source` declared on the relation
puts the whole thing on one lineage. `// data_test` is refused beside it: those
checks are probes the DuckDB executor splices around a managed write, so on a
warehouse relation — which the script writes itself, from any language — nothing
would run them, and a declarer would deploy green with its assertions silently
skipped. Assert on the relation with a dbt test in the project that reads it.
The `<warehouse>` segment is resolved at
deploy for the same reason a descriptor's `profile.warehouse` is: a name no
warehouse answers to is not a namespace, it strands the write on a node nothing
else reaches.
Known boundary, shared with every other runtime pipeline annotation: the record
is written from the normal execution path, and recording and cascading are decided
separately, so the routes off it differ.
* A **dedicated worker** never enters that path — it bypasses the record exactly
as it bypasses `// partitioned` resolution — while its job is still a top-level
`Script`, so the fan-out (which reads the deploy-time `asset` rows) runs. It
cascades and records nothing, leaving the relation with no last writer.
* A **flow runner** bypasses the path too, and is routed by `flow_step_id`, which
`is_eligible_kind` rejects. Neither record nor cascade.
* A **flow step running a deployed script** enters the path as a `Script` job, so
it records — and carries a `flow_step_id`, so it never cascades.
* A **flow step with an inline body** is a `FlowScript` job, which the recording
guard excludes along with previews: neither.
Fixing the recording half is one change for every runtime pipeline annotation,
not this one.
A `# on dbt://<relation>` subscription is held to the same relation a producer
is — a whole `<warehouse>/<schema>/<name>` under a configured warehouse, checked
by the validator the `// materialize` target goes through, since two spellings of
that rule would refuse and accept the same string. Beyond that it is refused in
exactly one shape: when every script that writes that relation is a dbt one. Nothing
produces it yet is NOT that shape — a subscriber may be deployed before its
producer, as for every other asset kind, and refusing there would break
deploy-order-independent syncs. A dbt script may neither subscribe nor declare a
`// materialize`: its graph ingest republishes that path's trigger and asset rows
wholesale, so either annotation would deploy something the dependency job then
silently removes — while the declared write would still stamp the relation on
every run.
The producer set is read as it stands committed, minus the deploying script's own
rows — those describe the version being replaced, so a script dropping its
`// materialize` while adding a subscription would otherwise count itself as the
producer that wakes it, which it could not be anyway (the dispatcher skips
self-loops).
What that leaves is a subscription accepted while it was live and later orphaned.
A dbt project that claims the relation afterwards names those edges in its own log
rather than leaving them silently dormant — the same "an edge that can never fire
is worse than saying so" the refusal is for, at the other point where it is
knowable. Both points that publish ownership warn: the deploy, and a run whose
static descriptor found its profile moved. An agent run publishes none — it is
forced to per-run models, so it stores a job-pinned snapshot and leaves workspace
ownership with the deployed graph — so it cannot orphan a subscription either.
Two orphanings are reported nowhere, and both are accepted rather than overlooked.
A native producer that drops its `// materialize` and leaves dbt alone on the
relation: the deploy that causes it does not touch the subscriber. And the
interleaving where a dbt ingest commits between a subscriber's producer check and
its own commit — the check sees no producer and accepts, the ingest's warning
query sees no trigger and says nothing. Closing the second means a per-relation
lock shared by the deploy path and the ingest, and the ingest takes
`script … FOR UPDATE` before its own advisory lock, so a deploy holding relation
locks first inverts that order into a deadlock across two subsystems — a worse
failure than the cosmetic edge it would prevent. Both are bounded the same way:
the next deploy of that project warns, and the canvas is where they show
meanwhile.
A plain READ still renders the consumer beside the model, which is what makes
the lineage one graph — but it is written in the script's own code, not in a
@@ -687,8 +772,8 @@ comment: the body parsers resolve an asset URI from a string literal
Python, TS/Bun/Deno, DuckDB or Ansible script is the read. Those four are the
languages with a body-asset parser; the native warehouse ones (snowflake,
bigquery, postgresql, mysql, mssql) declare no assets at all today, so a mart
they consume joins the graph only once that inference exists. Wiring the trigger up later means deciding what a
selective run should notify — that decision is the work, not the plumbing.
they consume joins the graph only once that inference exists — while a relation
one of them WRITES joins it now, through the annotation.
## Live per-model progress, and why only dbt-core 1.x has it
@@ -1308,17 +1393,22 @@ Against a real dbt project (jaffle_shop shape) and the local Postgres:
script reading one of the marts gets an edge to it.
6. **Shared node**: a native script that READS a mart renders as a reader of the
same node the dbt model writes — one node, not two islands. Declared with a
plain read (`# dbt://<mart>`), never `# on`: a `dbt://` subscription is
refused at deploy, because nothing but dbt writes a warehouse relation and a
dbt run does not dispatch (see "no cascade from dbt").
7. **Selection**: descriptor `select`/`exclude`, and a run-arg override, each
plain read (`# dbt://<mart>`), never `# on`: a subscription to a relation dbt
alone builds is refused at deploy, since a dbt run does not dispatch.
7. **Declared write**: a native `// materialize manual dbt://<relation>` script
and a dbt project reading that relation as a `source` render as one node; a
run of the script records its materialization and wakes a
`# on dbt://<relation>` subscriber — a subscription only that producer makes
wakeable, the dbt project reading the relation being no producer of it (see
"no cascade *from* dbt").
8. **Selection**: descriptor `select`/`exclude`, and a run-arg override, each
build only the expected subset.
8. **Dynamic descriptors**: a `{{ }}` placeholder in `vars` re-ingests the graph
9. **Dynamic descriptors**: a `{{ }}` placeholder in `vars` re-ingests the graph
from the run's own manifest, so a model that placeholder enables appears in
the same run that builds it.
9. **Both credential paths**: resource-rendered `profiles.yml`, and the project's
10. **Both credential paths**: resource-rendered `profiles.yml`, and the project's
own `profiles.yml` with env-var injection.
10. **Caching**: a second run reuses the cached `dbt_packages/` with no network
11. **Caching**: a second run reuses the cached `dbt_packages/` with no network
fetch.
Keep only tests that pin behavior a future change could break. Per AGENTS.md,
docs/ducklake-materialization.md
+4 -2
View File
@@ -606,8 +606,10 @@ how-to (extract-engine choice, cursor recipes, schema-drift handling, worked
examples) lives in windmilldocs `core_concepts/63_pipelines` → "Ingestion
(EL)"; this section records only what future feature work must not break.
- **`// materialize` is DuckDB-only** (deploy-rejected elsewhere, managed and
`manual` alike — `windmill-api-scripts/src/scripts.rs`), and the SDK
- **A `ducklake://` `// materialize` is DuckDB-only** (deploy-rejected
elsewhere, managed and `manual` alike — `windmill-api-scripts/src/scripts.rs`;
a `dbt://` warehouse-relation target is the one any language but dbt's own may
declare, and it is track-only — see `docs/dbt-runtime.md`), and the SDK
materialize helpers (`upsert_partition` / `upsertPartition`) build their SQL
inside the SDK, so the asset parsers cannot see the write. A polyglot node
that "writes the lake directly" therefore deploys with **no output edge** —
docs/reusable-ai-agents.md
+40 -8
View File
@@ -22,22 +22,54 @@ every workspace via the standard cached-resource-type sync, like other built-in
or flow expressions), so saving round-trips losslessly. Each host flow overrides what it
needs: `tool_inputs` stores per-tool overrides (a diff from the resource tool's own
transforms) that overlay onto the matching tools at runtime. Editing on a linked step edits
the flow's use of the agent; editing under the "Editing" banner edits the agent itself.
the flow's use of the agent; editing in the agent editor edits the agent itself.
In the flow editor, the AI agent step's **Step Input** tab shows a single read-only card
(*linked to <path>*, with the inherited brain + tools and an explanatory tooltip) plus
*Edit* (fork into the editable step, Save changes upserts back and re-links) and *Unlink*
(fork the resolved config — including any `tool_inputs` — back into the step as a one-off).
While editing, the step is the only copy of the edits: Cancel drops them and re-links (asking
first when there is something to drop), and the unsaved-changes badge opens a diff against the
deployed agent whose Discard changes is Cancel without the question. What a fork is an edit of,
and the deployed baseline the edits are judged against, live in `agentEditStore` (in memory), so
a reload brings the step back as a standalone agent with no path to save back to.
*Edit*, which opens the agent editor over the flow, and *Unlink* (fork the resolved config —
including any `tool_inputs` — back into the step as a one-off).
A linked agent's tools appear as display-only graph tool nodes (clicking one selects the
agent step); below the step's inputs, each tool gets a section with the standard schema-aware
input editors (prop picker included) and a read-only view of its code — edits persist into
`tool_inputs`.
## Drafts
The agent editor edits the resource through a **per-user resource draft** (`draft` table,
`item_kind = 'resource'`), autosaved by `useAgentDraft` and deployed by the editor's own Deploy
button. It is the same draft row the generic resource editor writes and the Review & Deploy page
lists, so an agent can be deployed from any of them.
A flow does not wait for that deploy to see the draft:
- Testing the flow, or a single linked step, runs the draft. `runFlowPreview` and `ModuleTest`
substitute each linked step for the standalone step the draft would run as
(`linkedAgentDrafts.ts`): `agent` cleared, the draft's brain as static input transforms, the
draft's tools on the step, and the step's own `user_message`/`user_attachments` kept on top —
the same overlay order `ai_executor.rs` applies to a linked step. `tool_inputs` is untouched,
since the worker overlays it in both branches.
- The step's linked card and the graph's tool nodes show the draft, with a *Draft* badge, so the
editor describes what a test would run. Read-only surfaces (the deployed flow page, the run
viewer) stay on the deployed agent: they resolve tools through `publishLinkedAgentTools` without
the draft flag.
- Deploying the flow lists every linked agent that has a draft in the confirmation dialog, beside
the draft triggers. Deploying one writes the resource and drops the draft; leaving one out keeps
its draft untouched, and the flow runs the agent as currently deployed. That is the one place
the two kinds differ: an undeployed draft trigger is deleted, because it belongs to the flow,
while an agent draft belongs to a resource other flows also use.
Because a draft is per-user, a flow test can behave differently for two people looking at the same
flow. That is the same contract as a flow draft, and deploying the agent is what makes it shared.
Inlining has a consequence worth knowing: a preview job's `raw_flow` then carries the agent's
config, where a linked step used to carry only the path and leave the resolution to the worker. So
an agent's prompt and tool set are readable by whoever can read that preview job, which is a wider
set than whoever can read the resource when the agent sits in a more restricted folder than the
flow. No credential travels with it — the provider stays a `$res:` reference, resolved at run time
as the runner. The agent editor's own test pane has inlined the same way since drafts existed;
closing the gap would mean the preview carrying a draft *reference* the worker resolves, rather
than the config.
Sharing works through standard resource folder permissions (save agents under `f/...`).
Only the agent's brain is interpolated when the step runs. A tool's own `$res:`/`$var:` defaults are
frontend/package-lock.json
Generated
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "@windmill-labs/components",
"version": "1.804.0",
"version": "1.805.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@windmill-labs/components",
"version": "1.804.0",
"version": "1.805.0",
"hasInstallScript": true,
"license": "AGPL-3.0",
"dependencies": {
frontend/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@windmill-labs/components",
"version": "1.804.0",
"version": "1.805.0",
"scripts": {
"dev": "vite dev",
"dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev",
frontend/src/lib/attachments/newTabModifier.dom.test.ts
+106
View File
@@ -0,0 +1,106 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { newTabModifier } from './newTabModifier.svelte'
const onPlatform = (userAgent: string) => vi.stubGlobal('navigator', { userAgent })
const LINUX = 'Mozilla/5.0 (X11; Linux x86_64)'
const MAC = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)'
const attached: (() => void)[] = []
/** Attach to a fresh element and return it with its cleanup, as `{@attach}` would. */
function pill() {
const node = document.createElement('span')
document.body.append(node)
const modifier = newTabModifier()
const cleanup = modifier.attach(node) as () => void
attached.push(cleanup)
const hover = (init: MouseEventInit = {}) =>
node.dispatchEvent(new MouseEvent('mouseenter', init))
const move = (init: MouseEventInit = {}) => node.dispatchEvent(new MouseEvent('mousemove', init))
const unhover = () => node.dispatchEvent(new MouseEvent('mouseleave'))
return { modifier, hover, move, unhover, cleanup }
}
const keydown = (init: KeyboardEventInit) =>
window.dispatchEvent(new KeyboardEvent('keydown', init))
describe('newTabModifier', () => {
// The window listeners outlive the DOM, so every case has to be torn down through the
// attachment rather than by emptying the body.
afterEach(() => {
attached.splice(0).forEach((cleanup) => cleanup())
document.body.replaceChildren()
vi.unstubAllGlobals()
})
// The hover event carries the live modifier state, so a modifier pressed before the pointer
// arrived (or while this window was unfocused) is picked up rather than read as false.
it('seeds from the hover event, per platform', () => {
onPlatform(LINUX)
const linux = pill()
linux.hover({ ctrlKey: true })
expect(linux.modifier.held).toBe(true)
onPlatform(MAC)
const mac = pill()
// macOS ctrl+click is a secondary click, so it must not read as a new-tab modifier.
mac.hover({ ctrlKey: true })
expect(mac.modifier.held).toBe(false)
mac.hover({ metaKey: true })
expect(mac.modifier.held).toBe(true)
})
// Editors and menus stop keydown propagation to keep their own shortcuts, so a bubble-phase
// listener would go blind whenever focus sits in one.
it('sees a keydown that a focused element stops from propagating', () => {
onPlatform(LINUX)
const { modifier, hover } = pill()
hover()
const input = document.createElement('input')
input.addEventListener('keydown', (e) => e.stopPropagation())
document.body.append(input)
input.dispatchEvent(
new KeyboardEvent('keydown', { key: 'Control', ctrlKey: true, bubbles: true })
)
expect(modifier.held).toBe(true)
})
// A modifier held across a keyboard app switch is cleared by the blur and delivers no keydown
// on the way back, while the pointer parked on the pill fires no fresh mouseenter either.
it('re-seeds from pointer movement after the window lost focus', () => {
onPlatform(LINUX)
const { modifier, hover, move } = pill()
hover({ ctrlKey: true })
window.dispatchEvent(new Event('blur'))
expect(modifier.held).toBe(false)
move({ ctrlKey: true })
expect(modifier.held).toBe(true)
})
it('stops tracking once unhovered', () => {
onPlatform(LINUX)
const { modifier, hover, unhover } = pill()
hover({ ctrlKey: true })
unhover()
expect(modifier.held).toBe(false)
keydown({ key: 'Control', ctrlKey: true })
expect(modifier.held).toBe(false)
})
it('stops tracking when the element is destroyed while hovered', () => {
onPlatform(LINUX)
const { modifier, hover, cleanup } = pill()
hover({ ctrlKey: true })
// Hovering again without leaving must not strand the first hover's listeners, which nothing
// would then hold a reference to.
hover({ ctrlKey: true })
cleanup()
expect(modifier.held).toBe(false)
keydown({ key: 'Control', ctrlKey: true })
expect(modifier.held).toBe(false)
})
})
frontend/src/lib/attachments/newTabModifier.svelte.ts
+66
View File
@@ -0,0 +1,66 @@
import type { Attachment } from 'svelte/attachments'
import { isMac } from '$lib/utils'
/**
* Tracks whether the modifier that turns a click into a new browser tab is held, but only while
* the attached element is hovered, which is the only moment the answer is used.
*/
export function newTabModifier() {
let held = $state(false)
// Only the modifier that actually yields a tab: shift opens a window, alt can start a
// download, and on macOS ctrl+click is a secondary click.
// Taken from each event rather than accumulated across keydown/keyup pairs, so a keyup lost to
// a focus change cannot strand the flag on.
const sync = (event: KeyboardEvent | MouseEvent) => {
held = isMac() ? event.metaKey : event.ctrlKey
}
const clear = () => {
held = false
}
const attach: Attachment<HTMLElement> = (node) => {
// One controller per hover: a mirrored remove list leaks any listener whose options drift.
let hover: AbortController | undefined
const leave = () => {
hover?.abort()
hover = undefined
clear()
}
const enter = (event: MouseEvent) => {
// Seeded from the hover itself: mouse events carry the same modifier flags as key events,
// so a modifier already held before the pointer arrived reads correctly.
sync(event)
// Re-entering without an intervening leave would strand the previous controller: nothing
// else references it, so its listeners could never be removed.
hover?.abort()
hover = new AbortController()
const { signal } = hover
// Same reason the hover seeds: a modifier held across a keyboard app switch delivers no
// keydown on the way back, so the pointer is all that is left to re-read it from.
node.addEventListener('mousemove', sync, { signal })
// Capture: editors and menus stopPropagation the keys they handle, hiding the modifier
// from a bubble-phase listener whenever focus sits in one.
window.addEventListener('keydown', sync, { capture: true, signal })
window.addEventListener('keyup', sync, { capture: true, signal })
// Not capture, unlike the two above: blur does not bubble but does reach the window while
// capturing, so it would fire for every element that loses focus.
window.addEventListener('blur', clear, { signal })
}
const life = new AbortController()
node.addEventListener('mouseenter', enter, { signal: life.signal })
node.addEventListener('mouseleave', leave, { signal: life.signal })
return () => {
life.abort()
leave()
}
}
return {
get held() {
return held
},
attach
}
}
frontend/src/lib/components/AppConnectInner.svelte
+68 -27
View File
@@ -41,6 +41,13 @@
import TextInput from './text_input/TextInput.svelte'
import { sameTopDomainOrigin } from '$lib/cookies'
import SyncResourceTypes from './SyncResourceTypes.svelte'
import {
alphabetical,
byPopularity,
hubResourceTypePicks,
localResourceTypeCounts,
recordHubResourceTypePick
} from './pickerPopularity'
import Label from './Label.svelte'
import ResourcePathHint from './ResourcePathHint.svelte'
@@ -332,6 +339,7 @@
export async function open(rt?: string) {
if (!rt) {
loadResourceTypes()
loadPopularity()
}
step = 1 //express && !manual ? 3 : 1
// The list is keyboard-driven from the search field, so it takes focus on open.
@@ -378,12 +386,29 @@
}
}
/**
* Orders the browse list: the types this workspace already has resources of lead, ranked
* among themselves by the hub's pick counts, then everything else on the same counts.
* `byPopularity` carries the full rule. Both signals are fetched, so the rows render
* alphabetically and re-sort when this lands.
*/
let popularity: (a: string, b: string) => number = $state(alphabetical)
async function loadPopularity() {
if (!effectiveWorkspace) return
const [hub, local] = await Promise.all([
hubResourceTypePicks(effectiveWorkspace),
localResourceTypeCounts(effectiveWorkspace)
])
popularity = byPopularity(hub, local)
}
async function loadConnects() {
if (!connects) {
try {
const list = (await OauthService.listOauthConnects())
.filter((x) => x.name != 'supabase_wizard')
.sort((a, b) => a.name.localeCompare(b.name))
const list = (await OauthService.listOauthConnects()).filter(
(x) => x.name != 'supabase_wizard'
)
connects = list.map((x) => x.name)
connectsInfo = Object.fromEntries(list.map((x) => [x.name, x]))
} catch (e) {
@@ -466,19 +491,17 @@
// providers — so any of them can also be connected with the user's own
// credentials or manually, not only via the shared instance setup (same as
// the authorization-code behavior).
connectsManual = availableRts
.map(
(x) =>
({
key: x,
...(apiTokenApps[x] ?? {
instructions: '',
img: undefined,
linkedSecret: undefined
})
}) as { key: string; img?: string; instructions: string[] }
)
.sort((a, b) => a.key.localeCompare(b.key))
connectsManual = availableRts.map(
(x) =>
({
key: x,
...(apiTokenApps[x] ?? {
instructions: '',
img: undefined,
linkedSecret: undefined
})
}) as { key: string; img?: string; instructions: string[] }
)
const filteredNativeLanguages = filteredConnectsManual?.filter(
(o) => nativeLanguagesCategory?.includes(o[0]) ?? false
)
@@ -952,6 +975,10 @@
}
})
}
// Saving is what "picking a type" means to the hub: reaching step 2 is still
// browsing. Both branches above count, `filling` included — an imported stub is
// a type taken into the workspace just the same.
recordHubResourceTypePick(effectiveWorkspace, resourceType)
dispatch('refresh', path)
dispatch('close')
sendUserToast(
@@ -972,6 +999,9 @@
if (step == 1) {
loadConnects()
loadResourceTypes()
// Opened on a specific type, `open()` skipped this; backing out to the browse
// list is the first time it is needed.
loadPopularity()
}
}
@@ -980,24 +1010,35 @@
let filteredConnects: { key: string }[] = $state([])
let filteredConnectsManual: { key: string; img?: string; instructions: string[] }[] = $state([])
// uFuzzy scores the name and the description as one string, so searching "google" ranks
// every type whose description mentions Google alongside the ones named after it. Re-sort
// on which field matched, keeping uFuzzy's order within a tier.
let searching = $derived(filter.trim() !== '')
// Searching, the query owns the order: uFuzzy scores the name and the description as one
// string, so "google" ranks every type whose description mentions Google alongside the
// ones named after it — re-sort on which field matched, keeping uFuzzy's order within a
// tier. Browsing, there is no query to rank against, so popularity orders the list.
const rank = (items: { key: string }[] | undefined) =>
items &&
sortResourceTypesByMatch(
items,
filter,
(x) => x.key,
(x) => resourceTypeDescriptions[x.key]
)
(searching
? sortResourceTypesByMatch(
items,
filter,
(x) => x.key,
(x) => resourceTypeDescriptions[x.key]
)
: // Both signals are keyed by resource type, and a sandbox client is a second row
// against one (`salesforce_sandbox` saves a `salesforce`), so it ranks on the
// parent's popularity. Its own key still breaks the tie the pair then have, or
// the two would order arbitrarily.
[...items].sort(
(a, b) =>
popularity(stripSandboxSuffix(a.key), stripSandboxSuffix(b.key)) ||
a.key.localeCompare(b.key)
))
let rankedConnects = $derived(rank(filteredConnects))
let rankedConnectsManual = $derived(
rank(filteredConnectsManual) as typeof filteredConnectsManual | undefined
)
let searching = $derived(filter.trim() !== '')
// Browsing, the "Others" list leads with the native database types. Searching, that
// grouping would outrank the search itself — `ms_sql_server` sorting under `mysql` on
// "sql" — so the ranked order stands on its own.
frontend/src/lib/components/DBManager.svelte
+97 -10
View File
@@ -12,8 +12,8 @@
import { Pane, Splitpanes } from 'svelte-splitpanes'
import { ClearableInput, Drawer, DrawerContent } from './common'
import { sendUserToast } from '$lib/toast'
import { type ColumnDef } from './apps/components/display/dbtable/utils'
import DBTable from './DBTable.svelte'
import { renderDbEqualityFilter, type ColumnDef } from './apps/components/display/dbtable/utils'
import DBTable, { type DbForeignKeyTarget, type DbRowFilter } from './DBTable.svelte'
import type { IDbSchemaOps, IDbTableOps } from './dbOps'
import DropdownV2 from './DropdownV2.svelte'
import ConfirmationModal from './common/confirmationModal/ConfirmationModal.svelte'
@@ -46,7 +46,12 @@
dbSupportsSchemas: boolean
databaseIsEmpty?: boolean
colDefs: Record<string, ColumnDef[]> | undefined
dbTableOpsFactory: (params: { colDefs: ColumnDef[]; tableKey: string }) => IDbTableOps
dbTableOpsFactory: (params: {
colDefs: ColumnDef[]
tableKey: string
/** Raw SQL predicate AND-ed into the reads (already escaped). */
whereClause?: string
}) => IDbTableOps
dbSchemaOps: IDbSchemaOps
refresh?: () => void
initialSchemaKey?: string
@@ -214,6 +219,83 @@
: selected.tableKey
)
// Set by "Go to row" on a foreign-keyed cell; pinned to the table it was
// created for so a schema change can't carry it onto an unrelated table.
let rowFilter: (DbRowFilter & { tableKey: string }) | undefined = $state()
let activeRowFilter = $derived(rowFilter?.tableKey === tableKey ? rowFilter : undefined)
let whereClause = $derived(
activeRowFilter
? renderDbEqualityFilter(activeRowFilter.column, activeRowFilter.value, dbType)
: undefined
)
function selectTable(schemaKey: string | undefined, table: string) {
rowFilter = undefined
selected = { schemaKey, tableKey: table }
}
/** Where a foreign key's `schema.table` target lives in the sidebar, or
* undefined when it cannot be opened from here. */
function resolveForeignKeyTarget(
targetTable: string
): { schemaKey: string; table: string } | undefined {
const parts = targetTable.split('.')
const table = parts[parts.length - 1]
const qualifier = parts.length > 1 ? parts.slice(0, -1).join('.') : undefined
// Without schema support the sidebar browses the connection's default
// schema only, and unqualified reads would hit a same-named local table.
if (!dbSupportsSchemas && qualifier && qualifier !== selected.schemaKey) return undefined
const schemaKey = dbSupportsSchemas && qualifier ? qualifier : selected.schemaKey
if (!schemaKey || !(table in (dbSchema.schema[schemaKey] ?? {}))) return undefined
return { schemaKey, table }
}
function goToRow(target: DbForeignKeyTarget) {
const resolved = resolveForeignKeyTarget(target.table)
if (!resolved) {
sendUserToast(`Table ${target.table} cannot be opened from this schema`, true)
return
}
if (renderDbEqualityFilter(target.column, target.value, dbType) === undefined) {
sendUserToast('This value cannot be used as a filter', true)
return
}
const { schemaKey, table } = resolved
selectTable(schemaKey, table)
rowFilter = {
tableKey: dbSupportsSchemas ? `${schemaKey}.${table}` : table,
column: target.column,
value: target.value
}
}
// The result carries the table it was fetched for: `resource` keeps the
// previous value while refetching, and a stale list would decorate the new
// table's same-named columns as foreign keys.
let foreignKeys = resource(
[() => selected.tableKey, () => selected.schemaKey, () => colDefs],
async ([table, schema], _prev, { signal }) => {
if (!table) return undefined
const forTableKey = dbSupportsSchemas && schema ? `${schema}.${table}` : table
const fks =
features?.foreignKeys === false
? []
: await dbSchemaOps.onFetchForeignKeys({ table, schema })
// A newer selection started meanwhile: an AbortError keeps this result
// out of `current`, where it would shadow the newer table's keys.
if (signal.aborted) throw new DOMException('Superseded', 'AbortError')
return { tableKey: forTableKey, foreignKeys: fks }
}
)
// Only keys whose target the sidebar can open get the "Go to row" affordance.
let currentForeignKeys = $derived.by(() => {
const fetched = foreignKeys.current
if (!fetched || fetched.tableKey !== tableKey) return undefined
return fetched.foreignKeys.filter(
(fk) => fk.targetTable && resolveForeignKeyTarget(fk.targetTable) !== undefined
)
})
let askingForConfirmation:
| (ConfirmationModal['$$prop_def'] & { onConfirm: () => void })
| undefined = $state()
@@ -395,14 +477,12 @@
role="button"
tabindex="0"
onclick={() => {
selected.schemaKey = schemaKey
selected.tableKey = tableKey
selectTable(schemaKey, tableKey)
toggleTableSelection(schemaKey, tableKey)
}}
onkeydown={(e) => {
if (e.key === 'Enter' || e.key === ' ') {
selected.schemaKey = schemaKey
selected.tableKey = tableKey
selectTable(schemaKey, tableKey)
toggleTableSelection(schemaKey, tableKey)
}
}}
@@ -468,7 +548,7 @@
<button
class={'w-full text-sm font-normal flex gap-2 items-center h-10 cursor-pointer pl-3 pr-1 ' +
(selected.tableKey === tableKey ? 'bg-surface-secondary' : 'hover:bg-surface-hover')}
onclick={() => (selected.tableKey = tableKey)}
onclick={() => selectTable(selected.schemaKey, tableKey)}
>
{#if asset}
<Star
@@ -541,8 +621,15 @@
</Pane>
<Pane class="p-3 pt-1">
{#if tableKey && colDefs?.[tableKey]?.length}
{@const dbTableOps = dbTableOpsFactory({ colDefs: colDefs[tableKey], tableKey })}
<DBTable {dbTableOps} bind:this={_dbTable} />
{@const dbTableOps = dbTableOpsFactory({ colDefs: colDefs[tableKey], tableKey, whereClause })}
<DBTable
{dbTableOps}
foreignKeys={currentForeignKeys}
onGoToRow={goToRow}
rowFilter={activeRowFilter}
onClearRowFilter={() => (rowFilter = undefined)}
bind:this={_dbTable}
/>
{:else if databaseIsEmpty}
<div class="h-full w-full center-center flex-col gap-4">
<span class="text-hint">Database is empty</span>
frontend/src/lib/components/DBManagerContent.svelte
+2 -1
View File
@@ -277,10 +277,11 @@
databaseIsEmpty={!Object.values(dbSchema.schema).flatMap((s) => Object.values(s)).length}
{dbSchema}
colDefs={colDefs.current}
dbTableOpsFactory={({ colDefs, tableKey }) =>
dbTableOpsFactory={({ colDefs, tableKey, whereClause }) =>
dbTableOpsWithPreviewScripts({
colDefs,
tableKey,
whereClause,
input: _input,
workspace: ws,
tag: workerTag
frontend/src/lib/components/DBTable.svelte
+133 -27
View File
@@ -1,11 +1,78 @@
<script lang="ts" module>
import type { ColDef, ITooltipComp, ITooltipParams } from 'ag-grid-community'
import type { TableEditorForeignKey } from './apps/components/display/dbtable/tableEditor'
import { mount, unmount } from 'svelte'
import DbForeignKeyTooltip from './DbForeignKeyTooltip.svelte'
export type DbRowFilter = { column: string; value: unknown }
export type DbForeignKeyTarget = { table: string; column: string; value: unknown }
type FkTooltipValue = { table: string; column: string; value: unknown }
/** AG Grid tooltip hosting the "Go to row" popover of a foreign-keyed cell. */
class ForeignKeyTooltip implements ITooltipComp {
private eGui = document.createElement('div')
private component: ReturnType<typeof mount> | undefined
init(params: ITooltipParams & { onGoToRow: (target: DbForeignKeyTarget) => void }) {
const target = params.value as FkTooltipValue
this.component = mount(DbForeignKeyTooltip, {
target: this.eGui,
props: {
targetTable: target.table,
targetColumn: target.column,
onGoToRow: () => {
params.hideTooltipCallback?.()
params.onGoToRow(target)
}
}
})
}
getGui() {
return this.eGui
}
destroy() {
if (this.component) unmount(this.component)
}
}
const hasForeignKeyValue = (value: unknown) =>
value !== null && value !== undefined && value !== ''
/** Single-column foreign keys only: a composite key has no one cell value to follow. */
function foreignKeyColDefs(
foreignKeys: TableEditorForeignKey[],
onGoToRow: (target: DbForeignKeyTarget) => void
): Record<string, Partial<ColDef>> {
const out: Record<string, Partial<ColDef>> = {}
for (const fk of foreignKeys) {
if (fk.columns.length !== 1) continue
const { sourceColumn, targetColumn } = fk.columns[0]
if (!sourceColumn || !targetColumn || !fk.targetTable) continue
const table = fk.targetTable
out[sourceColumn] = {
tooltipComponent: ForeignKeyTooltip,
tooltipComponentParams: { onGoToRow },
tooltipValueGetter: (p): FkTooltipValue | null =>
hasForeignKeyValue(p.value) ? { table, column: targetColumn, value: p.value } : null,
cellClass: (p) =>
hasForeignKeyValue(p.value)
? 'underline decoration-dotted decoration-tertiary underline-offset-2'
: ''
}
}
return out
}
</script>
<script lang="ts">
import { workspaceStore } from '$lib/stores'
import { sendUserToast } from '$lib/toast'
import { createGrid, type GridApi, type IDatasource } from 'ag-grid-community'
import { transformColumnDefs } from './apps/components/display/table/utils'
import DarkModeObserver from './DarkModeObserver.svelte'
import { Button } from './common'
import { Download } from 'lucide-svelte'
import { Badge, Button } from './common'
import { Download, X } from 'lucide-svelte'
import Popover from './Popover.svelte'
import DebouncedInput from './apps/components/helpers/DebouncedInput.svelte'
import InsertRowDrawerButton from './apps/components/display/InsertRowDrawerButton.svelte'
@@ -18,8 +85,14 @@
type Props = {
dbTableOps: IDbTableOps
/** Foreign keys of the displayed table; their cells get a "Go to row" popover. */
foreignKeys?: TableEditorForeignKey[]
onGoToRow?: (target: DbForeignKeyTarget) => void
/** Equality filter already applied by `dbTableOps`, shown as a dismissable chip. */
rowFilter?: DbRowFilter
onClearRowFilter?: () => void
}
let { dbTableOps }: Props = $props()
let { dbTableOps, foreignKeys, onGoToRow, rowFilter, onClearRowFilter }: Props = $props()
let [clientHeight, clientWidth, darkMode, firstRow, lastRow] = $state([0, 0, false, -1, -1])
let quicksearch = $state('')
@@ -78,6 +151,10 @@
}
}),
onViewportChanged: (e) => ([firstRow, lastRow] = [e.firstRow, e.lastRow]),
// Lets the pointer move from a foreign-keyed cell into its "Go to row" popover.
tooltipInteraction: true,
// AG Grid floors the show delay at 200ms whatever value is given.
tooltipShowDelay: 0,
cacheBlockSize: 100,
cacheOverflowSize: 10,
maxBlocksInCache: 20,
@@ -93,34 +170,48 @@
let prevUpdateKey: any = undefined
$effect(() => {
if (!$workspaceStore || !api) return
const key = { quicksearch, colDefs: dbTableOps.colDefs, refreshCount }
const key = { quicksearch, colDefs: dbTableOps.colDefs, refreshCount, rowFilter }
if (deepEqual(key, prevUpdateKey)) return
prevUpdateKey = key
untrack(() => updateGrid())
})
// Foreign keys arrive after the rows: refresh the column defs alone so the
// popovers appear without re-running the row and count queries.
let prevForeignKeys: TableEditorForeignKey[] | undefined = undefined
$effect(() => {
if (!api || foreignKeys === prevForeignKeys) return
prevForeignKeys = foreignKeys
untrack(() => api?.updateGridOptions({ columnDefs: buildColumnDefs() }))
})
function buildColumnDefs() {
const fkColDefs = foreignKeys && onGoToRow ? foreignKeyColDefs(foreignKeys, onGoToRow) : {}
return transformColumnDefs({
columnDefs: (dbTableOps.colDefs ?? []).map((c) =>
c.field && fkColDefs[c.field] ? { ...c, ...fkColDefs[c.field] } : c
),
...(dbTableOps.onDelete && {
onDelete: (values) => {
if (!$workspaceStore) return
dbTableOps
.onDelete?.({ values })
.then(() => {
refresh?.()
sendUserToast('Row deleted')
})
.catch((e) => {
sendUserToast(`Error deleting row: ${e?.message ?? e}`, true)
})
}
})
})
}
function updateGrid() {
dbTableOps.getCount({ quicksearch }).then((result) => (rowCount = result))
api?.purgeInfiniteCache()
api?.updateGridOptions({
datasource,
columnDefs: transformColumnDefs({
columnDefs: dbTableOps.colDefs ?? [],
...(dbTableOps.onDelete && {
onDelete: (values) => {
if (!$workspaceStore) return
dbTableOps
.onDelete?.({ values })
.then(() => {
refresh?.()
sendUserToast('Row deleted')
})
.catch((e) => {
sendUserToast(`Error deleting row: ${e?.message ?? e}`, true)
})
}
})
}),
columnDefs: buildColumnDefs(),
context: {
quicksearch
}
@@ -132,12 +223,27 @@
<div class="h-full relative flex flex-col">
<div class="flex py-2 h-12 justify-between gap-4">
<DebouncedInput
class="w-full max-w-[300px]"
type="text"
bind:value={quicksearch}
placeholder="Search..."
/>
<div class="flex w-full items-center gap-2">
<DebouncedInput
class="w-full max-w-[300px]"
type="text"
bind:value={quicksearch}
placeholder="Search..."
/>
{#if rowFilter}
<Badge rounded class="pr-0.5 text-xs" data-testid="db-row-filter-chip">
<span class="font-mono">{rowFilter.column} = {String(rowFilter.value)}</span>
<Button
iconOnly
unifiedSize="2xs"
variant="subtle"
startIcon={{ icon: X }}
title="Clear filter"
onClick={onClearRowFilter}
/>
</Badge>
{/if}
</div>
{#if dbTableOps.onInsert}
<InsertRowDrawerButton
columnDefs={dbTableOps.colDefs ?? []}
frontend/src/lib/components/DbForeignKeyTooltip.svelte
+23
View File
@@ -0,0 +1,23 @@
<script lang="ts">
import { ArrowRight } from 'lucide-svelte'
import Button from './common/button/Button.svelte'
type Props = {
targetTable: string
targetColumn: string
onGoToRow: () => void
}
let { targetTable, targetColumn, onGoToRow }: Props = $props()
</script>
<div
class="flex flex-col gap-1.5 rounded-md border bg-surface-tertiary p-2 text-xs shadow-md"
data-testid="db-fk-tooltip"
>
<div class="text-secondary">
References <span class="font-mono text-primary">{targetTable}.{targetColumn}</span>
</div>
<Button unifiedSize="xs" variant="default" startIcon={{ icon: ArrowRight }} onClick={onGoToRow}>
Go to row
</Button>
</div>
frontend/src/lib/components/FilterSearchbar.svelte
+32 -4
View File
@@ -112,17 +112,37 @@
}
/**
* Creates a URL-synced filter instance that automatically syncs with URL search parameters
* Creates a URL-synced filter instance that automatically syncs with URL search parameters.
*
* `initial` supplies defaults for keys the URL doesn't carry (a route segment, a persisted
* toggle); the returned `seed` re-applies them after a navigation has rewritten the query.
*/
export function useUrlSyncedFilterInstance<T extends FilterSchemaRec>(
schemaRec: T
): { val: Partial<FilterInstanceRec<T>> } {
schemaRec: T,
initial?: Partial<FilterInstanceRec<T>>
): {
val: Partial<FilterInstanceRec<T>>
seed: (values: Partial<FilterInstanceRec<T>>) => void
} {
// Build the Zod schema from the filter schema
const zodSchema = filterSchemaRecToZodSchema(schemaRec)
// Create URL-synced search params
const urlFilter = useSearchParams(zodSchema) as Record<string, unknown>
// A default has to arrive as a URL param: the URL→instance effect below drops whatever the
// URL lacks, so a value written to the instance is undone on the next sync. Going through
// urlFilter rather than straight to history keeps the search-param cells in step, so it
// does not matter whether a popstate follows.
function seed(values: Partial<FilterInstanceRec<T>>) {
const sp = new URLSearchParams(window.location.search)
for (const [key, value] of Object.entries(values) as [string, unknown][]) {
if (value === undefined || value === null || sp.has(key)) continue
urlFilter[key] = value instanceof Date ? value.toISOString() : value
}
}
if (initial) seed(initial)
// Create the filter instance object
const filterInstance: { val: Partial<FilterInstanceRec<T>> } = $state({ val: {} })
@@ -173,7 +193,15 @@
})
}
return filterInstance
return {
get val() {
return filterInstance.val
},
set val(v: Partial<FilterInstanceRec<T>>) {
filterInstance.val = v
},
seed
}
}
function filterToText<F extends FilterSchema>(filter: FilterInstance<F>, schema: F): string {
frontend/src/lib/components/FlowBuilder.svelte
+129 -21
View File
@@ -98,7 +98,18 @@
import { invalidateWorkspacePaths } from './PathNameAutocomplete.svelte'
import type { Trigger } from './triggers/utils'
import { deployTriggers, handleSelectTriggerFromKind } from './triggers/utils'
import DraftTriggersConfirmationModal from './common/confirmationModal/DraftTriggersConfirmationModal.svelte'
import DraftChangesConfirmationModal from './common/confirmationModal/DraftChangesConfirmationModal.svelte'
import {
agentDraftCanWrite,
linkedAgentPaths,
loadLinkedAgentDrafts,
type LinkedAgentDraft
} from './flows/linkedAgentDrafts'
import { agentDraftDeployRefusal } from './flows/agentDraft.svelte'
import { markAgentWritten } from './flows/agentEditorStore.svelte'
import { logReusableAgentUsage } from './flows/agentTelemetry'
import { deployDraft } from '$lib/utils_draft_deploy'
import { getUserExt } from '$lib/user'
import { Triggers } from './triggers/triggers.svelte'
import { StepsInputArgs } from './flows/stepsInputArgs.svelte'
import { aiChatManager } from './copilot/chat/AIChatManager.svelte'
@@ -192,8 +203,16 @@
let confirmCallback: () => void = $state(() => {}) // What happens when user clicks `override` in warning
let open: boolean = $state(false) // Is confirmation modal open
// Draft triggers confirmation modal
let draftTriggersModalOpen = $state(false)
// Draft changes (triggers + linked agents) confirmation modal
let draftChangesModalOpen = $state(false)
/** The unsaved agent drafts the pending deploy found. Loaded rather than derived: it takes a
* request per linked agent, so it is resolved when the deploy asks. */
let draftAgents = $state<LinkedAgentDraft[]>([])
let agentCanWrite = $state<Record<string, boolean>>({})
let agentRefusal = $state<Record<string, string | undefined>>({})
/** What the dialog's confirm hands back to `saveFlow`. */
type DraftChangesToDeploy = { triggers: Trigger[]; agents: LinkedAgentDraft[] }
// Top-bar responsive collapse. Measured via bind:clientWidth — we can't
// rely on viewport `md:` because the editor lives inside other panes
@@ -224,7 +243,7 @@
]
: []
)
let confirmDeploymentCallback: (triggersToDeploy: Trigger[]) => void = () => {}
let confirmDeploymentCallback: (toDeploy: DraftChangesToDeploy) => void = () => {}
// AI changes warning modal
let aiChangesWarningOpen = $state(false)
@@ -236,11 +255,57 @@
const job: Job | undefined = $derived(flowPreviewContent?.getJob())
let showJobStatus = $state(false)
async function handleDraftTriggersConfirmed(event: CustomEvent<{ selectedTriggers: Trigger[] }>) {
const { selectedTriggers } = event.detail
async function handleDraftChangesConfirmed(
event: CustomEvent<{ selectedTriggers: Trigger[]; selectedAgents: LinkedAgentDraft[] }>
) {
const { selectedTriggers, selectedAgents } = event.detail
// Continue with saving the flow
draftTriggersModalOpen = false
confirmDeploymentCallback(selectedTriggers)
draftChangesModalOpen = false
confirmDeploymentCallback({ triggers: selectedTriggers, agents: selectedAgents })
}
/** Deploy each selected agent's draft, the same way the Review & Deploy page deploys the same
* row: hand the path to `deployDraft` and let it promote whatever the draft holds, with no
* re-read to check it still matches what the dialog listed. Agents left out keep their draft
* untouched. */
async function deployAgentDrafts(agents: LinkedAgentDraft[]) {
const ws = opWorkspace
if (!ws) return
for (const listed of draftAgents) {
// Only the rows the dialog gave a choice on. A `Read-only` or `Invalid config` agent can
// never be selected, so counting it as "kept" would record a decision the user was never
// offered and bias the pair towards keeping.
const selectable = agentCanWrite[listed.path] !== false && !agentRefusal[listed.path]
if (selectable && !agents.some((a) => a.path === listed.path)) {
logReusableAgentUsage('draft_kept_on_deploy')
}
}
for (const agent of agents) {
// Writes the resource, deletes the draft row, and clears the local hint and the workspace
// drafts cache. A failure aborts the flow save the way a failing trigger does, rather than
// deploying a flow against agents that were meant to change with it.
const deployed = await deployDraft('resource', agent.path, ws, {
draftOnly: agent.noDeployed
})
if (!deployed.success) {
throw new Error(`Could not deploy agent ${agent.path}: ${deployed.error}`)
}
// Nothing was promoted: the draft had gone by the time the helper read it. Saying the agent
// deployed would be a lie about the one thing the toggle decides.
if (deployed.noop) {
throw new Error(
`The draft for ${agent.path} was deployed or discarded elsewhere while this deploy ran, so nothing was written for it.`
)
}
// `deployDraft` deletes the server row but leaves any in-memory cell for this key, and that
// cell is what `agentDraftState` prefers — a still-mounted holder would otherwise keep
// feeding a phantom draft to the cards and to the next deploy dialog. Local only: `remove`
// would POST a second delete, debounced and past the baseline the first one cleared.
UserDraft.forgetLocal('resource', agent.path, { workspace: ws })
// Every linked card and the graph key on this to refetch the agent they display.
markAgentWritten(ws, agent.path)
logReusableAgentUsage('draft_deployed_with_flow')
}
}
// Inside an AI session pane (SessionEditorTarget injects an aiChatManager via
@@ -429,18 +494,53 @@
deployedBy = flow.edited_by
}
async function saveFlow(deploymentMsg?: string, triggersToDeploy?: Trigger[]): Promise<void> {
if (!triggersToDeploy) {
// Check if there are draft triggers that need confirmation
async function saveFlow(deploymentMsg?: string, toDeploy?: DraftChangesToDeploy): Promise<void> {
if (!toDeploy) {
// Draft triggers and drafts on the agents this flow links: both are unsaved changes the
// deploy would otherwise leave behind, so they are confirmed together.
const draftTriggers = triggersState.triggers.filter((trigger) => trigger.draftConfig)
if (draftTriggers.length > 0) {
draftTriggersModalOpen = true
confirmDeploymentCallback = async (triggersToDeploy: Trigger[]) => {
await saveFlow(deploymentMsg, triggersToDeploy)
try {
draftAgents = [
...(
await loadLinkedAgentDrafts(linkedAgentPaths(flowStore.val.value), opWorkspace)
).values()
]
} catch (err: any) {
// This runs before the try below, and `withAIChangesWarning` invokes its callback without
// awaiting, so a rejection here would be unhandled: the button would do nothing at all,
// with no toast and no `onDeployError`. Report it the way the rest of the save does.
// Deploying anyway is not the fallback — this throws only when an agent's unsaved changes
// cannot be read, which is exactly when the dialog must not claim there are none.
onDeployError?.({ error: err })
sendUserToast(`The flow could not be saved: ${err?.body ?? err}`, true)
return
}
agentCanWrite = {}
agentRefusal = {}
if (draftAgents.length > 0) {
// One lookup for the whole list: an agent lives in a folder, and the groups and admin
// flag that answer for it are per workspace, so the nav user would answer for the wrong
// membership when a session editor operates on another workspace.
const user = await getUserExt(opWorkspace ?? '').catch(() => undefined)
agentCanWrite = Object.fromEntries(
draftAgents.map((a) => [a.path, agentDraftCanWrite(a, user ?? $userStore ?? undefined)])
)
// The path is passed, so a draft that renames the agent is refused here too: a rename is
// the resource editor's to deploy, and this dialog lists the agent under the path the
// flow links.
agentRefusal = Object.fromEntries(
draftAgents.map((a) => [a.path, agentDraftDeployRefusal(a.state, a.path)])
)
}
if (draftTriggers.length > 0 || draftAgents.length > 0) {
draftChangesModalOpen = true
confirmDeploymentCallback = async (confirmed: DraftChangesToDeploy) => {
await saveFlow(deploymentMsg, confirmed)
}
return
}
}
const triggersToDeploy = toDeploy?.triggers
loadingSave = true
try {
@@ -471,6 +571,11 @@
// loadingSave = false // del
// return
// Ahead of the flow itself, as the update branch deploys its triggers: an agent is a
// resource of its own, so the flow should land on top of the agent set it was tested
// against rather than the other way round.
await deployAgentDrafts(toDeploy?.agents ?? [])
// `newFlow` comes from the embedder, and updating a path that has no
// deployed flow 404s. Confirm with the server before taking the update
// branch so a first deploy still lands.
@@ -648,7 +753,7 @@
const resolved = linkedAgentToolsForScope(to)
for (const [moduleId, agentPath] of linkedAgentEntries(linkedAgentRefs)) {
if (resolved[moduleId] === undefined) {
publishLinkedAgentTools(agentPath, ws, to, moduleId)
publishLinkedAgentTools(agentPath, ws, to, moduleId, true)
}
}
}
@@ -716,7 +821,7 @@
// writing overrides against — the tool ids of the agent that was just replaced.
claimLinkedToolsFetch(scope, moduleId)
clearLinkedAgentTools(scope, moduleId)
publishLinkedAgentTools(agentPath, ws, scope, moduleId)
publishLinkedAgentTools(agentPath, ws, scope, moduleId, true)
}
publishedAgentByModule = next
})
@@ -1343,14 +1448,17 @@
currentValue={flowStore.val}
/>
<DraftTriggersConfirmationModal
bind:open={draftTriggersModalOpen}
<DraftChangesConfirmationModal
bind:open={draftChangesModalOpen}
draftTriggers={triggersState.triggers.filter((t) => t.draftConfig)}
{draftAgents}
{agentCanWrite}
{agentRefusal}
isFlow={true}
on:canceled={() => {
draftTriggersModalOpen = false
draftChangesModalOpen = false
}}
on:confirmed={handleDraftTriggersConfirmed}
on:confirmed={handleDraftChangesConfirmed}
/>
<AIChangesWarningModal bind:open={aiChangesWarningOpen} onConfirm={aiChangesConfirmCallback} />
frontend/src/lib/components/FlowGraphViewer.svelte
+18 -1
View File
@@ -67,6 +67,14 @@
const dispatch = createEventDispatcher()
// A bucket of this viewer's own, never the editor's. Both are keyed by (workspace, flow path), so
// a viewer mounted over the flow being edited — the version-history drawer, which renders the
// same path — would otherwise publish its deployed tools into the editor's bucket and replace the
// drafted tool nodes there. The editor's graph has to keep showing the tools a preview would
// actually run, and nothing republishes when the drawer closes. `FlowStatusViewerInner` scopes
// itself the same way, with `job:<id>`.
let linkedToolsPath = $derived(`view:${flow?.path ?? ''}`)
// This read-only viewer doesn't run initFlowState, so linked agents' tools would otherwise never
// resolve. Resolve them for display, keyed by module id. Best-effort: publishLinkedAgentTools
// swallows access errors and publishes [], so an inaccessible agent simply shows no tool nodes
@@ -80,7 +88,15 @@
for (const m of modules) {
const value = m?.value as { type?: string; agent?: string } | undefined
if (value?.type === 'aiagent' && value.agent) {
publishLinkedAgentTools(value.agent, ws, linkedToolsScope(ws, flow?.path), m.id)
// Without the draft: this viewer shows a deployed flow or a past run, both of which
// used the deployed agent.
publishLinkedAgentTools(
value.agent,
ws,
linkedToolsScope(ws, linkedToolsPath),
m.id,
false
)
}
}
})
@@ -101,6 +117,7 @@
earlyStop={flow?.value?.skip_expr !== undefined}
cache={flow?.value?.cache_ttl !== undefined}
path={flow?.path}
{linkedToolsPath}
{download}
minHeight={fillAvailableHeight ? Math.max(minHeight, availableHeight) : minHeight}
{workspace}
frontend/src/lib/components/FlowLoopIterationPreview.svelte
+17 -9
View File
@@ -5,6 +5,7 @@
import { createEventDispatcher, getContext } from 'svelte'
import type { FlowEditorContext } from './flows/types'
import { runFlowPreview } from './flows/utils.svelte'
import { sendUserToast } from '$lib/toast'
import SchemaForm from './SchemaForm.svelte'
import FlowStatusViewer from '../components/FlowStatusViewer.svelte'
import FlowProgressBar from './flows/FlowProgressBar.svelte'
@@ -101,15 +102,22 @@
// The preview flow holds only the loop body, so it inherits none of the flow's settings:
// carry the tag over so the iteration lands on the worker group the flow runs on.
const newFlow = { value: { modules }, summary: '', tag: flowStore.val.tag }
jobId = await runFlowPreview(
whileLoop ? withWhileLoopIter(args) : args,
newFlow,
$pathStore,
restartedFrom,
undefined,
undefined,
opWorkspace?.()
)
try {
jobId = await runFlowPreview(
whileLoop ? withWhileLoopIter(args) : args,
newFlow,
$pathStore,
restartedFrom,
undefined,
undefined,
opWorkspace?.()
)
} catch (err: any) {
// `runFlowPreview` resolves a linked agent's draft first and refuses when it cannot be read.
// Without this the rejection is unhandled and the button just does nothing.
sendUserToast(`Could not run preview: ${err?.body ?? err}`, true)
return
}
isRunning = true
}
frontend/src/lib/components/FlowStatusViewerInner.svelte
+3 -1
View File
@@ -294,7 +294,9 @@
untrack(() => {
for (const entry of refs ? refs.split('\u0001') : []) {
const [moduleId, agentPath] = entry.split('\u0000')
publishLinkedAgentTools(agentPath, ws, scope, moduleId)
// Without the draft: this viewer describes a run that has already happened, and it ran
// the deployed agent.
publishLinkedAgentTools(agentPath, ws, scope, moduleId, false)
}
})
})
frontend/src/lib/components/ModuleTest.svelte
+38 -2
View File
@@ -14,6 +14,14 @@
import JobLoader, { type Callbacks } from './JobLoader.svelte'
import { getStepHistoryLoaderContext } from './stepHistoryLoader.svelte'
import { loadSchemaFromModule } from './flows/flowInfers'
import {
inlineAgentDraft,
loadLinkedAgentDrafts,
normalizeAgentRef,
type LinkedAgentDraft
} from './flows/linkedAgentDrafts'
import { AGENT_FLOW_LOCAL_KEYS } from './flows/agentResourceUtils'
import { sendUserToast } from '$lib/toast'
interface Props {
mod: FlowModule
@@ -138,7 +146,35 @@
} else if (val.type == 'aiagent') {
const { schema } = await loadSchemaFromModule(mod, opWs)
const agentVal = val
// A linked step whose agent has an unsaved draft is tested as the draft, the same way the
// whole-flow preview and the agent editor's own test pane run it. `inlineAgentDraft`
// clears `agent` and moves the draft's brain and tools onto the step, so the branches
// below then treat it as a standalone agent.
let draft: LinkedAgentDraft | undefined
if (val.agent) {
const linked = normalizeAgentRef(val.agent)
try {
draft = (await loadLinkedAgentDrafts([linked], opWs)).get(linked)
} catch (err: any) {
// The load refuses when the agent's unsaved changes cannot be read, and this function's
// caller neither awaits nor catches: without this the rejection is unhandled and the
// button appears to do nothing, with the test already marked as started.
sendUserToast(`Could not run test: ${err?.body ?? err}`, true)
// Guarded like every other access to it here: the entry is only created for steps the
// panel is tracking, and this runs on a path where it may never have been.
if (modulesTestStates.states[mod.id]) {
modulesTestStates.states[mod.id].loading = false
}
return
}
}
const agentVal = draft ? inlineAgentDraft(val, draft.args) : val
// `args` is built from the whole AI agent schema whatever the step is, so on a linked step
// it carries every brain key as undefined even though the form renders only the flow-local
// ones (`flowLocalAgentSchema`). Overlaying those would shadow the brain the draft just
// supplied with nothing, so an inlined step takes only the inputs its form actually offers.
const formKeys = draft ? (AGENT_FLOW_LOCAL_KEYS as readonly string[]) : Object.keys(args)
// The test form only covers the schema it was given, and for a standalone agent that may be
// the flow-local one (the agent editor shows the brain in its own form, not here). Take the
@@ -150,7 +186,7 @@
? {}
: ((agentVal.input_transforms ?? {}) as Record<string, InputTransform>)),
...Object.fromEntries(
Object.keys(args).map((key) => [
formKeys.map((key) => [
key,
{
expr: `flow_input.${key}`,
frontend/src/lib/components/ResourceEditor.svelte
+107 -1
View File
@@ -2,7 +2,7 @@
import type { Schema } from '$lib/common'
import { ResourceService, WorkspaceService, type Resource, type ResourceType } from '$lib/gen'
import { canWrite } from '$lib/utils'
import { createEventDispatcher, untrack } from 'svelte'
import { createEventDispatcher, onDestroy, untrack } from 'svelte'
import { userStore, workspaceStore } from '$lib/stores'
import { sendUserToast } from '$lib/toast'
import { clearJsonSchemaResourceCache } from './schema/jsonSchemaResource.svelte'
@@ -13,7 +13,9 @@
import { getUserExt } from '$lib/user'
import type { UserExt } from '$lib/stores'
import { UserDraft, draftValuesEqual, type UserDraftHandle } from '$lib/userDraft.svelte'
import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte'
import { setLocalDraftHint } from '$lib/localDraftHints.svelte'
import { onUserInput } from '$lib/userDraftEditGate'
interface Props {
canSave?: boolean
@@ -108,6 +110,55 @@
workspaceSpecs.push({ ws, defaultValue })
}
// Gated per workspace until the user puts something into that workspace's
// form (see `onUserInput`): the autosave stays suspended and the deployed
// baseline absorbs whatever the form settles on. A workspace opened ON a
// saved draft keeps its baseline — that divergence is the user's own.
let userEdited: Record<string, boolean> = $state({})
let openedOnDraft: Record<string, boolean> = $state({})
const suspendedWorkspaces = new Set<string>()
function setGated(ws: string, gated: boolean): void {
if (!initialPath) return
if (gated === suspendedWorkspaces.has(ws)) return
if (gated) {
UserDraft.stopSync('resource', initialPath, { workspace: ws })
suspendedWorkspaces.add(ws)
} else {
UserDraft.restartSync('resource', initialPath, { workspace: ws })
suspendedWorkspaces.delete(ws)
}
}
// Nothing counts until this workspace's form is on screen, and while the
// schema is still arriving a precursor alone does not: it would open the gate
// just in time for the schema's materialized values to POST. `Path`, the
// labels and the description render above that skeleton and stay editable
// throughout, so a real value event still counts and keeps the edit.
onUserInput((kind) => {
if (!selected || !(selected in states)) return
if (kind === 'precursor' && loadingSchema) return
userEdited[selected] = true
})
$effect(() => {
const wss = Object.keys(states)
const edited = { ...userEdited }
const onDraft = { ...openedOnDraft }
untrack(() => {
// A workspace opened on a saved draft is never suspended — there is no
// phantom to prevent, and a write made while suspended is dropped for
// good. Without `onDraft` here this effect re-suspends it the moment its
// handle appears, undoing the decision made when it was opened.
for (const ws of wss) setGated(ws, !edited[ws] && !onDraft[ws])
})
})
// `stopSync` must be paired or the key stays unsynced for the session.
onDestroy(() => {
for (const ws of [...suspendedWorkspaces]) setGated(ws, false)
})
let isValid = $state(true)
let jsonError = $state('')
let perWsValid: Record<string, boolean> = $state({})
@@ -245,6 +296,13 @@
}
// Open with the saved draft if present, else the deployed.
const s: ResourceState = savedDraftState ?? deployedState
openedOnDraft[ws] = !!savedDraftState
// Gate BEFORE the handle is acquired: `stopSync` queues on a
// not-yet-live entry, and the form can settle before the effect
// above gets a chance to run. Only worth doing when no draft exists
// yet — where one does, there is no phantom to prevent and
// suspending could only drop a write.
if (!savedDraftState) setGated(ws, true)
ensureHandle(ws, s)
initialStates[ws] = structuredClone(deployedState)
// Draft-only paths (`no_deployed`) have no row — saving must
@@ -259,6 +317,47 @@
})
})
/** The schema can only ever write `args`. `path`, `labels`, `description` and
* `wsSpecific` are beyond its reach, so a difference in one of those is the
* user's — whatever event did or didn't reach the gate. Removing a label runs
* a click handler and emits nothing native, and would otherwise be absorbed. */
function differsOutsideArgs(a: ResourceState, b: ResourceState | undefined): boolean {
return !!b && !draftValuesEqual({ ...a, args: null }, { ...b, args: null })
}
// Absorb the form's settling writes into the deployed baseline while the
// selected workspace is gated, so they show up neither as the "unsaved
// changes" banner nor, once `discardIf` reads the baseline, as a draft.
// Only the selected workspace has a form rendered against it.
$effect(() => {
const ws = selected
if (!ws || !initialPath) return
if (userEdited[ws] || openedOnDraft[ws]) return
// `$state.snapshot` deep-reads, so nested `args` mutations re-run this.
const settled = states[ws]?.draft
? ($state.snapshot(states[ws].draft) as ResourceState)
: undefined
untrack(() => {
if (!settled) return
if (differsOutsideArgs(settled, initialStates[ws])) {
// An edit, not settling. This runs AFTER the write landed, and a write
// made while suspended is swallowed for good (the mirror advances its
// baseline either way), so un-suspend and push the value here rather
// than leaving it to whichever effect happens to run next.
userEdited[ws] = true
setGated(ws, false)
void UserDraftDbSyncer.save({
workspace: ws,
itemKind: 'resource',
path: initialPath,
value: settled
})
return
}
if (!draftValuesEqual(settled, initialStates[ws])) initialStates[ws] = settled
})
})
// Keep current.path bound to the outer `path` prop for consumers
$effect(() => {
if (current) path = current.path
@@ -292,6 +391,13 @@
}
export function discardLocalDraft(): void {
if (!selected) return
// Back to the deployed value with nothing of the user's left in it, so
// the gate closes again — otherwise the form settles on the schema's
// values a second time and the discarded draft comes straight back.
// `discard` POSTs the delete itself, so suspending first is safe.
openedOnDraft[selected] = false
userEdited[selected] = false
setGated(selected, true)
UserDraft.discard('resource', initialPath ?? '', initialStates[selected], {
workspace: selected
})
frontend/src/lib/components/RunsPage.svelte
+22 -15
View File
@@ -84,6 +84,8 @@
initialPath?: string
}
let { initialPath }: Props = $props()
let paths: string[] = $state([])
let usernames: string[] = $state([])
let folders: string[] = $state([])
@@ -100,25 +102,30 @@
let perPage = useLocalStorageValue('runs_per_page', 1000, 'number')
let showSchedulesStorage = useLocalStorageValue('runs_show_schedules', true, 'boolean')
let showFutureJobsStorage = useLocalStorageValue('runs_show_future_jobs', true, 'boolean')
let filters = useUrlSyncedFilterInstance(untrack(() => runsFilterSearchbarSchema))
function filterSeeds() {
return {
path: initialPath || undefined,
job_trigger_kind: showSchedulesStorage.val === false ? ('!schedule' as const) : undefined,
show_future_jobs: showFutureJobsStorage.val === false ? false : undefined
}
}
let { initialPath }: Props = $props()
let filters = useUrlSyncedFilterInstance(
untrack(() => runsFilterSearchbarSchema),
untrack(filterSeeds)
)
// `runs/[...path]` is a single route, so a navigation between its URLs — the sidebar's own
// "Runs" entry, `/runs/<path>` → `/runs`, Back — rewrites the query without remounting, and
// what was seeded at mount is gone. Re-apply it. Editing a filter writes with `replaceState`,
// which never reaches `page.url`, so a filter the user clears stays cleared.
$effect(() => {
page.url.href
untrack(() => filters.seed(filterSeeds()))
})
let batchRerunOptionsIsOpen = $state(false)
// Initialize path filter from route param if provided and not already set via query params
if (untrack(() => initialPath) && !filters.val.path) {
filters.val.path = untrack(() => initialPath)
}
// Apply persistent toggle values from local storage if URL doesn't specify them
if (!page.url.searchParams.has('job_trigger_kind') && showSchedulesStorage.val === false) {
filters.val.job_trigger_kind = '!schedule'
}
if (!page.url.searchParams.has('show_future_jobs') && showFutureJobsStorage.val === false) {
filters.val.show_future_jobs = false
}
// Sync toggle state back to local storage when filters change
$effect(() => {
if (!filters.val.job_trigger_kind || filters.val.job_trigger_kind === '!schedule') {
frontend/src/lib/components/ScriptBuilder.svelte
+2 -2
View File
@@ -102,7 +102,7 @@
import type { SavedAndModifiedValue } from './common/confirmationModal/unsavedTypes'
import DeployButton from './DeployButton.svelte'
import { type Trigger, deployTriggers, handleSelectTriggerFromKind } from './triggers/utils'
import DraftTriggersConfirmationModal from './common/confirmationModal/DraftTriggersConfirmationModal.svelte'
import DraftChangesConfirmationModal from './common/confirmationModal/DraftChangesConfirmationModal.svelte'
import { Triggers } from './triggers/triggers.svelte'
import type { ScriptBuilderProps } from './script_builder'
import WorkerTagSelect from './WorkerTagSelect.svelte'
@@ -1149,7 +1149,7 @@
currentValue={script}
/>
<DraftTriggersConfirmationModal
<DraftChangesConfirmationModal
bind:open={draftTriggersModalOpen}
draftTriggers={triggersState.triggers.filter((t) => t.draftConfig)}
on:canceled={() => {
frontend/src/lib/components/ScriptEditor.svelte
+1 -50
View File
@@ -15,7 +15,7 @@
import { copyToClipboard, emptySchema, sendUserToast } from '$lib/utils'
import Editor from './Editor.svelte'
import { inferArgs, inferAssets, inferAnsibleExecutionMode } from '$lib/infer'
import { parsePipelineAnnotations } from '$lib/components/assets/AssetGraph/parsePipelineAnnotations'
import { injectPartitionArg } from '$lib/scriptEditorSchema'
import { isWorkflowAsCode } from '$lib/components/graph/wacToFlow'
import WacDiagram from '$lib/components/graph/WacDiagram.svelte'
import { Pane, Splitpanes } from 'svelte-splitpanes'
@@ -1074,55 +1074,6 @@
}
}
// A `// partitioned` pipeline script is materialized one slice at a time and
// receives the slice as a runtime `partition` arg (the cascade injects it in
// production). It isn't a code parameter, so schema inference doesn't see it —
// surface it in the test form so a partitioned script can be run manually.
function injectPartitionArg(
s: any,
a: Record<string, any> | undefined,
l: string | undefined,
c: string
) {
try {
if (l !== 'duckdb' || !s?.properties) return
const part = parsePipelineAnnotations(c).partition
if (!part) return
// Date-based partition kinds render a date / datetime picker; a dynamic
// key is a free-form string.
const format =
part.kind === 'hourly'
? 'date-time'
: part.kind === 'daily' || part.kind === 'weekly' || part.kind === 'monthly'
? 'date'
: undefined
if (!s.properties['partition']) {
s.properties['partition'] = {
type: 'string',
...(format ? { format } : {}),
// ISO output so partition keys sort lexicographically (the date
// picker defaults to dd-MM-yyyy otherwise).
...(format === 'date' ? { dateFormat: 'yyyy-MM-dd' } : {}),
description:
part.kind === 'dynamic'
? 'Partition key value to materialize.'
: `Partition (${part.kind}) to materialize.`
}
if (Array.isArray(s.order) && !s.order.includes('partition')) {
s.order = ['partition', ...s.order]
}
}
// Pre-fill the *test* arg with the current slice for date kinds — a
// convenience default, kept on the args (not baked into the schema,
// where it would persist to the deployed script and go stale).
if (format && a && (a['partition'] == null || a['partition'] === '')) {
const now = new Date()
a['partition'] =
format === 'date' ? now.toISOString().slice(0, 10) : now.toISOString().slice(0, 16)
}
} catch (e) {}
}
async function inferModuleSchema() {
if (activeModuleTab === null) return
try {
frontend/src/lib/components/apps/components/display/dbtable/renderDbLiteral.test.ts
+48
View File
@@ -0,0 +1,48 @@
import { describe, expect, it } from 'vitest'
import { renderDbEqualityFilter, renderDbLiteral } from './utils'
describe('renderDbLiteral', () => {
it('doubles single quotes on every dialect', () => {
expect(renderDbLiteral("O'Brien", 'postgresql')).toBe("'O''Brien'")
expect(renderDbLiteral("O'Brien", 'mysql')).toBe("'O''Brien'")
})
it('doubles backslashes only where the dialect treats them as escapes', () => {
expect(renderDbLiteral('C:\\dir\\', 'postgresql')).toBe("'C:\\dir\\'")
expect(renderDbLiteral('C:\\dir\\', 'mysql')).toBe("'C:\\\\dir\\\\'")
expect(renderDbLiteral('C:\\dir\\', 'snowflake')).toBe("'C:\\\\dir\\\\'")
})
it('marks SQL Server strings as Unicode constants', () => {
expect(renderDbLiteral("Zoë's", 'ms_sql_server')).toBe("N'Zoë''s'")
})
it('renders numbers and booleans without quotes', () => {
expect(renderDbLiteral(42, 'postgresql')).toBe('42')
expect(renderDbLiteral(true, 'postgresql')).toBe('TRUE')
expect(renderDbLiteral(true, 'ms_sql_server')).toBe('1')
})
it('has no literal for values that cannot be compared safely', () => {
expect(renderDbLiteral(null, 'postgresql')).toBeUndefined()
expect(renderDbLiteral({ a: 1 }, 'postgresql')).toBeUndefined()
expect(renderDbLiteral(NaN, 'postgresql')).toBeUndefined()
})
})
describe('renderDbEqualityFilter', () => {
it('quotes the identifier per dialect', () => {
expect(renderDbEqualityFilter('user id', 'x', 'postgresql')).toBe(`"user id" = 'x'`)
expect(renderDbEqualityFilter('user id', 'x', 'ms_sql_server')).toBe(`[user id] = N'x'`)
expect(renderDbEqualityFilter('user id', 'x', 'mysql')).toBe("`user id` = 'x'")
expect(renderDbEqualityFilter('user id', null, 'postgresql')).toBeUndefined()
})
it('doubles a delimiter embedded in the identifier', () => {
expect(renderDbEqualityFilter('a"b', 1, 'postgresql')).toBe(`"a""b" = 1`)
expect(renderDbEqualityFilter('a"b', 1, 'snowflake')).toBe(`"a""b" = 1`)
expect(renderDbEqualityFilter('a"b', 1, 'duckdb')).toBe(`"a""b" = 1`)
expect(renderDbEqualityFilter('a]b', 1, 'ms_sql_server')).toBe(`[a]]b] = 1`)
expect(renderDbEqualityFilter('a`b', 1, 'mysql')).toBe('`a``b` = 1')
})
})
frontend/src/lib/components/apps/components/display/dbtable/utils.ts
+42 -9
View File
@@ -333,25 +333,58 @@ export function duckdbQuicksearchColumns(columnDefs: ColumnDef[]): string {
.join(', ')
}
/** Mirrors the backend's `render_db_quoted_identifier`, including doubling an
* embedded delimiter. */
export function renderDbQuotedIdentifier(identifier: string, dbType: DbType): string {
switch (dbType) {
case 'postgresql':
return `"${identifier}"` // PostgreSQL uses double quotes for identifiers
case 'ms_sql_server':
return `[${identifier}]` // MSSQL uses square brackets for identifiers
case 'mysql':
return `\`${identifier}\`` // MySQL uses backticks
case 'snowflake':
return `"${identifier}"` // Snowflake uses double quotes for identifiers
case 'bigquery':
return `\`${identifier}\`` // BigQuery uses backticks
case 'duckdb':
return `"${identifier}"` // DuckDB uses double quotes for identifiers
return `"${identifier.replace(/"/g, '""')}"`
case 'ms_sql_server':
return `[${identifier.replace(/]/g, ']]')}]`
case 'mysql':
case 'bigquery':
return `\`${identifier.replace(/`/g, '``')}\``
default:
throw new Error('Unsupported database type: ' + dbType)
}
}
/** Renders a cell value as a SQL literal. Returns undefined for values that
* have no safe literal form (null, objects, non-finite numbers). */
export function renderDbLiteral(value: unknown, dbType: DbType): string | undefined {
if (value === null || value === undefined) return undefined
if (typeof value === 'number') return Number.isFinite(value) ? String(value) : undefined
if (typeof value === 'bigint') return value.toString()
if (typeof value === 'boolean') {
if (dbType === 'ms_sql_server') return value ? '1' : '0'
return value ? 'TRUE' : 'FALSE'
}
if (typeof value !== 'string') return undefined
let escaped = value.replace(/'/g, "''")
// MySQL, Snowflake and BigQuery treat a backslash inside a string literal as
// an escape character.
if (dbType === 'mysql' || dbType === 'snowflake' || dbType === 'bigquery') {
escaped = escaped.replace(/\\/g, '\\\\')
}
// A plain constant is varchar on SQL Server and goes through the database
// code page; the N prefix keeps it Unicode against nvarchar columns.
return dbType === 'ms_sql_server' ? `N'${escaped}'` : `'${escaped}'`
}
/** `"column" = <literal>` predicate, or undefined when the value can't be
* rendered as a literal. */
export function renderDbEqualityFilter(
column: string,
value: unknown,
dbType: DbType
): string | undefined {
const literal = renderDbLiteral(value, dbType)
if (literal === undefined) return undefined
return `${renderDbQuotedIdentifier(column, dbType)} = ${literal}`
}
export function getLanguageByResourceType(name: string): ScriptLang {
const language = {
postgresql: 'postgresql',
frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte
+11 -5
View File
@@ -22,19 +22,22 @@
let {
workspace,
path,
editHref
path
}: {
workspace: string
path: string
/** Where the Edit button points (low-code vs raw editor). */
editHref: string
} = $props()
let app: any = $state(undefined)
let notExists = $state(false)
let noPermission = $state(false)
let canWriteApp = $state(false)
/** Raw vs low-code, read from the app itself rather than from the route:
* both kinds render here and either route serves either kind (links to a raw
* app point at /apps/get all over the app), so only the app can say which
* editor the Edit button must open. */
let isRawApp = $state(false)
let editHref = $derived(`${base}/${isRawApp ? 'apps_raw' : 'apps'}/edit/${path}?nodraft=true`)
let refresh: (() => void) | undefined
// The opaque iframe loads the dedicated cookieless, chrome-less viewer route.
@@ -103,11 +106,14 @@
}
}
// Edit button: determine write access on this real-origin page (cookie).
// Edit button: determine write access and which editor to open on this
// real-origin page (cookie). The sandboxed low-code app never loads on this
// page (it loads inside the opaque iframe), so `app` can't be the source.
async function loadPerms() {
try {
const lite: any = await AppService.getAppLiteByPath({ workspace, path })
canWriteApp = canWrite(lite?.path, lite?.extra_perms ?? {}, $userStore)
isRawApp = !!lite?.raw_app
} catch (_) {
canWriteApp = false
}
frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts
+12
View File
@@ -319,6 +319,18 @@ describe('resolveGraph', () => {
expect(assetTrigKeys(r, 'f/x/open')).toEqual(['ducklake:main.orders'])
})
// A relation a native `// materialize manual dbt://…` script writes wakes its
// subscribers, so hiding the edge would leave the author's own annotation off
// the canvas. The deploy refuses the ones that cannot fire.
it('draws an explicit dbt:// subscription as an unsaved trigger overlay', () => {
const liveAnnotations = {
scriptPath: 'f/x/open',
annotations: ann({ triggerAssets: [{ kind: 'dbt', path: 'main/analytics/orders' }] })
}
const r = resolveGraph(input({ liveAnnotations }))
expect(assetTrigKeys(r, 'f/x/open')).toEqual(['dbt:main/analytics/orders'])
})
it('open-script live annotations add unsaved triggers, deduped vs persisted', () => {
const base = baseGraph({
triggers: [
frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts
+10 -18
View File
@@ -77,23 +77,15 @@ function persistedNativeKinds(base: AssetGraphResponse, path: string): Set<strin
// `// pipeline`. Mirror of the backend `is_auto_trigger_kind`
// (windmill-common assets.rs): ducklake tables and s3 objects.
//
// `table` is excluded there and must stay excluded here: dbt is the only
// producer of a warehouse relation and a dbt run does not dispatch, so an edge
// derived from one draws a cascade arrow the deploy will never fire.
// `dbt` is excluded there and must stay excluded here: a warehouse relation is
// usually built by the dbt project that reads it, and a dbt run does not
// dispatch, so deriving an edge from every `dbt://` read would draw cascade
// arrows that mostly never fire. An EXPLICIT `// on dbt://…` is drawn — the
// relations a native `// materialize manual dbt://…` script writes do wake
// subscribers, and the deploy is what refuses the case that cannot (a relation
// dbt alone builds; `sole_dbt_producer` in windmill-common assets.rs).
const AUTO_TRIGGER_KINDS: ReadonlySet<AssetKind> = new Set(['ducklake', 's3object'])
/** Whether a subscription on this kind can ever fire once deployed.
*
* A `dbt://` one cannot: dbt is the only producer of a warehouse relation and
* a dbt run does not dispatch, so the deploy refuses `// on dbt://…` outright
* (`scripts.rs`). The editor must not draw an arrow the deploy will reject —
* applied to the EXPLICIT overlays; auto-derivation is already scoped by
* `AUTO_TRIGGER_KINDS`. Parsing is left alone so the Rust-parity test still
* compares like for like. */
function canTrigger(kind: AssetKind): boolean {
return kind !== 'dbt'
}
/** `kind:path` refs of a script's `// materialize` write target(s) (base +
* the scd2 `<dim>_current` companion), which the body `SELECT` doesn't express. */
function materializeWriteRefs(parsed: PipelineAnnotations): string[] {
@@ -381,7 +373,7 @@ function makeContext(input: ResolveGraphInput): ResolveContext {
const liveRefKeys = new Set<string>()
if (openIsSavedEdit) {
if (liveAnnotations.scriptPath === openPath) {
for (const a of liveAnnotations.annotations.triggerAssets.filter((a) => canTrigger(a.kind)))
for (const a of liveAnnotations.annotations.triggerAssets)
liveRefKeys.add(`${a.kind}:${a.path}`)
// The `// materialize <asset>` target is a declared *output*, but it
// lives in an annotation (not the SQL body), so neither triggerAssets
@@ -625,7 +617,7 @@ function seedDraftOverlays(acc: Accumulator, input: ResolveGraphInput) {
// stable when the user clicks off this draft. Live annotations
// (below) take over for the currently-open draft so keystroke
// edits still update in real time.
for (const a of parsed.triggerAssets.filter((a) => canTrigger(a.kind))) {
for (const a of parsed.triggerAssets) {
extraTriggers.push({
trigger_kind: 'asset',
asset_kind: a.kind,
@@ -704,7 +696,7 @@ function applyLiveBufferOverlay(acc: Accumulator, input: ResolveGraphInput, ctx:
for (let i = extraTriggers.length - 1; i >= 0; i--) {
if (extraTriggers[i].runnable_path === livePath) extraTriggers.splice(i, 1)
}
for (const a of liveAnnotations.annotations.triggerAssets.filter((a) => canTrigger(a.kind))) {
for (const a of liveAnnotations.annotations.triggerAssets) {
const key = `${a.kind}:${a.path}`
if (assetKeys.has(key)) continue
extraTriggers.push({
frontend/src/lib/components/assets/lib.ts
+5 -4
View File
@@ -96,10 +96,11 @@ export function formatAssetKind(asset: {
case 'volume':
return 'Volume'
case 'dbt':
// The SCHEME says dbt because dbt is the only thing that creates one;
// the PATH stays the relation, so a mart one project builds and the
// `source` the next reads land on one node — their dbt `unique_id`s
// differ where the relation does not (docs/dbt-runtime.md, decision 11).
// The SCHEME says dbt because dbt is what derives these relations; the
// PATH stays the relation, so a mart one project builds, the `source`
// the next reads, and a native `// materialize manual dbt://…` writer
// land on one node — their dbt `unique_id`s differ where the relation
// does not (docs/dbt-runtime.md, decision 11).
return 'dbt table'
}
}
frontend/src/lib/components/common/confirmationModal/DraftChangesConfirmationModal.svelte
+323
View File
@@ -0,0 +1,323 @@
<script lang="ts">
import ConfirmationModal from './ConfirmationModal.svelte'
import { createEventDispatcher, untrack } from 'svelte'
import type { Trigger } from '$lib/components/triggers/utils'
import DataTable from '$lib/components/table/DataTable.svelte'
import { twMerge } from 'tailwind-merge'
import TriggerLabel from '$lib/components/triggers/TriggerLabel.svelte'
import { triggerIconMap } from '$lib/components/triggers/utils'
import { Bot, Star } from 'lucide-svelte'
import ToggleButtonGroup from '../toggleButton-v2/ToggleButtonGroup.svelte'
import ToggleButton from '../toggleButton-v2/ToggleButton.svelte'
import { userStore } from '$lib/stores'
import Badge from '../badge/Badge.svelte'
import type { LinkedAgentDraft } from '$lib/components/flows/linkedAgentDrafts'
interface Props {
open?: boolean
draftTriggers?: Trigger[]
/** Saved agents this flow links to that have an unsaved draft. Scripts pass none: only a
* flow step can link an agent. */
draftAgents?: LinkedAgentDraft[]
/** Whether this user may write each listed agent's resource, keyed by path. */
agentCanWrite?: Record<string, boolean>
/** Why an agent cannot be deployed, keyed by path, from the same rule the agent editor's own
* Deploy button follows — so this dialog cannot offer a write that would be rejected. Decided
* by the caller: this is a generic dialog the script editor mounts too, and agent validation
* has no business in its bundle. */
agentRefusal?: Record<string, string | undefined>
isFlow?: boolean
}
let {
open = $bindable(false),
draftTriggers = [],
draftAgents = [],
agentCanWrite = {},
agentRefusal = {},
isFlow = false
}: Props = $props()
let selectedTriggers: Trigger[] = $state(untrack(() => draftTriggers))
let selectedAgents: LinkedAgentDraft[] = $state([])
const dispatch = createEventDispatcher<{
canceled: void
confirmed: { selectedTriggers: Trigger[]; selectedAgents: LinkedAgentDraft[] }
}>()
function toggleTrigger(trigger: Trigger, selected: 'discard' | 'deploy') {
if (selected === 'discard') {
if (trigger.isDraft) {
selectedTriggers = selectedTriggers.filter((t) => !t.isDraft || t.id !== trigger.id)
} else {
selectedTriggers = selectedTriggers.filter(
(t) => t.isDraft || t.type !== trigger.type || t.path !== trigger.path
)
}
} else if (!isSelected(selectedTriggers, trigger)) {
selectedTriggers = [...selectedTriggers, trigger]
}
}
function isSelected(triggers: Trigger[], trigger: Trigger): boolean {
if (trigger.isDraft) {
return triggers.some((t) => t.id === trigger.id)
} else {
return triggers.some((t) => t.path === trigger.path && t.type === trigger.type)
}
}
function toggleAgent(agent: LinkedAgentDraft, selected: 'discard' | 'deploy') {
if (selected === 'discard') {
selectedAgents = selectedAgents.filter((a) => a.path !== agent.path)
} else if (!selectedAgents.some((a) => a.path === agent.path)) {
selectedAgents = [...selectedAgents, agent]
}
}
function checkSavePermissions(trigger: Trigger) {
// Creating http trigger is forbidden for non-admin users
const adminOnly =
trigger.type === 'http' &&
!$userStore?.is_admin &&
!$userStore?.is_super_admin &&
trigger.isDraft
const invalidConfig = !trigger.draftConfig?.canSave
return invalidConfig ? 'invalid-config' : adminOnly ? 'admin-only' : 'deploy'
}
function checkAgentPermissions(agent: LinkedAgentDraft): {
state: 'deploy' | 'read-only' | 'invalid-config'
reason?: string
} {
if (agentCanWrite[agent.path] === false) {
return { state: 'read-only' }
}
const refusal = agentRefusal[agent.path]
return refusal ? { state: 'invalid-config', reason: refusal } : { state: 'deploy' }
}
$effect(() => {
if (!open) return
selectedTriggers = [...draftTriggers].filter((t) => checkSavePermissions(t) === 'deploy')
selectedAgents = [...draftAgents].filter((a) => checkAgentPermissions(a).state === 'deploy')
})
const runnable = $derived(isFlow ? 'flow' : 'script')
// Named after what is actually listed, so the title is not a second, vaguer copy of the section
// headings below it. "Unsaved changes detected" is taken by the leave-the-page guard, which means
// the opposite of this dialog: there, unlisted work is about to be lost.
const title = $derived.by(() => {
const triggers = draftTriggers.length > 0
const agents = draftAgents.length > 0
if (triggers && agents) return 'Draft triggers and agents detected'
if (agents) return 'Draft agents detected'
return 'Draft triggers detected'
})
</script>
<ConfirmationModal
{open}
{title}
confirmationText={isFlow ? 'Deploy Flow' : 'Deploy Script'}
type="reload"
showIcon={false}
on:canceled={() => dispatch('canceled')}
on:confirmed={() => dispatch('confirmed', { selectedTriggers, selectedAgents })}
>
<div class="flex flex-col w-full gap-8 pb-4">
{#if draftTriggers.length > 0}
<div class="flex flex-col gap-2">
<div class="text-secondary text-sm">
{`Your ${runnable} has draft triggers. Select which draft triggers to deploy with the ${runnable}. Undeployed draft triggers will be permanently deleted.`}
</div>
<div class={draftTriggers.length > 5 ? 'h-[300px]' : ''}>
<DataTable size="sm" tableFixed={true}>
<thead>
<tr class="bg-gray-50 dark:bg-gray-700 text-secondary dark:text-gray-300 text-xs">
<th class="text-left py-2 px-4">Triggers to deploy</th>
<th class="w-32 text-center py-2 px-1 justify-center"> </th>
</tr>
</thead>
<tbody>
{#each draftTriggers as trigger}
{@const SvelteComponent = triggerIconMap[trigger.type]}
{@const permission = checkSavePermissions(trigger)}
{@const isSelectedTrigger = isSelected(selectedTriggers, trigger)}
<tr
class={twMerge(
'transition-colors h-12 border-t border-gray-200 dark:border-gray-700 whitespace-nowrap',
permission === 'deploy' ? 'hover:bg-surface-hover ' : ''
)}
>
<td class={twMerge('text-center py-1 px-4')}>
<div class="flex flex-row items-center gap-2">
<div class="relative flex justify-center items-center">
<SvelteComponent
size={14}
class={isSelectedTrigger ? 'text-accent' : 'text-hint'}
/>
{#if trigger.isPrimary}
<Star size={8} class="absolute -mt-3 ml-3 text-accent" />
{/if}
</div>
<!-- `TriggerLabel` emits its badges as bare siblings, so the gap between them
is the container's to set, as the other call sites do. -->
<div class="flex grow min-w-0 items-center gap-2 text-left">
<TriggerLabel {trigger} discard={!isSelectedTrigger} />
</div>
</div>
</td>
<td class="text-left py-1">
{#if permission === 'deploy'}
<div class="flex justify-start">
<ToggleButtonGroup
class="w-fit h-fit"
selected={isSelectedTrigger ? 'deploy' : 'discard'}
on:selected={(e) => toggleTrigger(trigger, e.detail)}
>
{#snippet children({ item })}
<ToggleButton
label={!trigger.isDraft && trigger.draftConfig ? 'Reset' : 'Discard'}
value={'discard'}
{item}
small
class="data-[state=on]:text-white data-[state=on]:bg-red-400 justify-center"
/>
<ToggleButton
label={!trigger.isDraft && trigger.draftConfig ? 'Update' : 'Deploy'}
value={'deploy'}
{item}
small
class="data-[state=on]:bg-surface-accent-primary data-[state=on]:text-white justify-center"
/>
{/snippet}
</ToggleButtonGroup>
</div>
{:else if permission === 'admin-only'}
<Badge color="red">Admin only</Badge>
{:else if permission === 'invalid-config'}
<Badge color="red">Invalid config</Badge>
{/if}
</td>
</tr>
{/each}
</tbody>
</DataTable>
</div>
</div>
{/if}
{#if draftAgents.length > 0}
<div class="flex flex-col gap-2">
<div class="text-secondary text-sm">
Saved agents this flow uses have unsaved changes. Select which ones to deploy with the
flow. An agent kept as a draft stays editable, and the flow runs the agent as currently
deployed.
</div>
<div class={draftAgents.length > 5 ? 'h-[300px]' : ''}>
<DataTable size="sm" tableFixed={true}>
<thead>
<tr class="bg-gray-50 dark:bg-gray-700 text-secondary dark:text-gray-300 text-xs">
<th class="text-left py-2 px-4">Agents to deploy</th>
<!-- Wider than the trigger table's: "Keep as draft" does not fit its w-32. -->
<th class="w-48 text-center py-2 px-1 justify-center"> </th>
</tr>
</thead>
<tbody>
{#each draftAgents as agent (agent.path)}
{@const permission = checkAgentPermissions(agent)}
{@const isSelectedAgent = selectedAgents.some((a) => a.path === agent.path)}
<tr
class={twMerge(
'transition-colors border-t border-gray-200 dark:border-gray-700',
permission.state === 'deploy' ? 'hover:bg-surface-hover ' : ''
)}
>
<!-- No fixed height, unlike the trigger table's `h-12` row: a never-deployed agent
carries a second line of warning under the path, so the row grows with it. -->
<td class="py-2 px-4">
<div class="flex flex-row items-center gap-2">
<Bot size={14} class={isSelectedAgent ? 'text-accent' : 'text-hint'} />
<div class="flex grow min-w-0 flex-col text-left">
<div class="flex items-center gap-2 min-w-0">
<!-- Dimmed rather than struck through: keeping a draft destroys nothing, so the
trigger table's deletion styling would say the wrong thing. -->
<span
class={twMerge(
'truncate text-xs',
isSelectedAgent ? '' : 'text-tertiary'
)}
title={agent.path}
>
{agent.path}
</span>
{#if agent.noDeployed}
<Badge small color="indigo">Never deployed</Badge>
{/if}
</div>
{#if agent.noDeployed && !isSelectedAgent}
<!-- The section's promise — the flow falls back to the deployed agent — does not
hold for one that has never been deployed. There is nothing to fall back to,
so the flow would land naming a path that does not resolve. -->
<span class="text-xs text-red-600 dark:text-red-400 whitespace-normal">
Never deployed, so the flow will not run until this agent is deployed.
</span>
{/if}
</div>
</div>
</td>
<!-- Right-aligned, unlike the trigger column: this one is `w-48` to fit "Keep as
draft", so left-aligned content would float in the slack instead of lining up
with the trigger rows above it. -->
<td class="text-right py-1 pr-2">
{#if permission.state === 'deploy'}
<div class="flex justify-end">
<ToggleButtonGroup
class="w-fit h-fit"
selected={isSelectedAgent ? 'deploy' : 'discard'}
on:selected={(e) => toggleAgent(agent, e.detail)}
>
{#snippet children({ item })}
<ToggleButton
label="Keep as draft"
value={'discard'}
{item}
small
class="justify-center"
/>
<ToggleButton
label={agent.noDeployed ? 'Deploy' : 'Update'}
value={'deploy'}
{item}
small
class="data-[state=on]:bg-surface-accent-primary data-[state=on]:text-white justify-center"
/>
{/snippet}
</ToggleButtonGroup>
</div>
{:else if permission.state === 'read-only'}
<Badge color="red" title="You do not have write access to this agent">
Read-only
</Badge>
{:else}
<Badge color="red" title={permission.reason}>Invalid config</Badge>
{/if}
</td>
</tr>
{/each}
</tbody>
</DataTable>
</div>
</div>
{/if}
</div>
</ConfirmationModal>
frontend/src/lib/components/common/confirmationModal/DraftTriggersConfirmationModal.svelte
-168
View File
@@ -1,168 +0,0 @@
<script lang="ts">
import ConfirmationModal from './ConfirmationModal.svelte'
import { createEventDispatcher, untrack } from 'svelte'
import type { Trigger } from '$lib/components/triggers/utils'
import DataTable from '$lib/components/table/DataTable.svelte'
import { twMerge } from 'tailwind-merge'
import TriggerLabel from '$lib/components/triggers/TriggerLabel.svelte'
import { triggerIconMap } from '$lib/components/triggers/utils'
import { Star } from 'lucide-svelte'
import ToggleButtonGroup from '../toggleButton-v2/ToggleButtonGroup.svelte'
import ToggleButton from '../toggleButton-v2/ToggleButton.svelte'
import { userStore } from '$lib/stores'
import Badge from '../badge/Badge.svelte'
interface Props {
open?: boolean
draftTriggers?: Trigger[]
isFlow?: boolean
}
let { open = $bindable(false), draftTriggers = [], isFlow = false }: Props = $props()
let selectedTriggers: Trigger[] = $state(untrack(() => draftTriggers))
const dispatch = createEventDispatcher<{
canceled: void
confirmed: { selectedTriggers: Trigger[] }
}>()
function toggleTrigger(trigger: Trigger, selected: 'discard' | 'deploy') {
if (selected === 'discard') {
if (trigger.isDraft) {
selectedTriggers = selectedTriggers.filter((t) => !t.isDraft || t.id !== trigger.id)
} else {
selectedTriggers = selectedTriggers.filter(
(t) => t.isDraft || t.type !== trigger.type || t.path !== trigger.path
)
}
} else if (!isSelected(selectedTriggers, trigger)) {
selectedTriggers = [...selectedTriggers, trigger]
}
}
function isSelected(triggers: Trigger[], trigger: Trigger): boolean {
if (trigger.isDraft) {
return triggers.some((t) => t.id === trigger.id)
} else {
return triggers.some((t) => t.path === trigger.path && t.type === trigger.type)
}
}
function checkSavePermissions(trigger: Trigger) {
// Creating http trigger is forbidden for non-admin users
const adminOnly =
trigger.type === 'http' &&
!$userStore?.is_admin &&
!$userStore?.is_super_admin &&
trigger.isDraft
const invalidConfig = !trigger.draftConfig?.canSave
return invalidConfig ? 'invalid-config' : adminOnly ? 'admin-only' : 'deploy'
}
$effect(() => {
open &&
(selectedTriggers = [...draftTriggers].filter((t) => checkSavePermissions(t) === 'deploy'))
})
</script>
<ConfirmationModal
{open}
title="Draft triggers detected !"
confirmationText={isFlow ? 'Deploy Flow' : 'Deploy Script'}
type="reload"
showIcon={false}
on:canceled={() => dispatch('canceled')}
on:confirmed={() => dispatch('confirmed', { selectedTriggers })}
>
<div class="flex flex-col w-full gap-8 pb-4">
<div class="text-secondary text-sm">
{`${isFlow ? 'Your flow' : 'Your script'} has draft triggers. Select which draft triggers to deploy with the ${isFlow ? 'flow' : 'script'}. Undeployed
draft triggers will be permanently deleted.`}
</div>
<div class={draftTriggers.length > 5 ? 'h-[300px]' : ''}>
<DataTable size="sm" tableFixed={true}>
<thead>
<tr class="bg-gray-50 dark:bg-gray-700 text-secondary dark:text-gray-300 text-xs">
<th class="text-left py-2 px-4">Triggers to deploy</th>
<th class="w-32 text-center py-2 px-1 justify-center"> </th>
</tr>
</thead>
<tbody>
{#each draftTriggers as trigger}
{@const SvelteComponent = triggerIconMap[trigger.type]}
{@const permission = checkSavePermissions(trigger)}
{@const isSelectedTrigger = isSelected(selectedTriggers, trigger)}
<tr
class={twMerge(
'transition-colors h-12 border-t border-gray-200 dark:border-gray-700 whitespace-nowrap',
permission === 'deploy' ? 'hover:bg-surface-hover ' : ''
)}
>
<td class={twMerge('text-center py-1 px-4')}>
<div class="flex flex-row items-center gap-2">
<div class="relative flex justify-center items-center">
<SvelteComponent
size={14}
class={isSelectedTrigger ? 'text-accent' : 'text-hint'}
/>
{#if trigger.isPrimary}
<Star size={8} class="absolute -mt-3 ml-3 text-accent" />
{/if}
</div>
<div class="flex grow min-w-0 items-center text-left">
<TriggerLabel {trigger} discard={!isSelectedTrigger} />
</div>
</div>
</td>
<td class="text-left py-1">
{#if permission === 'deploy'}
<div class="flex justify-start">
<ToggleButtonGroup
class="w-fit h-fit"
selected={isSelectedTrigger ? 'deploy' : 'discard'}
on:selected={(e) => toggleTrigger(trigger, e.detail)}
>
{#snippet children({ item })}
<ToggleButton
label={!trigger.isDraft && trigger.draftConfig ? 'Reset' : 'Discard'}
value={'discard'}
{item}
small
class="data-[state=on]:text-white data-[state=on]:bg-red-400 justify-center"
/>
<ToggleButton
label={!trigger.isDraft && trigger.draftConfig ? 'Update' : 'Deploy'}
value={'deploy'}
{item}
small
class="data-[state=on]:bg-surface-accent-primary data-[state=on]:text-white justify-center"
/>
{/snippet}
</ToggleButtonGroup>
</div>
{:else if permission === 'admin-only'}
<Badge color="red">Admin only</Badge>
{:else if permission === 'invalid-config'}
<Badge color="red">Invalid config</Badge>
{/if}
</td>
</tr>
{/each}
{#if draftTriggers.length === 0}
<tr>
<td colspan="3" class="text-center py-6 text-gray-500 dark:text-gray-400 text-sm">
No draft triggers found
</td>
</tr>
{/if}
</tbody>
</DataTable>
</div>
</div>
</ConfirmationModal>
frontend/src/lib/components/copilot/chat/LinkRenderer.svelte
+11 -2
View File
@@ -3,6 +3,7 @@
import { ExternalLink, PanelRight } from 'lucide-svelte'
import { Button } from '$lib/components/common'
import RowIcon from '$lib/components/common/table/RowIcon.svelte'
import { newTabModifier } from '$lib/attachments/newTabModifier.svelte'
import {
hasToolDisplayActionHandler,
runToolDisplayAction
@@ -44,6 +45,8 @@
const previewAction = $derived(available?.type === 'open_item_preview' ? available : undefined)
const drawerAction = $derived(available?.type === 'open_created_resource' ? available : undefined)
const modifier = newTabModifier()
const hint = $derived(
previewAction ? `Open ${wmPath} in the preview panel` : `Open ${wmPath} in a new tab`
)
@@ -67,7 +70,11 @@
{#if href}
{#if wmKind}
<span class="group inline-flex items-baseline">
<!-- Only a preview pill can change icon, so only it is worth tracking the modifier for. -->
<span
class="group inline-flex items-baseline"
{@attach previewAction ? modifier.attach : undefined}
>
<a
{href}
target={previewAction ? undefined : '_blank'}
@@ -85,7 +92,9 @@
<span
class="absolute inset-0 flex items-center justify-center text-tertiary opacity-0 transition-opacity group-hover:opacity-100"
>
{#if previewAction}
<!-- Narrower than the modifier list `onclick` bails on: only the modifier that
really opens a tab may show the icon for one. -->
{#if previewAction && !modifier.held}
<PanelRight size={12} />
{:else}
<ExternalLink size={11} />
frontend/src/lib/components/copilot/chat/pipeline/core.ts
+2 -2
View File
@@ -338,8 +338,8 @@ export function getPipelinePromptSection(ctx: PipelineContext): string {
Data Pipeline editor (ACTIVE):
- The user has the /pipeline/${ctx.folder} editor open. A pipeline is a DAG of scripts (nodes) connected by storage assets (DuckLake tables, data tables, S3 objects, volumes, resources) and execution triggers.
- Annotations are top-of-file comments in the NODE'S OWN comment syntax: \`--\` for SQL (duckdb/postgresql), \`#\` for python3/bash, \`//\` for bun/TS. The \`//\` shown below is the TS form — translate it (a \`// pipeline\` line in a SQL node is a syntax error that won't deploy).
- A script becomes a pipeline node when its source starts with the \`// pipeline\` annotation. Declare execution-DAG inputs with \`// on <asset-uri | schedule | webhook | email | kafka | mqtt | amqp | nats | postgres | sqs | gcp | data_upload>\` (e.g. \`// on ducklake://main/orders\`). Outputs are inferred from what the body writes (wmill SDK calls / SQL CREATE TABLE / writeS3File); declare a managed output with \`// materialize <asset-uri>\`. Optional badges: \`// partitioned <daily|hourly|weekly|monthly|dynamic>\`, \`// freshness <duration>\`, \`// tag <name>\`, \`// retry <count> [delay]\`, \`// data_test <kind> ...\`, \`// measure <name> = <agg> [where <pred>]\`, \`// dimension <name> = <expr>\`.
- \`materialize\` (the managed output): \`// materialize <asset-uri>\` means the runtime writes the node's output table FOR you — write the body as a single SELECT and the runtime wraps it in the create/replace, so do NOT also write your own CREATE TABLE / INSERT. IMPORTANT: \`// materialize\` is **DuckDB-only** and its target MUST be a DuckLake table (\`ducklake://<name>/<table>\`) — deploy rejects it on any other language or target. For a \`python3\`/\`bun\`/\`postgresql\` node, do NOT use \`// materialize\`; write the output via the SDK instead (e.g. \`wmill.writeS3File(...)\`, a \`CREATE TABLE\` in postgresql, or \`wmill.databaseUrlFromResource\`/ducklake helpers) and let the output be inferred. Reach for \`duckdb\` when a node should materialize a DuckLake table. Write strategy: with no option it REPLACES the whole table each run (full refresh; the only mode whose output columns may change); \`// materialize <uri> append\` INSERT-appends rows (incremental); \`// materialize <uri> key=<col>\` merges/upserts on \`<col>\`. \`// materialize manual <uri>\` opts OUT of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. \`materialize\` is paired with partitioning for incremental pipelines: a \`// partitioned <daily|hourly|weekly|monthly|dynamic>\` node runs once per partition (append/merge into a fixed-schema table), and the \`{partition}\` token — usable in any asset URI AND in the body SQL — is substituted with the current partition's IDENTITY string at run time. To filter the source to the active slice on a time grain, use the runtime-injected macro: \`WHERE wm_partition(<ts_col>) = {partition}\`. \`wm_partition(ts)\` buckets a timestamp with the exact identity format the runtime used (daily/hourly/weekly/monthly), so it always matches and you never hand-write a \`strftime\` format. Do NOT write \`= TIMESTAMP {partition}\`: the identity string is not a valid timestamp literal for hourly/weekly/monthly and errors at runtime. For \`dynamic\` partitioning the identity is your caller-supplied key (not a timestamp, no macro), so filter on it directly: \`WHERE <your_key_col> = {partition}\`. \`materialize\` is an output DECLARATION on the node — it is not a command; there is no "materialize run".
- A script becomes a pipeline node when its source starts with the \`// pipeline\` annotation. Declare execution-DAG inputs with \`// on <asset-uri | schedule | webhook | email | kafka | mqtt | amqp | nats | postgres | sqs | gcp | data_upload>\` (e.g. \`// on ducklake://main/orders\`). Outputs are inferred from what the body writes (wmill SDK calls / SQL CREATE TABLE / writeS3File); declare a managed output with \`// materialize <asset-uri>\`. Optional badges: \`// partitioned <daily|hourly|weekly|monthly|dynamic>\`, \`// freshness <duration>\`, \`// tag <name>\`, \`// retry <count> [delay]\`, \`// data_test <kind> ...\` (managed DuckLake targets only — deploy rejects it beside a \`dbt://\` target), \`// measure <name> = <agg> [where <pred>]\`, \`// dimension <name> = <expr>\`.
- \`materialize\` (the managed output): a managed \`// materialize ducklake://<name>/<table>\` means the runtime writes the node's output table FOR you — write the body as a single SELECT and the runtime wraps it in the create/replace, so do NOT also write your own CREATE TABLE / INSERT. The \`dbt://\` target below is the opposite: the node writes its own DDL and none of the write strategies apply to it. IMPORTANT: a MANAGED \`// materialize\` is **DuckDB-only** and its target MUST be a DuckLake table (\`ducklake://<name>/<table>\`) — deploy rejects a \`ducklake://\` target on any other language. For a \`python3\`/\`bun\`/\`postgresql\` node writing the lake, do NOT use \`// materialize\`; write the output via the SDK instead (e.g. \`wmill.writeS3File(...)\`, a \`CREATE TABLE\` in postgresql, or \`wmill.databaseUrlFromResource\`/ducklake helpers) and let the output be inferred. Reach for \`duckdb\` when a node should materialize a DuckLake table. The one target any language BUT DBT'S OWN may declare (a dbt project's writes come from its manifest, so \`// materialize\` on a dbt script is rejected at deploy) is a WAREHOUSE RELATION: \`// materialize manual dbt://<warehouse>/<schema>/<name>\`, with \`<warehouse>\` a warehouse the workspace configures under Settings → dbt. \`manual\` is its only mode — nothing generates warehouse DDL, so the node issues its own write and the annotation records the outcome. Use it on an ingestion node a dbt project reads as a \`source\`: the declared relation and the dbt model become ONE graph node, and a downstream \`// on dbt://<warehouse>/<schema>/<name>\` fires when that node completes. Write strategy: with no option it REPLACES the whole table each run (full refresh; the only mode whose output columns may change); \`// materialize <uri> append\` INSERT-appends rows (incremental); \`// materialize <uri> key=<col>\` merges/upserts on \`<col>\`. \`// materialize manual <uri>\` opts OUT of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. \`materialize\` is paired with partitioning for incremental pipelines: a \`// partitioned <daily|hourly|weekly|monthly|dynamic>\` node runs once per partition (append/merge into a fixed-schema table), and the \`{partition}\` token — usable in any asset URI AND in the body SQL — is substituted with the current partition's IDENTITY string at run time. To filter the source to the active slice on a time grain, use the runtime-injected macro: \`WHERE wm_partition(<ts_col>) = {partition}\`. \`wm_partition(ts)\` buckets a timestamp with the exact identity format the runtime used (daily/hourly/weekly/monthly), so it always matches and you never hand-write a \`strftime\` format. Do NOT write \`= TIMESTAMP {partition}\`: the identity string is not a valid timestamp literal for hourly/weekly/monthly and errors at runtime. For \`dynamic\` partitioning the identity is your caller-supplied key (not a timestamp, no macro), so filter on it directly: \`WHERE <your_key_col> = {partition}\`. \`materialize\` is an output DECLARATION on the node — it is not a command; there is no "materialize run".
- \`measure\` / \`dimension\` (declared metrics): on a node that materializes a DuckLake table, \`// measure <name> = <aggregate> [where <predicate>]\` names the canonical way to aggregate that table (e.g. \`// measure revenue = sum(amount) where not is_refund\`), and \`// dimension <name> = <expr>\` names a way to slice it (e.g. \`// dimension region = region\`, \`// dimension month = date_trunc('month', ordered_at)\`). They execute nothing: they are catalogued at deploy so the editor and other agents can reuse the definition instead of re-deriving it and silently disagreeing. Keep the predicate in the \`where\` clause rather than folding it into the aggregate: it is rendered as \`<agg> FILTER (WHERE <pred>)\`, which is what lets two measures with different predicates sit under one GROUP BY. DuckLake-only, and only meaningful next to \`// materialize\`. Declare one when a number carries a judgement call someone else would get wrong (refunds excluded, test rows dropped, which column is the amount); do NOT blanket every table with measures, an obvious \`count(*)\` earns nothing. To USE a metric another node declares, read that node with read_pipeline_node and reuse its exact expression rather than guessing it.
- Use get_pipeline_graph to see the current nodes/assets/triggers, and read_pipeline_node before editing one.
- Every node of this pipeline lives at \`f/${ctx.folder}/<node_name>\` — \`${ctx.folder}\` is the folder name and \`f/\` is the owner prefix every workspace path carries, so write it exactly once (never \`f/f/…\`, and never a bare \`<node_name>\`).
frontend/src/lib/components/dbOps.ts
+49 -35
View File
@@ -14,6 +14,7 @@ import { WorkspaceService } from '$lib/gen'
import { pendingMigrations } from './workspaceSettings/datatableMigrationUtils'
import {
buildTableEditorValues,
type TableEditorForeignKey,
type TableEditorValues
} from './apps/components/display/dbtable/tableEditor'
import { type AlterTableValues } from './apps/components/display/dbtable/queries/alterTable'
@@ -250,6 +251,10 @@ export type IDbSchemaOps = {
schema?: string
colDefs: TableMetadata
}) => Promise<TableEditorValues>
onFetchForeignKeys: (params: {
table: string
schema?: string
}) => Promise<TableEditorForeignKey[]>
}
/** Thrown by a schema op when the user declines the out-of-order run warning.
@@ -396,6 +401,48 @@ export function dbSchemaOpsWithPreviewScripts({
}
}
/** Resolves to [] when the database has no foreign key introspection
* (BigQuery) or the query fails: callers treat foreign keys as optional. */
async function fetchForeignKeys({
table,
schema
}: {
table: string
schema?: string
}): Promise<TableEditorForeignKey[]> {
if (dbType === 'bigquery') return []
try {
const fkContent = makeMarker('FOREIGN_KEYS', { table, schema })
const fkResult = await runScriptAndPollResult({
workspace,
requestBody: { args: dbArg, content: fkContent, language, tag }
})
let rawForeignKeys: RawForeignKey[]
if (dbType === 'snowflake') {
rawForeignKeys = transformSnowflakeForeignKeys(fkResult as any[])
} else {
rawForeignKeys = fkResult as RawForeignKey[]
if (rawForeignKeys && Array.isArray(rawForeignKeys)) {
rawForeignKeys = rawForeignKeys.map((fk) => {
const lowerFk: any = {}
Object.keys(fk).forEach((key) => {
lowerFk[key.toLowerCase()] = fk[key]
})
return lowerFk
})
}
}
if (rawForeignKeys && Array.isArray(rawForeignKeys)) {
return transformForeignKeys(rawForeignKeys)
}
} catch (e) {
console.warn('Failed to fetch foreign keys:', e)
}
return []
}
return {
onDelete: async ({ tableKey, schema }) => {
const content = makeMarker('DROP_TABLE', { table: tableKey, schema })
@@ -454,44 +501,11 @@ export function dbSchemaOpsWithPreviewScripts({
const downContent = makeMarker('CREATE_SCHEMA', { schema })
await applyDdl(migrationName('drop_schema', schema), content, downContent)
},
onFetchForeignKeys: fetchForeignKeys,
onFetchTableEditorDefinition: async ({ table, schema, colDefs }) => {
let foreignKeys: import('./apps/components/display/dbtable/tableEditor').TableEditorForeignKey[] =
[]
const foreignKeys = await fetchForeignKeys({ table, schema })
let pk_constraint_name: string | undefined
// Fetch foreign keys (not supported for BigQuery)
if (dbType !== 'bigquery') {
try {
const fkContent = makeMarker('FOREIGN_KEYS', { table, schema })
const fkResult = await runScriptAndPollResult({
workspace,
requestBody: { args: dbArg, content: fkContent, language, tag }
})
let rawForeignKeys: RawForeignKey[]
if (dbType === 'snowflake') {
rawForeignKeys = transformSnowflakeForeignKeys(fkResult as any[])
} else {
rawForeignKeys = fkResult as RawForeignKey[]
if (rawForeignKeys && Array.isArray(rawForeignKeys)) {
rawForeignKeys = rawForeignKeys.map((fk) => {
const lowerFk: any = {}
Object.keys(fk).forEach((key) => {
lowerFk[key.toLowerCase()] = fk[key]
})
return lowerFk
})
}
}
if (rawForeignKeys && Array.isArray(rawForeignKeys)) {
foreignKeys = transformForeignKeys(rawForeignKeys)
}
} catch (e) {
console.warn('Failed to fetch foreign keys:', e)
}
}
// Fetch primary key constraint name (not supported for BigQuery/MySQL)
if (dbType !== 'bigquery' && dbType !== 'mysql') {
try {
frontend/src/lib/components/flows/agentDraft.svelte.ts
+110 -67
View File
@@ -6,6 +6,7 @@ import { sendUserToast } from '$lib/toast'
import { canWrite } from '$lib/utils'
import { userStore } from '$lib/stores'
import { getUserExt } from '$lib/user'
import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte'
import { useTriggerDraftSync, type TriggerDraftSync } from '../triggers/useTriggerDraftSync.svelte'
import { logReusableAgentUsage } from './agentTelemetry'
import {
@@ -84,6 +85,95 @@ export interface AgentResourceState {
wsSpecific: boolean
}
/**
* Why writing this draft to its resource would be refused, if it would. Shared with the flow's
* deploy dialog, which lists the drafts of the agents a flow links: an agent the editor's own
* Deploy button rejects must not be deployable from a flow either.
*
* `currentPath` is the path the draft is being deployed to; pass undefined where the caller has
* none of its own to compare against.
*/
export function agentDraftDeployRefusal(
state: AgentResourceState,
currentPath: string | undefined
): string | undefined {
// The editor offers only values, but a transform can arrive from a step that was forked
// before this existed, or from the generic resource editor: say so rather than writing it.
const transformValued = transformValuedBrainKeys(state.args)
if (transformValued.length > 0) {
const fields = transformValued.map((key) => AGENT_BRAIN_LABELS[key] ?? key)
const many = fields.length > 1
return `${fields.join(', ')} ${many ? 'are' : 'is'} set to an expression or an AI-filled value, which a saved agent cannot store. Replace ${many ? 'them' : 'it'} with a plain value before deploying.`
}
// The resource endpoint takes any JSON, so nothing downstream stops an agent that cannot run:
// the worker needs a provider to call and rejects a tool whose name it cannot pass to the
// model. Deploying one would break every flow linking it, so it is refused here.
const blocked = agentConfigRunError(state.args)
if (blocked) {
return blocked
}
// Renaming is not the agent editor's to do: moving the resource leaves every step that links to
// it naming a path that no longer exists, and reconciling those is a feature of its own. A
// renamed path can still reach here, the generic editor writing the same draft row and offering
// a path field, so refuse it rather than performing half of a rename.
if (currentPath && state.path !== currentPath) {
return `This draft renames the agent to ${state.path}. Deploy it from the resource editor instead.`
}
// Only a draft naming another type: the load refuses a resource that is not an agent, while a
// draft the generic resource editor wrote names no type at all and inherits the loaded one.
if (state.resource_type && state.resource_type !== 'ai_agent') {
return `This draft is a ${state.resource_type} resource, not an agent.`
}
return undefined
}
/**
* Write an agent to its resource from the state the editor holds, which can be ahead of the
* persisted draft row: the form stays editable while a deploy is in flight. Surfaces that deploy
* the row itself go through `deployDraft` instead.
*
* `notAnAgent` separates the one failure that invalidates the caller's whole view of the path, its
* holding something else now, from a write that merely failed.
*/
type AgentWriteResult = { ok: true } | { ok: false; error: string; notAnAgent?: true }
async function writeAgentResource(
workspace: string,
state: AgentResourceState,
noDeployed: boolean
): Promise<AgentWriteResult> {
const body = {
path: state.path,
value: state.args,
description: state.description,
labels: state.labels,
ws_specific: state.wsSpecific
}
try {
if (noDeployed) {
// A create needs a type, and every caller proved this path is an agent before offering it.
await ResourceService.createResource({
workspace,
requestBody: { ...body, resource_type: state.resource_type ?? 'ai_agent' }
})
} else {
// The type the caller proved is as old as its own load, and an update carries no type of
// its own: were the path deleted and recreated as something else meanwhile, this write
// would put an agent config inside that resource. Reading it again narrows the window to
// the request rather than to however long the editor or the dialog stayed open.
const current = await ResourceService.getResource({ workspace, path: state.path })
const refused = agentEditorRefusal(state.path, current.resource_type)
if (refused) {
return { ok: false, error: refused, notAnAgent: true }
}
await ResourceService.updateResource({ workspace, path: state.path, requestBody: body })
}
} catch (err) {
return { ok: false, error: `Could not save agent: ${err}` }
}
return { ok: true }
}
export interface AgentDraftOptions {
/** The `ai_agent` resource being edited. */
path: () => string | undefined
@@ -219,6 +309,16 @@ export function useAgentDraft(opts: AgentDraftOptions): AgentDraftHandle {
// config before the autosave lands.
state =
((r as any).draft as AgentResourceState | undefined) ?? structuredClone(deployedState)
// Adopt the row's timestamp as this tab's baseline. Without it the first save from
// each tab goes out with no `last_sync`, which the backend treats as unconditional
// and so silently overwrites another tab's newer draft. It also clears any parked
// conflict or failure for the key: a conflict is deliberately sticky (the retry
// keeps the same baseline), and nothing else mounts a resolver for `resource`
// drafts, so re-opening the agent is the only place it can be resolved.
UserDraftDbSyncer.recordRemoteSync(
{ workspace: ws, itemKind: 'resource', path },
(r as { draft_saved_at?: string }).draft_saved_at
)
loading = false
await sync.maybeRestore()
},
@@ -237,42 +337,9 @@ export function useAgentDraft(opts: AgentDraftOptions): AgentDraftHandle {
const ws = opts.workspace()
const s = state
if (!ws || !s) return false
// The editor offers only values, but a transform can arrive from a step that was forked
// before this existed, or from the generic resource editor: say so rather than writing it.
const transformValued = transformValuedBrainKeys(s.args)
if (transformValued.length > 0) {
const fields = transformValued.map((key) => AGENT_BRAIN_LABELS[key] ?? key)
const many = fields.length > 1
sendUserToast(
`${fields.join(', ')} ${many ? 'are' : 'is'} set to an expression or an AI-filled value, which a saved agent cannot store. Replace ${many ? 'them' : 'it'} with a plain value before deploying.`,
true
)
return false
}
// The resource endpoint takes any JSON, so nothing downstream stops an agent that cannot run:
// the worker needs a provider to call and rejects a tool whose name it cannot pass to the
// model. Deploying one would break every flow linking it, so it is refused here.
const blocked = agentConfigRunError(s.args)
if (blocked) {
sendUserToast(blocked, true)
return false
}
// Renaming is not this editor's to do: moving the resource leaves every step that links to it
// naming a path that no longer exists, and reconciling those is a feature of its own. A
// renamed path can still reach here, the generic editor writing the same draft row and
// offering a path field, so refuse it rather than performing half of a rename.
const currentPath = opts.path()
if (currentPath && s.path !== currentPath) {
sendUserToast(
`This draft renames the agent to ${s.path}. Deploy it from the resource editor instead.`,
true
)
return false
}
// Only a draft naming another type: the load refuses a resource that is not an agent, while a
// draft the generic resource editor wrote names no type at all and inherits the loaded one.
if (s.resource_type && s.resource_type !== 'ai_agent') {
sendUserToast(`This draft is a ${s.resource_type} resource, not an agent.`, true)
const refused = agentDraftDeployRefusal(s, opts.path())
if (refused) {
sendUserToast(refused, true)
return false
}
// The form stays editable while the request is in flight, so everything below works from a
@@ -280,39 +347,15 @@ export function useAgentDraft(opts: AgentDraftOptions): AgentDraftHandle {
// made during the request as saved, and the banner would clear on a value the server never
// received; against the snapshot it stays a draft, which is what it is.
const submitted = structuredClone($state.snapshot(s)) as AgentResourceState
const body = {
path: submitted.path,
value: submitted.args,
description: submitted.description,
labels: submitted.labels,
ws_specific: submitted.wsSpecific
}
try {
if (noDeployed) {
await ResourceService.createResource({
workspace: ws,
// A create needs a type, and the load proved this path is an agent before opening.
requestBody: { ...body, resource_type: submitted.resource_type ?? 'ai_agent' }
})
const written = await writeAgentResource(ws, submitted, noDeployed)
if (!written.ok) {
// A path that is no longer an agent tears this editor down; anything else is a plain error
// the user can retry from the form as it stands.
if (written.notAnAgent) {
refuse(written.error)
} else {
// The type this editor proved is as old as the load, and an update carries no type of
// its own: were the path deleted and recreated as something else meanwhile, this write
// would put an agent config inside that resource. Reading it again narrows the window
// to the request rather than to however long the editor stayed open.
const current = await ResourceService.getResource({ workspace: ws, path: submitted.path })
const refused = agentEditorRefusal(submitted.path, current.resource_type)
if (refused) {
refuse(refused)
return false
}
await ResourceService.updateResource({
workspace: ws,
path: submitted.path,
requestBody: body
})
sendUserToast(written.error, true)
}
} catch (err) {
sendUserToast(`Could not save agent: ${err}`, true)
return false
}
// The counter the step card's write-back used to report, from the surface that now owns the
frontend/src/lib/components/flows/agentEditorStore.svelte.ts
+24
View File
@@ -6,6 +6,8 @@
* resources page. Module-level rather than a context value because what opens it — a step's card,
* a list row — unmounts the moment the selection moves.
*/
import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte'
export interface AgentEditorTarget {
path: string
/** The workspace the opener operates on; the nav workspace when absent. */
@@ -61,3 +63,25 @@ export function markAgentWritten(workspace: string | undefined, path: string) {
export function agentWriteCount(workspace: string | undefined, path: string | undefined): number {
return agentWrites[writeKey(workspace, path)] ?? 0
}
/** How many times each agent's DRAFT has been saved, for the surfaces that display an agent by
* fetching it. A draft write moves no deployed version, so `agentWriteCount` never sees it, and a
* card keyed on that alone would keep describing the config a test no longer runs. */
let agentDraftSaves = $state<Record<string, number>>({})
/** Every writer in this document goes through the draft syncer — this editor, the generic resource
* editor — so one subscription answers for them all, firing when the write lands rather than on
* each keystroke. `resource` is the item kind agent draft rows use. In-memory: a save in another
* tab never arrives here, so a card lags it until reload, while what a test runs is read live. */
UserDraftDbSyncer.onAnySaved(({ workspace, itemKind, path }) => {
if (itemKind !== 'resource') return
const key = writeKey(workspace, path)
agentDraftSaves[key] = (agentDraftSaves[key] ?? 0) + 1
})
export function agentDraftSaveCount(
workspace: string | undefined,
path: string | undefined
): number {
return agentDraftSaves[writeKey(workspace, path)] ?? 0
}
frontend/src/lib/components/flows/agentTelemetry.ts
+4
View File
@@ -13,6 +13,10 @@ export type ReusableAgentEvent =
| 'linked'
/** A linked step was forked back into a standalone agent. */
| 'unlinked'
/** A linked agent's unsaved draft was deployed alongside the flow that uses it. */
| 'draft_deployed_with_flow'
/** A linked agent's unsaved draft was left as a draft when its flow was deployed. */
| 'draft_kept_on_deploy'
export function logReusableAgentUsage(event: ReusableAgentEvent): void {
logFeatureUsage('ai_agent', 'reusable', { key: event })
frontend/src/lib/components/flows/content/AgentEditorHost.svelte
+1 -13
View File
@@ -29,6 +29,7 @@
inputTransformsToAgentConfig,
type AIAgentConfig
} from '../agentResourceUtils'
import { agentArgsToTransforms } from '../linkedAgentDrafts'
import { AGENT_TOOLS_ROW } from '../agentFormFields'
import { toolDisplayName, type AgentTool } from '../agentToolUtils'
import { useAgentDraft } from '../agentDraft.svelte'
@@ -223,19 +224,6 @@
})
})
/** Every argument the resource carries, as a static transform. `tools` is the roster rather than
* a field, so it rides on the module's own key instead. Not only the keys the form renders: a
* run reads them all, and an agent holding its own `user_message` answers with it when nothing
* overrides it, so a test here has to run the configuration a linked step would. */
function agentArgsToTransforms(args: AIAgentConfig): Record<string, InputTransform> {
const it: Record<string, InputTransform> = {}
for (const [key, value] of Object.entries(args ?? {})) {
if (key === 'tools' || value === undefined) continue
it[key] = { type: 'static', value } as InputTransform
}
return it
}
/** Everything the form does not model. `inputTransformsToAgentConfig` rebuilds the value from
* `AGENT_BRAIN_KEYS` alone, so a key this editor never renders — one a newer backend added, or
* the `user_message` default a resource may carry, which the runtime does read when the step
frontend/src/lib/components/flows/content/AgentEditorModal.svelte
+3 -1
View File
@@ -183,7 +183,9 @@
const moduleIds = new Set(linkedModulesForAgent(scope, path))
moduleIds.add(at.host.moduleId)
return Promise.all(
[...moduleIds].map((moduleId) => publishLinkedAgentTools(path, at.ws, scope, moduleId))
// With the draft: a deploy leaves none, but a version restore leaves the draft standing and
// it is still what a test of the host step would run.
[...moduleIds].map((moduleId) => publishLinkedAgentTools(path, at.ws, scope, moduleId, true))
)
}
frontend/src/lib/components/flows/content/AgentResourceBar.svelte
+91 -16
View File
@@ -4,7 +4,7 @@
import Badge from '$lib/components/common/badge/Badge.svelte'
import Path from '$lib/components/Path.svelte'
import TextInput from '$lib/components/text_input/TextInput.svelte'
import { ResourceService, type InputTransform } from '$lib/gen'
import { ResourceService, type InputTransform, type Resource } from '$lib/gen'
import { workspaceStore } from '$lib/stores'
import { sendUserToast } from '$lib/toast'
import { Bot, ChevronDown, ChevronUp, Save, Unlink, Pencil } from 'lucide-svelte'
@@ -19,14 +19,24 @@
type AIAgentConfig,
type AgentTool
} from '../agentResourceUtils'
import { agentWriteCount, markAgentWritten, openAgentEditor } from '../agentEditorStore.svelte'
import {
agentDraftSaveCount,
agentWriteCount,
markAgentWritten,
openAgentEditor
} from '../agentEditorStore.svelte'
import {
setLinkedAgentTools,
clearLinkedAgentTools,
linkedModulesForAgent,
linkedToolsScope
} from '../linkedAgentToolsStore.svelte'
import { logReusableAgentUsage } from '../agentTelemetry'
import { claimLinkedToolsFetch } from '../flowState'
import { AgentDraftUnavailable, fetchAgentWithDraft } from '../linkedAgentDrafts'
import type { AgentResourceState } from '../agentDraft.svelte'
import { getLocalDraftHint } from '$lib/localDraftHints.svelte'
import Tooltip from '$lib/components/meltComponents/Tooltip.svelte'
import type { AgentTool as AgentToolStrict } from '../agentToolUtils'
import { resource } from 'runed'
import { untrack } from 'svelte'
@@ -65,6 +75,10 @@
// deploy from the agent editor mounted alongside it. Both reads below key on it, so neither
// keeps naming the config and version a write has just replaced.
let writes = $derived(agentWriteCount(ws, agent))
// Draft saves as well, for the link fetch: the card shows what a test of this step would run,
// and that is the draft. Only the deploy moves `writes`, so without this the card would keep
// describing the config the agent held before it was edited.
let draftSaves = $derived(agentDraftSaveCount(ws, agent))
let saveDrawer: Drawer | undefined = $state()
let newPath = $state('')
@@ -75,14 +89,18 @@
type LinkedInfo = {
// What this result was fetched for. runed's resource neither aborts nor tags a superseded
// request, so a slow fetch can land after a newer one: every consumer gates on these matching
// the current (ws, agent, writes). `writes` is what covers a refetch of the *same* link after
// a deploy — without it a pre-deploy response is indistinguishable from the current one, and
// accepting it republishes the tools the deploy just replaced.
// the current (ws, agent, writes, draftSaves). `writes` is what covers a refetch of the *same*
// link after a deploy — without it a pre-deploy response is indistinguishable from the current
// one, and accepting it republishes the tools the deploy just replaced. `draftSaves` does the
// same for a draft save, which the card follows just as closely.
ws?: string
path?: string
writes: number
draftSaves: number
config: AIAgentConfig
tools: AgentTool[]
/** The config shown came from the agent's unsaved draft rather than the deployed resource. */
fromDraft: boolean
providerPath?: string
providerOk: boolean
}
@@ -90,14 +108,37 @@
// A linked agent is rigid and read-only: its brain and tools come from the resource. We
// load them here for display, and probe the provider resource so we can warn when it isn't
// accessible in this workspace (the user then needs to unlink/fork or gain access).
// The draft when there is one, since that is what a test of this step runs.
let linkedResource = resource(
() => ({ ws, path: agent, writes }),
async ({ ws, path, writes }): Promise<LinkedInfo> => {
() => ({ ws, path: agent, writes, draftSaves }),
async ({ ws, path, writes, draftSaves }): Promise<LinkedInfo> => {
if (!ws || !path) {
return { ws, path, writes, config: {}, tools: [], providerOk: true }
return {
ws,
path,
writes,
draftSaves,
config: {},
tools: [],
fromDraft: false,
providerOk: true
}
}
let response: Resource
let draft: AgentResourceState | undefined
try {
;({ response, draft } = await fetchAgentWithDraft(path, ws))
} catch (err) {
// Only the DRAFT was unreadable. This card is a display, so fall back to the deployed
// agent rather than rendering one with no brain and no tools, which reads as "the agent
// is empty" while the Draft badge still says it has unsaved changes. Same fallback the
// graph's tool nodes take; the paths that run or deploy the draft still refuse.
if (!(err instanceof AgentDraftUnavailable)) throw err
response = await ResourceService.getResource({ workspace: ws, path })
}
const cfg = (draft?.args ?? response.value ?? {}) as AIAgentConfig & {
provider?: { resource?: string }
}
const res = await ResourceService.getResource({ workspace: ws, path })
const cfg = (res.value ?? {}) as AIAgentConfig & { provider?: { resource?: string } }
const tools = (cfg.tools ?? []) as AgentTool[]
const providerRef = cfg.provider?.resource
const providerPath =
@@ -116,8 +157,10 @@
ws,
path,
writes,
draftSaves,
config: cfg,
tools,
fromDraft: draft != undefined,
providerPath,
providerOk
}
@@ -129,7 +172,13 @@
let loadedInfo = $state<LinkedInfo | undefined>(undefined)
$effect(() => {
const current = linkedResource.current
if (current && current.ws === ws && current.path === agent && current.writes === writes) {
if (
current &&
current.ws === ws &&
current.path === agent &&
current.writes === writes &&
current.draftSaves === draftSaves
) {
loadedInfo = current
}
})
@@ -140,6 +189,12 @@
let brainParams = $derived(summarizeAgentBrain(linkedInfo?.config))
let providerPath = $derived(linkedInfo?.providerPath)
let providerOk = $derived(linkedInfo?.providerOk ?? true)
// The hint flips on the first keystroke in the agent editor, so the badge does not wait for the
// debounced autosave and the refetch behind it; the fetched answer covers a draft written
// elsewhere, which no editor here has published an opinion about.
let hasDraft = $derived(
getLocalDraftHint(ws, 'resource', agent ?? '') ?? linkedInfo?.fromDraft ?? false
)
/** The agent the card is about: the one this step links to, or the one being edited. */
let cardPath = $derived(agent)
// The version eval runs are recorded against. The resource does not hold it; its newest history
@@ -190,9 +245,18 @@
}
const loaded = linkedInfo
if (loaded) {
claimLinkedToolsFetch(toolScope, moduleId)
// linkedResource types tools loosely; they are the same resource tools the store holds.
setLinkedAgentTools(toolScope, moduleId, loaded.tools as AgentToolStrict[], agent)
// Every step of this flow linking this agent, not just this one. Tools belong to the agent,
// so the sibling steps show the same set, and only the selected step mounts this card:
// without them a draft saved from here leaves their nodes on what the flow load resolved,
// while a test of those steps runs the draft. Claimed like this card's own publish, so a
// sibling's in-flight fetch cannot land afterwards and put the old tools back.
const modules = new Set(linkedModulesForAgent(toolScope, agent))
modules.add(moduleId)
for (const id of modules) {
claimLinkedToolsFetch(toolScope, id)
// linkedResource types tools loosely; they are the same resource tools the store holds.
setLinkedAgentTools(toolScope, id, loaded.tools as AgentToolStrict[], agent)
}
publishedFor = agent
} else if (publishedFor !== undefined && publishedFor !== agent) {
// The link moved and the new agent hasn't resolved, so the stored tools are the old one's.
@@ -358,13 +422,15 @@
// `tools` is one array per module value, so it identifies the step itself — the path alone
// would not, since a replacement can carry the same link.
const stepMarker = tools
const res = await ResourceService.getResource({ workspace: ws, path })
// The draft, like the card above and like a test of this step: forking the deployed value
// while the card displays a drafted prompt would hand back something the user never saw.
const { response, draft } = await fetchAgentWithDraft(path, ws)
// The module may have been replaced while the fetch was in flight (undo, session drafts);
// applying a stale fork would overwrite the restored state.
if (agent !== path || tools !== stepMarker) {
return false
}
const cfg = (res.value ?? {}) as AIAgentConfig
const cfg = (draft?.args ?? response.value ?? {}) as AIAgentConfig
// Preserve the flow-local inputs already wired in the step.
const local: Record<string, InputTransform> = {}
for (const key of AGENT_FLOW_LOCAL_KEYS) {
@@ -448,6 +514,15 @@
v{version}
</Badge>
{/if}
{#if hasDraft}
<Tooltip class="inline-flex items-center shrink-0">
<Badge small color="indigo">Draft</Badge>
{#snippet text()}
This agent has unsaved changes. Testing this flow runs the draft, and deploying the
flow offers to deploy it.
{/snippet}
</Tooltip>
{/if}
</div>
<div class="flex items-center gap-1 shrink-0">
{#if brainParams.length > 0 || inheritedTools.length > 0}
frontend/src/lib/components/flows/flowState.ts
+29 -5
View File
@@ -5,6 +5,7 @@ import { get } from 'svelte/store'
import { workspaceStore } from '$lib/stores'
import { isFlowModuleTool, agentToolToFlowModule, type AgentTool } from './agentToolUtils'
import { linkedToolsScope, setLinkedAgentTools } from './linkedAgentToolsStore.svelte'
import { fetchAgentWithDraft, normalizeAgentRef } from './linkedAgentDrafts'
import { loadFlowModuleState } from './flowStateUtils.svelte'
import { emptyFlowModuleState } from './utils.svelte'
import type { StateStore } from '$lib/utils'
@@ -90,7 +91,9 @@ async function mapFlowModule(
// the graph can render its tool nodes. They are display-only (their inputs are edited in
// the step panel, which infers schemas itself), so no per-tool module state is loaded —
// resource tool ids are not flow-unique and must not key into the flow state.
await publishLinkedAgentTools(agentRef, workspace, scope, flowModule.id)
// Drafts included: every caller of `initFlowState` is a flow editor, where the graph has
// to show the tools a test would run. Read-only viewers publish for themselves.
await publishLinkedAgentTools(agentRef, workspace, scope, flowModule.id, true)
} else {
// Shape-checked because `tools` is JSON-authored: throwing here would skip the agent's
// own state below, leaving it with no schema rather than with no tool schemas.
@@ -119,11 +122,17 @@ export async function publishLinkedAgentTools(
agentRef: string,
workspace: string | undefined,
scope: string,
moduleId: string
moduleId: string,
/** Resolve from the agent's unsaved draft when there is one. Editors pass true so the graph
* shows the tool set a test would run; read-only viewers pass false, since a run they are
* displaying used the deployed agent. Required rather than defaulted: an editor call site that
* forgets it republishes the deployed tools over the drafted ones, which reads as the graph
* spontaneously reverting. */
withDraft: boolean
) {
const genKey = `${scope}:${moduleId}`
const gen = claimLinkedToolsFetch(scope, moduleId)
const tools = await resolveLinkedAgentTools(agentRef, workspace)
const tools = await resolveLinkedAgentTools(agentRef, workspace, withDraft)
if (linkedToolFetchGen.get(genKey) === gen) {
setLinkedAgentTools(scope, moduleId, tools, agentRef)
}
@@ -155,12 +164,27 @@ export function claimLinkedToolsFetch(scope: string, moduleId: string): number {
// resource is missing or inaccessible so a broken link never stalls the flow load.
export async function resolveLinkedAgentTools(
agentRef: string,
workspace?: string
workspace: string | undefined,
withDraft: boolean
): Promise<AgentTool[]> {
const ws = workspace ?? get(workspaceStore)
if (!ws) return []
const path = agentRef.replace(/^\$res:/, '').replace(/^res:\/\//, '')
const path = normalizeAgentRef(agentRef)
try {
if (withDraft) {
try {
const { response, draft } = await fetchAgentWithDraft(path, ws)
const value = (draft?.args ?? response.value) as { tools?: AgentTool[] } | undefined
return (value?.tools ?? []) as AgentTool[]
} catch {
// The draft read failed for any reason. This is a display, not a run, so fall through to
// the deployed tools rather than showing an agent with none: an empty node list reads as
// "the agent lost its tools" instead of "we could not reach the server". The paths that
// act on a draft — the previews and the deploy dialog — surface the failure instead.
// Not rethrowing anything here: the outer catch turns every throw into `[]`, so a
// rethrow would skip the very fallback this exists for.
}
}
const res = await ResourceService.getResource({ workspace: ws, path })
return ((res.value as { tools?: AgentTool[] } | undefined)?.tools ?? []) as AgentTool[]
} catch {
frontend/src/lib/components/flows/linkedAgentDrafts.test.ts
+148
View File
@@ -0,0 +1,148 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import {
inlineAgentDraft,
inlineAgentDrafts,
loadLinkedAgentDrafts,
type LinkedAgentDraft
} from './linkedAgentDrafts'
import { ResourceService, type FlowModule, type FlowValue } from '$lib/gen'
type AiAgentValue = Extract<FlowModule['value'], { type: 'aiagent' }>
function linkedStep(input_transforms: Record<string, any>): AiAgentValue {
return {
type: 'aiagent',
agent: 'f/team/support',
tool_inputs: { t1: { query: { type: 'javascript', expr: 'flow_input.q' } } },
input_transforms
} as unknown as AiAgentValue
}
describe('inlineAgentDraft', () => {
// The overlay order is the worker's: the resource brain first, the step's flow-local inputs on
// top. Reversing it would run the agent author's own `user_message` instead of the flow's.
it('keeps the step wired to the flow while the brain comes from the draft', () => {
const inlined = inlineAgentDraft(
linkedStep({
user_message: { type: 'javascript', expr: 'flow_input.question' },
user_attachments: { type: 'static', value: [] }
}),
{
provider: { kind: 'openai', model: 'gpt-4o', resource: '$res:f/team/openai' },
system_prompt: 'answer in french',
user_message: 'the default the agent carries',
tools: [{ id: 't1', summary: 'search' }]
} as any
)
expect(inlined.agent).toBeUndefined()
expect(inlined.tools).toEqual([{ id: 't1', summary: 'search' }])
expect(inlined.input_transforms).toEqual({
provider: {
type: 'static',
value: { kind: 'openai', model: 'gpt-4o', resource: '$res:f/team/openai' }
},
system_prompt: { type: 'static', value: 'answer in french' },
user_message: { type: 'javascript', expr: 'flow_input.question' },
user_attachments: { type: 'static', value: [] }
})
// Host bindings are the step's, not the agent's, and the worker overlays them either way.
expect(inlined.tool_inputs).toEqual({
t1: { query: { type: 'javascript', expr: 'flow_input.q' } }
})
})
// A linked step carries only the flow-local inputs, but one persisted before linking existed can
// still hold stale brain transforms. They must not shadow the draft the test is meant to run.
it('drops brain transforms the step still carries', () => {
const inlined = inlineAgentDraft(
linkedStep({
user_message: { type: 'static', value: 'hi' },
system_prompt: { type: 'static', value: 'stale' }
}),
{ system_prompt: 'from the draft' } as any
)
expect(inlined.input_transforms).toEqual({
system_prompt: { type: 'static', value: 'from the draft' },
user_message: { type: 'static', value: 'hi' }
})
})
})
describe('inlineAgentDrafts', () => {
// The index is keyed on the bare path while a step may name its agent `$res:`-prefixed, and the
// walk has to reach inside branches and loops. Miss either and every preview silently runs the
// deployed agent — the failure this whole path exists to prevent, and a silent one.
it('reaches a $res:-prefixed link nested in a branch', () => {
const value = {
modules: [
{
id: 'b',
value: {
type: 'branchone',
default: [],
branches: [
{
modules: [
{
id: 'inner',
value: {
type: 'aiagent',
agent: '$res:f/team/support',
tools: [],
input_transforms: { user_message: { type: 'static', value: 'hi' } }
}
}
]
}
]
}
}
]
} as unknown as FlowValue
const drafts = new Map<string, LinkedAgentDraft>([
['f/team/support', { args: { system_prompt: 'drafted' } } as unknown as LinkedAgentDraft]
])
const inner = (inlineAgentDrafts(value, drafts).modules[0].value as any).branches[0].modules[0]
expect(inner.value.agent).toBeUndefined()
expect(inner.value.input_transforms.system_prompt).toEqual({
type: 'static',
value: 'drafted'
})
// The input the flow supplies survives the rewrite.
expect(inner.value.input_transforms.user_message).toEqual({ type: 'static', value: 'hi' })
})
})
// A link the user cannot resolve is an ordinary state and must not block the flow; anything else is
// an outage, and answering "no draft" to one would silently test or deploy against the deployed
// agent while the editor shows the draft.
describe('loadLinkedAgentDrafts error handling', () => {
function failWith(status: number | undefined) {
return async () => {
const err: Error & { status?: number } = new Error('boom')
err.status = status
throw err
}
}
beforeEach(() => {
vi.restoreAllMocks()
})
it.each([401, 403, 404])('treats %i as no draft', async (status) => {
vi.spyOn(ResourceService, 'getResource').mockImplementation(failWith(status) as any)
await expect(loadLinkedAgentDrafts(['f/team/support'], 'ws')).resolves.toEqual(new Map())
})
it.each([500, undefined])('propagates %s rather than reporting no draft', async (status) => {
vi.spyOn(ResourceService, 'getResource').mockImplementation(failWith(status) as any)
await expect(loadLinkedAgentDrafts(['f/team/support'], 'ws')).rejects.toThrow(
'Could not load the agent f/team/support'
)
})
})
frontend/src/lib/components/flows/linkedAgentDrafts.ts
+229
View File
@@ -0,0 +1,229 @@
import {
ResourceService,
type FlowModule,
type FlowValue,
type InputTransform,
type Resource
} from '$lib/gen'
import { UserDraft } from '$lib/userDraft.svelte'
import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte'
import { canWrite } from '$lib/utils'
import type { UserExt } from '$lib/stores'
import { dfs } from './dfs'
import { flowLocalInputs, type AIAgentConfig } from './agentResourceUtils'
import type { AgentResourceState } from './agentDraft.svelte'
import type { AgentTool } from './agentToolUtils'
/** A step names its agent bare or as `$res:<path>`/`res://<path>`; all three are the same agent,
* and a draft index has to answer for a lookup written any of those ways. Same normalization as
* `linkedAgentToolsStore`, and as the `trim_start_matches` the worker applies. */
export function normalizeAgentRef(agentRef: string): string {
return agentRef.replace(/^\$res:/, '').replace(/^res:\/\//, '')
}
/** Every `ai_agent` resource this flow links to, deduped. `dfs` walks agent tool nodes as well as
* branches and loops, so a nested linked agent tool is included. */
export function linkedAgentPaths(value: FlowValue | undefined): string[] {
if (!value?.modules) return []
const paths = new Set<string>()
for (const module of dfs(value.modules, (m) => m)) {
const v = module?.value as { type?: string; agent?: string } | undefined
if (v?.type === 'aiagent' && v.agent) {
paths.add(normalizeAgentRef(v.agent))
}
}
return [...paths]
}
/**
* The unsaved draft for an agent, freshest first: the cell an open agent editor is writing, then
* what a `get_draft` response carried.
*
* Only the live cell is reliably current, and only while an editor holds it: `releaseEntry` drops
* the cached write at refcount 0 on purpose, so once the agent editor closes the persisted row is
* the sole answer. Read through `fetchAgentWithDraft`, which settles that row first.
*/
export function agentDraftState(
response: { draft?: unknown },
path: string,
workspace: string | undefined
): AgentResourceState | undefined {
const live = UserDraft.get<AgentResourceState>('resource', path, { workspace })
return live ?? (response.draft as AgentResourceState | undefined)
}
/** A refusal that already names the agent and says what is wrong with it, so a caller wrapping it
* would only repeat itself. */
export class AgentDraftUnavailable extends Error {}
/**
* An agent's resource together with the draft a run of it would use.
*
* The flush is what makes the answer current. Autosave is debounced by 1.5s (10s ceiling), and
* closing the agent editor releases the in-memory cell without cancelling that pending POST — so
* testing or deploying right after closing would otherwise read a row the last edits have not
* reached yet. `flush` replays the parked save and is a no-op when there is none.
*/
export async function fetchAgentWithDraft(
path: string,
workspace: string
): Promise<{ response: Resource; draft: AgentResourceState | undefined }> {
const query = { workspace, itemKind: 'resource' as const, path }
await UserDraftDbSyncer.flush(query)
// `flush` resolves whether or not the save actually landed: `postSave` catches network and
// server errors into its failure map, and answers a conflicting write by parking a snapshot,
// returning normally in both cases. The row about to be read is then older than the edit still
// held in the browser, and nothing downstream could tell. Running that row is a test of the
// wrong agent; deploying it is worse, because the deploy deletes the draft and takes the newer
// edit with it. Neither is recoverable from here, so refuse the read.
const failure = UserDraftDbSyncer.getState(query).failureMessage
if (failure) {
throw new AgentDraftUnavailable(`The unsaved changes to ${path} could not be saved: ${failure}`)
}
if (UserDraftDbSyncer.getConflict(query).conflict) {
throw new AgentDraftUnavailable(
`The unsaved changes to ${path} could not be saved because it was edited elsewhere. Open the agent to resolve it.`
)
}
const response = await ResourceService.getResource({ workspace, path, getDraft: true })
return { response, draft: agentDraftState(response, path, workspace) }
}
/** One linked agent whose resource the user has an unsaved draft for. */
export interface LinkedAgentDraft {
path: string
/** The draft's resource value: what a run of this agent would use. */
args: AIAgentConfig
/** The whole draft row, as the resource editors write it — the deploy payload. */
state: AgentResourceState
/** No deployed row at this path, so deploying has to create rather than update. */
noDeployed: boolean
/** Of the deployed resource, for `agentDraftCanWrite`. */
extraPerms: Record<string, boolean>
}
/** Whether `user` may write this agent's resource. Split from the load so that resolving the
* drafts of a whole flow costs no `whoami` — only the deploy dialog needs the answer, and it
* looks the user up once for every agent it lists. */
export function agentDraftCanWrite(draft: LinkedAgentDraft, user: UserExt | undefined): boolean {
return canWrite(draft.path, draft.extraPerms, user)
}
/** A link that cannot resolve for the user rather than because something went wrong: the agent was
* deleted, or sits in a folder they cannot read. Both are ordinary states of a rigid link, and
* neither should stop the caller — the flow still tests and deploys, against the deployed agent.
* Every other failure is an outage, and answering "no draft" to one would quietly run or deploy
* the wrong configuration, which is the whole thing this module exists to prevent. */
function isExpectedLinkFailure(err: unknown): boolean {
const status = (err as { status?: number } | null | undefined)?.status
return status === 401 || status === 403 || status === 404
}
/**
* The unsaved draft of every given `ai_agent` path, for the paths that have one.
*
* Throws when a path fails to load for any reason other than being missing or unreadable, so a
* caller cannot mistake an outage for an agent with nothing unsaved.
*/
export async function loadLinkedAgentDrafts(
paths: string[],
workspace: string | undefined
): Promise<Map<string, LinkedAgentDraft>> {
const out = new Map<string, LinkedAgentDraft>()
if (!workspace || paths.length === 0) return out
await Promise.all(
paths.map(async (path) => {
let response: Resource
let draft: AgentResourceState | undefined
try {
;({ response, draft } = await fetchAgentWithDraft(path, workspace))
} catch (err) {
if (isExpectedLinkFailure(err)) return
if (err instanceof AgentDraftUnavailable) throw err
throw new Error(`Could not load the agent ${path}: ${err}`)
}
if (!draft) return
out.set(path, {
path,
args: (draft.args ?? {}) as AIAgentConfig,
state: draft,
noDeployed: Boolean((response as { no_deployed?: boolean }).no_deployed),
extraPerms: response.extra_perms ?? {}
})
})
)
return out
}
/**
* Every argument a saved agent carries, as a static input transform. Not only the keys the agent
* form renders: a run reads them all, and an agent holding its own `user_message` answers with it
* when nothing overrides it. `tools` is the step's own roster rather than an input, so it rides on
* the module's `tools` key instead.
*/
export function agentArgsToTransforms(args: AIAgentConfig): Record<string, InputTransform> {
const it: Record<string, InputTransform> = {}
for (const [key, value] of Object.entries(args ?? {})) {
if (key === 'tools' || value === undefined) continue
it[key] = { type: 'static', value } as InputTransform
}
return it
}
type AiAgentValue = Extract<FlowModule['value'], { type: 'aiagent' }>
/**
* The standalone step a linked step's draft would run as: the draft's brain and tools inlined, with
* the step's own flow-local inputs kept on top.
*
* The overlay order is the worker's (`ai_executor.rs`): its linked branch interpolates the whole
* resource brain and only then writes `user_message`/`user_attachments` back from the step's own
* args. `tool_inputs` stays untouched — the worker overlays it onto the tools in both branches, so
* an inlined step keeps the host flow's tool bindings.
*/
export function inlineAgentDraft(value: AiAgentValue, args: AIAgentConfig): AiAgentValue {
const { agent: _agent, ...rest } = value
return {
...rest,
tools: (args.tools ?? []) as AgentTool[],
input_transforms: {
...agentArgsToTransforms(args),
...flowLocalInputs(value.input_transforms as Record<string, InputTransform>)
}
} as AiAgentValue
}
/**
* Replace every linked agent step that has a draft with the draft's own configuration, so a preview
* runs what the agent editor is showing rather than the deployed resource. Returns a new value: the
* flow editor hands its live store object to previews.
*/
export function inlineAgentDrafts(
value: FlowValue,
drafts: Map<string, LinkedAgentDraft>
): FlowValue {
if (drafts.size === 0) return value
// JSON rather than `structuredClone`: the flow editor's value is a Svelte `$state` proxy, which
// `structuredClone` refuses outright. A flow value is JSON by definition — it is about to be
// posted as one — so the round trip loses nothing this preview would have carried.
const next = JSON.parse(JSON.stringify(value)) as FlowValue
for (const module of dfs(next.modules ?? [], (m) => m)) {
const v = module?.value as AiAgentValue | undefined
if (v?.type !== 'aiagent' || !v.agent) continue
const draft = drafts.get(normalizeAgentRef(v.agent))
if (!draft) continue
module.value = inlineAgentDraft(v, draft.args)
}
return next
}
/** Load the drafts this flow's linked agents have and inline them. The whole substitution, for a
* caller holding nothing but the value it is about to preview. */
export async function withAgentDrafts(
value: FlowValue,
workspace: string | undefined
): Promise<FlowValue> {
const paths = linkedAgentPaths(value)
if (paths.length === 0) return value
return inlineAgentDrafts(value, await loadLinkedAgentDrafts(paths, workspace))
}
frontend/src/lib/components/flows/linkedToolsFetchGuard.test.ts
+7 -5
View File
@@ -8,7 +8,9 @@ vi.mock('./agentToolUtils', () => ({
isFlowModuleTool: () => false,
agentToolToFlowModule: (t: unknown) => t
}))
vi.mock('$lib/stores', () => ({ workspaceStore: { subscribe: (f: (v: string) => void) => (f('ws'), () => {}) } }))
vi.mock('$lib/stores', () => ({
workspaceStore: { subscribe: (f: (v: string) => void) => (f('ws'), () => {}) }
}))
import {
claimLinkedToolsFetch,
@@ -38,8 +40,8 @@ describe('linked tools fetch guard', () => {
)
.mockResolvedValueOnce({ value: { tools: [tool('new')] } } as never)
const stale = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step')
await publishLinkedAgentTools('f/a/new', 'ws', scope, 'step')
const stale = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step', false)
await publishLinkedAgentTools('f/a/new', 'ws', scope, 'step', false)
release?.({ value: { tools: [tool('old')] } })
await stale
@@ -56,7 +58,7 @@ describe('linked tools fetch guard', () => {
() => new Promise((r) => (release = r)) as ReturnType<typeof ResourceService.getResource>
)
const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step')
const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step', false)
setLinkedAgentTools(scope, 'step', [tool('kept')], 'u/admin/a')
invalidateLinkedToolsFetches(scope)
@@ -73,7 +75,7 @@ describe('linked tools fetch guard', () => {
() => new Promise((r) => (release = r)) as ReturnType<typeof ResourceService.getResource>
)
const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step')
const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step', false)
claimLinkedToolsFetch(scope, 'step')
setLinkedAgentTools(scope, 'step', [tool('direct')], 'u/admin/a')
release?.({ value: { tools: [tool('stale')] } })
frontend/src/lib/components/flows/pickers/PickHubScript.svelte
+85 -73
View File
@@ -8,8 +8,13 @@
import { IntegrationService, ScriptService, type HubScriptKind } from '$lib/gen'
import { Loader2 } from 'lucide-svelte'
import TextInput from '$lib/components/text_input/TextInput.svelte'
import { disableHubStore } from '$lib/stores'
import { disableHubStore, workspaceStore } from '$lib/stores'
import { logHubScriptPick } from '$lib/utils/featureUsage'
import {
alphabetical,
byPopularity,
localCountsByIntegration
} from '$lib/components/pickerPopularity'
interface Props {
kind?: HubScriptKind & string
@@ -46,19 +51,25 @@
}[] = $state([])
let allApps: string[] = $state([])
let popularity: (a: string, b: string) => number = $state(alphabetical)
let apps: string[] = $derived.by(() =>
filter.length > 0 ? Array.from(new Set(items?.map((x) => x.app) ?? [])).sort() : allApps
filter.length > 0
? Array.from(new Set(items?.map((x) => x.app) ?? [])).sort(popularity)
: allApps
)
async function getAllApps(filterKind: typeof kind) {
if ($disableHubStore) return
try {
hubNotAvailable = false
allApps = (
await IntegrationService.listHubIntegrations({
kind: filterKind
})
).map((x) => x.name)
// Independent reads, so they share one round trip before first paint.
const [integrations, local] = await Promise.all([
IntegrationService.listHubIntegrations({ kind: filterKind }),
$workspaceStore ? localCountsByIntegration($workspaceStore) : {}
])
const hubPicks = Object.fromEntries(integrations.map((x) => [x.name, x.picks ?? 0]))
popularity = byPopularity(hubPicks, local)
allApps = integrations.map((x) => x.name).sort(popularity)
} catch (err) {
console.error('Hub is not available')
allApps = []
@@ -154,72 +165,73 @@
{#if $disableHubStore}
<!-- Hub disabled, show nothing -->
{:else}
<div class="w-full flex items-center gap-2">
{@render children?.()}
<div class="relative w-full">
<TextInput
inputProps={{
placeholder: 'Search Hub Scripts'
}}
bind:value={filter}
class="grow !pr-9"
{size}
/>
{#if loading}
<Loader2 class="animate-spin text-gray-400 absolute right-2 top-1" />
{/if}
</div>
</div>
{#if hubNotAvailable}
<Alert type="warning" title="Hub not available">
Could not connect to the Windmill Hub. If you are in a closed environment, you can disable the Hub in the <a href="/#superadmin-settings?tab=private_hub">instance settings</a>.
</Alert>
{:else if (items.length > 0 && apps.length > 0) || !loading}
<ListFilters {syncQuery} filters={apps} bind:selectedFilter={appFilter} resourceType />
{#if items.length == 0}
<NoItemFound />
{:else}
<ul class="divide-y border rounded-md bg-surface-tertiary">
{#each items as item (item.path)}
<li class="flex flex-row w-full">
<button
class="p-4 gap-4 flex flex-row grow hover:bg-surface-hover transition-all items-center"
onclick={() => handlePick(item)}
>
<div class="flex items-center gap-4">
<div class="flex justify-center items-center">
{#if item['app'] in APP_TO_ICON_COMPONENT}
{@const SvelteComponent = APP_TO_ICON_COMPONENT[item['app']]}
<SvelteComponent height={18} width={18} />
{/if}
</div>
<div class="w-full text-left">
<div class="text-emphasis flex-wrap text-xs font-semibold mb-1">
{item.summary ?? ''}
</div>
<div class="text-secondary text-2xs font-normal">
{item.path}
</div>
</div>
</div>
{#if kind !== 'script'}
<Badge color="gray" baseClass="border">{capitalize(kind)}</Badge>
{/if}
</button>
</li>
{/each}
</ul>
{/if}
{#if items.length == 20}
<div class="text-primary text-xs font-normal py-4">
There are more items than being displayed. Refine your search.
<div class="w-full flex items-center gap-2">
{@render children?.()}
<div class="relative w-full">
<TextInput
inputProps={{
placeholder: 'Search Hub Scripts'
}}
bind:value={filter}
class="grow !pr-9"
{size}
/>
{#if loading}
<Loader2 class="animate-spin text-gray-400 absolute right-2 top-1" />
{/if}
</div>
</div>
{#if hubNotAvailable}
<Alert type="warning" title="Hub not available">
Could not connect to the Windmill Hub. If you are in a closed environment, you can disable the
Hub in the <a href="/#superadmin-settings?tab=private_hub">instance settings</a>.
</Alert>
{:else if (items.length > 0 && apps.length > 0) || !loading}
<ListFilters {syncQuery} filters={apps} bind:selectedFilter={appFilter} resourceType />
{#if items.length == 0}
<NoItemFound />
{:else}
<ul class="divide-y border rounded-md bg-surface-tertiary">
{#each items as item (item.path)}
<li class="flex flex-row w-full">
<button
class="p-4 gap-4 flex flex-row grow hover:bg-surface-hover transition-all items-center"
onclick={() => handlePick(item)}
>
<div class="flex items-center gap-4">
<div class="flex justify-center items-center">
{#if item['app'] in APP_TO_ICON_COMPONENT}
{@const SvelteComponent = APP_TO_ICON_COMPONENT[item['app']]}
<SvelteComponent height={18} width={18} />
{/if}
</div>
<div class="w-full text-left">
<div class="text-emphasis flex-wrap text-xs font-semibold mb-1">
{item.summary ?? ''}
</div>
<div class="text-secondary text-2xs font-normal">
{item.path}
</div>
</div>
</div>
{#if kind !== 'script'}
<Badge color="gray" baseClass="border">{capitalize(kind)}</Badge>
{/if}
</button>
</li>
{/each}
</ul>
{/if}
{#if items.length == 20}
<div class="text-primary text-xs font-normal py-4">
There are more items than being displayed. Refine your search.
</div>
{/if}
{:else}
{#each Array(10).fill(0) as _}
<Skeleton layout={[0.5, [4]]} />
{/each}
{/if}
{:else}
{#each Array(10).fill(0) as _}
<Skeleton layout={[0.5, [4]]} />
{/each}
{/if}
{/if}
frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte
+16 -5
View File
@@ -44,12 +44,17 @@
import { Circle, ExternalLink } from 'lucide-svelte'
import Popover from '$lib/components/Popover.svelte'
import { usePromise } from '$lib/svelte5Utils.svelte'
import { disableHubStore, hubBaseUrlStore, userStore } from '$lib/stores'
import { disableHubStore, hubBaseUrlStore, userStore, workspaceStore } from '$lib/stores'
import { get } from 'svelte/store'
import Button from '$lib/components/common/button/Button.svelte'
import { Alert } from '$lib/components/common'
import type { FlowBuilderWhitelabelCustomUi } from '$lib/components/custom_ui'
import { logHubScriptPick } from '$lib/utils/featureUsage'
import {
alphabetical,
byPopularity,
localCountsByIntegration
} from '$lib/components/pickerPopularity'
let customUi: undefined | FlowBuilderWhitelabelCustomUi = getContext('customUi')
@@ -94,9 +99,10 @@
}: Props = $props()
let allApps: string[] = $state([])
let popularity: (a: string, b: string) => number = $state(alphabetical)
$effect(() => {
if (filter.length > 0) {
apps = Array.from(new Set(items?.map((x) => x.app) ?? [])).sort()
apps = Array.from(new Set(items?.map((x) => x.app) ?? [])).sort(popularity)
} else {
apps = allApps
}
@@ -106,9 +112,14 @@
if ($disableHubStore) return
try {
hubNotAvailable = false
allApps = (await listHubIntegrationsCached({ kind: filterKind, refreshCount })).map(
(x) => x.name
)
// Independent reads, so they share one round trip before first paint.
const [integrations, local] = await Promise.all([
listHubIntegrationsCached({ kind: filterKind, refreshCount }),
$workspaceStore ? localCountsByIntegration($workspaceStore) : {}
])
const hubPicks = Object.fromEntries(integrations.map((x) => [x.name, x.picks ?? 0]))
popularity = byPopularity(hubPicks, local)
allApps = integrations.map((x) => x.name).sort(popularity)
} catch (err) {
console.error('Failed to fetch hub integrations:', err)
allApps = []
frontend/src/lib/components/flows/utils.svelte.ts
+12 -4
View File
@@ -17,6 +17,7 @@ import { get } from 'svelte/store'
import type { FlowModuleState } from './flowState'
import { type PickableProperties, dfs } from './previousResults'
import { forEachFlowModule } from './dfs'
import { withAgentDrafts } from './linkedAgentDrafts'
import { NEVER_TESTED_THIS_FAR } from './models'
import { sendUserToast } from '$lib/toast'
import type { ExtendedOpenFlow } from './types'
@@ -187,6 +188,12 @@ export function jobsToResults(jobs: Job[]) {
})
}
/**
* Run the flow the editor currently holds. A step linked to a saved agent runs that agent's
* unsaved draft when there is one (`withAgentDrafts`), so testing exercises what the agent editor
* is showing rather than the deployed resource — the same rule the agent editor's own test pane
* follows. The value passed in is left alone; only what goes to the server is substituted.
*/
export async function runFlowPreview(
args: Record<string, any>,
flow: OpenFlow & { tag?: string },
@@ -198,14 +205,15 @@ export async function runFlowPreview(
// editor; falls back to the navigation workspace for full-page previews.
workspace?: string
) {
const newFlow = flow
const ws = workspace ?? get(workspaceStore) ?? ''
const value = await withAgentDrafts(flow.value, ws)
return await JobService.runFlowPreview({
workspace: workspace ?? get(workspaceStore) ?? '',
workspace: ws,
requestBody: {
args,
value: newFlow.value,
value,
path: path,
tag: newFlow.tag,
tag: flow.tag,
restarted_from: restartedFrom,
temp_script_refs: tempScriptRefs
},
frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte
+28 -22
View File
@@ -7,7 +7,7 @@
import Popover from '$lib/components/meltComponents/Popover.svelte'
import AssetNode from './AssetNode.svelte'
import type { FlowGraphAssetContext } from '$lib/components/flows/types'
import { getContext } from 'svelte'
import { getContext, untrack } from 'svelte'
import { assetEq } from '$lib/components/assets/lib'
import { getNodeColorClasses } from '../../util'
@@ -24,15 +24,17 @@
data.overflowedAssets.some((asset) => assetEq(flowGraphAssetsCtx?.val.selectedAsset, asset))
)
let wasOpenedBecauseOfExternalSelected = false
// Open while a sibling asset node is hovered and one of the hidden assets is the same asset.
let openedByHover = $state(false)
$effect(() => {
if (includesSelected && !isOpen) {
isOpen = true
wasOpenedBecauseOfExternalSelected = true
}
if (wasOpenedBecauseOfExternalSelected && !includesSelected) {
if (includesSelected) {
if (!untrack(() => isOpen)) {
isOpen = true
openedByHover = true
}
} else if (untrack(() => openedByHover)) {
isOpen = false
wasOpenedBecauseOfExternalSelected = false
openedByHover = false
}
})
const colors = $derived(getNodeColorClasses(undefined, includesSelected))
@@ -41,9 +43,17 @@
<NodeWrapper nodeId={id}>
{#snippet children({ darkMode })}
<!-- svelte-ignore a11y_no_static_element_interactions -->
<!-- A hover-opened popover must stay passive. It is rendered in place, so it can
land on top of the hovered asset node: taking the pointer there ends that hover,
which closes the popover and re-opens it as soon as the cursor is back on the
node. Moving focus into it would likewise steal focus from wherever the user is. -->
<Popover
portal={null}
usePointerDownOutside
disableFocusTrap
openFocus={null}
closeFocus={null}
contentClasses={openedByHover ? 'pointer-events-none' : ''}
bind:isOpen
class={twMerge(
'!w-full text-2xs font-normal h-6 pr-0.5 flex justify-center items-center rounded-md text-primary drop-shadow-base',
@@ -55,21 +65,17 @@
placement="top"
>
{#snippet trigger()}
+{data.overflowedAssets.length}
{/snippet}
+{data.overflowedAssets.length}
{/snippet}
{#snippet content()}
<ul>
{#each data.overflowedAssets as asset}
<li class="w-48">
<AssetNode data={{ asset, displayedAccessType: data.displayedAccessType }} />
</li>
{/each}
</ul>
{/snippet}
<ul>
{#each data.overflowedAssets as asset}
<li class="w-48">
<AssetNode data={{ asset, displayedAccessType: data.displayedAccessType }} />
</li>
{/each}
</ul>
{/snippet}
</Popover>
{/snippet}
</NodeWrapper>

Some files were not shown because too many files have changed in this diff Show More