starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-21 00:02:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into remove-iframe-editors

Browse Source 
# Conflicts:
#	frontend/src/lib/components/flows/conversations/FlowChat.svelte
This commit is contained in:
Diego Imbert
2026-09-18 14:10:40 +02:00
parent 83778f571f 041ac95dc4
commit fffd8b5217
156 changed files with 9137 additions and 849 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.release-please-manifest.json
+1 -1
View File
@@ -1,3 +1,3 @@
{
".": "1.813.0"
".": "1.814.0"
}
CHANGELOG.md
+26
View File
@@ -1,5 +1,31 @@
# Changelog
## [1.814.0](https://github.com/windmill-labs/windmill/compare/v1.813.0...v1.814.0) (2026-09-17)
### Features
* **ai-chat:** add list_workers and list_data_metrics global tools ([#11143](https://github.com/windmill-labs/windmill/issues/11143)) ([e954d33](https://github.com/windmill-labs/windmill/commit/e954d33613e4ff5027667eb8f646615d9bbd499d))
* **ai-chat:** merge get_job_logs and get_flow_run_details into get_run ([#11172](https://github.com/windmill-labs/windmill/issues/11172)) ([5bb37ca](https://github.com/windmill-labs/windmill/commit/5bb37ca3388666fba72c55534e37f37bb3e9299e))
* allow git sync auto-pull, promotion and PRs on Pro licenses ([#11173](https://github.com/windmill-labs/windmill/issues/11173)) ([02e47de](https://github.com/windmill-labs/windmill/commit/02e47de8b4c4f3f54753aabf8c67bc8e71ffb957))
* badge chat-input flows on the home list ([#11164](https://github.com/windmill-labs/windmill/issues/11164)) ([3d08197](https://github.com/windmill-labs/windmill/commit/3d0819718221f885b61e73d02b43dcc853c7d02a))
* collect flow conversations and agent memory once their last message goes ([#11178](https://github.com/windmill-labs/windmill/issues/11178)) ([23c24a9](https://github.com/windmill-labs/windmill/commit/23c24a9688d4c8c462f53221334d538280f16bca))
* flow chat model picker on a shared model-settings component ([#11187](https://github.com/windmill-labs/windmill/issues/11187)) ([189793c](https://github.com/windmill-labs/windmill/commit/189793c2e4db7f1c853695ebcc895c1ec82ed19f))
* keep flow inputs and seed the agent when chat mode is enabled ([#11177](https://github.com/windmill-labs/windmill/issues/11177)) ([68f2248](https://github.com/windmill-labs/windmill/commit/68f2248018fc218a090bf939e1eb22ff97d5bc22))
* let plan mode search and read connected mcp servers ([#11205](https://github.com/windmill-labs/windmill/issues/11205)) ([5371519](https://github.com/windmill-labs/windmill/commit/5371519f0f5ce7750982dcdb374dca72115902e7))
* let test_run_flow name the conversation of a chat-mode test run ([#11198](https://github.com/windmill-labs/windmill/issues/11198)) ([6e1ef93](https://github.com/windmill-labs/windmill/commit/6e1ef93f329cb396ffc3df3304d592e8fa0e0e71))
* managed memory with an inherited or custom memory id per step ([#11118](https://github.com/windmill-labs/windmill/issues/11118)) ([c297ed0](https://github.com/windmill-labs/windmill/commit/c297ed0052d998fb8f063faa2a36c6eb03e327be))
* render the flow chat through the shared session chat components ([#11175](https://github.com/windmill-labs/windmill/issues/11175)) ([a9ec0ae](https://github.com/windmill-labs/windmill/commit/a9ec0aec3ac0c6b0f7919d0eb2168816923826d7))
* show flow step detail inside the graph tab on narrow detail layouts ([#11168](https://github.com/windmill-labs/windmill/issues/11168)) ([64dffe6](https://github.com/windmill-labs/windmill/commit/64dffe6106ad6a55b61a423c855a4b5b0cef533e))
* store mcp tool call, result and reasoning on flow conversation rows ([#11176](https://github.com/windmill-labs/windmill/issues/11176)) ([a571117](https://github.com/windmill-labs/windmill/commit/a571117f3fd2cef14c920770645c60ee358fdfdd))
* tell test flow conversations from deployed ones and rename a chat ([#11179](https://github.com/windmill-labs/windmill/issues/11179)) ([4eab995](https://github.com/windmill-labs/windmill/commit/4eab995cf7cf091a5e4640da4cb77e0921bb7fdf))
### Bug Fixes
* disable a schedule whose cron has no run left instead of panicking ([#11195](https://github.com/windmill-labs/windmill/issues/11195)) ([381d447](https://github.com/windmill-labs/windmill/commit/381d4470ef699ea82283742132e56556b95d2bd2))
* skip expiry notifications for app embed and SDK tokens ([#11169](https://github.com/windmill-labs/windmill/issues/11169)) ([9d348f8](https://github.com/windmill-labs/windmill/commit/9d348f84c7830f36b6153472556fd70e3d84cd24))
## [1.813.0](https://github.com/windmill-labs/windmill/compare/v1.812.0...v1.813.0) (2026-09-16)
backend/.sqlx/query-06abbf945bee93349ff88f64906b96ea1e853ef202510281427cfa9beeff81b3.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT 1 AS one FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "one",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null
]
},
"hash": "06abbf945bee93349ff88f64906b96ea1e853ef202510281427cfa9beeff81b3"
}
backend/.sqlx/query-06ce02cd7ce2f5a57355153edb573c242f9ba758db66e9a5e16f30e3e1494201.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace_settings ws\n SET datatable = (\n SELECT jsonb_set(ws.datatable, '{datatables}', jsonb_object_agg(\n dt.key,\n CASE WHEN dt.value->'reference'->>'workspace_id' = $1\n AND dt.value->'reference'->>'datatable' = $2\n THEN jsonb_set(dt.value, '{reference,datatable}', to_jsonb($3::text))\n ELSE dt.value END\n ))\n FROM jsonb_each(ws.datatable->'datatables') dt\n )\n WHERE EXISTS (\n SELECT 1 FROM jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) d\n WHERE d.value->'reference'->>'workspace_id' = $1\n AND d.value->'reference'->>'datatable' = $2\n )",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "06ce02cd7ce2f5a57355153edb573c242f9ba758db66e9a5e16f30e3e1494201"
}
backend/.sqlx/query-0d10e0fa5cf4033c7d93c9ed56be8209046007917f44da954eccf2188e5bff1f.json
Generated
+29
View File
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "SELECT permissioned_as, permissioned_as_email FROM v2_job\n WHERE id = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "permissioned_as",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "permissioned_as_email",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
false,
false
]
},
"hash": "0d10e0fa5cf4033c7d93c9ed56be8209046007917f44da954eccf2188e5bff1f"
}
backend/.sqlx/query-297c7a40dfce729d44aa37bc7c65560517bd25e40c0752a00467829191e2eb98.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace_settings ws\n SET datatable = (\n SELECT jsonb_set(ws.datatable, '{datatables}', jsonb_object_agg(\n dt.key,\n CASE WHEN dt.value->'reference'->>'workspace_id' = $2\n THEN jsonb_set(dt.value, '{reference,workspace_id}', to_jsonb($1::text))\n ELSE dt.value END\n ))\n FROM jsonb_each(ws.datatable->'datatables') dt\n )\n WHERE jsonb_typeof(ws.datatable->'datatables') = 'object'\n AND ws.datatable::text LIKE '%\"reference\"%'",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "297c7a40dfce729d44aa37bc7c65560517bd25e40c0752a00467829191e2eb98"
}
backend/.sqlx/query-2a391cc1bfcd2f75b46144a394c01237e09c3060da88170f1f6e06468309d213.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE capture_config SET server_id = NULL, last_server_ping = NULL\n WHERE workspace_id = $1 AND trigger_kind = 'postgres'\n AND (trigger_config->>'postgres_resource_path' = $2\n OR trigger_config->>'postgres_resource_path' LIKE $3)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "2a391cc1bfcd2f75b46144a394c01237e09c3060da88170f1f6e06468309d213"
}
backend/.sqlx/query-334dbcd48fb59c96c62c2705ab2d1ce716cd52417f487cc1a8dd376017b2db7d.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = $1)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Name"
]
},
"nullable": [
null
]
},
"hash": "334dbcd48fb59c96c62c2705ab2d1ce716cd52417f487cc1a8dd376017b2db7d"
}
backend/.sqlx/query-4692810d2be817bbb5de9b476d68d695941bd4fb5ccef393e4da522ed479d601.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE postgres_trigger SET server_id = NULL, last_server_ping = NULL\n WHERE workspace_id = $1\n AND (postgres_resource_path = $2 OR postgres_resource_path LIKE $3)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "4692810d2be817bbb5de9b476d68d695941bd4fb5ccef393e4da522ed479d601"
}
backend/.sqlx/query-5048e21546f9710697100100e1255ab103979433bc386d7c89d0e30db12bfd57.json
Generated
+29
View File
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "SELECT ws.workspace_id AS \"workspace_id!\", dt.key AS \"datatable!\"\n FROM workspace_settings ws\n CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt\n WHERE dt.value->'reference'->>'workspace_id' = $1\n AND dt.value->'reference'->>'datatable' = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id!",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "datatable!",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
null
]
},
"hash": "5048e21546f9710697100100e1255ab103979433bc386d7c89d0e30db12bfd57"
}
backend/.sqlx/query-538dd1779874e4003932d7f17750239c625f85e25b3364bf2edf566f518c8ee2.json
Generated
+29
View File
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "SELECT ws.workspace_id AS \"workspace_id!\", dt.key AS \"datatable!\"\n FROM workspace_settings ws\n CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt\n WHERE dt.value->'reference'->>'workspace_id' = $1\n AND dt.value->'reference'->>'datatable' = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id!",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "datatable!",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
null
]
},
"hash": "538dd1779874e4003932d7f17750239c625f85e25b3364bf2edf566f518c8ee2"
}
backend/.sqlx/query-58e5cfe9eb87bda9f7de87c403861b6e7b9d35a41594681e2a92a87359e6a018.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT username FROM usr WHERE workspace_id = $1 AND email = $2 AND disabled = false",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "username",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "58e5cfe9eb87bda9f7de87c403861b6e7b9d35a41594681e2a92a87359e6a018"
}
backend/.sqlx/query-6f9fb5d72f486358fa25d6887bd69b93910e028f140c07048f2c1c8d63ee6909.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO global_settings (name, value) VALUES ($1, $2)\n ON CONFLICT (name) DO UPDATE SET value = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Jsonb"
]
},
"nullable": []
},
"hash": "6f9fb5d72f486358fa25d6887bd69b93910e028f140c07048f2c1c8d63ee6909"
}
backend/.sqlx/query-71ee2cb6661cca1fa4d8874a7f6d368347c59f36fd87df6dc7996152ccb84af0.json
Generated
+38
View File
@@ -0,0 +1,38 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id, name, enabled, pwd FROM datatable_role",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "enabled",
"type_info": "Bool"
},
{
"ordinal": 3,
"name": "pwd",
"type_info": "Text"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
false,
false,
true
]
},
"hash": "71ee2cb6661cca1fa4d8874a7f6d368347c59f36fd87df6dc7996152ccb84af0"
}
backend/.sqlx/query-79799b5a2e499df6c28e286c42b9ad2db940c2455ab19cc95e5198baf96d5629.json
Generated
+29
View File
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT ws.workspace_id AS \"workspace_id!\", dt.key AS \"datatable!\"\n FROM workspace_settings ws\n JOIN workspace w ON w.id = ws.workspace_id AND w.deleted = false\n CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt\n WHERE ws.workspace_id <> $1\n AND dt.value->'database'->>'resource_type' = 'instance'\n AND dt.value->'database'->>'resource_path' = $2\n ORDER BY ws.workspace_id, dt.key\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id!",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "datatable!",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
null
]
},
"hash": "79799b5a2e499df6c28e286c42b9ad2db940c2455ab19cc95e5198baf96d5629"
}
backend/.sqlx/query-86af9d51a158ea5cb6161461ecddf2a63695f8cbf8af648da5a0a77a5b9d02ba.json
Generated
+17
View File
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO datatable_role (id, name, enabled, pwd) VALUES ($1, $2, $3, $4)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Bool",
"Text"
]
},
"nullable": []
},
"hash": "86af9d51a158ea5cb6161461ecddf2a63695f8cbf8af648da5a0a77a5b9d02ba"
}
backend/.sqlx/query-975e404ac3a6258bb8220e122e3de094c7ab23330fdbc74d6e4ad472ddd3c820.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO global_settings (name, value) VALUES ($1, $2)\n ON CONFLICT (name) DO UPDATE SET value = $2, updated_at = now()",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Jsonb"
]
},
"nullable": []
},
"hash": "975e404ac3a6258bb8220e122e3de094c7ab23330fdbc74d6e4ad472ddd3c820"
}
backend/.sqlx/query-9f663180166f53d117e794f3f3a5723a0a43db163ecca7d5a63d4e74ab1d3be1.json
Generated
+20
View File
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "SELECT pg_advisory_xact_lock(hashtext('datatable_role_catalog'))",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "pg_advisory_xact_lock",
"type_info": "Void"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "9f663180166f53d117e794f3f3a5723a0a43db163ecca7d5a63d4e74ab1d3be1"
}
backend/.sqlx/query-a57de2bb0442a5ee8a607cd63cfcf675de175796184f620cb4b09670c8b0b19f.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE capture_config SET server_id = NULL, last_server_ping = NULL\n WHERE workspace_id = $1 AND trigger_kind = 'postgres'\n AND (trigger_config->>'postgres_resource_path' = $2\n OR trigger_config->>'postgres_resource_path' LIKE $3)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "a57de2bb0442a5ee8a607cd63cfcf675de175796184f620cb4b09670c8b0b19f"
}
backend/.sqlx/query-b42af37fb474bea4c5419b0a46d9eadfe384013ab970ccf9c5effd1c78321b7c.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT ws.datatable->'datatables' FROM workspace_settings ws\n WHERE ws.workspace_id = $1 FOR UPDATE",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null
]
},
"hash": "b42af37fb474bea4c5419b0a46d9eadfe384013ab970ccf9c5effd1c78321b7c"
}
backend/.sqlx/query-b9842d2d8abf382bd82d8fa1de012373638be391f884f81dc387ffc465badac6.json
Generated
+20
View File
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "SELECT jsonb_object_keys(value->'databases') FROM global_settings\n WHERE name = 'custom_instance_pg_databases'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "jsonb_object_keys",
"type_info": "Text"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "b9842d2d8abf382bd82d8fa1de012373638be391f884f81dc387ffc465badac6"
}
backend/.sqlx/query-c1d026c886799dabc39ce73e1fe09ccb175c7271df75d67aa9c72ad6f825a992.json
Generated
+28
View File
@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM usr WHERE email = $1 RETURNING username, workspace_id",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "username",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "workspace_id",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false
]
},
"hash": "c1d026c886799dabc39ce73e1fe09ccb175c7271df75d67aa9c72ad6f825a992"
}
backend/.sqlx/query-c5451ea9d9fa5146af242d1ee8c19ebd65b80e7ed9f29b9fb2e03767c2aa94ba.json
Generated
+28
View File
@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "SELECT ws.workspace_id AS \"workspace_id!\", dt.key AS \"datatable!\"\n FROM workspace_settings ws\n CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt\n WHERE dt.value->'reference'->>'workspace_id' = $1\n ORDER BY ws.workspace_id, dt.key",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id!",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "datatable!",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
null
]
},
"hash": "c5451ea9d9fa5146af242d1ee8c19ebd65b80e7ed9f29b9fb2e03767c2aa94ba"
}
backend/.sqlx/query-c85d362fe2e652d4ac01a35bf470e80b993020a2ff5dcb5849dc570d52798587.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM datatable_role WHERE id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "c85d362fe2e652d4ac01a35bf470e80b993020a2ff5dcb5849dc570d52798587"
}
backend/.sqlx/query-d48ca62c86b1af7a9dd2450c1c28dc45020a2a553d8874c49f9eafedea5a9d40.json
Generated
+24
View File
@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "SELECT dt.key AS \"datatable!\"\n FROM workspace_settings ws\n CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt\n WHERE ws.workspace_id = $1\n AND dt.key <> $2\n AND NOT dt.value ? 'permissions'\n AND dt.value->'database'->>'resource_type' = 'instance'\n AND dt.value->'database'->>'resource_path' = $3\n ORDER BY dt.key",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "datatable!",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "d48ca62c86b1af7a9dd2450c1c28dc45020a2a553d8874c49f9eafedea5a9d40"
}
backend/.sqlx/query-d5fb5dde6300862f978739a3d9249fc2b3e7697c0da7d3195398933d3d81aadf.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT datatable FROM workspace_settings WHERE workspace_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "datatable",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
true
]
},
"hash": "d5fb5dde6300862f978739a3d9249fc2b3e7697c0da7d3195398933d3d81aadf"
}
backend/.sqlx/query-da5057c86607327bafc2942c218025ca9181a0c396405984d87e422e129521c1.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace_settings ws\n SET datatable = (\n SELECT jsonb_set(ws.datatable, '{datatables}', jsonb_object_agg(\n dt.key,\n CASE WHEN dt.value->'reference'->>'workspace_id' = $1\n AND dt.value->'reference'->>'datatable' = $2\n THEN jsonb_set(dt.value, '{reference,datatable}', to_jsonb($3::text))\n ELSE dt.value END\n ))\n FROM jsonb_each(ws.datatable->'datatables') dt\n )\n WHERE jsonb_typeof(ws.datatable->'datatables') = 'object'\n AND ws.datatable::text LIKE '%\"reference\"%'",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "da5057c86607327bafc2942c218025ca9181a0c396405984d87e422e129521c1"
}
backend/.sqlx/query-dc8dfc37559e9b6713bde48155f48b5a2c7b8199eace1508e102b60d1ff40c04.json
Generated
+20
View File
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "SELECT value->'roles' FROM global_settings WHERE name = 'custom_instance_pg_databases'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "dc8dfc37559e9b6713bde48155f48b5a2c7b8199eace1508e102b60d1ff40c04"
}
backend/.sqlx/query-e159b2ff15633f85e839ee4fe1ec2ecd11caf228ea8d0f52ad66def595644250.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE postgres_trigger SET server_id = NULL, last_server_ping = NULL\n WHERE workspace_id = $1\n AND (postgres_resource_path = $2 OR postgres_resource_path LIKE $3)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "e159b2ff15633f85e839ee4fe1ec2ecd11caf228ea8d0f52ad66def595644250"
}
backend/.sqlx/query-e2061df65ffd4a72146c4ca316829265289c8d6f625ac272655c88e1ad0b1745.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT datatable FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "datatable",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
true
]
},
"hash": "e2061df65ffd4a72146c4ca316829265289c8d6f625ac272655c88e1ad0b1745"
}
backend/.sqlx/query-ebaf3ed3097621da59dd201b5a4b9d1f440692f183c7c378f59e4b73f1c6e241.json
Generated
+17
View File
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace_settings\n SET datatable = jsonb_set(\n jsonb_set(\n datatable #- ARRAY['datatables', $2, 'reference'],\n ARRAY['datatables', $2, 'database'], $3::jsonb),\n ARRAY['datatables', $2, 'forked_from'], $4::jsonb\n )\n WHERE workspace_id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Jsonb",
"Jsonb"
]
},
"nullable": []
},
"hash": "ebaf3ed3097621da59dd201b5a4b9d1f440692f183c7c378f59e4b73f1c6e241"
}
backend/.sqlx/query-f3ee09fb17955ca8d886f446d397063c4094546a7807343b570b823796372cef.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT workspace_id FROM workspace_settings WHERE datatable::text LIKE $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false
]
},
"hash": "f3ee09fb17955ca8d886f446d397063c4094546a7807343b570b823796372cef"
}
backend/.sqlx/query-f4adc9e26ebfebce18a29fb2c21bf06394cacb8a9699a608327b097e0ac1363e.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace_settings\n SET datatable = CASE WHEN $3::jsonb = 'null'::jsonb\n THEN datatable #- ARRAY['datatables', $2, 'permissions']\n ELSE jsonb_set(datatable, ARRAY['datatables', $2, 'permissions'], $3::jsonb)\n END\n WHERE workspace_id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Jsonb"
]
},
"nullable": []
},
"hash": "f4adc9e26ebfebce18a29fb2c21bf06394cacb8a9699a608327b097e0ac1363e"
}
backend/.sqlx/query-fcb34e643b888122766e115a01394ab31ac856252aaa76c75ea27a447009c363.json
Generated
+17
View File
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE datatable_role SET name = $2, enabled = $3, pwd = $4 WHERE id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Varchar",
"Bool",
"Text"
]
},
"nullable": []
},
"hash": "fcb34e643b888122766e115a01394ab31ac856252aaa76c75ea27a447009c363"
}
backend/Cargo.lock
Generated
+125 -125
View File
@@ -728,7 +728,7 @@ dependencies = [
"futures-lite 2.6.1",
"parking",
"polling 3.11.0",
"rustix 1.1.4",
"rustix 1.1.5",
"slab",
"windows-sys 0.61.2",
]
@@ -873,7 +873,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.5",
"syn 3.0.6",
]
[[package]]
@@ -1976,7 +1976,7 @@ dependencies = [
"prettyplease 0.3.0",
"proc-macro2",
"quote",
"syn 3.0.5",
"syn 3.0.6",
]
[[package]]
@@ -2000,7 +2000,7 @@ dependencies = [
"proc-macro-crate",
"proc-macro2",
"quote",
"syn 3.0.5",
"syn 3.0.6",
]
[[package]]
@@ -2144,7 +2144,7 @@ checksum = "6a1f896587b6f2c069c73d2f0913e2d590c3990285cd2f0b6aa02b786b4c679c"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.5",
"syn 3.0.6",
]
[[package]]
@@ -2164,9 +2164,9 @@ dependencies = [
[[package]]
name = "bytes-str"
version = "0.2.8"
version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "577d2bf5650f8554d5a372af5ac93535110a0fc75b3e702bb853369febf227c2"
checksum = "4dde6d05e75a31ec9610eb6446a6f0a10dd30ff5100d720fee4c7c7a9008b5ba"
dependencies = [
"bytes",
"serde",
@@ -2338,9 +2338,9 @@ dependencies = [
[[package]]
name = "cfg-if"
version = "1.0.4"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
[[package]]
name = "cfg_aliases"
@@ -2454,7 +2454,7 @@ dependencies = [
"heck",
"proc-macro2",
"quote",
"syn 3.0.5",
"syn 3.0.6",
]
[[package]]
@@ -3047,7 +3047,7 @@ dependencies = [
"proc-macro2",
"quote",
"strsim 0.11.1",
"syn 3.0.5",
"syn 3.0.6",
]
[[package]]
@@ -3102,7 +3102,7 @@ checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785"
dependencies = [
"darling_core 0.24.1",
"quote",
"syn 3.0.5",
"syn 3.0.6",
]
[[package]]
@@ -4580,7 +4580,7 @@ checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab"
dependencies = [
"libc",
"option-ext",
"redox_users 0.5.2",
"redox_users 0.5.3",
"windows-sys 0.61.2",
]
@@ -4603,7 +4603,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.5",
"syn 3.0.6",
]
[[package]]
@@ -4850,7 +4850,7 @@ checksum = "a65863d15a4ce2888bd2f0f543cc963d3879c3a022c8ee43f6141d479a3ac815"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.5",
"syn 3.0.6",
]
[[package]]
@@ -5198,7 +5198,7 @@ version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4"
dependencies = [
"rustix 1.1.4",
"rustix 1.1.5",
"windows-sys 0.59.0",
]
@@ -5319,7 +5319,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.5",
"syn 3.0.6",
]
[[package]]
@@ -9438,7 +9438,7 @@ dependencies = [
"concurrent-queue",
"hermit-abi 0.5.3",
"pin-project-lite",
"rustix 1.1.4",
"rustix 1.1.5",
"windows-sys 0.61.2",
]
@@ -9574,7 +9574,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0"
dependencies = [
"proc-macro2",
"syn 3.0.5",
"syn 3.0.6",
]
[[package]]
@@ -10235,11 +10235,10 @@ dependencies = [
[[package]]
name = "redox_users"
version = "0.5.2"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac"
checksum = "60dc65c0ff1a7ae1294b0c67b9f14baf70b644404010370171787bfac1038fc0"
dependencies = [
"getrandom 0.2.17",
"libredox",
"thiserror 2.0.20",
]
@@ -10261,7 +10260,7 @@ checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.5",
"syn 3.0.6",
]
[[package]]
@@ -10580,7 +10579,7 @@ dependencies = [
"proc-macro2",
"quote",
"serde_json",
"syn 3.0.5",
"syn 3.0.6",
]
[[package]]
@@ -10798,9 +10797,9 @@ dependencies = [
[[package]]
name = "rustix"
version = "1.1.4"
version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d"
dependencies = [
"bitflags 2.13.2",
"errno",
@@ -11217,7 +11216,7 @@ dependencies = [
"proc-macro2",
"quote",
"serde_derive_internals 0.30.0",
"syn 3.0.5",
"syn 3.0.6",
]
[[package]]
@@ -11416,7 +11415,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.5",
"syn 3.0.6",
]
[[package]]
@@ -11438,7 +11437,7 @@ checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.5",
"syn 3.0.6",
]
[[package]]
@@ -11492,7 +11491,7 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.5",
"syn 3.0.6",
]
[[package]]
@@ -11560,7 +11559,7 @@ dependencies = [
"darling 0.24.1",
"proc-macro2",
"quote",
"syn 3.0.5",
"syn 3.0.6",
]
[[package]]
@@ -12708,9 +12707,9 @@ dependencies = [
[[package]]
name = "syn"
version = "3.0.5"
version = "3.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9"
checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee"
dependencies = [
"proc-macro2",
"quote",
@@ -12745,7 +12744,7 @@ checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.5",
"syn 3.0.6",
]
[[package]]
@@ -13036,7 +13035,7 @@ dependencies = [
"fastrand 2.5.0",
"getrandom 0.4.3",
"once_cell",
"rustix 1.1.4",
"rustix 1.1.5",
"windows-sys 0.61.2",
]
@@ -13055,7 +13054,7 @@ version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874"
dependencies = [
"rustix 1.1.4",
"rustix 1.1.5",
"windows-sys 0.61.2",
]
@@ -13115,7 +13114,7 @@ checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.5",
"syn 3.0.6",
]
[[package]]
@@ -14075,7 +14074,7 @@ checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.5",
"syn 3.0.6",
]
[[package]]
@@ -14179,9 +14178,9 @@ checksum = "81b79ad29b5e19de4260020f8919b443b2ef0277d242ce532ec7b7a2cc8b6007"
[[package]]
name = "unicode-ident"
version = "1.0.24"
version = "1.0.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
[[package]]
name = "unicode-normalization"
@@ -14546,7 +14545,7 @@ dependencies = [
"bumpalo",
"proc-macro2",
"quote",
"syn 3.0.5",
"syn 3.0.6",
"wasm-bindgen-shared",
]
@@ -14589,7 +14588,7 @@ checksum = "8c89dcab8b516b6b603baca9d550b7282d68fcc7f367e3956cff7ebf406a3f12"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.5",
"syn 3.0.6",
]
[[package]]
@@ -14794,7 +14793,7 @@ dependencies = [
[[package]]
name = "windmill"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"async-nats",
@@ -14882,7 +14881,7 @@ dependencies = [
[[package]]
name = "windmill-ai"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"async-stream",
"async-trait",
@@ -14916,7 +14915,7 @@ dependencies = [
[[package]]
name = "windmill-alerting"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14929,7 +14928,7 @@ dependencies = [
[[package]]
name = "windmill-api"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"argon2",
@@ -15069,7 +15068,7 @@ dependencies = [
[[package]]
name = "windmill-api-agent-workers"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15092,7 +15091,7 @@ dependencies = [
[[package]]
name = "windmill-api-assets"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15109,7 +15108,7 @@ dependencies = [
[[package]]
name = "windmill-api-auth"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15135,7 +15134,7 @@ dependencies = [
[[package]]
name = "windmill-api-client"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"reqwest 0.12.28",
"serde",
@@ -15145,7 +15144,7 @@ dependencies = [
[[package]]
name = "windmill-api-configs"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15162,7 +15161,7 @@ dependencies = [
[[package]]
name = "windmill-api-debug"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"axum 0.8.9",
"base64 0.22.1",
@@ -15184,7 +15183,7 @@ dependencies = [
[[package]]
name = "windmill-api-embeddings"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15207,7 +15206,7 @@ dependencies = [
[[package]]
name = "windmill-api-flow-conversations"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15223,7 +15222,7 @@ dependencies = [
[[package]]
name = "windmill-api-flows"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15245,7 +15244,7 @@ dependencies = [
[[package]]
name = "windmill-api-groups"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15267,7 +15266,7 @@ dependencies = [
[[package]]
name = "windmill-api-inputs"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15281,7 +15280,7 @@ dependencies = [
[[package]]
name = "windmill-api-integration-tests"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"async-nats",
@@ -15316,7 +15315,7 @@ dependencies = [
[[package]]
name = "windmill-api-jobs"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15341,7 +15340,7 @@ dependencies = [
[[package]]
name = "windmill-api-npm-proxy"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15369,7 +15368,7 @@ dependencies = [
[[package]]
name = "windmill-api-openapi"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15391,7 +15390,7 @@ dependencies = [
[[package]]
name = "windmill-api-schedule"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15411,7 +15410,7 @@ dependencies = [
[[package]]
name = "windmill-api-scripts"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15449,7 +15448,7 @@ dependencies = [
[[package]]
name = "windmill-api-settings"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15471,13 +15470,14 @@ dependencies = [
"windmill-ai",
"windmill-alerting",
"windmill-api-auth",
"windmill-audit",
"windmill-common",
"windmill-object-store",
]
[[package]]
name = "windmill-api-sse"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"lazy_static",
"serde",
@@ -15489,7 +15489,7 @@ dependencies = [
[[package]]
name = "windmill-api-users"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"argon2",
"axum 0.8.9",
@@ -15513,7 +15513,7 @@ dependencies = [
[[package]]
name = "windmill-api-workers"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15527,7 +15527,7 @@ dependencies = [
[[package]]
name = "windmill-api-workspaces"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15562,7 +15562,7 @@ dependencies = [
[[package]]
name = "windmill-audit"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"chrono",
"lazy_static",
@@ -15576,7 +15576,7 @@ dependencies = [
[[package]]
name = "windmill-autoscaling"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15595,7 +15595,7 @@ dependencies = [
[[package]]
name = "windmill-common"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"aes-gcm",
"aho-corasick",
@@ -15702,7 +15702,7 @@ dependencies = [
[[package]]
name = "windmill-dep-map"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"chrono",
"futures",
@@ -15722,7 +15722,7 @@ dependencies = [
[[package]]
name = "windmill-git-sync"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"regex",
"serde",
@@ -15739,7 +15739,7 @@ dependencies = [
[[package]]
name = "windmill-indexer"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"astral-tokio-tar",
@@ -15766,7 +15766,7 @@ dependencies = [
[[package]]
name = "windmill-jseval"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"futures",
@@ -15783,7 +15783,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"itertools 0.14.0",
"lazy_static",
@@ -15799,7 +15799,7 @@ dependencies = [
[[package]]
name = "windmill-mcp"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15820,7 +15820,7 @@ dependencies = [
[[package]]
name = "windmill-native-triggers"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15851,7 +15851,7 @@ dependencies = [
[[package]]
name = "windmill-oauth"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"arc-swap",
@@ -15876,7 +15876,7 @@ dependencies = [
[[package]]
name = "windmill-object-store"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"async-stream",
@@ -15911,7 +15911,7 @@ dependencies = [
[[package]]
name = "windmill-operator"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"futures",
@@ -15929,7 +15929,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"convert_case 0.6.0",
"serde",
@@ -15938,7 +15938,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -15950,7 +15950,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"serde_json",
@@ -15962,7 +15962,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"gosyn",
@@ -15974,7 +15974,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -15986,7 +15986,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"serde_json",
@@ -15998,7 +15998,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"nu-parser",
@@ -16009,7 +16009,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -16020,7 +16020,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -16032,7 +16032,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-asset"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -16043,7 +16043,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -16065,7 +16065,7 @@ dependencies = [
[[package]]
name = "windmill-parser-r"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"serde_json",
@@ -16077,7 +16077,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ruby"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16091,7 +16091,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"convert_case 0.6.0",
@@ -16108,7 +16108,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16121,7 +16121,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql-asset"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"serde",
@@ -16133,7 +16133,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16151,7 +16151,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts-asset"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"serde-wasm-bindgen",
@@ -16167,7 +16167,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wac"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -16183,7 +16183,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16197,7 +16197,7 @@ dependencies = [
[[package]]
name = "windmill-queue"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -16236,7 +16236,7 @@ dependencies = [
[[package]]
name = "windmill-runtime-nativets"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"const_format",
@@ -16276,7 +16276,7 @@ dependencies = [
[[package]]
name = "windmill-sql-datatype-parser-wasm"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"getrandom 0.3.4",
"wasm-bindgen",
@@ -16287,7 +16287,7 @@ dependencies = [
[[package]]
name = "windmill-store"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -16322,7 +16322,7 @@ dependencies = [
[[package]]
name = "windmill-test-utils"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16346,7 +16346,7 @@ dependencies = [
[[package]]
name = "windmill-trigger"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16379,7 +16379,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-amqp"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16406,7 +16406,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-azure"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16439,7 +16439,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-email"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16459,7 +16459,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-gcp"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16493,7 +16493,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-http"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16529,7 +16529,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-kafka"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16552,7 +16552,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-mqtt"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16576,7 +16576,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-nats"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"async-nats",
@@ -16600,7 +16600,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-postgres"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16635,7 +16635,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-sqs"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16663,7 +16663,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-websocket"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16688,7 +16688,7 @@ dependencies = [
[[package]]
name = "windmill-types"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"bitflags 2.13.2",
@@ -16707,7 +16707,7 @@ dependencies = [
[[package]]
name = "windmill-worker"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"async-once-cell",
@@ -16825,7 +16825,7 @@ dependencies = [
[[package]]
name = "windmill-worker-volumes"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"bytes",
"futures",
@@ -17458,7 +17458,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156"
dependencies = [
"libc",
"rustix 1.1.4",
"rustix 1.1.5",
]
[[package]]
@@ -17519,7 +17519,7 @@ checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.5",
"syn 3.0.6",
"synstructure 0.14.0",
]
@@ -17560,7 +17560,7 @@ checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.5",
"syn 3.0.6",
"synstructure 0.14.0",
]
@@ -17616,7 +17616,7 @@ checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.5",
"syn 3.0.6",
]
[[package]]
backend/Cargo.toml
+2 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.813.0"
version = "1.814.0"
authors.workspace = true
edition.workspace = true
@@ -88,7 +88,7 @@ members = [
exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"]
[workspace.package]
version = "1.813.0"
version = "1.814.0"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
d252afcc80e77fcc4f9a2a346b80908c8605a6c0
7e338e4dabf91689bfd7fb0333c6534040b17b59
backend/migrations/20260908142148_datatable_role_catalog.down.sql
+18
View File
@@ -0,0 +1,18 @@
-- Refuse while the catalog holds anything. Each row is a live Postgres login with a password
-- only this table carries, so dropping it would leave credentials on the cluster that Windmill can
-- no longer disable, delete or even name — and re-applying could not recreate them, because the
-- role names would already be taken. Cleaning them up here is not an option either: dropping a
-- role means reassigning what it owns in *every* instance database, and a migration runs in one.
--
-- Delete the roles through instance settings first; that path does the cluster work.
LOCK TABLE datatable_role IN ACCESS EXCLUSIVE MODE;
DO $$
BEGIN
IF EXISTS (SELECT 1 FROM datatable_role) THEN
RAISE EXCEPTION 'Cannot roll back: % data table role(s) still exist as Postgres logins. Delete them in instance settings first, which drops them from the cluster.',
(SELECT count(*) FROM datatable_role);
END IF;
END $$;
DROP TABLE IF EXISTS datatable_role;
backend/migrations/20260908142148_datatable_role_catalog.up.sql
+21
View File
@@ -0,0 +1,21 @@
-- The instance's data table role catalog: one row per Postgres login Windmill created for data
-- table access.
--
-- A table rather than a `global_settings` key, because the value is a set of live cluster
-- credentials and that table has generic read, list, write and CLI round-trip paths that know
-- nothing about what they are carrying. Every one of them is a way to leak the passwords or to
-- overwrite the catalog with a copy that has none, and a row nothing generic touches has none of
-- those. One row per role also makes two concurrent creates two inserts rather than a
-- read-modify-write over one document.
CREATE TABLE datatable_role (
id VARCHAR(50) PRIMARY KEY,
-- The Postgres role name, verbatim. Unique because it is the cluster's own key.
name VARCHAR(63) NOT NULL UNIQUE,
enabled BOOLEAN NOT NULL DEFAULT true,
-- Generated by Windmill, never entered by anyone, and never leaves the server.
pwd TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
GRANT ALL ON datatable_role TO windmill_user;
GRANT ALL ON datatable_role TO windmill_admin;
backend/parsers/windmill-parser-wasm/Cargo.lock
Generated
+24 -24
View File
@@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windmill-common"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"aho-corasick",
"anyhow",
@@ -6274,7 +6274,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"proc-macro2",
"quote",
@@ -6286,7 +6286,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"convert_case",
"serde",
@@ -6295,7 +6295,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6307,7 +6307,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"serde_json",
@@ -6319,7 +6319,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"gosyn",
@@ -6331,7 +6331,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6343,7 +6343,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"serde_json",
@@ -6355,7 +6355,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"nu-parser",
@@ -6366,7 +6366,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -6377,7 +6377,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -6389,7 +6389,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-asset"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -6400,7 +6400,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -6422,7 +6422,7 @@ dependencies = [
[[package]]
name = "windmill-parser-r"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"serde_json",
@@ -6434,7 +6434,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ruby"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6448,7 +6448,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"convert_case",
@@ -6465,7 +6465,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6478,7 +6478,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql-asset"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"serde",
@@ -6490,7 +6490,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6508,7 +6508,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts-asset"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"serde-wasm-bindgen",
@@ -6524,7 +6524,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wac"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -6540,7 +6540,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wasm"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"getrandom 0.2.17",
@@ -6572,7 +6572,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6586,7 +6586,7 @@ dependencies = [
[[package]]
name = "windmill-types"
version = "1.813.0"
version = "1.814.0"
dependencies = [
"anyhow",
"bitflags",
backend/parsers/windmill-parser-wasm/Cargo.toml
+1 -1
View File
@@ -12,7 +12,7 @@ resolver = "2"
members = ["."]
[workspace.package]
version = "1.813.0"
version = "1.814.0"
edition = "2021"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
backend/parsers/windmill-parser/src/asset_parser.rs
+33 -1
View File
@@ -730,7 +730,12 @@ pub fn parse_asset_syntax(
s: &str,
enable_default_syntax: bool,
) -> Option<(AssetKind, Cow<'_, str>)> {
if enable_default_syntax && s == "datatable" {
// `datatable` and `datatable?role=analyst` both name the default data table: the role picks
// which Postgres login the connection is made as, not which data table is read.
if enable_default_syntax
&& s.strip_prefix("datatable")
.is_some_and(|rest| rest.is_empty() || rest.starts_with('?'))
{
return Some((AssetKind::DataTable, Cow::Borrowed("main")));
} else if enable_default_syntax && s == "ducklake" {
return Some((AssetKind::Ducklake, Cow::Borrowed("main")));
@@ -741,6 +746,14 @@ pub fn parse_asset_syntax(
if *kind == AssetKind::Dbt {
return Some((*kind, Cow::Owned(canonicalize_table_asset_path(suffix))));
}
// Same reasoning as above, for the explicit form. Specific to data tables: a
// `Resource`'s `?table=` is part of what it names, and stripping it would merge two
// different assets.
if *kind == AssetKind::DataTable {
if let Some((path, _role)) = suffix.split_once('?') {
return Some((*kind, Cow::Borrowed(path)));
}
}
// The suffix is kept verbatim. For S3 the path encodes the storage:
// `s3://<storage>/<key>`, with an EMPTY storage segment for the
// workspace default — so `s3:///key` yields `/key` (leading slash
@@ -1692,6 +1705,25 @@ fn parse_trigger_spec(s: &str) -> Option<TriggerSpec> {
mod pipeline_annotation_tests {
use super::*;
#[test]
fn a_datatable_role_is_not_part_of_the_asset_it_names() {
// The role picks which Postgres login the connection is made as, so two references that
// differ only by role are the same asset and must land on one graph node.
assert_eq!(
parse_asset_syntax("datatable://sales?role=analytics", false),
Some((AssetKind::DataTable, Cow::Borrowed("sales")))
);
assert_eq!(
parse_asset_syntax("datatable?role=analytics", true),
Some((AssetKind::DataTable, Cow::Borrowed("main")))
);
// A resource's `?table=` is part of what it names, so it is kept.
assert_eq!(
parse_asset_syntax("$res:f/db/pg?table=users", false),
Some((AssetKind::Resource, Cow::Borrowed("f/db/pg?table=users")))
);
}
#[test]
fn s3_path_keeps_storage_distinction() {
// An S3 asset path is `<storage>/<key>` with an empty storage segment
backend/tests/list_jobs.rs
+60
View File
@@ -1199,3 +1199,63 @@ async fn test_wm_labels_from_result_merged_with_static_labels(
Ok(())
}
/// `tag` lives only on `v2_job`, which count_jobs joins only when `tags` is set.
#[sqlx::test(fixtures("base"))]
async fn test_count_completed_jobs_tags_filter(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let client = windmill_api_client::create_client(
&format!("http://localhost:{port}"),
"SECRET_TOKEN".to_string(),
);
for (ws, tag, status) in [
("test-workspace", "deno", "success"),
("test-workspace", "deno", "failure"),
("test-workspace", "python3", "success"),
("other-workspace", "deno", "success"),
] {
let id = uuid::Uuid::new_v4();
sqlx::query("INSERT INTO v2_job (id, workspace_id, tag) VALUES ($1, $2, $3)")
.bind(id)
.bind(ws)
.bind(tag)
.execute(&db)
.await?;
sqlx::query(
"INSERT INTO v2_job_completed (id, workspace_id, status, duration_ms) VALUES ($1, $2, $3::job_status, 0)",
)
.bind(id)
.bind(ws)
.bind(status)
.execute(&db)
.await?;
}
for (query, expected) in [
("", 3),
("tags=deno", 2),
("tags=deno&success=true", 1),
("tags=deno,python3&completed_after_s_ago=3600", 3),
] {
let response = client
.client()
.get(format!(
"{}/w/test-workspace/jobs/completed/count_jobs?{query}",
client.baseurl()
))
.send()
.await?;
assert!(
response.status().is_success(),
"{query}: {}",
response.text().await?
);
assert_eq!(response.json::<i64>().await?, expected, "{query}");
}
Ok(())
}
backend/windmill-api-groups/src/folders.rs
+8
View File
@@ -800,6 +800,14 @@ async fn delete_folder(
not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?;
// See the same call in `delete_group`: a freed name must not stay in a tenant list.
windmill_common::workspaces::remove_datatable_tenant_in_workspace(
&mut tx,
&w_id,
&format!("f/{name}"),
)
.await?;
let del = sqlx::query_scalar!(
"DELETE FROM folder WHERE name = $1 AND workspace_id = $2 RETURNING 1",
name,
backend/windmill-api-groups/src/groups.rs
+9
View File
@@ -797,6 +797,15 @@ async fn delete_group(
}
not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?;
// A tenant list names a principal, so a freed name must not linger in one: a later group
// reusing it would silently inherit the data table access this one had.
windmill_common::workspaces::remove_datatable_tenant_in_workspace(
&mut tx,
&w_id,
&format!("g/{name}"),
)
.await?;
sqlx::query!(
"DELETE FROM usr_to_group WHERE group_ = $1 AND workspace_id = $2",
name,
backend/windmill-api-integration-tests/tests/datatable_roles.rs
+1247
View File
File diff suppressed because it is too large Load Diff
backend/windmill-api-integration-tests/tests/fixtures/datatable_roles.sql
Vendored
+45
View File
@@ -0,0 +1,45 @@
-- A data table under roles in `test-workspace`, and a fork whose entry points at it rather than
-- carrying a copy. `test-user-2` is a non-admin of the parent and an admin of the fork: the shape
-- the pointer exists for.
-- Empty registry: role provisioning grants CONNECT on every database named here, and the data
-- table's `dt_main` is a name in workspace settings, not a database that exists.
INSERT INTO global_settings (name, value) VALUES
('custom_instance_pg_databases', '{"user_pwd": "pw", "databases": {}}'::jsonb)
ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value;
INSERT INTO datatable_role (id, name, enabled, pwd) VALUES ('role1', 'analytics', true, 'pw');
UPDATE workspace_settings SET datatable = '{
"datatables": {
"main": {
"database": {"resource_type": "instance", "resource_path": "dt_main"},
"permissions": {
"default_role": "role1",
"roles": {
"admin": {"tenants": []},
"role1": {"tenants": ["u/test-user-2", "g/analysts", "f/finance"]}
}
}
}
}
}'::jsonb WHERE workspace_id = 'test-workspace';
INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES
('test-workspace', 'analysts', 'Analysts', '{}');
INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) VALUES
('test-workspace', 'finance', 'finance', '{}', '{}');
INSERT INTO workspace (id, name, owner, parent_workspace_id) VALUES
('wm-fork-dt', 'fork of test-workspace', 'test2@windmill.dev', 'test-workspace');
INSERT INTO workspace_key (workspace_id, kind, key) VALUES ('wm-fork-dt', 'cloud', 'test-key');
INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES
('wm-fork-dt', 'all', 'All users', '{}');
INSERT INTO usr (workspace_id, email, username, is_admin, role) VALUES
('wm-fork-dt', 'test2@windmill.dev', 'test-user-2', true, 'Admin');
INSERT INTO workspace_settings (workspace_id, datatable) VALUES ('wm-fork-dt', '{
"datatables": {
"main": {"reference": {"workspace_id": "test-workspace", "datatable": "main"}}
}
}'::jsonb);
backend/windmill-api-settings/Cargo.toml
+2 -1
View File
@@ -11,7 +11,7 @@ path = "src/lib.rs"
[features]
default = []
enterprise = ["license"]
private = ["windmill-common/private"]
private = ["windmill-common/private", "windmill-audit/private"]
parquet = ["windmill-common/parquet", "windmill-object-store/parquet"]
license = ["dep:rsa"]
@@ -19,6 +19,7 @@ license = ["dep:rsa"]
windmill-ai = { workspace = true, default-features = false }
windmill-alerting.workspace = true
windmill-api-auth.workspace = true
windmill-audit.workspace = true
windmill-common = { workspace = true, default-features = false }
axum.workspace = true
anyhow.workspace = true
backend/windmill-api-settings/src/datatable_roles_oss.rs
+44
View File
@@ -0,0 +1,44 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! Where the data table role catalog endpoints come from: the enterprise implementation, or a
//! refusal. Roles are an Enterprise Edition feature; see `windmill_common::datatable_roles_oss`.
#[cfg(all(feature = "private", feature = "enterprise"))]
pub(crate) use crate::datatable_roles_ee::{
create_datatable_role, delete_datatable_role, list_datatable_roles, update_datatable_role,
};
#[cfg(not(all(feature = "private", feature = "enterprise")))]
pub(crate) use ce::*;
// The routes stay registered so the API has one shape; each answers after authentication, before
// anything is read.
#[cfg(not(all(feature = "private", feature = "enterprise")))]
mod ce {
use windmill_api_auth::ApiAuthed;
use windmill_common::{
datatable_roles_oss::datatable_roles_unavailable as unavailable, error::Result,
};
pub(crate) async fn list_datatable_roles(_authed: ApiAuthed) -> Result<String> {
Err(unavailable())
}
pub(crate) async fn create_datatable_role(_authed: ApiAuthed) -> Result<String> {
Err(unavailable())
}
pub(crate) async fn update_datatable_role(_authed: ApiAuthed) -> Result<String> {
Err(unavailable())
}
pub(crate) async fn delete_datatable_role(_authed: ApiAuthed) -> Result<String> {
Err(unavailable())
}
}
backend/windmill-api-settings/src/lib.rs
+13
View File
@@ -17,6 +17,9 @@ mod audit_logs_s3;
mod audit_logs_s3_backfill;
#[cfg(feature = "parquet")]
mod background_task;
#[cfg(all(feature = "private", feature = "enterprise"))]
mod datatable_roles_ee;
mod datatable_roles_oss;
#[cfg(feature = "private")]
mod ee;
pub mod ee_oss;
@@ -151,6 +154,16 @@ pub fn global_service() -> Router {
"/list_custom_instance_pg_databases",
post(list_custom_instance_pg_databases),
)
.route(
"/datatable_roles",
get(datatable_roles_oss::list_datatable_roles)
.post(datatable_roles_oss::create_datatable_role),
)
.route(
"/datatable_roles/{id}",
post(datatable_roles_oss::update_datatable_role)
.delete(datatable_roles_oss::delete_datatable_role),
)
.route(
"/refresh_custom_instance_user_pwd",
post(refresh_custom_instance_user_pwd),
backend/windmill-api-users/src/users.rs
+29 -3
View File
@@ -1703,14 +1703,25 @@ async fn delete_user(
.await?;
windmill_common::user_drafts::delete_drafts_of_email(&mut *tx, &email_to_delete).await?;
let usernames = sqlx::query_scalar!(
"DELETE FROM usr WHERE email = $1 RETURNING username",
let memberships = sqlx::query!(
"DELETE FROM usr WHERE email = $1 RETURNING username, workspace_id",
&email_to_delete
)
.fetch_all(&mut *tx)
.await?;
for username in usernames {
for row in memberships {
let username = row.username;
// A tenant list names a principal of its workspace, so the name has to be freed in every
// workspace this account belonged to: a later account taking the username would otherwise
// inherit the data table access it had.
windmill_common::workspaces::remove_datatable_tenant_in_workspace(
&mut tx,
&row.workspace_id,
&format!("u/{username}"),
)
.await?;
sqlx::query!("DELETE FROM password WHERE email = $1", &email_to_delete)
.execute(&mut *tx)
.await?;
@@ -2456,6 +2467,15 @@ pub async fn delete_workspace_user_internal(
tx: &mut Transaction<'_, Postgres>,
authed: Option<&ApiAuthed>, // None for system operations
) -> Result<()> {
// Same reasoning as the `extra_perms` sweep below: a freed username must not stay named
// anywhere that grants access, tenant lists included.
windmill_common::workspaces::remove_datatable_tenant_in_workspace(
tx,
w_id,
&format!("u/{username_to_delete}"),
)
.await?;
// ---- Clean up extra_perms referencing this user ----
let extra_perms_tables = [
"script",
@@ -3965,6 +3985,12 @@ async fn leave_workspace(
) -> Result<String> {
forbid_job_token_account_destruction(&authed)?;
let mut tx = db.begin().await?;
windmill_common::workspaces::remove_datatable_tenant_in_workspace(
&mut tx,
&w_id,
&format!("u/{}", authed.username),
)
.await?;
sqlx::query!(
"DELETE FROM usr WHERE workspace_id = $1 AND username = $2",
&w_id,
backend/windmill-api-workspaces/src/datatable_migrations.rs
+119 -6
View File
@@ -30,6 +30,7 @@ use windmill_api_auth::{require_super_admin, ApiAuthed};
use windmill_api_jobs::run_wait_result_internal;
use windmill_audit::audit_oss::audit_log;
use windmill_audit::ActionKind;
use windmill_common::datatable_roles::ADMIN_DATATABLE_ROLE;
use windmill_common::db::UserDB;
use windmill_common::error::{pg_error_message, Error, JsonResult, Result};
use windmill_common::jobs::{JobPayload, RawCode};
@@ -38,7 +39,11 @@ use windmill_common::runnable_settings::{ConcurrencySettingsWithCustom, Debounci
use windmill_common::scripts::ScriptLang;
use windmill_common::users::username_to_permissioned_as;
use windmill_common::worker::to_raw_value;
use windmill_common::workspaces::get_datatable_resource_from_db_unchecked;
use windmill_common::worker::SqlAnnotations;
use windmill_common::workspaces::{
ensure_can_use_datatable_role, ensure_datatable_admin_access,
get_datatable_resource_from_db_unchecked, resolve_governing_datatable, DatatableAccess,
};
use windmill_common::{PgDatabase, DB};
use windmill_git_sync::{
handle_deployment_metadata, handle_deployment_metadata_batch, DeployedObject,
@@ -86,6 +91,42 @@ pub(crate) fn routes() -> Router {
)
}
/// Refuse a migration whose role this caller may not use, before a job is pushed or a version
/// recorded.
///
/// A migration that declares `-- role <name>` runs as that role, so the caller has to be one of its
/// tenants. One that declares none runs as `admin` and reaches every object in the database
/// whatever the roles grant, so it is for the admins of the workspace that governs the data table
/// — a fork can run a migration under a role it holds, never a migration under `admin`.
///
/// The executor re-checks the role when it resolves the connection, so this is not the boundary. It
/// is what makes the refusal legible: which migration, and which role.
async fn ensure_migration_role_allowed(
db: &DB,
w_id: &str,
datatable_name: &str,
authed: &ApiAuthed,
sql: &str,
timestamp: i64,
name: &str,
) -> Result<()> {
let context = format!("Migration {timestamp} ({name})");
let access = DatatableAccess::Authed(authed.to_authed_ref());
match SqlAnnotations::datatable_role(sql)? {
Some(role) => {
ensure_can_use_datatable_role(db, w_id, datatable_name, Some(&role), &access, &context)
.await
}
None => ensure_datatable_admin_access(db, w_id, datatable_name, &access)
.await
.map_err(|e| {
Error::NotAuthorized(format!(
"{context} declares no role, so it would run as admin. {e}"
))
}),
}
}
#[derive(Serialize)]
struct AppliedMigration {
version: i64,
@@ -128,7 +169,18 @@ async fn datatable_database_arg(
.await?
.ok_or_else(|| Error::internal_err(format!("datatable {datatable_name} not found")))?;
Ok(to_raw_value(&format!("datatable://{datatable_name}")))
// `?role=admin` rather than a bare reference, so a migration that declares no `-- role` runs
// as the connection that owns the schema instead of falling through to the data table's
// default role — which is what `ensure_migration_role_allowed` gated it as, and which is the
// only role a DDL statement can be expected to succeed under. A migration that does declare a
// role overrides this: the annotation wins over the reference.
//
// A legacy name containing `?` cannot be migrated through this reference: the appended query
// makes it neither an exact name nor a parseable one. Accepted on purpose, since such names can
// no longer be created and none are expected to carry migrations.
Ok(to_raw_value(&format!(
"datatable://{datatable_name}?role={ADMIN_DATATABLE_ROLE}"
)))
}
/// Run a migration's SQL as a normal Windmill `postgresql` job, permissioned as
@@ -384,6 +436,11 @@ async fn run_datatable_migrations(
Path((w_id, datatable_name)): Path<(String, String)>,
Query(query): Query<RunDatatableMigrationsQuery>,
) -> JsonResult<RunDatatableMigrationsResult> {
// Before the admin connection is opened at all: the bookkeeping below is created and read
// through it, so a caller no role covers must be refused here rather than after the fact.
crate::datatable_permissions::ensure_reaches_datatable(&db, &w_id, &datatable_name, &authed)
.await?;
audit_log(
&db,
&authed,
@@ -440,6 +497,16 @@ async fn run_datatable_migrations(
if applied_versions.contains(&m.timestamp) {
continue;
}
ensure_migration_role_allowed(
&db,
&w_id,
&datatable_name,
&authed,
&m.code_up,
m.timestamp,
&m.name,
)
.await?;
run_datatable_migration_job(&db, &user_db, &authed, &w_id, &database_arg, &m.code_up)
.await
.map_err(|e| {
@@ -506,6 +573,11 @@ async fn rollback_datatable_migrations(
Path((w_id, datatable_name)): Path<(String, String)>,
Query(query): Query<RollbackDatatableMigrationsQuery>,
) -> JsonResult<RollbackDatatableMigrationsResult> {
// Before the admin connection is opened at all: the bookkeeping below is created and read
// through it, so a caller no role covers must be refused here rather than after the fact.
crate::datatable_permissions::ensure_reaches_datatable(&db, &w_id, &datatable_name, &authed)
.await?;
audit_log(
&db,
&authed,
@@ -588,6 +660,17 @@ async fn rollback_datatable_migrations(
))
})?;
ensure_migration_role_allowed(
&db,
&w_id,
&datatable_name,
&authed,
&code_down,
version,
&definition.name,
)
.await?;
let database_arg = datatable_database_arg(&db, &w_id, &datatable_name).await?;
run_datatable_migration_job(&db, &user_db, &authed, &w_id, &database_arg, &code_down)
.await
@@ -748,10 +831,15 @@ async fn read_applied_datatable_versions(
/// List a data table's migrations annotated with whether each has been applied.
async fn datatable_migrations_status(
_authed: ApiAuthed,
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path((w_id, datatable_name)): Path<(String, String)>,
) -> JsonResult<DatatableMigrationsStatusResult> {
// Reads `_wm_migrations` through the data table's admin connection, so it answers to the same
// question as running one: may you reach this data table at all.
crate::datatable_permissions::ensure_reaches_datatable(&db, &w_id, &datatable_name, &authed)
.await?;
let enabled = datatable_migrations_enabled(&db, &w_id, &datatable_name).await?;
if !enabled {
return Ok(Json(DatatableMigrationsStatusResult {
@@ -1431,6 +1519,15 @@ async fn generate_initial_datatable_migration(
Extension(db): Extension<DB>,
Path((w_id, datatable_name)): Path<(String, String)>,
) -> JsonResult<DatatableMigration> {
// Returns a `pg_dump` of the whole schema and writes into the data table's own bookkeeping, so
// it answers to the workspace that governs it rather than to whoever is asking.
ensure_datatable_admin_access(
&db,
&w_id,
&datatable_name,
&DatatableAccess::Authed(authed.to_authed_ref()),
)
.await?;
validate_datatable_path_segment(&datatable_name)?;
ensure_datatable_migrations_enabled(&db, &w_id, &datatable_name).await?;
@@ -1601,9 +1698,21 @@ pub(crate) struct DatatableRename {
pub(crate) to: String,
}
async fn resolve_datatable_pg(db: &DB, w_id: &str, datatable: &str) -> Result<PgDatabase> {
/// The database whose `_wm_migrations` a rename or delete of `datatable` in `w_id` should touch —
/// `None` when that is somebody else's.
///
/// A fork's entry points at the workspace that governs the data table, so renaming or removing it
/// changes what the fork calls the data table and nothing more. Following the pointer here would
/// let a fork admin relabel or wipe the *governing* workspace's migration bookkeeping through
/// their own settings form, and the parent would then re-run every migration from zero.
async fn resolve_datatable_pg(db: &DB, w_id: &str, datatable: &str) -> Result<Option<PgDatabase>> {
let governing = resolve_governing_datatable(db, w_id, datatable).await?;
if governing.workspace_id != w_id {
return Ok(None);
}
let db_resource = get_datatable_resource_from_db_unchecked(db, w_id, datatable).await?;
serde_json::from_value(db_resource)
.map(Some)
.map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))
}
@@ -1621,7 +1730,9 @@ fn ignore_missing_wm_migrations(e: tokio_postgres::Error) -> Result<()> {
/// Drop a data table's rows from its own database's `_wm_migrations`.
async fn remote_forget_datatable_migrations(db: &DB, w_id: &str, datatable: &str) -> Result<()> {
let pg_db = resolve_datatable_pg(db, w_id, datatable).await?;
let Some(pg_db) = resolve_datatable_pg(db, w_id, datatable).await? else {
return Ok(());
};
let (client, connection) = pg_db.connect(Some(db)).await?;
tokio::spawn(async move {
let _ = connection.await;
@@ -1646,7 +1757,9 @@ async fn remote_rename_datatable_migrations(
from: &str,
to: &str,
) -> Result<()> {
let pg_db = resolve_datatable_pg(db, w_id, resolve_by).await?;
let Some(pg_db) = resolve_datatable_pg(db, w_id, resolve_by).await? else {
return Ok(());
};
let (client, connection) = pg_db.connect(Some(db)).await?;
tokio::spawn(async move {
let _ = connection.await;
backend/windmill-api-workspaces/src/datatable_permissions.rs
+67
View File
@@ -0,0 +1,67 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! Who may connect to a data table as which role.
//!
//! The decision lives on the data table entry of the workspace that governs it, which is not
//! necessarily the workspace asking: a fork's entry points at its parent's, and everything here
//! resolves through that pointer first. Nothing in this module runs SQL against the data table —
//! a save is tenant lists and a default, and the Postgres roles themselves are the instance
//! catalog's business.
use axum::{routing::get, Router};
use windmill_api_auth::ApiAuthed;
use windmill_common::error::Result;
use windmill_common::workspaces::GoverningDatatable;
use windmill_common::DB;
use crate::datatable_permissions_oss as roles;
pub(crate) fn routes() -> Router {
Router::new()
.route(
"/datatable_permissions/{datatable_name}",
get(roles::get_datatable_permissions).post(roles::set_datatable_permissions),
)
.route(
"/datatable_usable_roles/{datatable_name}",
get(roles::list_usable_datatable_roles),
)
}
/// Administering a data table — its permissions, its migrations that declare no role, its exports
/// — is for the admins of the workspace that governs it. A fork can use the data table; it never
/// administers it.
// The gate for whatever administers a data table under roles, which the routes of this module alone
// do not always reach.
#[allow(dead_code)]
pub(crate) async fn ensure_governs_datatable(
db: &DB,
authed: &ApiAuthed,
w_id: &str,
governing: &GoverningDatatable,
) -> Result<()> {
roles::ensure_governs_datatable(db, authed, w_id, governing).await
}
/// Refuse a caller that no tenant of this data table covers.
///
/// The bookkeeping endpoints below open the data table's `admin` connection to read or create
/// `_wm_migrations` before they know which migration will run — so without this, someone covered
/// by no role at all can still force admin-backed reads and writes on a database they may not
/// touch. It asks only "may you reach this data table as anything"; which role a given migration
/// runs as is still decided per migration, and by the executor after that.
pub(crate) async fn ensure_reaches_datatable(
db: &DB,
w_id: &str,
datatable_name: &str,
authed: &ApiAuthed,
) -> Result<()> {
roles::ensure_reaches_datatable(db, w_id, datatable_name, authed).await
}
backend/windmill-api-workspaces/src/datatable_permissions_oss.rs
+73
View File
@@ -0,0 +1,73 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! Where the data table permissions endpoints and their gates come from: the enterprise
//! implementation, or a refusal. Roles are an Enterprise Edition feature; see
//! `windmill_common::datatable_roles_oss`.
#[cfg(all(feature = "private", feature = "enterprise"))]
pub(crate) use crate::datatable_permissions_ee::{
ensure_governs_datatable, ensure_reaches_datatable, get_datatable_permissions,
list_usable_datatable_roles, set_datatable_permissions,
};
#[cfg(not(all(feature = "private", feature = "enterprise")))]
pub(crate) use ce::*;
#[cfg(not(all(feature = "private", feature = "enterprise")))]
mod ce {
use windmill_api_auth::ApiAuthed;
use windmill_common::{
datatable_roles_oss::datatable_roles_unavailable as unavailable,
error::Result,
workspaces::{resolve_governing_datatable, GoverningDatatable},
DB,
};
/// Nobody administers a data table's roles without them.
#[allow(dead_code)]
pub(crate) async fn ensure_governs_datatable(
_db: &DB,
_authed: &ApiAuthed,
_w_id: &str,
_governing: &GoverningDatatable,
) -> Result<()> {
Err(unavailable())
}
/// A data table not under roles is reached as it was before roles existed. One under roles is
/// refused: no role of it can be connected as.
pub(crate) async fn ensure_reaches_datatable(
db: &DB,
w_id: &str,
datatable_name: &str,
_authed: &ApiAuthed,
) -> Result<()> {
let governing = resolve_governing_datatable(db, w_id, datatable_name).await?;
if governing.datatable.permissions.is_none() {
Ok(())
} else {
Err(unavailable())
}
}
// The routes stay registered so the API has one shape; each answers after authentication,
// before anything is read.
pub(crate) async fn get_datatable_permissions(_authed: ApiAuthed) -> Result<String> {
Err(unavailable())
}
pub(crate) async fn set_datatable_permissions(_authed: ApiAuthed) -> Result<String> {
Err(unavailable())
}
pub(crate) async fn list_usable_datatable_roles(_authed: ApiAuthed) -> Result<String> {
Err(unavailable())
}
}
backend/windmill-api-workspaces/src/lib.rs
+5
View File
@@ -2,6 +2,8 @@
pub mod ai_session_backups;
pub mod data_metrics;
pub mod datatable_migrations;
pub mod datatable_permissions;
pub mod datatable_permissions_oss;
pub mod deployment_requests;
pub mod workspaces;
pub mod workspaces_extra;
@@ -9,3 +11,6 @@ pub mod workspaces_oss;
#[cfg(feature = "private")]
pub mod workspaces_ee;
#[cfg(all(feature = "private", feature = "enterprise"))]
pub mod datatable_permissions_ee;
backend/windmill-api-workspaces/src/workspaces.rs
+616 -80
View File
@@ -45,10 +45,12 @@ use windmill_common::workspaces::GitRepositorySettings;
#[cfg(feature = "enterprise")]
use windmill_common::workspaces::WorkspaceDeploymentUISettings;
use windmill_common::workspaces::{
check_deploy_rules, check_user_against_rule, get_datatable_resource_from_db_unchecked,
check_deploy_rules, check_user_against_rule, get_datatable_resource_from_db,
get_datatable_resource_from_db_unchecked, parse_datatable_ref_for, resolve_governing_datatable,
validate_dev_workspace_id, validate_fork_workspace_id, validate_workspace_name, DataTable,
DataTableCatalogResourceType, DataTableForkBehavior, ProtectionRuleKind, ProtectionRules,
ProtectionRuleset, RuleCheckResult, WorkspaceGitSyncSettings, DEV_WORKSPACE_LOCK_RULE_NAME,
DataTableCatalogResourceType, DataTableForkBehavior, DatatableAccess, GoverningDatatable,
ProtectionRuleKind, ProtectionRules, ProtectionRuleset, RuleCheckResult,
WorkspaceGitSyncSettings, DEV_WORKSPACE_LOCK_RULE_NAME,
};
use windmill_common::workspaces::{Ducklake, DucklakeCatalogResourceType};
use windmill_common::PgDatabase;
@@ -141,6 +143,7 @@ pub fn workspaced_service() -> Router {
get(test_datatable_connection),
)
.merge(crate::datatable_migrations::routes())
.merge(crate::datatable_permissions::routes())
.route("/git_sync_enabled", get(get_git_sync_enabled))
.route("/git_sync_deploy_mode", get(get_git_sync_deploy_mode))
.route("/edit_git_sync_config", post(edit_git_sync_config))
@@ -1140,6 +1143,8 @@ async fn get_settings(
if let Some(git_sync) = settings.git_sync.as_mut() {
redact_git_sync_webhook_secrets(git_sync);
}
settings.datatable =
windmill_common::workspaces::strip_datatable_permissions(settings.datatable.take());
Ok(Json(settings))
}
@@ -1176,8 +1181,10 @@ async fn get_public_settings(
.await
.map_err(|e| Error::internal_err(format!("getting public settings: {e:#}")))?;
let settings = not_found_if_none(settings, "workspace settings", &w_id)?;
let mut settings = not_found_if_none(settings, "workspace settings", &w_id)?;
tx.commit().await?;
settings.datatable =
windmill_common::workspaces::strip_datatable_permissions(settings.datatable.take());
Ok(Json(settings))
}
@@ -2184,6 +2191,12 @@ struct DataTableListItem {
name: String,
resource_type: String,
resource_path: String,
/// The workspace whose entry governs this one, when it is not this workspace — a fork pointing
/// at its parent. Its permissions apply here, and only its admins may edit them.
#[serde(skip_serializing_if = "Option::is_none")]
governing_workspace_id: Option<String>,
/// Whether the governing entry is under roles.
permissioned: bool,
}
async fn list_datatables(
@@ -2191,26 +2204,29 @@ async fn list_datatables(
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
) -> JsonResult<Vec<DataTableListItem>> {
let config = sqlx::query_scalar!(
"SELECT datatable->'datatables' FROM workspace_settings WHERE workspace_id = $1",
&w_id
)
.fetch_one(&db)
.await?;
// A pointer entry owns no database, so what it resolves to is the only truthful answer here.
// One that resolves to nothing — a pointer whose workspace was deleted — is dropped rather than
// listed with a database it does not have; what happened is named where it is actionable
// instead: by the delete that stranded it, and by any attempt to use it.
let resolved =
windmill_common::workspaces::resolve_workspace_governing_datatables(&db, &w_id).await?;
let items: Vec<DataTableListItem> = match config {
Some(val) => {
let map: HashMap<String, DataTable> = serde_json::from_value(val).unwrap_or_default();
map.into_iter()
.map(|(name, dt)| DataTableListItem {
name,
resource_type: dt.database.resource_type.as_ref().to_string(),
resource_path: dt.database.resource_path,
})
.collect()
}
None => vec![],
};
let mut items = Vec::with_capacity(resolved.len());
for (name, governing) in resolved {
let database = governing
.datatable
.database
.as_ref()
.expect("a governing entry owns a database");
items.push(DataTableListItem {
name,
resource_type: database.resource_type.as_ref().to_string(),
resource_path: database.resource_path.clone(),
governing_workspace_id: (governing.workspace_id != w_id)
.then(|| governing.workspace_id.clone()),
permissioned: governing.datatable.permissions.is_some(),
});
}
Ok(Json(items))
}
@@ -2298,6 +2314,15 @@ async fn test_datatable_connection(
Path((w_id, datatable_name)): Path<(String, String)>,
) -> JsonResult<DataTableConnectionCheck> {
require_admin(authed.is_admin, &authed.username)?;
// Reports what the admin connection can do, so it answers to the workspace that governs the
// data table rather than to whichever one is asking.
windmill_common::workspaces::ensure_datatable_admin_access(
&db,
&w_id,
&datatable_name,
&DatatableAccess::Authed(authed.to_authed_ref()),
)
.await?;
let db_resource = get_datatable_resource_from_db_unchecked(&db, &w_id, &datatable_name).await?;
let pg_db: PgDatabase = serde_json::from_value(db_resource)
@@ -2385,7 +2410,7 @@ async fn test_datatable_connection(
}
async fn list_datatable_schemas(
_authed: ApiAuthed,
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
) -> JsonResult<Vec<DataTableSchema>> {
@@ -2393,7 +2418,7 @@ async fn list_datatable_schemas(
let mut results = Vec::new();
for datatable_name in datatable_names {
let schema = match get_datatable_schema(&db, &w_id, &datatable_name).await {
let schema = match get_datatable_schema(&db, &authed, &w_id, &datatable_name).await {
Ok(schemas) => DataTableSchema { datatable_name, schemas, error: None },
Err(e) => DataTableSchema {
datatable_name,
@@ -2408,7 +2433,7 @@ async fn list_datatable_schemas(
}
async fn list_datatable_tables(
_authed: ApiAuthed,
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
) -> JsonResult<Vec<DataTableTables>> {
@@ -2416,7 +2441,7 @@ async fn list_datatable_tables(
let mut results = Vec::new();
for datatable_name in datatable_names {
let tables = match get_datatable_tables(&db, &w_id, &datatable_name).await {
let tables = match get_datatable_tables(&db, &authed, &w_id, &datatable_name).await {
Ok(schemas) => DataTableTables { datatable_name, schemas, error: None },
Err(e) => DataTableTables {
datatable_name,
@@ -2431,13 +2456,14 @@ async fn list_datatable_tables(
}
async fn get_datatable_table_schema(
_authed: ApiAuthed,
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Query(query): Query<GetDataTableSchemaQuery>,
) -> JsonResult<DataTableTableSchema> {
let columns = get_datatable_table_columns(
&db,
&authed,
&w_id,
&query.datatable_name,
&query.schema_name,
@@ -2469,13 +2495,39 @@ async fn list_datatable_names(db: &DB, w_id: &str) -> Result<Vec<String>> {
.collect())
}
async fn get_datatable_schema(db: &DB, w_id: &str, datatable_name: &str) -> Result<SchemaMap> {
// Get the datatable resource (connection credentials)
let db_resource = get_datatable_resource_from_db_unchecked(db, w_id, datatable_name).await?;
/// Connect to a data table as the caller, not as `admin`: the role they named, or the data table's
/// default. A data table not under roles resolves as `admin`, exactly as it did before roles.
///
/// Every schema-browsing query below then reports what this Postgres role can actually reach,
/// which is why they filter on `has_schema_privilege` — `pg_catalog` is world-readable, so an
/// unfiltered listing would name schemas the connection cannot even enter.
async fn resolve_datatable_pg_as_caller(
db: &DB,
authed: &ApiAuthed,
w_id: &str,
datatable_name: &str,
) -> Result<PgDatabase> {
let db_resource = get_datatable_resource_from_db(
db,
w_id,
datatable_name,
// The data table's default role. Browsing has no way to name another one yet; when the
// database manager grows a role picker it passes the pick through here.
None,
DatatableAccess::Authed(authed.to_authed_ref()),
)
.await?;
serde_json::from_value(db_resource)
.map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))
}
// Parse the resource as PgDatabase
let pg_db: PgDatabase = serde_json::from_value(db_resource)
.map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?;
async fn get_datatable_schema(
db: &DB,
authed: &ApiAuthed,
w_id: &str,
datatable_name: &str,
) -> Result<SchemaMap> {
let pg_db = resolve_datatable_pg_as_caller(db, authed, w_id, datatable_name).await?;
// Connect to the datatable database
let (client, connection) = pg_db.connect(Some(db)).await?;
@@ -2495,6 +2547,7 @@ async fn get_datatable_schema(db: &DB, w_id: &str, datatable_name: &str) -> Resu
FROM pg_namespace
WHERE nspname NOT IN ('information_schema', 'pg_toast', 'pg_catalog')
AND nspname NOT LIKE 'pg_%'
AND has_schema_privilege(oid, 'USAGE')
ORDER BY nspname
"#,
&[],
@@ -2562,10 +2615,13 @@ async fn get_datatable_schema(db: &DB, w_id: &str, datatable_name: &str) -> Resu
Ok(schema_map)
}
async fn get_datatable_tables(db: &DB, w_id: &str, datatable_name: &str) -> Result<TableListMap> {
let db_resource = get_datatable_resource_from_db_unchecked(db, w_id, datatable_name).await?;
let pg_db: PgDatabase = serde_json::from_value(db_resource)
.map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?;
async fn get_datatable_tables(
db: &DB,
authed: &ApiAuthed,
w_id: &str,
datatable_name: &str,
) -> Result<TableListMap> {
let pg_db = resolve_datatable_pg_as_caller(db, authed, w_id, datatable_name).await?;
let (client, connection) = pg_db.connect(Some(db)).await?;
tokio::spawn(async move {
@@ -2581,6 +2637,7 @@ async fn get_datatable_tables(db: &DB, w_id: &str, datatable_name: &str) -> Resu
FROM pg_namespace
WHERE nspname NOT IN ('information_schema', 'pg_toast', 'pg_catalog')
AND nspname NOT LIKE 'pg_%'
AND has_schema_privilege(oid, 'USAGE')
ORDER BY nspname
"#,
&[],
@@ -2629,6 +2686,7 @@ async fn get_datatable_tables(db: &DB, w_id: &str, datatable_name: &str) -> Resu
async fn get_datatable_table_columns(
db: &DB,
authed: &ApiAuthed,
w_id: &str,
datatable_name: &str,
schema_name: &str,
@@ -2641,9 +2699,7 @@ async fn get_datatable_table_columns(
)));
}
let db_resource = get_datatable_resource_from_db_unchecked(db, w_id, datatable_name).await?;
let pg_db: PgDatabase = serde_json::from_value(db_resource)
.map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?;
let pg_db = resolve_datatable_pg_as_caller(db, authed, w_id, datatable_name).await?;
let (client, connection) = pg_db.connect(Some(db)).await?;
tokio::spawn(async move {
@@ -2853,7 +2909,10 @@ mod tests {
}
/// Resolve a source string to PgDatabase credentials with user-scoped permission checks.
/// For `datatable://name`: accessible to everyone (variables are resolved internally).
///
/// For `datatable://name`: the **admin** connection, so it is gated on admin reach. Every caller
/// copies, dumps or drops a whole database, and a dump taken under a restricted role would be a
/// silently truncated copy rather than an error — which is worse than refusing.
/// For `$res:path`: uses UserDB (row-level security) to verify the user can see the resource,
/// then interpolates `$var:` references in the resource value.
pub(crate) async fn resolve_pg_source_checked(
@@ -2864,6 +2923,13 @@ pub(crate) async fn resolve_pg_source_checked(
source: &str,
) -> Result<PgDatabase> {
let db_resource = if let Some(name) = source.strip_prefix("datatable://") {
windmill_common::workspaces::ensure_datatable_admin_access(
db,
w_id,
name,
&DatatableAccess::Authed(authed.to_authed_ref()),
)
.await?;
get_datatable_resource_from_db_unchecked(db, w_id, name).await?
} else if let Some(path) = source.strip_prefix("$res:") {
let db_with_authed = windmill_common::db::DbWithOptAuthed::from_authed(
@@ -2904,22 +2970,13 @@ pub(crate) async fn resolve_pg_source_checked(
/// Whether the data table `name` is backed by the Windmill instance's own PostgreSQL
/// rather than a user resource.
pub(crate) async fn is_instance_datatable(db: &DB, w_id: &str, name: &str) -> Result<bool> {
let config = sqlx::query_scalar!(
"SELECT datatable->'datatables'->$2 FROM workspace_settings WHERE workspace_id = $1",
w_id,
name
)
.fetch_optional(db)
.await?
.flatten();
Ok(config
.and_then(|v| {
v.get("database")
.and_then(|d| d.get("resource_type"))
.and_then(|r| r.as_str())
.map(|s| s == "instance")
})
.unwrap_or(false))
// Resolved rather than read: a pointer entry owns no database of its own, so only the entry it
// lands on can answer. A name that resolves to nothing keeps the historical `false`.
Ok(resolve_governing_datatable(db, w_id, name)
.await
.ok()
.and_then(|g| g.datatable.database)
.is_some_and(|d| d.resource_type == DataTableCatalogResourceType::Instance))
}
/// Same, for the `datatable://<name>` / `$res:<path>` form the import endpoints take.
@@ -3250,6 +3307,14 @@ async fn create_pg_database(
) -> Result<String> {
windmill_common::validate_dbname(&req.target_dbname)?;
// The copy this database is for is refused a call later, and nothing collects an instance
// database that no data table entry names. Refuse here too, so the clone stops before one
// exists rather than leaving an empty registered `wm_fork_…` behind.
if let Some(reference) = req.source.strip_prefix("datatable://") {
let (name, _) = parse_datatable_ref_for(&db, &w_id, reference).await?;
ensure_datatable_is_clonable(&db, &w_id, &name).await?;
}
// Non-superadmin: restrict dbname to wm_fork_ prefix
if !windmill_api_auth::is_super_admin_authed(&db, &authed).await? {
if !req.target_dbname.starts_with("wm_fork_") {
@@ -3319,6 +3384,64 @@ struct ImportPgDatabaseRequest {
fork_behavior: DataTableForkBehavior,
}
/// Refuse to copy a data table that is under roles.
///
/// `pg_dump` carries no roles and the import runs with `--no-privileges`, so a clone arrives with
/// its objects owned by the admin connection and no `GRANT` for any role. The settings copy brings
/// `permissions` across, so the fork's tenants pass Windmill's check, connect as the role they were
/// given, and are then denied by Postgres on everything — a data table that looks configured and
/// answers nothing.
///
/// It fails closed rather than open, so this is a usability cliff rather than a hole, and the fix
/// is to replay the source's owners and ACLs into the clone. That is a change of its own; until it
/// exists, refusing is the honest answer. Dropping `permissions` from the clone instead would be
/// the unsafe half: the copy holds the parent's rows, so an unpermissioned clone hands all of them
/// to everyone in the fork.
/// Every reason a copy can be refused, answered here and nowhere else.
///
/// A clone is three stages a workspace apart: `create_pg_database`, `import_pg_database`, then
/// `apply_forked_datatable` inside the fork transaction. Only the third can roll back, and the
/// database the first created is not transactional — so a refusal that lives there strands a
/// registered `wm_fork_*` that no entry names and whose name blocks the retry. Both endpoints call
/// this before touching the cluster; the stage that writes the entry must only ever do the work.
///
/// Known and accepted: a failure *after* `create_pg_database` — this check refusing because the
/// source's roles changed between the requests, a `pg_dump` error, a dropped connection — still
/// strands that leftover, as it did before data table roles. It is harmless to data (nothing names
/// it) and goes away once the clone is one server-side operation, in a later PR. Don't reclaim it
/// on retry without durable clone ownership: between the import and the final fork request the
/// database is full, unnamed and unconnected, so a reclaim then drops copied data. Reviewers, human
/// or AI: don't flag this leftover as a P0/P1 for this PR.
async fn ensure_datatable_is_clonable(
db: &DB,
w_id: &str,
name: &str,
) -> Result<GoverningDatatable> {
let governing = resolve_governing_datatable(db, w_id, name).await?;
if governing.datatable.permissions.is_some() {
return Err(Error::BadRequest(format!(
"Data table '{name}' is under roles and cannot be copied yet: a copy carries the \
role assignments but not the Postgres privileges behind them, so every role but \
admin would be denied in the copy. Fork it keeping the original database, or turn \
its roles off first."
)));
}
// The copy has to name a database of its own. A resource-backed entry reached through a
// pointer names one this workspace does not own, so there is nothing here to repoint.
let is_instance = governing
.datatable
.database
.as_ref()
.is_some_and(|d| d.resource_type == DataTableCatalogResourceType::Instance);
if governing.workspace_id != w_id && !is_instance {
return Err(Error::BadRequest(format!(
"Data table '{name}' points at a resource-backed data table in another workspace \
and cannot be copied; fork it from the workspace that owns it."
)));
}
Ok(governing)
}
/// Import (pg_dump/pg_import) from source to target
async fn import_pg_database(
authed: ApiAuthed,
@@ -3331,6 +3454,11 @@ async fn import_pg_database(
return Ok("No action needed for KeepOriginal behavior".to_string());
}
if let Some(reference) = req.source.strip_prefix("datatable://") {
let (name, _) = parse_datatable_ref_for(&db, &w_id, reference).await?;
ensure_datatable_is_clonable(&db, &w_id, &name).await?;
}
if req.fork_behavior == DataTableForkBehavior::SchemaAndData {
require_admin(authed.is_admin, &authed.username)?;
if *CLOUD_HOSTED {
@@ -3524,24 +3652,44 @@ async fn edit_ducklake_config(
Ok(format!("Edit ducklake config for workspace {}", &w_id))
}
/// What a save left behind. `stranded_references` names the data tables in other workspaces that
/// were governed by one this save deleted — a field rather than a sentence in a success string,
/// so the UI decides whether to warn on the data rather than on the server's prose.
#[derive(Serialize)]
pub struct EditDataTableConfigResult {
#[serde(skip_serializing_if = "Vec::is_empty")]
stranded_references: Vec<StrandedReference>,
}
#[derive(Serialize)]
pub struct StrandedReference {
workspace_id: String,
datatable: String,
}
async fn edit_datatable_config(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
ApiAuthed { is_admin, username, .. }: ApiAuthed,
Json(mut new_config): Json<EditDataTableConfig>,
) -> Result<String> {
) -> JsonResult<EditDataTableConfigResult> {
require_admin(is_admin, &username)?;
let is_superadmin = require_super_admin(&db, &authed).await.is_ok();
let mut tx = db.begin().await?;
// Read under the row lock this transaction will write with. `permissions`, `reference` and
// `forked_from` are carried across from what this read returns, so a permissions save
// committing between the read and the whole-document write below would be silently rolled back
// by it.
let old_datatables: HashMap<String, DataTable> = serde_json::from_value(
sqlx::query_scalar!(
"SELECT ws.datatable->'datatables' FROM workspace_settings ws WHERE ws.workspace_id = $1",
"SELECT ws.datatable->'datatables' FROM workspace_settings ws
WHERE ws.workspace_id = $1 FOR UPDATE",
&w_id
)
.fetch_one(&db)
.fetch_one(&mut *tx)
.await?
.unwrap_or(serde_json::Value::Null),
)
@@ -3563,6 +3711,58 @@ async fn edit_datatable_config(
crate::datatable_migrations::validate_datatable_path_segment(&r.from)?;
crate::datatable_migrations::validate_new_datatable_name(&r.to)?;
}
// A rename is a claim about what this save is doing, and other workspaces' pointers are
// rewritten from it — so the claim has to match the configuration it describes, or a caller
// can move every fork of one data table onto another by asserting a rename that did not
// happen. The shape below is what "these old keys became those new keys" actually means.
{
let old_keys = &old_datatables;
let new_keys = &new_config.settings.datatables;
let froms: std::collections::HashSet<&str> =
new_config.renames.iter().map(|r| r.from.as_str()).collect();
let tos: std::collections::HashSet<&str> =
new_config.renames.iter().map(|r| r.to.as_str()).collect();
if froms.len() != new_config.renames.len() {
return Err(Error::BadRequest(
"A data table is renamed twice in one save".to_string(),
));
}
if tos.len() != new_config.renames.len() {
return Err(Error::BadRequest(
"Two data tables are renamed to the same name in one save".to_string(),
));
}
for r in &new_config.renames {
if !old_keys.contains_key(&r.from) {
return Err(Error::BadRequest(format!(
"Cannot rename data table '{}': this workspace has no such data table",
r.from
)));
}
if !new_keys.contains_key(&r.to) {
return Err(Error::BadRequest(format!(
"Cannot rename data table '{}' to '{}': the save does not contain '{}'",
r.from, r.to, r.to
)));
}
// The source has to be gone, or gone-and-reoccupied by another rename — which is what
// a swap is. Without this, `main -> decoy` passes against a save that keeps both, and
// every fork of `main` silently follows onto a different data table.
if new_keys.contains_key(&r.from) && !tos.contains(r.from.as_str()) {
return Err(Error::BadRequest(format!(
"Data table '{}' is renamed to '{}' but the save still contains '{}'",
r.from, r.to, r.from
)));
}
// And the target has to be free, or freed by another rename.
if old_keys.contains_key(&r.to) && !froms.contains(r.to.as_str()) {
return Err(Error::BadRequest(format!(
"Cannot rename data table '{}' to '{}': '{}' already exists",
r.from, r.to, r.to
)));
}
}
}
// Map new name -> old name so a renamed data table inherits the previous
// flag instead of being treated as brand new.
@@ -3583,18 +3783,52 @@ async fn edit_datatable_config(
.get(name.as_str())
.copied()
.unwrap_or(name.as_str());
dt.migrations_enabled = match old_datatables.get(lookup) {
let old = old_datatables.get(lookup);
dt.migrations_enabled = match old {
Some(old) => old.migrations_enabled,
None => {
// Keyed by how the substrate is serialized into `workspace_settings`,
// so these line up with the `datatable_configured` adoption counts.
created_substrates.push(match dt.database.resource_type {
DataTableCatalogResourceType::Instance => "instance",
DataTableCatalogResourceType::Postgresql => "postgresql",
created_substrates.push(match dt.database.as_ref().map(|d| d.resource_type) {
Some(DataTableCatalogResourceType::Instance) => "instance",
Some(DataTableCatalogResourceType::Postgresql) => "postgresql",
None => "reference",
});
Some(true)
}
};
// Carried across from the stored entry rather than taken from the request. `permissions`
// is an access decision, edited through its own endpoint; `reference` is what makes a fork
// answer to the workspace that governs its data table, and letting a save clear it would
// hand the fork the database outright. `forked_from` is the clone stamp the fork flow
// writes: whether an entry has one is carried the same way, since it is what marks the
// database droppable, but the schema baseline inside it is the diff view's to advance.
dt.permissions = old.and_then(|old| old.permissions.clone());
dt.reference = old.and_then(|old| old.reference.clone());
dt.forked_from = match old.and_then(|old| old.forked_from.as_ref()) {
Some(stored) => Some(dt.forked_from.take().unwrap_or_else(|| stored.clone())),
None => None,
};
// Carrying the block onto a resource-backed entry would produce a data table the chokepoint
// refuses on every job — a save that succeeds and breaks everything afterwards. Refuse it
// instead: turning roles off first is one step, and it keeps discarding an access decision
// something somebody chose rather than a side effect of moving a database.
if dt.permissions.is_some()
&& dt
.database
.as_ref()
.is_some_and(|d| d.resource_type != DataTableCatalogResourceType::Instance)
{
return Err(Error::BadRequest(format!(
"Data table '{name}' is under roles, which only a data table on the instance \
database can be. Turn its roles off before moving it to a PostgreSQL resource."
)));
}
// A pointer names no database of its own, so the form's empty `database` is correct there.
if dt.reference.is_some() {
dt.database = None;
}
windmill_common::workspaces::validate_datatable_shape(name, dt)?;
}
let args_for_audit = format!("{:?}", new_config.settings);
@@ -3609,16 +3843,23 @@ async fn edit_datatable_config(
)
.await?;
// Check that non-superadmins are not abusing Instance databases
// Check that non-superadmins are not abusing Instance databases, which reach a database this
// workspace does not own. Pointing an entry at another workspace's data table is not checked
// here because it cannot be requested at all: `reference` is overwritten from the stored entry
// above, for every caller.
if !is_superadmin {
for (name, dt) in new_config.settings.datatables.iter() {
if dt.database.resource_type == DataTableCatalogResourceType::Instance {
let old_dt = old_datatables.get(name);
if old_dt.is_none()
|| old_dt.unwrap().database.resource_type
!= DataTableCatalogResourceType::Instance
|| old_dt.unwrap().database.resource_path != dt.database.resource_path
{
let old_dt = old_datatables.get(name);
if dt
.database
.as_ref()
.is_some_and(|d| d.resource_type == DataTableCatalogResourceType::Instance)
{
let unchanged = old_dt.and_then(|o| o.database.as_ref()).is_some_and(|o| {
o.resource_type == DataTableCatalogResourceType::Instance
&& Some(&o.resource_path) == dt.database.as_ref().map(|d| &d.resource_path)
});
if !unchanged {
return Err(Error::BadRequest(
"Only superadmins can create or modify data tables with Instance databases"
.to_string(),
@@ -3628,6 +3869,84 @@ async fn edit_datatable_config(
}
}
// Worked out from the locked entries rather than taken from `deleted_datatables`: a settings
// sync sends the whole map without that list, and dropping a governing entry strands every
// fork pointing at it all the same.
let removed: Vec<String> = old_datatables
.keys()
.filter(|name| {
!new_config.settings.datatables.contains_key(*name)
&& !new_config.renames.iter().any(|r| &r.from == *name)
})
.cloned()
.collect();
// A database under roles is reached only through an entry that carries them. Roles follow an
// entry through a declared rename alone, and a settings sync never declares one, so an entry
// without roles that newly points at such a database — a name added, or an existing one
// repointed — would answer everyone there as `admin`. That holds whichever workspace governs it.
let newly_pointed: Vec<(&String, &str)> = new_config
.settings
.datatables
.iter()
.filter(|(_, dt)| dt.permissions.is_none())
.filter_map(|(name, dt)| {
let db = dt
.database
.as_ref()
.filter(|d| d.resource_type == DataTableCatalogResourceType::Instance)?;
let lookup = rename_src
.get(name.as_str())
.copied()
.unwrap_or(name.as_str());
let repointed = old_datatables
.get(lookup)
.and_then(|old| old.database.as_ref())
.is_none_or(|old_db| {
old_db.resource_type != db.resource_type
|| old_db.resource_path != db.resource_path
});
repointed.then_some((name, db.resource_path.as_str()))
})
.collect();
// Another workspace turning roles on for the same database holds only its own settings row, so
// without this the scan below could read past its uncommitted write.
windmill_common::datatable_roles::lock_instance_databases_governance(
&mut *tx,
newly_pointed.iter().map(|(_, dbname)| *dbname),
)
.await?;
let governed_elsewhere: Vec<String> = if newly_pointed.is_empty() {
vec![]
} else {
sqlx::query_scalar(
"SELECT DISTINCT dt.value->'database'->>'resource_path' FROM workspace_settings ws
CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt
WHERE ws.workspace_id <> $1 AND dt.value ? 'permissions'
AND dt.value->'database'->>'resource_type' = 'instance'",
)
.bind(&w_id)
.fetch_all(&mut *tx)
.await?
};
for (name, dbname) in newly_pointed {
let governed_here = old_datatables.values().any(|old| {
old.permissions.is_some()
&& old.database.as_ref().is_some_and(|d| {
d.resource_type == DataTableCatalogResourceType::Instance
&& d.resource_path == dbname
})
});
if governed_here || governed_elsewhere.iter().any(|g| g == dbname) {
return Err(Error::BadRequest(format!(
"Data table '{name}' would point at database '{dbname}', which a data table under \
roles uses, without carrying those roles: everyone reaching '{name}' would connect \
there as `admin`. Rename the data table under roles from the data table settings, \
which carries its roles, or turn its roles off first."
)));
}
}
let config: serde_json::Value = serde_json::to_value(new_config.settings)
.map_err(|err| Error::internal_err(err.to_string()))?;
@@ -3649,6 +3968,44 @@ async fn edit_datatable_config(
)
.await?;
// A fork points at a data table by name, so a rename here has to follow or every fork's entry
// resolves to nothing. In two passes through a temporary name, like the migration cascade one
// layer down: applied in order, `sa -> sb` then `sb -> sa` would move what pointed at `sa` all
// the way back to `sa`, and `A -> B`, `B -> C` would carry `A`'s pointers to `C`. Each pointer
// moves once, from what it named before this save. Inside the transaction: the rename and the
// pointers that name it are one change, and half of it is a fork whose jobs stop.
for (i, r) in new_config.renames.iter().enumerate() {
repoint_datatable_references(&mut tx, &w_id, &r.from, &format!("__wm_rename_tmp/{i}"))
.await?;
}
for (i, r) in new_config.renames.iter().enumerate() {
repoint_datatable_references(&mut tx, &w_id, &format!("__wm_rename_tmp/{i}"), &r.to)
.await?;
}
// A deletion cannot be followed the same way — there is nothing to point at any more. Read who
// is left stranded so the caller is told, the way deleting a workspace does.
let mut stranded: Vec<StrandedReference> = Vec::new();
for name in &removed {
let rows = sqlx::query!(
r#"SELECT ws.workspace_id AS "workspace_id!", dt.key AS "datatable!"
FROM workspace_settings ws
CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt
WHERE dt.value->'reference'->>'workspace_id' = $1
AND dt.value->'reference'->>'datatable' = $2"#,
&w_id,
name,
)
.fetch_all(&mut *tx)
.await?;
stranded.extend(
rows.into_iter().map(|r| StrandedReference {
workspace_id: r.workspace_id,
datatable: r.datatable,
}),
);
}
tx.commit().await?;
for substrate in created_substrates {
@@ -3663,7 +4020,9 @@ async fn edit_datatable_config(
)
.await?;
Ok(format!("Edit datatable config for workspace {}", &w_id))
Ok(Json(EditDataTableConfigResult {
stranded_references: stranded,
}))
}
#[derive(Deserialize)]
@@ -7545,13 +7904,144 @@ async fn snapshot_datatable_schema(
.map_err(|e| Error::internal_err(format!("Failed to serialize schema: {}", e)))
}
/// Turn every data table the fork chose to keep into a pointer at the parent's entry.
///
/// `clone_workspace_data` copies `workspace_settings` wholesale, so a kept data table arrives as a
/// byte-identical copy naming the parent's database — including the parent's `permissions`, which
/// a fork admin could then edit to widen their own access to it. A pointer has nothing local to
/// edit: the parent's entry stays the only place the decision lives.
///
/// The cloned data tables are skipped: they own a fresh database of their own, and they keep the
/// copied `permissions` as their starting point, which they then govern.
async fn point_kept_datatables_at_parent(
tx: &mut Transaction<'_, Postgres>,
parent_w_id: &str,
forked_w_id: &str,
cloned: &[ForkedDatatableInfo],
) -> Result<()> {
let settings: Option<serde_json::Value> = sqlx::query_scalar!(
"SELECT datatable FROM workspace_settings WHERE workspace_id = $1",
forked_w_id
)
.fetch_optional(&mut **tx)
.await?
.flatten();
let Some(mut settings) = settings else {
return Ok(());
};
let Some(datatables) = settings
.get_mut("datatables")
.and_then(|d| d.as_object_mut())
else {
return Ok(());
};
let mut changed = false;
for (name, entry) in datatables.iter_mut() {
if cloned.iter().any(|c| &c.name == name) {
continue;
}
let dt: DataTable = match serde_json::from_value(entry.clone()) {
Ok(dt) => dt,
Err(_) => continue,
};
// Already a pointer: the parent was itself a fork, and its entry names the workspace that
// governs. Following it from here is the same answer, so leave it alone.
if dt.reference.is_some() {
continue;
}
// Only instance databases. A resource-backed data table names a resource, and the settings
// clone gave the fork its own copy of that resource in its own workspace — pointing at the
// parent's entry would silently move the fork onto the parent's resource instead.
if dt
.database
.as_ref()
.is_none_or(|d| d.resource_type != DataTableCatalogResourceType::Instance)
{
continue;
}
*entry = serde_json::to_value(DataTable {
database: None,
reference: Some(windmill_common::workspaces::DataTableReference {
workspace_id: parent_w_id.to_string(),
datatable: name.clone(),
}),
forked_from: None,
migrations_enabled: dt.migrations_enabled,
permissions: None,
})
.map_err(|e| Error::internal_err(format!("serializing data table '{name}': {e}")))?;
changed = true;
}
if changed {
sqlx::query!(
"UPDATE workspace_settings SET datatable = $1 WHERE workspace_id = $2",
settings,
forked_w_id
)
.execute(&mut **tx)
.await?;
}
Ok(())
}
/// Move every pointer in any workspace that names `(w_id, from)` to `(w_id, to)`.
///
/// `EXISTS` rather than a `LIKE` over the whole document: the update rewrites the row, so matching
/// every workspace that holds any pointer would rewrite rows to a byte-identical value and hold an
/// exclusive lock on them until commit.
async fn repoint_datatable_references(
tx: &mut Transaction<'_, Postgres>,
w_id: &str,
from: &str,
to: &str,
) -> Result<()> {
sqlx::query!(
r#"UPDATE workspace_settings ws
SET datatable = (
SELECT jsonb_set(ws.datatable, '{datatables}', jsonb_object_agg(
dt.key,
CASE WHEN dt.value->'reference'->>'workspace_id' = $1
AND dt.value->'reference'->>'datatable' = $2
THEN jsonb_set(dt.value, '{reference,datatable}', to_jsonb($3::text))
ELSE dt.value END
))
FROM jsonb_each(ws.datatable->'datatables') dt
)
WHERE EXISTS (
SELECT 1 FROM jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) d
WHERE d.value->'reference'->>'workspace_id' = $1
AND d.value->'reference'->>'datatable' = $2
)"#,
w_id,
from,
to,
)
.execute(&mut **tx)
.await?;
Ok(())
}
async fn apply_forked_datatable(
db: &DB,
tx: &mut Transaction<'_, Postgres>,
authed: &ApiAuthed,
parent_w_id: &str,
forked_w_id: &str,
fdt: &ForkedDatatableInfo,
) -> Result<()> {
// Cloning reads the parent's whole schema as admin and hands the copy to the fork, so it is
// for the workspace that governs the data table — a fork can use one, never duplicate it.
windmill_common::workspaces::ensure_datatable_admin_access(
db,
parent_w_id,
&fdt.name,
&DatatableAccess::Authed(authed.to_authed_ref()),
)
.await?;
let governing = ensure_datatable_is_clonable(db, parent_w_id, &fdt.name).await?;
windmill_common::validate_dbname(&fdt.new_dbname)?;
if !fdt.new_dbname.starts_with("wm_fork_") {
return Err(Error::BadRequest(format!(
@@ -7583,25 +8073,46 @@ async fn apply_forked_datatable(
let dt: DataTable = serde_json::from_value(config_val)
.map_err(|e| Error::internal_err(format!("Failed to parse datatable config: {}", e)))?;
if dt.database.resource_type == DataTableCatalogResourceType::Instance {
// Instance: update resource_path to the new dbname
// A clone owns its copy, so the fork's entry has to be terminal. When the parent was itself a
// fork the settings clone hands down a pointer instead, and what it points at is what the copy
// was taken from. `ensure_datatable_is_clonable` already settled that this shape can be
// cloned, so there is nothing left to refuse here — by now the database exists and is filled.
let database = match dt.database.clone() {
Some(database) => database,
None => governing.datatable.database.clone().ok_or_else(|| {
Error::internal_err(format!(
"Data table '{}' resolves to an entry that owns no database",
fdt.name
))
})?,
};
if database.resource_type == DataTableCatalogResourceType::Instance {
// The whole `database` object, not just its `resource_path`: a pointer entry has none to
// patch. `reference` goes with it — exactly one of the two may be set.
let new_database = serde_json::json!({
"resource_type": "instance",
"resource_path": &fdt.new_dbname,
});
sqlx::query!(
r#"UPDATE workspace_settings
SET datatable = jsonb_set(
jsonb_set(datatable, ARRAY['datatables', $2, 'database', 'resource_path'], to_jsonb($3::text)),
jsonb_set(
datatable #- ARRAY['datatables', $2, 'reference'],
ARRAY['datatables', $2, 'database'], $3::jsonb),
ARRAY['datatables', $2, 'forked_from'], $4::jsonb
)
WHERE workspace_id = $1"#,
forked_w_id,
&fdt.name,
&fdt.new_dbname,
new_database,
forked_from,
)
.execute(&mut **tx)
.await?;
} else {
// Resource: update the resource's dbname and mark as ws_specific
let resource_path = &dt.database.resource_path;
let resource_path = &database.resource_path;
sqlx::query!(
r#"UPDATE resource
SET value = jsonb_set(value, '{dbname}', to_jsonb($3::text))
@@ -8038,6 +8549,14 @@ async fn create_workspace_fork(
.execute(&mut *tx)
.await?;
// The pointers this fork writes to the parent's data tables stay invisible until it commits, so
// a rename of one of them cannot carry them. Holding the parent's settings row makes such a
// rename wait for this commit, and makes the copy below read one that committed first.
sqlx::query("SELECT 1 FROM workspace_settings WHERE workspace_id = $1 FOR SHARE")
.bind(&parent_workspace_id)
.execute(&mut *tx)
.await?;
// Clone all data from the parent workspace using Rust implementation
if let Err(e) =
clone_workspace_data(&mut tx, &db, &parent_workspace_id, &forked_id, &authed).await
@@ -8075,9 +8594,18 @@ async fn create_workspace_fork(
// Update forked datatable settings to point to new databases
for fdt in &nw.forked_datatables {
apply_forked_datatable(&db, &mut tx, &parent_workspace_id, &forked_id, fdt).await?;
apply_forked_datatable(&db, &mut tx, &authed, &parent_workspace_id, &forked_id, fdt)
.await?;
}
point_kept_datatables_at_parent(
&mut tx,
&parent_workspace_id,
&forked_id,
&nw.forked_datatables,
)
.await?;
// The settings clone copies the source's ducklake config verbatim — including a parent
// fork's own `fork_behavior` stamps. Sharing is a per-fork-creation choice, never
// inherited: reset any cloned stamps first, then apply this fork's requested list.
@@ -8880,6 +9408,14 @@ async fn leave_workspace(
) -> Result<String> {
windmill_api_auth::forbid_job_token_account_destruction(&authed)?;
let mut tx = db.begin().await?;
// The membership is what made `u/<username>` mean this person. Leaving it behind in a tenant
// list would hand their data table access back on rejoin, or to whoever takes the name next.
windmill_common::workspaces::remove_datatable_tenant_in_workspace(
&mut tx,
&w_id,
&format!("u/{}", authed.username),
)
.await?;
sqlx::query!(
"DELETE FROM usr WHERE workspace_id = $1 AND email = $2",
&w_id,
backend/windmill-api-workspaces/src/workspaces_extra.rs
+66 -5
View File
@@ -492,6 +492,30 @@ pub(crate) async fn change_workspace_id(
.fetch_all(&mut *tx)
.await?;
// A fork's data table entry names the workspace that governs it by id, so the rename has to
// follow there too — anywhere, not just in the reparented children: a detached workspace can
// point at this one without being its fork. Left behind, the pointer resolves to the archived
// shell and every job through it stops.
info!("Re-pointing data table references to the new workspace id");
sqlx::query!(
r#"UPDATE workspace_settings ws
SET datatable = (
SELECT jsonb_set(ws.datatable, '{datatables}', jsonb_object_agg(
dt.key,
CASE WHEN dt.value->'reference'->>'workspace_id' = $2
THEN jsonb_set(dt.value, '{reference,workspace_id}', to_jsonb($1::text))
ELSE dt.value END
))
FROM jsonb_each(ws.datatable->'datatables') dt
)
WHERE jsonb_typeof(ws.datatable->'datatables') = 'object'
AND ws.datatable::text LIKE '%"reference"%'"#,
&rw.new_id,
&old_id,
)
.execute(&mut *tx)
.await?;
info!("Updating workspace_protection_rule table");
sqlx::query!(
"UPDATE workspace_protection_rule SET workspace_id = $1 WHERE workspace_id = $2",
@@ -971,6 +995,22 @@ pub(crate) async fn delete_workspace(
// but the destructive cleanup itself runs only after the commit below: a delete that
// fails mid-way must never leave a live workspace with its fork data destroyed and no
// registry row to retry from. Read-only: nothing is dropped here.
// Read before the delete: another workspace's data table entry can point at one of this
// workspace's, and deleting the workspace it names leaves that pointer resolving to nothing.
// Nothing sweeps them — turning them back into copies would hand each fork the database
// outright — so the deleter is told which data tables they just stranded.
let stranded_pointers = sqlx::query!(
r#"SELECT ws.workspace_id AS "workspace_id!", dt.key AS "datatable!"
FROM workspace_settings ws
CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt
WHERE dt.value->'reference'->>'workspace_id' = $1
ORDER BY ws.workspace_id, dt.key"#,
&w_id,
)
.fetch_all(&db)
.await
.unwrap_or_default();
let fork_ducklake_cleanups = prepare_fork_ducklake_cleanups(&db, &w_id, None)
.await
.unwrap_or_else(|e| {
@@ -1289,7 +1329,23 @@ pub(crate) async fn delete_workspace(
tracing::warn!("failed to broadcast fork lineage change: {e:#}");
}
Ok(format!("Deleted workspace {}", &w_id))
if stranded_pointers.is_empty() {
Ok(format!("Deleted workspace {}", &w_id))
} else {
let stranded = stranded_pointers
.iter()
.map(|r| format!("{}/{}", r.workspace_id, r.datatable))
.collect::<Vec<_>>()
.join(", ");
Ok(format!(
concat!(
"Deleted workspace {}. These data tables were governed by it and no longer ",
"resolve: {}. Their databases still exist; a superadmin can point them at ",
"another workspace's data table."
),
&w_id, stranded
))
}
}
#[derive(Deserialize)]
@@ -1343,15 +1399,20 @@ pub async fn drop_forked_datatable_databases(
let mut errors: Vec<String> = Vec::new();
for dt_name in &req.datatable_names {
let dt = match datatables.get(dt_name) {
Some(dt) if dt.forked_from.is_some() => dt,
// Only a clone is droppable, and a clone is terminal by construction: a kept data table is
// a pointer at the parent's database, which this fork does not own.
let database = match datatables.get(dt_name) {
Some(dt) if dt.forked_from.is_some() => match dt.database.as_ref() {
Some(database) => database,
None => continue,
},
_ => continue,
};
if dt.database.resource_type
if database.resource_type
== windmill_common::workspaces::DataTableCatalogResourceType::Instance
{
let db_to_drop = &dt.database.resource_path;
let db_to_drop = &database.resource_path;
if !db_to_drop.starts_with("wm_fork_") {
errors.push(format!(
"Refusing to drop instance database '{}' for datatable://{}: name does not start with 'wm_fork_'",
backend/windmill-api/openapi.yaml
+266 -4
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.813.0
version: 1.814.0
title: Windmill API
contact:
@@ -1588,6 +1588,89 @@ paths:
additionalProperties:
$ref: "#/components/schemas/CustomInstanceDb"
/settings/datatable_roles:
get:
summary: list the instance's data table roles
operationId: listInstanceDatatableRoles
tags:
- setting
responses:
"200":
description: the instance role catalog
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/InstanceDatatableRole"
post:
summary: create a data table role on the instance's Postgres cluster
operationId: createInstanceDatatableRole
tags:
- setting
requestBody:
required: true
content:
application/json:
schema:
type: object
required: [name]
properties:
name:
type: string
responses:
"200":
description: the created role
content:
application/json:
schema:
$ref: "#/components/schemas/InstanceDatatableRole"
/settings/datatable_roles/{id}:
post:
summary: rename a data table role or turn its login on and off
operationId: updateInstanceDatatableRole
tags:
- setting
parameters:
- in: path
name: id
required: true
schema:
type: string
requestBody:
required: true
content:
application/json:
schema:
type: object
properties:
name:
type: string
enabled:
type: boolean
responses:
"200":
description: the updated role
content:
application/json:
schema:
$ref: "#/components/schemas/InstanceDatatableRole"
delete:
summary: drop a data table role from the cluster and from every workspace that named it
operationId: deleteInstanceDatatableRole
tags:
- setting
parameters:
- in: path
name: id
required: true
schema:
type: string
responses:
"200":
description: deleted
/settings/setup_custom_instance_pg_database/{name}:
post:
summary: Runs CREATE DATABASE on the Windmill Postgres and grants access to the custom_instance_user
@@ -5141,7 +5224,7 @@ paths:
type: array
items:
type: object
required: [name, resource_type, resource_path]
required: [name, resource_type, resource_path, permissioned]
properties:
name:
type: string
@@ -5150,6 +5233,97 @@ paths:
enum: [postgres, instance]
resource_path:
type: string
governing_workspace_id:
type: string
permissioned:
type: boolean
/w/{workspace}/workspaces/datatable_permissions/{datatable_name}:
get:
summary: get who may connect to a data table as which role
operationId: getDatatablePermissions
tags:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: datatable_name
in: path
required: true
schema:
type: string
responses:
"200":
description: the data table's roles and their tenants
content:
application/json:
schema:
$ref: "#/components/schemas/DatatablePermissions"
post:
summary: set who may connect to a data table as which role
operationId: setDatatablePermissions
tags:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: datatable_name
in: path
required: true
schema:
type: string
requestBody:
required: true
content:
application/json:
schema:
type: object
required: [permissioned]
properties:
permissioned:
type: boolean
default_role:
type: string
roles:
type: array
items:
$ref: "#/components/schemas/DatatableRoleTenants"
responses:
"200":
description: status
content:
text/plain:
schema:
type: string
/w/{workspace}/workspaces/datatable_usable_roles/{datatable_name}:
get:
summary: list the data table roles the caller may connect as
operationId: listUsableDatatableRoles
tags:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: datatable_name
in: path
required: true
schema:
type: string
responses:
"200":
description: usable roles
content:
application/json:
schema:
type: object
required: [permissioned, roles, default_role]
properties:
permissioned:
type: boolean
roles:
type: array
items:
type: string
default_role:
type: string
/w/{workspace}/workspaces/list_datatable_schemas:
get:
@@ -5331,7 +5505,22 @@ paths:
description: status
content:
application/json:
schema: {}
schema:
type: object
properties:
stranded_references:
description: >-
Data tables in other workspaces that were governed by one this save deleted
and no longer resolve.
type: array
items:
type: object
required: [workspace_id, datatable]
properties:
workspace_id:
type: string
datatable:
type: string
/w/{workspace}/workspaces/run_datatable_migrations/{datatable_name}:
post:
@@ -33400,6 +33589,66 @@ components:
- ducklake
- datatable
InstanceDatatableRole:
type: object
required: [id, name, enabled]
properties:
id:
type: string
name:
type: string
enabled:
type: boolean
DatatableRoleTenants:
type: object
required: [id, tenants]
properties:
id:
type: string
name:
type: string
tenants:
type: array
items:
type: string
DatatablePermissions:
type: object
required: [supported, permissioned, default_role, roles, editable, available_roles]
properties:
supported:
type: boolean
description: >-
Whether this data table can be put under roles at all. Only one backed by the
instance database can: a role is a login on that cluster.
permissioned:
type: boolean
default_role:
type: string
roles:
type: array
items:
$ref: "#/components/schemas/DatatableRoleTenants"
governing_workspace_id:
type: string
editable:
type: boolean
available_roles:
type: array
items:
$ref: "#/components/schemas/InstanceDatatableRole"
ungoverned_reachers:
type: array
items:
type: object
required: [workspace_id, datatable]
properties:
workspace_id:
type: string
datatable:
type: string
CustomInstanceDb:
type: object
required:
@@ -35448,9 +35697,11 @@ components:
type: object
additionalProperties:
type: object
required: [database]
properties:
database:
description: >-
Set on an entry that owns its database. Absent on a fork's entry, which points at
another workspace's data table instead.
type: object
properties:
resource_type:
@@ -35462,6 +35713,17 @@ components:
type: string
required:
- resource_type
reference:
description: >-
The workspace and data table that govern this one. Server-owned: written by fork
creation, and carried across a settings save whatever the request says.
type: object
required: [workspace_id, datatable]
properties:
workspace_id:
type: string
datatable:
type: string
migrations_enabled:
type: boolean
description: Whether the SQL migrations feature is opted in for this data table
backend/windmill-api/src/capture.rs
+6
View File
@@ -567,6 +567,9 @@ async fn set_config(
};
let mut tx = user_db.begin(&authed).await?;
if matches!(nc.trigger_kind, TriggerKind::Postgres) {
windmill_common::datatable_roles::lock_datatable_streams(&mut *tx, false).await?;
}
sqlx::query!(
r#"
@@ -614,6 +617,9 @@ async fn ping_config(
)>,
) -> Result<()> {
let mut tx = user_db.begin(&authed).await?;
if matches!(trigger_kind, TriggerKind::Postgres) {
windmill_common::datatable_roles::lock_datatable_streams(&mut *tx, false).await?;
}
sqlx::query!(
r#"
backend/windmill-api/src/jobs.rs
+51 -28
View File
@@ -4369,12 +4369,12 @@ async fn count_completed_jobs_detail(
Query(query): Query<CountCompletedJobsQuery>,
) -> error::JsonResult<i64> {
let mut sqlb = SqlBuilder::select_from("v2_job_completed");
//FOR RLS
sqlb.join("v2_job USING (id)");
sqlb.field("COUNT(*) as count");
// Filtering on v2_job.workspace_id instead would keep the planner off
// ix_job_workspace_id_completed_at_all and scan the whole retention window.
if !(w_id == "admins" && query.all_workspaces.unwrap_or(false)) {
sqlb.and_where_eq("v2_job.workspace_id", "?".bind(&w_id));
sqlb.and_where_eq("v2_job_completed.workspace_id", "?".bind(&w_id));
}
if let Some(after_s_ago) = query.completed_after_s_ago {
@@ -4393,6 +4393,7 @@ async fn count_completed_jobs_detail(
}
if let Some(tags) = query.tags {
sqlb.join("v2_job USING (id)");
sqlb.and_where_in(
"v2_job.tag",
&tags.split(",").map(|t| quote(t)).collect::<Vec<_>>(),
@@ -4400,7 +4401,19 @@ async fn count_completed_jobs_detail(
}
let sql = sqlb.sql()?;
let stats = sqlx::query_scalar::<_, i64>(&sql).fetch_one(&db).await?;
let mut tx = db.begin().await?;
set_list_jobs_statement_timeout(&mut tx).await?;
let stats = sqlx::query_scalar::<_, i64>(&sql)
.fetch_one(&mut *tx)
.await
.map_err(|e| {
list_jobs_timeout_error(
e,
"Counting completed jobs",
"Lower completed_after_s_ago or narrow the filters.",
)
})?;
tx.commit().await?;
Ok(Json(stats))
}
@@ -4429,6 +4442,33 @@ lazy_static::lazy_static! {
.unwrap_or(30);
}
/// A client that gives up does not cancel its query, so without this bound every retry of a
/// slow filter stacks another scan running until the connection-wide 5min timeout.
async fn set_list_jobs_statement_timeout(tx: &mut Transaction<'_, Postgres>) -> error::Result<()> {
let timeout_secs = *LIST_JOBS_STATEMENT_TIMEOUT_SECS;
if timeout_secs > 0 {
sqlx::query(&format!("SET LOCAL statement_timeout = '{timeout_secs}s'"))
.execute(&mut **tx)
.await?;
}
Ok(())
}
fn list_jobs_timeout_error(e: sqlx::Error, action: &str, hint: &str) -> Error {
let timeout_secs = *LIST_JOBS_STATEMENT_TIMEOUT_SECS;
match e {
sqlx::Error::Database(ref db_err)
if timeout_secs > 0 && db_err.code().as_deref() == Some("57014") =>
{
Error::Generic(
StatusCode::BAD_REQUEST,
format!("{action} took more than {timeout_secs}s and was stopped. {hint}"),
)
}
e => e.into(),
}
}
async fn list_jobs(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
@@ -4545,32 +4585,14 @@ async fn list_jobs(
};
// tracing::info!("sql: {}", &sql);
let mut tx: Transaction<'_, Postgres> = user_db.begin(&authed).await?;
// A client that gives up does not cancel its query, so without this bound every retry of a
// slow filter stacks another scan running until the connection-wide 5min timeout.
let timeout_secs = *LIST_JOBS_STATEMENT_TIMEOUT_SECS;
if timeout_secs > 0 {
sqlx::query(&format!("SET LOCAL statement_timeout = '{timeout_secs}s'"))
.execute(&mut *tx)
.await?;
}
set_list_jobs_statement_timeout(&mut tx).await?;
let jobs: Vec<UnifiedJob> = sqlx::query_as(&sql)
.fetch_all(&mut *tx)
.warn_after_seconds_with_sql(5, format!("list_jobs: {}", sql))
.await
.map_err(|e| match e {
sqlx::Error::Database(ref db_err)
if timeout_secs > 0 && db_err.code().as_deref() == Some("57014") =>
{
Error::Generic(
StatusCode::BAD_REQUEST,
format!(
"Listing jobs took more than {timeout_secs}s and was stopped. Set a start date or narrow the filters."
),
)
}
e => e.into(),
.map_err(|e| {
list_jobs_timeout_error(e, "Listing jobs", "Set a start date or narrow the filters.")
})?;
tx.commit().await?;
@@ -8343,8 +8365,9 @@ pub async fn run_wait_result_flow_by_version(
/// job lives, in particular DuckDB, which runs in-process in the worker.
///
/// What it does permit is any statement against the workspace's data tables, writes and DDL
/// included: the helper's body is an unrestricted SQL template and data tables carry no
/// per-user ACL. Narrowing that is a separate decision from this exemption.
/// included: the helper's body is an unrestricted SQL template. What that reaches is the
/// operator's own data table role — the preview job is permissioned as them, so the executor
/// resolves it under their tenancy like any other job.
///
/// The database argument is only half the target: the executor honors a `-- database`
/// directive in the SQL over it, and `-- s3` redirects the result set, so both are refused.
@@ -8576,7 +8599,7 @@ async fn run_inline_preview_script(
#[cfg(not(feature = "run_inline"))]
async fn run_inline_preview_script() -> error::Result<Response> {
Err(error::Error::InternalErr(
"inline preview requires the worker feature".to_string(),
"inline preview requires the run_inline feature on the worker".to_string(),
))
}
backend/windmill-api/src/users.rs
+11
View File
@@ -352,6 +352,17 @@ async fn update_username_in_workpsace<'c>(
new_username: &str,
w_id: &str,
) -> error::Result<()> {
// ---- data table tenants ----
// Tenants name the user, so the rename has to follow here too; a list left naming the old
// username silently drops the access instead of moving it.
windmill_common::workspaces::rename_datatable_tenant_in_workspace(
tx,
w_id,
&format!("u/{old_username}"),
&format!("u/{new_username}"),
)
.await?;
// ---- instance and workspace users ----
sqlx::query!(
"UPDATE usr SET username = $1 WHERE email = $2",
backend/windmill-api/src/workspaces_export.rs
+2 -2
View File
@@ -1639,7 +1639,7 @@ pub(crate) async fn tarball_workspace(
mute_critical_alerts: row.mute_critical_alerts,
color: row.color.clone(),
operator_settings: row.operator_settings.clone(),
datatable: row.datatable.clone(),
datatable: windmill_common::workspaces::strip_datatable_permissions(row.datatable.clone()),
slack_team_id: row.slack_team_id.clone(),
slack_name: row.slack_name.clone(),
slack_command_script: row.slack_command_script.clone(),
@@ -1703,7 +1703,7 @@ pub(crate) async fn tarball_workspace(
mute_critical_alerts: row.mute_critical_alerts,
color: row.color,
operator_settings: row.operator_settings,
datatable: row.datatable,
datatable: windmill_common::workspaces::strip_datatable_permissions(row.datatable),
slack_team_id: row.slack_team_id,
slack_name: row.slack_name,
slack_command_script: row.slack_command_script,
backend/windmill-common/src/datatable_roles.rs
+362
View File
@@ -0,0 +1,362 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! The instance's data table role catalog.
//!
//! A data table role is a real Postgres login role on the Windmill cluster, named exactly as the
//! user named it, shared by every instance database. Windmill decides who may ask for a role (the
//! per-data-table tenant lists in [`crate::workspaces`]); Postgres decides what the role may then
//! touch. The catalog here is only the first half's vocabulary plus the cluster provisioning.
//!
//! Entries are keyed by a generated id so a rename moves nothing else: tenants name the id.
use std::collections::BTreeMap;
use crate::{
error::{Error, Result},
DB,
};
/// The connection every data table resolved to before roles existed (`custom_instance_user`). It
/// owns every pre-existing object, so it is a reserved name rather than a catalog entry: never
/// created, renamed or dropped.
pub const ADMIN_DATATABLE_ROLE: &str = "admin";
/// The login the admin connection uses, and the role every created role is granted to — that
/// membership is what later lets it `ALTER ... OWNER TO` a role and drop it.
pub const CUSTOM_INSTANCE_USER: &str = "custom_instance_user";
/// One catalog entry, as stored in `datatable_role`. The password is per role and instance-wide;
/// it belongs to the instance, not to any workspace's settings.
/// No `Serialize`/`Deserialize`: the catalog is rows now, and a derived `Serialize` would emit
/// `pwd` — the same way out for a credential that the hand-written `Debug` below closes on the log
/// side.
#[derive(Clone)]
pub struct InstanceDatatableRole {
/// The Postgres role name, verbatim.
pub name: String,
pub enabled: bool,
/// Absent only for a role whose provisioning did not finish; resolving as it then errors
/// rather than falling back to admin.
///
/// A plain string rather than a `StringOrSecretRef` like the instance user's password: that
/// one is a secret ref because an operator supplies it and may want it to come from their own
/// backend, while this one is minted here and never entered by anyone, so there is nothing for
/// a ref to point at. Encrypting generated secrets at rest is a separate change that would
/// take the replication password with it.
pub pwd: Option<String>,
}
/// Hand-written so `{:?}` on a catalog cannot put a live Postgres password in a log line or an
/// audit record. Everything else about the entry is safe to print.
impl std::fmt::Debug for InstanceDatatableRole {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("InstanceDatatableRole")
.field("name", &self.name)
.field("enabled", &self.enabled)
.field("pwd", &self.pwd.as_ref().map(|_| "<redacted>"))
.finish()
}
}
pub type DatatableRoleCatalog = BTreeMap<String, InstanceDatatableRole>;
/// Names Postgres or Windmill already owns. `admin` is excluded because it never reaches the
/// cluster as a role name at all — it resolves to `custom_instance_user`.
fn is_reserved_role_name(name: &str) -> bool {
let lower = name.to_ascii_lowercase();
lower == ADMIN_DATATABLE_ROLE
|| lower == "postgres"
|| lower == "public"
|| lower.starts_with("pg_")
|| lower.starts_with("windmill_")
|| lower.starts_with("custom_instance_")
}
/// The charset is what makes every downstream interpolation safe: the name reaches Postgres as a
/// quoted identifier, a `-- role <name>` annotation, and a `?role=` query parameter.
pub fn validate_role_name(name: &str) -> Result<()> {
if name.is_empty() || name.len() > 63 {
return Err(Error::BadRequest(format!(
"Invalid data table role name '{name}': it must be between 1 and 63 characters"
)));
}
if !name
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-')
{
return Err(Error::BadRequest(format!(
"Invalid data table role name '{name}': only letters, digits, '_' and '-' are allowed"
)));
}
if is_reserved_role_name(name) {
return Err(Error::BadRequest(format!(
"'{name}' is reserved and cannot be used as a data table role name"
)));
}
Ok(())
}
/// A double-quoted Postgres identifier. Doubling `"` is Postgres's own escaping inside one, so this
/// quotes any name — schema, table or role. Role names are validated as well
/// ([`validate_role_name`]) because they also travel unquoted, in `-- role <name>` and `?role=`.
pub fn quote_ident(name: &str) -> String {
format!("\"{}\"", name.replace('"', "\"\""))
}
/// Serialize the mutations that are not already serialized by the row itself.
///
/// A create is an insert and a delete is a delete, which Postgres orders for us — the unique index
/// on `name` is what makes two concurrent creates of the same name one winner and one error. What
/// still needs it is the window between the cluster DDL and the row: `CREATE ROLE` is not visible
/// to another transaction's `pg_roles` check until commit, so without this two creates of the same
/// name both pass their existence check and one fails on the index having already made the login.
/// Held for the transaction, so the DDL has to run on that same transaction to be covered.
pub async fn lock_role_catalog(tx: &mut sqlx::Transaction<'_, sqlx::Postgres>) -> Result<()> {
sqlx::query!("SELECT pg_advisory_xact_lock(hashtext('datatable_role_catalog'))")
.execute(&mut **tx)
.await?;
Ok(())
}
/// A replication stream reads every row whatever a data table's roles grant. Turning roles on looks
/// for streams holding this exclusive; whatever can start a Postgres trigger or capture streaming
/// holds it shared on the transaction that commits it. So either the look sees the stream, or the
/// stream's listener connects after roles are committed and refuses. Held for the transaction.
pub async fn lock_datatable_streams(conn: &mut sqlx::PgConnection, exclusive: bool) -> Result<()> {
let lock = if exclusive {
"pg_advisory_xact_lock"
} else {
"pg_advisory_xact_lock_shared"
};
sqlx::query(&format!("SELECT {lock}(hashtext('datatable_streams'))"))
.execute(conn)
.await?;
Ok(())
}
/// Whether an instance database is reached only through entries under roles is decided by two
/// writes that lock different workspaces' settings rows: turning roles on for one entry, and a
/// settings save pointing an entry without roles at the database. Each holds this for every
/// database it decides on, so neither reads past the other's uncommitted write. Held for the
/// transaction; the names are locked in sorted order so two holders cannot deadlock.
pub async fn lock_instance_databases_governance<'a>(
conn: &mut sqlx::PgConnection,
dbnames: impl IntoIterator<Item = &'a str>,
) -> Result<()> {
let dbnames: std::collections::BTreeSet<&str> = dbnames.into_iter().collect();
for dbname in dbnames {
sqlx::query("SELECT pg_advisory_xact_lock(hashtext('datatable_instance_database:' || $1))")
.bind(dbname)
.execute(&mut *conn)
.await?;
}
Ok(())
}
/// Disclosure: returns every role's stored Postgres password in plaintext. Any server path that
/// has to resolve or name a role may call it — including handlers open to a workspace member, who
/// need the names — but callers MUST NOT let `pwd` reach a response, a log line, an audit record
/// or an export. Nothing about who may call it: the credential is the whole risk, and `Debug` is
/// hand-written to redact it for the same reason.
pub async fn read_role_catalog(db: &DB) -> Result<DatatableRoleCatalog> {
crate::datatable_roles_oss::read_role_catalog(db).await
}
/// As [`read_role_catalog`], reading inside the caller's transaction so the value is the one
/// [`lock_role_catalog`] is protecting. Same disclosure contract.
pub async fn read_role_catalog_tx(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
) -> Result<DatatableRoleCatalog> {
crate::datatable_roles_oss::read_role_catalog_tx(tx).await
}
/// Record a role, in the caller's transaction so it commits with the `CREATE ROLE` it describes.
///
/// Authorization: writes a generated Postgres credential. Callers MUST restrict this to superadmin
/// paths and MUST hold [`lock_role_catalog`] on `tx`.
pub async fn insert_role_catalog_entry(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
id: &str,
role: &InstanceDatatableRole,
) -> Result<()> {
crate::datatable_roles_oss::insert_role_catalog_entry(tx, id, role).await
}
/// Update a role's recorded name, login flag and password. Same contract as
/// [`insert_role_catalog_entry`].
pub async fn update_role_catalog_entry(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
id: &str,
role: &InstanceDatatableRole,
) -> Result<()> {
crate::datatable_roles_oss::update_role_catalog_entry(tx, id, role).await
}
/// Forget a role. Same contract as [`insert_role_catalog_entry`]; run it in the transaction that
/// drops the cluster login, so the two cannot disagree.
pub async fn delete_role_catalog_entry(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
id: &str,
) -> Result<()> {
crate::datatable_roles_oss::delete_role_catalog_entry(tx, id).await
}
/// Resolve the role a caller named to its catalog id. A disabled role is an error rather than a
/// silent fallback: the caller asked for something the instance deliberately turned off.
pub fn role_id_by_name<'a>(catalog: &'a DatatableRoleCatalog, name: &str) -> Result<&'a str> {
let entry = catalog
.iter()
.find(|(_, role)| role.name == name)
.ok_or_else(|| {
Error::NotFound(format!(
"'{name}' is not a data table role of this instance. Defined roles: {}.",
catalog
.values()
.map(|r| r.name.as_str())
.collect::<Vec<_>>()
.join(", ")
))
})?;
if !entry.1.enabled {
return Err(Error::BadRequest(format!(
"Data table role '{name}' is disabled on this instance"
)));
}
Ok(entry.0.as_str())
}
/// Every instance database the registry knows about. Role provisioning has to reach all of them:
/// a role that cannot `CONNECT` to a database is refused by Postgres before any grant matters.
///
/// Authorization: checks nothing, and names every instance database across all workspaces. Callers
/// MUST be superadmin-gated or keep the names server-side; never return them to a workspace caller.
pub async fn registered_instance_databases(db: &DB) -> Result<Vec<String>> {
crate::datatable_roles_oss::registered_instance_databases(db).await
}
/// `CONNECT` on `dbname` for every enabled role, and none for `PUBLIC`. Run at role creation, at
/// database creation, and lazily whenever an instance data table is administered, so a database
/// provisioned before a role existed is repaired rather than left silently unreachable.
///
/// Authorization: rewrites a database's ACL with the server's own credentials and checks nothing.
/// Callers MUST have authorized administration of `dbname` — superadmin, or an admin of the
/// workspace governing a data table on it.
pub async fn converge_connect_grants(db: &DB, dbname: &str) -> Result<()> {
crate::datatable_roles_oss::converge_connect_grants(db, dbname).await
}
/// As [`converge_connect_grants`], with a catalog the caller already read. Same contract.
pub async fn converge_connect_grants_with(
db: &DB,
dbname: &str,
catalog: &DatatableRoleCatalog,
) -> Result<()> {
crate::datatable_roles_oss::converge_connect_grants_with(db, dbname, catalog).await
}
/// `CREATE ROLE <name> LOGIN PASSWORD ...; GRANT <name> TO custom_instance_user`, and `CONNECT` on
/// every registered database. No privileges beyond that — an admin grants them through SQL or the
/// ACL editor.
///
/// Authorization: creates a cluster-wide Postgres login. Callers MUST restrict this to superadmin
/// paths, and MUST hold [`lock_role_catalog`] on the same transaction.
pub async fn create_instance_role(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
name: &str,
password: &str,
) -> Result<()> {
crate::datatable_roles_oss::create_instance_role(tx, name, password).await
}
/// Authorization: alters a cluster-wide Postgres login. Callers MUST restrict this to superadmin
/// paths, and MUST hold [`lock_role_catalog`] on the same transaction.
pub async fn set_instance_role_login(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
name: &str,
enabled: bool,
) -> Result<()> {
crate::datatable_roles_oss::set_instance_role_login(tx, name, enabled).await
}
/// A rename discards an md5-hashed password, so the caller has to hand over a fresh one.
///
/// Authorization: renames a cluster-wide Postgres login. Callers MUST restrict this to superadmin
/// paths, and MUST hold [`lock_role_catalog`] on the same transaction.
pub async fn rename_instance_role(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
from: &str,
to: &str,
password: &str,
) -> Result<()> {
crate::datatable_roles_oss::rename_instance_role(tx, from, to, password).await
}
/// A role owning anything in any database blocks its own `DROP ROLE`, and both its objects and the
/// privileges granted to it are only visible from inside each database — hence the pass over the
/// registry. An unreachable database aborts the whole delete: dropping the role while one database
/// still holds objects owned by it leaves those objects owned by a numeric OID nobody can name.
///
/// Each pass runs as the instance's own Postgres user rather than `custom_instance_user`, which
/// owns the databases and can therefore revoke a grant whoever made it. `custom_instance_user`
/// could only undo what it granted itself, so a privilege planted by an operator in psql — the
/// ordinary way privileges reach a role — would survive and block the drop.
///
/// Authorization: drops a cluster-wide Postgres login and reassigns everything it owns. Callers
/// MUST restrict this to superadmin paths, and MUST hold [`lock_role_catalog`] on `tx`.
///
/// The per-database passes open their own connections and cannot join `tx`; the lock is what keeps
/// a concurrent mutation out while they run. Only the final `DROP ROLE` is on `tx`, so it commits
/// or rolls back with the catalog write that forgets the role. Those passes commit as they go, so
/// callers MUST have disabled the role in an earlier committed transaction: a failure part-way
/// then leaves a disabled role to retry, not an enabled one already stripped in some databases.
pub async fn drop_instance_role(
db: &DB,
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
name: &str,
) -> Result<()> {
crate::datatable_roles_oss::drop_instance_role(db, tx, name).await
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn role_names_are_validated() {
assert!(validate_role_name("analytics").is_ok());
assert!(validate_role_name("read-only_2").is_ok());
assert!(validate_role_name("").is_err());
assert!(validate_role_name(&"a".repeat(64)).is_err());
assert!(validate_role_name("has space").is_err());
assert!(validate_role_name("quote\"injection").is_err());
// Reserved, case-insensitively.
assert!(validate_role_name("admin").is_err());
assert!(validate_role_name("Postgres").is_err());
assert!(validate_role_name("pg_read_all_data").is_err());
assert!(validate_role_name("windmill_user").is_err());
assert!(validate_role_name("custom_instance_user").is_err());
}
#[test]
fn a_disabled_role_is_an_error_not_a_fallback() {
let mut catalog = DatatableRoleCatalog::new();
catalog.insert(
"id1".to_string(),
InstanceDatatableRole {
name: "analytics".to_string(),
enabled: false,
pwd: Some("x".to_string()),
},
);
assert!(role_id_by_name(&catalog, "analytics").is_err());
assert!(role_id_by_name(&catalog, "nope").is_err());
catalog.get_mut("id1").unwrap().enabled = true;
assert_eq!(role_id_by_name(&catalog, "analytics").unwrap(), "id1");
}
}
backend/windmill-common/src/datatable_roles_oss.rs
+208
View File
@@ -0,0 +1,208 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! Where data table roles come from: the enterprise implementation, or a refusal.
//!
//! Roles are an Enterprise Edition feature. An edition without them creates, grants and connects
//! as none, and a data table saved under roles — by an enterprise build, before a downgrade — is
//! refused rather than resolved as `admin`. A data table not under roles, asked for no role,
//! resolves as it always has. `private` alone is not that edition: community builds carry it.
use crate::error::Error;
/// What every roles path answers without the Enterprise Edition.
pub fn datatable_roles_unavailable() -> Error {
Error::BadRequest("Data table roles are a Windmill Enterprise Edition feature".to_string())
}
#[cfg(all(feature = "private", feature = "enterprise"))]
pub(crate) use crate::datatable_roles_ee::{
can_use_datatable_role, can_use_datatable_role_in_governing_workspace, converge_connect_grants,
converge_connect_grants_with, create_instance_role, delete_role_catalog_entry,
drop_instance_role, ensure_can_use_datatable_role, ensure_datatable_admin_access,
ensure_instance_db_grant_options_unchecked, forget_datatable_role_everywhere,
insert_role_catalog_entry, read_role_catalog, read_role_catalog_tx,
registered_instance_databases, rename_instance_role, resolve_datatable_role_connection,
set_instance_role_login, update_role_catalog_entry,
};
#[cfg(not(all(feature = "private", feature = "enterprise")))]
pub(crate) use ce::*;
#[cfg(not(all(feature = "private", feature = "enterprise")))]
mod ce {
use super::datatable_roles_unavailable as unavailable;
use crate::{
datatable_roles::{DatatableRoleCatalog, InstanceDatatableRole},
db::AuthedRef,
error::Result,
workspaces::{
resolve_governing_datatable, DataTableRoleTenants, DatatableAccess, GoverningDatatable,
},
DB,
};
type Tx<'a> = sqlx::Transaction<'a, sqlx::Postgres>;
pub(crate) async fn read_role_catalog(_db: &DB) -> Result<DatatableRoleCatalog> {
Err(unavailable())
}
pub(crate) async fn read_role_catalog_tx(_tx: &mut Tx<'_>) -> Result<DatatableRoleCatalog> {
Err(unavailable())
}
pub(crate) async fn insert_role_catalog_entry(
_tx: &mut Tx<'_>,
_id: &str,
_role: &InstanceDatatableRole,
) -> Result<()> {
Err(unavailable())
}
pub(crate) async fn update_role_catalog_entry(
_tx: &mut Tx<'_>,
_id: &str,
_role: &InstanceDatatableRole,
) -> Result<()> {
Err(unavailable())
}
pub(crate) async fn delete_role_catalog_entry(_tx: &mut Tx<'_>, _id: &str) -> Result<()> {
Err(unavailable())
}
pub(crate) async fn registered_instance_databases(_db: &DB) -> Result<Vec<String>> {
Err(unavailable())
}
/// Nothing to converge: with no roles to admit, an instance database keeps the `CONNECT`
/// grants it was created with, `PUBLIC`'s included, as it did before roles existed.
pub(crate) async fn converge_connect_grants(_db: &DB, _dbname: &str) -> Result<()> {
Ok(())
}
/// As [`converge_connect_grants`].
pub(crate) async fn converge_connect_grants_with(
_db: &DB,
_dbname: &str,
_catalog: &DatatableRoleCatalog,
) -> Result<()> {
Ok(())
}
pub(crate) async fn create_instance_role(
_tx: &mut Tx<'_>,
_name: &str,
_password: &str,
) -> Result<()> {
Err(unavailable())
}
pub(crate) async fn set_instance_role_login(
_tx: &mut Tx<'_>,
_name: &str,
_enabled: bool,
) -> Result<()> {
Err(unavailable())
}
pub(crate) async fn rename_instance_role(
_tx: &mut Tx<'_>,
_from: &str,
_to: &str,
_password: &str,
) -> Result<()> {
Err(unavailable())
}
pub(crate) async fn drop_instance_role(_db: &DB, _tx: &mut Tx<'_>, _name: &str) -> Result<()> {
Err(unavailable())
}
pub(crate) async fn ensure_instance_db_grant_options_unchecked(
_db: &DB,
_dbname: &str,
) -> Result<()> {
Err(unavailable())
}
/// No tenant list covers anyone: there is no role to connect as.
pub(crate) fn can_use_datatable_role(
_tenants: &DataTableRoleTenants,
_authed: &AuthedRef<'_>,
) -> bool {
false
}
pub(crate) async fn can_use_datatable_role_in_governing_workspace(
_db: &DB,
_governing_w_id: &str,
_w_id: &str,
_tenants: &DataTableRoleTenants,
_access: &DatatableAccess<'_>,
) -> Result<bool> {
Err(unavailable())
}
/// Reached only for a data table under roles or a caller naming a role: both are refused.
pub(crate) async fn resolve_datatable_role_connection(
_db: &DB,
_w_id: &str,
_name: &str,
_governing: &GoverningDatatable,
_db_resource: serde_json::Value,
_role: Option<&str>,
_access: DatatableAccess<'_>,
) -> Result<serde_json::Value> {
Err(unavailable())
}
/// A data table not under roles, asked for no role or for `admin`, is not a role decision and
/// passes, as it did before roles existed. Anything else is refused.
pub(crate) async fn ensure_can_use_datatable_role(
db: &DB,
w_id: &str,
name: &str,
role: Option<&str>,
_access: &DatatableAccess<'_>,
_context: &str,
) -> Result<()> {
let governing = resolve_governing_datatable(db, w_id, name).await?;
if governing.datatable.permissions.is_none()
&& role.is_none_or(|r| r == crate::datatable_roles::ADMIN_DATATABLE_ROLE)
{
Ok(())
} else {
Err(unavailable())
}
}
/// A data table not under roles is the `admin` connection for anyone who reaches it, as before
/// roles existed. One under roles is refused.
pub(crate) async fn ensure_datatable_admin_access(
db: &DB,
w_id: &str,
name: &str,
_access: &DatatableAccess<'_>,
) -> Result<()> {
let governing = resolve_governing_datatable(db, w_id, name).await?;
if governing.datatable.permissions.is_none() {
Ok(())
} else {
Err(unavailable())
}
}
pub(crate) async fn forget_datatable_role_everywhere(
_tx: &mut Tx<'_>,
_role_id: &str,
) -> Result<()> {
Err(unavailable())
}
}
backend/windmill-common/src/lib.rs
+47 -11
View File
@@ -37,6 +37,10 @@ pub mod bench;
pub mod cache;
pub mod client;
pub mod data_metrics;
pub mod datatable_roles;
#[cfg(all(feature = "private", feature = "enterprise"))]
mod datatable_roles_ee;
pub mod datatable_roles_oss;
pub mod db;
#[cfg(all(feature = "enterprise", feature = "private"))]
mod db_entra_ee;
@@ -1514,6 +1518,41 @@ pub async fn drop_custom_instance_database(db: &DB, dbname: &str) -> error::Resu
Ok(())
}
/// What `custom_instance_user` holds on an instance database.
///
/// `WITH GRANT OPTION` throughout: this is the connection every data table resolves to as `admin`,
/// and it is the one that hands privileges to data table roles. Postgres refuses to let a role pass
/// on a privilege it does not itself hold with grant option, so without these an admin could own
/// the database and still be unable to grant `SELECT` on it to `analytics`.
pub(crate) fn instance_db_grants(dbname: &str) -> String {
format!(
"GRANT CONNECT ON DATABASE \"{dbname}\" TO custom_instance_user WITH GRANT OPTION;
GRANT CREATE ON DATABASE \"{dbname}\" TO custom_instance_user WITH GRANT OPTION;
DO $$ BEGIN
IF EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = 'public') THEN
GRANT USAGE ON SCHEMA public TO custom_instance_user WITH GRANT OPTION;
GRANT CREATE ON SCHEMA public TO custom_instance_user WITH GRANT OPTION;
ALTER DEFAULT PRIVILEGES IN SCHEMA public
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO custom_instance_user;
END IF;
END $$;"
)
}
/// Re-apply [`instance_db_grants`] to an instance database provisioned before data table roles
/// existed, whose grants carry no grant option. Connects as the instance's own Postgres user —
/// the database and `public` schema owner — since only it can hand out an option it holds.
///
/// Authorization: reaches an instance database with the server's own credentials and checks
/// nothing. Callers MUST have authorized administration of `dbname` — superadmin, or an admin of
/// the workspace governing a data table on it.
pub async fn ensure_instance_db_grant_options_unchecked(
db: &DB,
dbname: &str,
) -> error::Result<()> {
crate::datatable_roles_oss::ensure_instance_db_grant_options_unchecked(db, dbname).await
}
/// Create a custom instance database: CREATE DATABASE, grant permissions, register in global_settings.
/// The `tag` is stored in global_settings metadata (e.g. "datatable" or "ducklake").
pub async fn create_custom_instance_database(
@@ -1553,17 +1592,7 @@ pub async fn create_custom_instance_database(
let (client, connection) = new_pg_creds.connect(Some(db)).await?;
let join_handle = tokio::spawn(async move { connection.await });
if let Err(e) = client
.batch_execute(&format!(
"GRANT CONNECT ON DATABASE \"{dbname}\" TO custom_instance_user;
GRANT USAGE ON SCHEMA public TO custom_instance_user;
GRANT CREATE ON SCHEMA public TO custom_instance_user;
GRANT CREATE ON DATABASE \"{dbname}\" TO custom_instance_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA public
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO custom_instance_user;"
))
.await
{
if let Err(e) = client.batch_execute(&instance_db_grants(dbname)).await {
tracing::warn!(
"Failed to grant permissions on '{}': {}. Continuing.",
dbname,
@@ -1592,6 +1621,13 @@ pub async fn create_custom_instance_database(
.execute(db)
.await?;
// A data table role can only reach a database it may CONNECT to, and PUBLIC's default CONNECT
// would otherwise let every role in regardless of what this instance defines. Best-effort: a
// failure here leaves the database usable as `admin`, and the next role change repairs it.
if let Err(e) = crate::datatable_roles::converge_connect_grants(db, dbname).await {
tracing::warn!("Could not set CONNECT grants on instance database '{dbname}': {e}");
}
tracing::info!("Created custom instance database '{}'", dbname);
Ok(())
}
backend/windmill-common/src/worker.rs
+127
View File
@@ -1082,6 +1082,83 @@ pub struct SqlAnnotations {
pub raw_output: bool,
}
impl SqlAnnotations {
/// The data table role a query declares as `-- role <name>`, if any. Only meaningful against a
/// `datatable://` database that is under roles; absent means the data table's default role.
///
/// Hand-written rather than derived because the value matters, not just the presence, and
/// because the executor needs it before it knows the connection is a data table at all. Like
/// every annotation it lives in the leading comment block.
///
/// A leading comment whose first word is `role` is an annotation *attempt*, and a malformed
/// one is an error. The alternative — ignoring what does not parse — resolves the query to the
/// data table's default role instead, so a typo silently runs it under a login the author did
/// not choose, which is the opposite of what naming a role is for. Only callers that already
/// know the target is a `datatable://` reference ever run this, so ordinary SQL keeps its
/// comments.
pub fn datatable_role(code: &str) -> error::Result<Option<String>> {
for line in code.lines() {
let line = line.trim();
if line.is_empty() {
continue;
}
if !line.starts_with("--") {
break;
}
// The keyword may be followed by whitespace, `:` or `=` — `role x`, `role: x`,
// `role=x`, `Role = x` all open an attempt, while `rolexyz` does not. Each accepted
// separator is one spelling that would otherwise take the `continue` below and run the
// query as the data table's default role, which is the silence this exists to remove.
let body = line[2..].trim_start();
let Some(after) = body
.get(..4)
.filter(|kw| kw.eq_ignore_ascii_case("role"))
.map(|_| &body[4..])
else {
continue;
};
if !after.is_empty()
&& !after.starts_with(char::is_whitespace)
&& !after.starts_with([':', '='])
{
continue;
}
// Past this point the line is an attempt to name a role, so a malformed one is an
// error rather than a miss. Falling through would run the query as the data table's
// default role — quietly, and under a login the author did not choose.
let after = after.trim_start();
let after = after.strip_prefix([':', '=']).unwrap_or(after);
let mut tokens = after.split_whitespace();
let role = tokens
.next()
.map(|role| role.strip_suffix(';').unwrap_or(role));
let rest = tokens.next();
match (role, rest) {
(Some(role), None)
if !role.is_empty()
&& role.len() <= 63
&& role
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') =>
{
return Ok(Some(role.to_string()));
}
_ => {
return Err(error::Error::BadRequest(format!(
"Malformed data table role annotation: `{line}`. Write it as \
`-- role <name>` on a line of its own, where <name> is letters, digits, \
'_' or '-'. A comment in the leading block that starts with the word \
'role' is read as this annotation; move it below the first statement if \
it is prose."
)));
}
}
}
Ok(None)
}
}
#[annotations("#")]
pub struct BashAnnotations {
pub docker: bool,
@@ -2653,6 +2730,56 @@ mod tests {
use super::*;
use std::collections::HashMap;
#[test]
fn datatable_role_is_read_from_the_leading_comment_block() {
let role = |code| SqlAnnotations::datatable_role(code);
assert_eq!(
role("-- role analytics\nSELECT 1").unwrap(),
Some("analytics".to_string())
);
// Blank lines and other annotations before it are fine.
assert_eq!(
role("\n-- prepare\n-- role read_only\nSELECT 1").unwrap(),
Some("read_only".to_string())
);
// Past the first statement it is an ordinary comment, not an annotation.
assert_eq!(role("SELECT 1;\n-- role analytics").unwrap(), None);
assert_eq!(role("SELECT 1").unwrap(), None);
// Unambiguous intent is honoured: the keyword matches case-insensitively, a trailing
// semicolon is a habit carried over from SQL rather than a different role, and the colon
// spelling is the one most likely to be typed.
for accepted in [
"-- Role operator\nSELECT 1",
"-- role operator;\nSELECT 1",
"-- role: operator\nSELECT 1",
"-- role:operator\nSELECT 1",
"-- role=operator\nSELECT 1",
"-- Role = operator\nSELECT 1",
] {
assert_eq!(
role(accepted).unwrap(),
Some("operator".to_string()),
"not honoured: {accepted}"
);
}
// Anything else opening with the word is refused rather than resolved to the default role:
// the whole point of naming one is to not run as something else.
for near_miss in [
"-- role operator -- why\nSELECT 1",
"-- role an;alytics\nSELECT 1",
"-- role\nSELECT 1",
"-- role:\nSELECT 1",
"-- role based access is handled below\nSELECT 1",
] {
assert!(role(near_miss).is_err(), "silently ignored: {near_miss}");
}
// A word that merely starts with the keyword is not an attempt.
assert_eq!(role("-- rolebased notes\nSELECT 1").unwrap(), None);
}
fn matcher(id: &str) -> WorkspaceMatcher {
WorkspaceMatcher { id: id.to_string(), include_forks: false }
}
backend/windmill-common/src/workspaces.rs
+771 -46
View File
@@ -6,6 +6,7 @@ use serde::{Deserialize, Serialize};
use strum::AsRefStr;
use crate::{
datatable_roles::{ADMIN_DATATABLE_ROLE, CUSTOM_INSTANCE_USER},
error::{self, to_anyhow, Error, Result},
get_database_url,
secret_backend::{get_secret_value, is_external_stored_value},
@@ -1292,9 +1293,17 @@ impl Default for DataTableForkBehavior {
}
}
#[derive(Deserialize, Serialize, Debug)]
#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct DataTable {
pub database: DataTableDatabase,
/// Set on a *terminal* entry — one that owns its database. Mutually exclusive with
/// [`DataTable::reference`]; [`validate_datatable_shape`] is the one place that enforces it.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub database: Option<DataTableDatabase>,
/// Set on a *pointer* entry — one that names another workspace's entry and owns nothing.
/// A keep-original fork gets one of these instead of a copy of the parent's entry, so there is
/// nothing local for a fork admin to widen.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub reference: Option<DataTableReference>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub forked_from: Option<DataTableForkedFrom>,
/// Whether the SQL-migrations feature is opted in for this data table.
@@ -1302,22 +1311,85 @@ pub struct DataTable {
/// when migrations already exist (see `datatable_migrations_enabled`).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub migrations_enabled: Option<bool>,
/// Who may connect as which role. Absent = unpermissioned: every caller connects as `admin`,
/// which is how data tables behaved before roles existed. Only meaningful on a terminal entry;
/// a pointer is governed by what it points at.
///
/// Never leaves the instance: stripped from the workspace export and ignored on import, since
/// tenants are workspace-scoped names and syncing them would make repo write access a second
/// door onto the access decision.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub permissions: Option<DataTablePermissions>,
}
#[derive(Deserialize, Serialize, Debug)]
/// A pointer at another workspace's data table entry.
#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
pub struct DataTableReference {
pub workspace_id: String,
pub datatable: String,
}
/// The access decision for one data table: which role a caller gets, and who may ask for each.
#[derive(Deserialize, Serialize, Debug, Clone, Default)]
pub struct DataTablePermissions {
/// A role id from the instance catalog, or `admin`. Absent = `admin`.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub default_role: Option<String>,
/// Keyed by instance role id, plus the reserved `admin` key. A role absent from this map
/// cannot be used on this data table at all, whatever the instance catalog says.
#[serde(default)]
pub roles: std::collections::BTreeMap<String, DataTableRoleTenants>,
}
impl DataTablePermissions {
pub fn default_role(&self) -> &str {
self.default_role.as_deref().unwrap_or(ADMIN_DATATABLE_ROLE)
}
}
#[derive(Deserialize, Serialize, Debug, Clone, Default)]
pub struct DataTableRoleTenants {
/// `u/<user>`, `g/<group>`, `f/<folder>`, or `*` for every member.
#[serde(default)]
pub tenants: Vec<String>,
}
/// Every member of the governing workspace.
pub const DATATABLE_TENANT_WILDCARD: &str = "*";
/// How deep a chain of pointer entries may go before it is called a loop. Data tables are not
/// expected to chain at all — a fork points at its parent — so this only has to be generous
/// enough to survive a fork of a fork.
const DATATABLE_REFERENCE_MAX_DEPTH: usize = 20;
/// Exactly one of `database` and `reference` must be set. Called wherever an entry is persisted,
/// so nothing downstream has to handle an entry that is both or neither.
pub fn validate_datatable_shape(name: &str, dt: &DataTable) -> Result<()> {
match (&dt.database, &dt.reference) {
(Some(_), None) | (None, Some(_)) => Ok(()),
(Some(_), Some(_)) => Err(Error::BadRequest(format!(
"Data table '{name}' both owns a database and points at another one"
))),
(None, None) => Err(Error::BadRequest(format!(
"Data table '{name}' names neither a database nor another data table"
))),
}
}
#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct DataTableForkedFrom {
/// Schema snapshot at fork time
#[serde(default, skip_serializing_if = "Option::is_none")]
pub schema: Option<serde_json::Value>,
}
#[derive(Deserialize, Serialize, Debug)]
#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct DataTableDatabase {
pub resource_type: DataTableCatalogResourceType,
pub resource_path: String,
}
#[derive(Deserialize, Serialize, Debug, PartialEq)]
#[derive(Deserialize, Serialize, Debug, PartialEq, Clone, Copy)]
#[serde(rename_all = "lowercase")]
#[derive(AsRefStr)]
#[strum(serialize_all = "lowercase")]
@@ -1353,37 +1425,13 @@ fn datatable_not_found_error(name: &str, datatables: Option<&serde_json::Value>)
))
}
pub async fn get_datatable_resource_from_db_unchecked(
db: &DB,
w_id: &str,
name: &str,
) -> Result<serde_json::Value> {
get_datatable_resource_inner(db, w_id, name, false).await
}
/// Same as [`get_datatable_resource_from_db_unchecked`] but for postgres trigger
/// connections: custom-instance datatables resolve to
/// `custom_instance_replication_user` rather than `custom_instance_user`. BYO-postgres
/// datatables resolve to the user's own resource unchanged; configuring it for
/// replication there is the user's responsibility.
/// Read one workspace's data table entry, without following a pointer.
///
/// Authorization: like its `_unchecked` sibling, returns resolved connection
/// credentials and performs no authorization — callers MUST have already authorized
/// access to the datatable (e.g. the trigger's own create-time check).
pub async fn get_datatable_replication_resource_from_db_unchecked(
db: &DB,
w_id: &str,
name: &str,
) -> Result<serde_json::Value> {
get_datatable_resource_inner(db, w_id, name, true).await
}
async fn get_datatable_resource_inner(
db: &DB,
w_id: &str,
name: &str,
replication: bool,
) -> Result<serde_json::Value> {
/// Disclosure: this is the primitive [`resolve_governing_datatable`] calls on every path, so it is
/// deliberately open to anything that has to resolve a data table, including for a workspace the
/// caller does not belong to. What it returns is not: callers MUST NOT put `permissions` into a
/// response, an export or a log — it names the governing workspace's users, groups and folders.
pub async fn read_datatable_entry(db: &DB, w_id: &str, name: &str) -> Result<DataTable> {
let datatables = sqlx::query_scalar!(
r#"
SELECT ws.datatable->'datatables' AS datatables
@@ -1401,37 +1449,590 @@ async fn get_datatable_resource_inner(
.and_then(|d| d.get(name))
.filter(|v| !v.is_null())
.ok_or_else(|| datatable_not_found_error(name, datatables.as_ref()))?;
let datatable = serde_json::from_value::<DataTable>(datatable.clone())?;
Ok(serde_json::from_value::<DataTable>(datatable.clone())?)
}
let db_resource = if datatable.database.resource_type == DataTableCatalogResourceType::Instance
{
/// The terminal entry a reference chain lands on: the workspace that governs the data table, the
/// entry name there, and the entry itself. A terminal entry resolves to itself.
///
/// Every decision downstream — which database to connect to, whose `permissions` apply, whose
/// members tenants are evaluated against, who may administer it — is taken on this, never on the
/// entry the caller named.
///
/// Authorization: resolving deliberately crosses into the governing workspace, so it answers for a
/// workspace the caller may not belong to and checks nothing itself. It is the input to the
/// checks, not one of them: callers MUST pass what it returns to
/// [`can_use_datatable_role_in_governing_workspace`] or [`ensure_datatable_admin_access`] before
/// acting on it, and MUST NOT return its `permissions` or `workspace_id` to a caller from
/// elsewhere without gating on the answer.
pub struct GoverningDatatable {
pub workspace_id: String,
pub name: String,
pub datatable: DataTable,
}
impl GoverningDatatable {
/// Backed by the Windmill instance's own Postgres, which is the only substrate data table
/// roles apply to.
pub fn is_instance(&self) -> bool {
self.datatable
.database
.as_ref()
.is_some_and(|d| d.resource_type == DataTableCatalogResourceType::Instance)
}
}
pub async fn resolve_governing_datatable(
db: &DB,
w_id: &str,
name: &str,
) -> Result<GoverningDatatable> {
let mut workspace_id = w_id.to_string();
let mut name = name.to_string();
let mut hops = 0;
for _ in 0..DATATABLE_REFERENCE_MAX_DEPTH {
let datatable = read_datatable_entry(db, &workspace_id, &name)
.await
.map_err(|e| {
if hops == 0 {
e
} else {
// A pointer outlives the workspace it names: deleting one only nulls the fork
// lineage, it does not sweep the entries that pointed at it. Say which one is
// gone rather than reporting a data table this workspace never had.
Error::NotFound(format!(
"Data table '{name}' of workspace '{workspace_id}' governs this one and no \
longer exists. A superadmin can point this data table somewhere else."
))
}
})?;
hops += 1;
validate_datatable_shape(&name, &datatable)?;
match &datatable.reference {
None => return Ok(GoverningDatatable { workspace_id, name, datatable }),
Some(reference) => {
workspace_id = reference.workspace_id.clone();
name = reference.datatable.clone();
}
}
}
Err(Error::BadRequest(format!(
"Data table '{name}' points at another data table through more than \
{DATATABLE_REFERENCE_MAX_DEPTH} hops; the chain is likely a loop"
)))
}
/// Every entry of a workspace resolved as [`resolve_governing_datatable`] resolves one, in stored
/// order, reading the settings rows one pointer hop at a time rather than once per entry. An entry
/// that does not resolve — malformed, a dangling pointer, a loop — is left out. Same authorization
/// contract as the single resolution: it checks nothing.
pub async fn resolve_workspace_governing_datatables(
db: &DB,
w_id: &str,
) -> Result<Vec<(String, GoverningDatatable)>> {
type Entries =
std::collections::HashMap<String, std::collections::HashMap<String, serde_json::Value>>;
async fn load(db: &DB, workspaces: &[String], entries: &mut Entries) -> Result<Vec<String>> {
let rows: Vec<(String, String, serde_json::Value)> = sqlx::query_as(
"SELECT ws.workspace_id, dt.key, dt.value FROM workspace_settings ws
CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt
WHERE ws.workspace_id = ANY($1)",
)
.bind(workspaces)
.fetch_all(db)
.await?;
for ws in workspaces {
entries.entry(ws.clone()).or_default();
}
let mut keys = Vec::with_capacity(rows.len());
for (ws, key, value) in rows {
keys.push(key.clone());
entries.entry(ws).or_default().insert(key, value);
}
Ok(keys)
}
let mut entries = Entries::new();
let listed = load(db, &[w_id.to_string()], &mut entries).await?;
// (index into `listed`, workspace, entry name) still to be followed.
let mut cursors: Vec<(usize, String, String)> = listed
.iter()
.enumerate()
.map(|(i, name)| (i, w_id.to_string(), name.clone()))
.collect();
let mut resolved: Vec<(usize, GoverningDatatable)> = vec![];
for _ in 0..DATATABLE_REFERENCE_MAX_DEPTH {
let mut next = vec![];
for (i, ws, name) in cursors.drain(..) {
let Some(value) = entries
.get(&ws)
.and_then(|m| m.get(&name))
.filter(|v| !v.is_null())
else {
continue;
};
let Ok(datatable) = serde_json::from_value::<DataTable>(value.clone()) else {
continue;
};
if validate_datatable_shape(&name, &datatable).is_err() {
continue;
}
match &datatable.reference {
None => {
resolved.push((i, GoverningDatatable { workspace_id: ws, name, datatable }))
}
Some(reference) => next.push((
i,
reference.workspace_id.clone(),
reference.datatable.clone(),
)),
}
}
if next.is_empty() {
break;
}
let to_load: Vec<String> = next
.iter()
.map(|(_, ws, _)| ws.clone())
.filter(|ws| !entries.contains_key(ws))
.collect::<std::collections::BTreeSet<_>>()
.into_iter()
.collect();
if !to_load.is_empty() {
load(db, &to_load, &mut entries).await?;
}
cursors = next;
}
resolved.sort_by_key(|(i, _)| *i);
Ok(resolved
.into_iter()
.map(|(i, governing)| (listed[i].clone(), governing))
.collect())
}
/// Build the `admin` connection for a governing entry: `custom_instance_user` for an instance
/// database, the user's own resource for a BYO-postgres one.
async fn resolve_datatable_connection_unchecked(
db: &DB,
governing: &GoverningDatatable,
replication: bool,
) -> Result<serde_json::Value> {
let database = governing
.datatable
.database
.as_ref()
.expect("a governing entry owns a database");
if database.resource_type == DataTableCatalogResourceType::Instance {
let mut pg_creds = PgDatabase::parse_uri(&get_database_url().await?.as_str().await)?;
pg_creds.dbname = datatable.database.resource_path.clone();
pg_creds.dbname = database.resource_path.clone();
if replication {
pg_creds.user = Some("custom_instance_replication_user".to_string());
pg_creds.password = Some(get_custom_pg_instance_replication_password(&db).await?);
} else {
pg_creds.user = Some("custom_instance_user".to_string());
pg_creds.user = Some(CUSTOM_INSTANCE_USER.to_string());
pg_creds.password = Some(get_custom_pg_instance_password(&db).await?);
}
serde_json::to_value(&pg_creds)
.map_err(|e| Error::internal_err(format!("Error serializing pg creds: {}", e)))?
.map_err(|e| Error::internal_err(format!("Error serializing pg creds: {}", e)))
} else {
// Name the data table too: the caller asked for one by name, and a bare
// "resource f/x/y does not exist" leaves them to work out which one points at it.
transform_json_unchecked(
&serde_json::Value::String(format!("$res:{}", datatable.database.resource_path)),
w_id,
&serde_json::Value::String(format!("$res:{}", database.resource_path)),
&governing.workspace_id,
db,
)
.await
.map_err(|e| match e {
Error::NotFound(m) => Error::NotFound(format!("data table {name}: {m}")),
Error::NotFound(m) => Error::NotFound(format!("data table {}: {m}", governing.name)),
e => e,
})?
})
}
}
/// Resolve a data table to connection credentials **without authorizing anything**: always the
/// `admin` connection.
///
/// Authorization: callers MUST have authorized access already. Anything that acts for a user or a
/// job wants [`get_datatable_resource_from_db`] instead.
pub async fn get_datatable_resource_from_db_unchecked(
db: &DB,
w_id: &str,
name: &str,
) -> Result<serde_json::Value> {
let governing = resolve_governing_datatable(db, w_id, name).await?;
resolve_datatable_connection_unchecked(db, &governing, false).await
}
/// Same as [`get_datatable_resource_from_db_unchecked`] but for postgres trigger
/// connections: custom-instance datatables resolve to
/// `custom_instance_replication_user` rather than `custom_instance_user`. BYO-postgres
/// datatables resolve to the user's own resource unchanged; configuring it for
/// replication there is the user's responsibility.
///
/// Authorization: a replication connection reads every row whatever the roles grant, so no role or
/// admin check makes it safe. Callers MUST refuse a data table under roles outright — the Postgres
/// trigger crate's `ensure_not_under_roles` — and turning roles on is refused while one streams.
pub async fn get_datatable_replication_resource_from_db_unchecked(
db: &DB,
w_id: &str,
name: &str,
) -> Result<serde_json::Value> {
let governing = resolve_governing_datatable(db, w_id, name).await?;
resolve_datatable_connection_unchecked(db, &governing, true).await
}
/// The identity a resolution is made for. `Unchecked` is for callers that authorized already;
/// everything else is checked against the governing entry's tenants.
pub enum DatatableAccess<'a> {
/// Reaches every role. For callers that already authorized, or that have no user at all.
Unchecked,
Authed(crate::db::AuthedRef<'a>),
/// A job's owner, without reading the job row — only fetched if the data table turns out to
/// be permissioned.
PermissionedAs {
permissioned_as: &'a str,
email: &'a str,
},
/// A job identified by id; its owner is read from `v2_job`. For agent workers and anything
/// else that authenticates as infrastructure rather than as the job's user.
Job(uuid::Uuid),
/// No identity established. Unpermissioned data tables resolve as before; permissioned ones
/// are refused, so a caller predating this feature fails closed.
NoIdentity,
}
/// Does one tenant list cover this identity? Admins of the governing workspace pass everything —
/// they can edit the tenant lists anyway, so refusing them would only be theatre.
pub fn can_use_datatable_role(
tenants: &DataTableRoleTenants,
authed: &crate::db::AuthedRef<'_>,
) -> bool {
crate::datatable_roles_oss::can_use_datatable_role(tenants, authed)
}
/// Evaluate a tenant list **as a member of the governing workspace**, whoever is calling.
///
/// A caller reaching a data table through a pointer is a member of some other workspace, and being
/// its admin means nothing here — that is the whole point of the pointer. They are looked up in
/// the governing workspace by email and evaluated there, or refused when they are not a member.
/// A `g/` or `f/` permissioned-as from a foreign workspace is refused outright: those names are
/// defined per workspace and mean nothing outside the one that defined them.
pub async fn can_use_datatable_role_in_governing_workspace(
db: &DB,
governing_w_id: &str,
w_id: &str,
tenants: &DataTableRoleTenants,
access: &DatatableAccess<'_>,
) -> Result<bool> {
crate::datatable_roles_oss::can_use_datatable_role_in_governing_workspace(
db,
governing_w_id,
w_id,
tenants,
access,
)
.await
}
/// Resolve a data table to connection credentials for one identity.
///
/// This is the chokepoint: everything that opens a connection to a data table on someone's behalf
/// goes through it. `role` is the role name the caller asked for — the `-- role` annotation, the
/// `?role=` on a `datatable://` reference, or `None` for the data table's default.
///
/// The resolved role **logs in as itself**. Never `SET ROLE`: a script could `RESET ROLE` its way
/// back to admin.
pub async fn get_datatable_resource_from_db(
db: &DB,
w_id: &str,
name: &str,
role: Option<&str>,
access: DatatableAccess<'_>,
) -> Result<serde_json::Value> {
let governing = resolve_governing_datatable(db, w_id, name).await?;
let db_resource = resolve_datatable_connection_unchecked(db, &governing, false).await?;
// Not under roles and asked for none, or for `admin` by name: the `admin` connection, as before
// roles existed, in every edition. Anything else is a role decision. Every migration names
// `admin` explicitly, so an edition without roles must not treat that as one.
if governing.datatable.permissions.is_none() && role.is_none_or(|r| r == ADMIN_DATATABLE_ROLE) {
return Ok(db_resource);
}
crate::datatable_roles_oss::resolve_datatable_role_connection(
db,
w_id,
name,
&governing,
db_resource,
role,
access,
)
.await
}
/// Would the chokepoint accept this identity connecting as this role? Answers without resolving
/// credentials, for callers that want to refuse early and say which thing was refused.
///
/// Not the security boundary — [`get_datatable_resource_from_db`] re-checks when it actually opens
/// the connection. This is what turns "permission denied for table x" into a message naming the
/// migration and the role.
pub async fn ensure_can_use_datatable_role(
db: &DB,
w_id: &str,
name: &str,
role: Option<&str>,
access: &DatatableAccess<'_>,
context: &str,
) -> Result<()> {
crate::datatable_roles_oss::ensure_can_use_datatable_role(db, w_id, name, role, access, context)
.await
}
/// Gate the operations that see the whole database whatever the roles grant: a migration that
/// declares no role, exports, and editing the permissions themselves. Not replication, which a
/// data table under roles refuses whoever asks (see `ensure_not_under_roles`). Passing
/// means the caller could have connected as `admin` anyway.
pub async fn ensure_datatable_admin_access(
db: &DB,
w_id: &str,
name: &str,
access: &DatatableAccess<'_>,
) -> Result<()> {
crate::datatable_roles_oss::ensure_datatable_admin_access(db, w_id, name, access).await
}
/// Rewrite the `permissions` of every data table entry of one workspace, in the caller's
/// transaction. `change` reports whether it touched anything; the row is only written when
/// something did.
///
/// Authorization: writes an access decision for any workspace named, with an arbitrary mutation,
/// and checks nothing. It exists for the cascades below — the transaction that frees or renames a
/// principal — so callers MUST be the operation that made the principal change, and MUST run in
/// its transaction. Anything editing a decision on purpose belongs in the permissions endpoint,
/// which is gated on the workspace that governs the data table.
///
/// The tenant lists name principals of this workspace, so anything that frees or renames one has
/// to come through here in the same transaction that frees it — otherwise a `u/alice` reused by a
/// later account silently inherits her access.
pub async fn update_datatable_permissions_in_workspace<F>(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
w_id: &str,
change: F,
) -> Result<()>
where
F: Fn(&mut DataTablePermissions) -> bool,
{
let Some(mut settings) = sqlx::query_scalar!(
"SELECT datatable FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE",
w_id
)
.fetch_optional(&mut **tx)
.await?
.flatten() else {
return Ok(());
};
Ok(db_resource)
let Some(datatables) = settings
.get_mut("datatables")
.and_then(|d| d.as_object_mut())
else {
return Ok(());
};
let mut touched = false;
for entry in datatables.values_mut() {
let Some(permissions) = entry.get("permissions").filter(|p| !p.is_null()) else {
continue;
};
let Ok(mut permissions) =
serde_json::from_value::<DataTablePermissions>(permissions.clone())
else {
continue;
};
if change(&mut permissions) {
entry["permissions"] = serde_json::to_value(&permissions)
.map_err(|e| Error::internal_err(format!("serializing permissions: {e}")))?;
touched = true;
}
}
if touched {
sqlx::query!(
"UPDATE workspace_settings SET datatable = $1 WHERE workspace_id = $2",
settings,
w_id
)
.execute(&mut **tx)
.await?;
}
Ok(())
}
/// Drop a freed principal (`u/alice`, `g/analysts`, `f/finance`) from every tenant list of one
/// workspace. Same contract as [`update_datatable_permissions_in_workspace`]: for the transaction
/// that frees the principal, not for editing a decision.
pub async fn remove_datatable_tenant_in_workspace(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
w_id: &str,
tenant: &str,
) -> Result<()> {
update_datatable_permissions_in_workspace(tx, w_id, |permissions| {
let mut touched = false;
for role in permissions.roles.values_mut() {
let before = role.tenants.len();
role.tenants.retain(|t| t != tenant);
touched |= role.tenants.len() != before;
}
touched
})
.await
}
/// Follow a renamed principal through every tenant list of one workspace. Same contract as
/// [`update_datatable_permissions_in_workspace`].
pub async fn rename_datatable_tenant_in_workspace(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
w_id: &str,
old: &str,
new: &str,
) -> Result<()> {
update_datatable_permissions_in_workspace(tx, w_id, |permissions| {
let mut touched = false;
for role in permissions.roles.values_mut() {
let mut role_touched = false;
for tenant in role.tenants.iter_mut() {
if tenant == old {
*tenant = new.to_string();
role_touched = true;
}
}
if role_touched {
// The rename can collide with a name already in the list, and the two need not be
// adjacent — `Vec::dedup` only collapses neighbours, so it would leave the pair.
let mut seen = std::collections::HashSet::new();
role.tenants.retain(|t| seen.insert(t.clone()));
touched = true;
}
}
touched
})
.await
}
/// Strip a deleted instance role from every workspace that had tenanted it, so nothing is left
/// naming a role that no longer exists.
///
/// Authorization: reaches every workspace on the instance. Callers MUST be the superadmin path
/// dropping the role from the cluster — it exists to follow that, not to edit tenants.
///
/// Takes that path's transaction rather than opening its own: run afterwards, a failure part-way
/// leaves the catalog row already gone, so the retry answers `NotFound` while some workspaces
/// still name a role nothing can connect as. In the transaction, the cluster drop, the catalog row
/// and every tenant list commit together or not at all. A data table whose default role was the deleted one falls
/// back to `admin` — the one role that is always present.
pub async fn forget_datatable_role_everywhere(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
role_id: &str,
) -> Result<()> {
crate::datatable_roles_oss::forget_datatable_role_everywhere(tx, role_id).await
}
/// Drop the `permissions` block from a `workspace_settings.datatable` value before it leaves the
/// server.
///
/// Who may connect as which role is an access decision, not configuration, and its tenants name
/// principals of one workspace — `g/analysts` in dev is a different group from `g/analysts` in
/// prod. Shipping it would both mean nothing at the far end and turn a settings push into a way to
/// widen access, so the decision stays where it was made. [`DataTable`] deserializes fine without
/// it, and the settings-editing endpoint carries the stored block across untouched.
pub fn strip_datatable_permissions(
datatable: Option<serde_json::Value>,
) -> Option<serde_json::Value> {
let mut datatable = datatable?;
if let Some(entries) = datatable
.get_mut("datatables")
.and_then(|d| d.as_object_mut())
{
for entry in entries.values_mut() {
if let Some(entry) = entry.as_object_mut() {
entry.remove("permissions");
}
}
}
Some(datatable)
}
/// As [`parse_datatable_ref`], except that an entry whose stored name itself contains `?` — which
/// names could before they were restricted — resolves by that exact name, without a role. It is
/// looked up first, so `sales?role=x` never reaches a different entry than the one stored so.
///
/// Authorization: checks nothing, and its answer reveals whether `w_id` stores that exact name.
/// Callers MUST already act for `w_id` — a job of it, or a caller authenticated into it — and
/// MUST still pass the name to [`get_datatable_resource_from_db`] or an admin-access check.
pub async fn parse_datatable_ref_for(
db: &DB,
w_id: &str,
reference: &str,
) -> Result<(String, Option<String>)> {
if reference.contains('?') {
let exists = sqlx::query_scalar::<_, Option<bool>>(
"SELECT (datatable->'datatables') ? $2 FROM workspace_settings WHERE workspace_id = $1",
)
.bind(w_id)
.bind(reference)
.fetch_optional(db)
.await?
.flatten()
.unwrap_or(false);
if exists {
return Ok((reference.to_string(), None));
}
}
let (name, role) = parse_datatable_ref(reference)?;
Ok((name.to_string(), role.map(str::to_string)))
}
/// Split a `datatable://` reference into its name and the role its query string names.
///
/// A query string that does not parse is an error rather than an absent role. Falling back would
/// resolve the reference to the data table's default role, so `?Role=analytics` or a mistyped
/// `?role=` would quietly connect as something the caller did not ask for — the same trap as a
/// malformed `-- role` annotation, and `role` is the only parameter a reference takes.
pub fn parse_datatable_ref(reference: &str) -> Result<(&str, Option<&str>)> {
let (name, query) = reference.split_once('?').unwrap_or((reference, ""));
let mut role = None;
for param in query.split('&').filter(|p| !p.is_empty()) {
let (key, value) = param.split_once('=').unwrap_or((param, ""));
if !key.eq_ignore_ascii_case("role") {
return Err(Error::BadRequest(format!(
"Data table reference '{name}' carries an unknown parameter '{key}'. \
The only one it takes is `?role=<name>`."
)));
}
if role.is_some() {
return Err(Error::BadRequest(format!(
"Data table reference '{name}' names a role more than once."
)));
}
if value.is_empty() || !is_datatable_role_name(value) {
return Err(Error::BadRequest(format!(
"Data table reference '{name}' has a malformed role '{value}'. Write it as \
`?role=<name>`, where <name> is letters, digits, '_' or '-'."
)));
}
role = Some(value);
}
Ok((name, role))
}
fn is_datatable_role_name(role: &str) -> bool {
!role.is_empty()
&& role.len() <= 63
&& role
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-')
}
#[derive(Deserialize, Serialize, Debug)]
@@ -2642,6 +3243,130 @@ async fn transform_json_unchecked(
mod tests {
use super::*;
fn tenants(list: &[&str]) -> DataTableRoleTenants {
DataTableRoleTenants { tenants: list.iter().map(|t| t.to_string()).collect() }
}
#[cfg(all(feature = "private", feature = "enterprise"))]
#[test]
fn a_tenant_list_covers_users_groups_folders_and_the_wildcard() {
let groups = vec!["analysts".to_string()];
let folders = vec![("finance".to_string(), true, false)];
let scopes = None;
let token_prefix = None;
let is_admin = false;
let is_operator = false;
let authed = crate::db::AuthedRef {
email: "alice@windmill.dev",
username: "alice",
is_admin: &is_admin,
is_operator: &is_operator,
groups: &groups,
folders: &folders,
scopes: &scopes,
token_prefix: &token_prefix,
};
assert!(can_use_datatable_role(&tenants(&["u/alice"]), &authed));
assert!(can_use_datatable_role(&tenants(&["g/analysts"]), &authed));
assert!(can_use_datatable_role(&tenants(&["f/finance"]), &authed));
assert!(can_use_datatable_role(&tenants(&["*"]), &authed));
assert!(!can_use_datatable_role(&tenants(&[]), &authed));
assert!(!can_use_datatable_role(
&tenants(&["u/bob", "g/ops"]),
&authed
));
// A bare name is not a principal: only the three prefixes and the wildcard match.
assert!(!can_use_datatable_role(&tenants(&["alice"]), &authed));
// An admin of the governing workspace reaches every role: they can edit the lists anyway.
let is_admin = true;
let admin = crate::db::AuthedRef { is_admin: &is_admin, ..authed };
assert!(can_use_datatable_role(&tenants(&[]), &admin));
}
#[cfg(not(all(feature = "private", feature = "enterprise")))]
#[test]
fn without_the_enterprise_edition_no_tenant_list_covers_anyone() {
let groups = vec![];
let folders = vec![];
let scopes = None;
let token_prefix = None;
let is_admin = true;
let is_operator = false;
let admin = crate::db::AuthedRef {
email: "alice@windmill.dev",
username: "alice",
is_admin: &is_admin,
is_operator: &is_operator,
groups: &groups,
folders: &folders,
scopes: &scopes,
token_prefix: &token_prefix,
};
assert!(!can_use_datatable_role(&tenants(&["*"]), &admin));
assert!(!can_use_datatable_role(&tenants(&["u/alice"]), &admin));
}
#[test]
fn a_datatable_ref_splits_off_its_role() {
assert_eq!(parse_datatable_ref("sales").unwrap(), ("sales", None));
assert_eq!(
parse_datatable_ref("sales?role=analytics").unwrap(),
("sales", Some("analytics"))
);
// The key matches case-insensitively, the way the `-- role` annotation does.
assert_eq!(
parse_datatable_ref("sales?Role=analytics").unwrap(),
("sales", Some("analytics"))
);
// A query string that does not parse is refused rather than read as "no role": resolving
// it to the data table's default would connect as a login the caller never asked for.
for malformed in [
"sales?role=",
"sales?role=an;alytics",
"sales?x=1&role=analytics",
"sales?role=a&role=b",
] {
assert!(
parse_datatable_ref(malformed).is_err(),
"silently ignored: {malformed}"
);
}
}
#[test]
fn an_entry_owns_a_database_or_points_at_one_but_never_both() {
let terminal = DataTable {
database: Some(DataTableDatabase {
resource_type: DataTableCatalogResourceType::Instance,
resource_path: "dt_main".to_string(),
}),
reference: None,
forked_from: None,
migrations_enabled: None,
permissions: None,
};
assert!(validate_datatable_shape("main", &terminal).is_ok());
let pointer = DataTable {
database: None,
reference: Some(DataTableReference {
workspace_id: "prod".to_string(),
datatable: "main".to_string(),
}),
..terminal.clone()
};
assert!(validate_datatable_shape("main", &pointer).is_ok());
let both = DataTable { database: terminal.database.clone(), ..pointer.clone() };
assert!(validate_datatable_shape("main", &both).is_err());
let neither = DataTable { database: None, reference: None, ..terminal.clone() };
assert!(validate_datatable_shape("main", &neither).is_err());
}
#[test]
fn test_parse_fork_branch() {
// Generated fork (`wm-fork-abc`) and dev workspace (`staging`) forms.
backend/windmill-trigger-postgres/src/handler.rs
+28 -2
View File
@@ -21,11 +21,12 @@ use windmill_common::{
use windmill_git_sync::DeployedObject;
use windmill_api_auth::{check_scopes, ApiAuthed};
use windmill_trigger::{Trigger, TriggerCrud, TriggerData};
use windmill_trigger::{Trigger, TriggerCrud, TriggerData, TriggerMode};
use super::{
check_if_valid_publication_for_postgres_version, create_logical_replication_slot,
create_pg_publication, drop_publication, generate_random_string, get_default_pg_connection,
create_pg_publication, drop_publication, ensure_not_under_roles, generate_random_string,
get_default_pg_connection,
mapper::{Mapper, MappingInfo},
PostgresConfig, PostgresConfigRequest, PostgresPublicationReplication, PostgresTrigger,
PublicationData, Relations, Slot, SlotList, TableToTrack, TemplateScript, TestPostgresConfig,
@@ -64,6 +65,29 @@ impl TriggerCrud for PostgresTrigger {
DeployedObject::PostgresTrigger { path, parent_path }
}
async fn validate_config(
&self,
db: &DB,
config: &Self::TriggerConfigRequest,
workspace_id: &str,
) -> Result<()> {
ensure_not_under_roles(db, workspace_id, &config.postgres_resource_path).await
}
async fn authorize_set_trigger_mode(
&self,
_authed: &ApiAuthed,
tx: &mut PgConnection,
_workspace_id: &str,
_path: &str,
mode: &TriggerMode,
) -> Result<()> {
if *mode != TriggerMode::Disabled {
windmill_common::datatable_roles::lock_datatable_streams(tx, false).await?;
}
Ok(())
}
async fn create_trigger(
&self,
db: &DB,
@@ -72,6 +96,7 @@ impl TriggerCrud for PostgresTrigger {
w_id: &str,
trigger: TriggerData<Self::TriggerConfigRequest>,
) -> Result<()> {
windmill_common::datatable_roles::lock_datatable_streams(&mut *tx, false).await?;
let resolved_edited_by = trigger.base.resolve_edited_by(authed);
let resolved_permissioned_as = trigger.base.resolve_permissioned_as(authed);
let Self::TriggerConfigRequest {
@@ -161,6 +186,7 @@ impl TriggerCrud for PostgresTrigger {
path: &str,
trigger: TriggerData<Self::TriggerConfigRequest>,
) -> Result<()> {
windmill_common::datatable_roles::lock_datatable_streams(&mut *tx, false).await?;
let resolved_edited_by = trigger.base.resolve_edited_by(authed);
let resolved_permissioned_as = trigger.base.resolve_permissioned_as(authed);
let Self::TriggerConfigRequest {
backend/windmill-trigger-postgres/src/lib.rs
+30
View File
@@ -374,6 +374,35 @@ pub async fn get_raw_postgres_connection(
Ok(client)
}
/// A replication stream reads every row of every table whatever the data table's roles grant, so
/// the two don't mix: a data table under roles takes no triggers or captures, and roles cannot be
/// turned on while one is enabled on it.
///
/// Authorization: checks nothing, and its refusal says whether `w_id`'s data table is under roles.
/// Callers MUST have established that the caller may manage triggers in `w_id` first.
pub(crate) async fn ensure_not_under_roles(
db: &DB,
w_id: &str,
postgres_resource_path: &str,
) -> Result<()> {
let Some(datatable_name) = postgres_resource_path.strip_prefix("datatable://") else {
return Ok(());
};
if windmill_common::workspaces::resolve_governing_datatable(db, w_id, datatable_name)
.await?
.datatable
.permissions
.is_some()
{
return Err(Error::BadRequest(format!(
"Data table '{datatable_name}' is under roles, and a Postgres trigger or capture \
cannot read one: a replication stream sees every row whatever the roles grant. \
Turn its roles off to stream it."
)));
}
Ok(())
}
pub async fn resolve_postgres_resource(
authed: &ApiAuthed,
user_db: Option<UserDB>,
@@ -382,6 +411,7 @@ pub async fn resolve_postgres_resource(
w_id: &str,
) -> Result<Postgres> {
if let Some(datatable_name) = postgres_resource_path.strip_prefix("datatable://") {
ensure_not_under_roles(db, w_id, postgres_resource_path).await?;
// Trigger connections (publication/slot management + logical replication) run
// as the dedicated replication user on custom-instance databases.
let resource_value =
backend/windmill-trigger-postgres/src/listener.rs
+12 -3
View File
@@ -20,7 +20,8 @@ use windmill_common::{
use windmill_trigger::{listener::ListeningTrigger, trigger_helpers::TriggerJobArgs, Listener};
use super::{
drop_publication, get_default_pg_connection, get_raw_postgres_connection,
drop_publication, ensure_not_under_roles, get_default_pg_connection,
get_raw_postgres_connection,
handler::drop_logical_replication_slot,
relation::RelationConverter,
replication_message::{
@@ -135,8 +136,8 @@ impl PostgresSimpleClient {
/// Resolves the Postgres resource, validates that the configured publication and
/// replication slot still exist, and opens a fresh logical replication stream.
///
/// Returns `Error::BadConfig` when the publication or slot is missing (an
/// unrecoverable misconfiguration). Any other error is treated as transient
/// Returns `Error::BadConfig` when the publication or slot is missing, or the
/// data table is under roles (unrecoverable misconfigurations). Any other error is treated as transient
/// (connection refused, network interruption, ...) and is retried by the caller.
/// The resource is re-resolved on every call so credential rotations are picked
/// up across reconnections.
@@ -149,6 +150,14 @@ async fn connect_logical_replication_stream(
let PostgresConfig { postgres_resource_path, publication_name, replication_slot_name, .. } =
trigger_config;
// Retrying cannot lift roles, so this disables the trigger like a missing slot does.
ensure_not_under_roles(db, workspace_id, postgres_resource_path)
.await
.map_err(|e| match e {
Error::BadRequest(msg) => Error::BadConfig(msg),
e => e,
})?;
let database = resolve_postgres_resource(
authed,
Some(UserDB::new(db.clone())),
backend/windmill-worker/src/agent_workers.rs
+11 -2
View File
@@ -64,16 +64,25 @@ pub async fn get_ducklake_from_agent_http(
.await
}
/// An agent worker authenticates as infrastructure, not as the job's user, so the job id travels
/// with the request: the server reads the job's owner from it and evaluates the data table's
/// tenants against them. A worker predating this sends neither, and the server fails it closed on
/// a data table under roles.
#[allow(dead_code)]
pub async fn get_datatable_resource_from_agent_http(
client: &HttpClient,
name: &str,
w_id: &str,
role: Option<&str>,
job_id: &uuid::Uuid,
) -> anyhow::Result<serde_json::Value> {
let role_query = role
.map(|r| format!("&role={}", urlencoding::encode(r)))
.unwrap_or_default();
client
.get(&format!(
"/api/w/{}/agent_workers/get_datatable_resource/{}",
w_id, &name
"/api/w/{}/agent_workers/get_datatable_resource/{}?job_id={}{}",
w_id, &name, job_id, role_query
))
.await
}
backend/windmill-worker/src/duckdb_executor.rs
+110 -26
View File
@@ -13,8 +13,8 @@ use windmill_common::error::{to_anyhow, Error, Result};
use windmill_common::utils::sanitize_string_from_password;
use windmill_common::worker::{get_memory, to_raw_value, Connection, SqlResultCollectionStrategy};
use windmill_common::workspaces::{
get_datatable_resource_from_db_unchecked, get_ducklake_from_db_unchecked,
strip_fork_reserved_attach_args, DucklakeCatalogResourceType,
get_datatable_resource_from_db, get_ducklake_from_db_unchecked,
strip_fork_reserved_attach_args, DatatableAccess, DucklakeCatalogResourceType,
};
use windmill_common::PgDatabase;
use windmill_object_store::S3_PROXY_LAST_ERRORS_CACHE;
@@ -1494,13 +1494,9 @@ pub async fn do_duckdb(
.await?
{
probe_blocks.extend(q);
} else if let Some(q) = transform_attach_datatable(
&query_block,
conn,
&mut hidden_passwords,
&job.workspace_id,
)
.await?
} else if let Some(q) =
transform_attach_datatable(&query_block, conn, &mut hidden_passwords, job)
.await?
{
probe_blocks.extend(q);
} else {
@@ -1575,13 +1571,9 @@ pub async fn do_duckdb(
.await?
{
v.extend(ducklake_query);
} else if let Some(datatable_query) = transform_attach_datatable(
&query_block,
conn,
&mut hidden_passwords,
&job.workspace_id,
)
.await?
} else if let Some(datatable_query) =
transform_attach_datatable(&query_block, conn, &mut hidden_passwords, job)
.await?
{
v.extend(datatable_query);
} else {
@@ -2609,33 +2601,79 @@ fn fork_defer_statements(
Ok(stmts)
}
struct AttachedDatatable<'a> {
/// The data table reference, query string included; a bare `datatable` is `main`.
reference: String,
alias: &'a str,
}
/// `ATTACH 'datatable[://<name>][?role=<role>]' AS <alias>`. A bare `datatable` names the default
/// data table, so the role query string has to be accepted with and without an explicit name. The
/// reference is split only once the workspace can be read, because a stored name may contain `?`.
fn parse_attach_datatable(query: &str) -> Option<AttachedDatatable<'_>> {
lazy_static::lazy_static! {
static ref RE: regex::Regex = regex::Regex::new(
r"(?i)ATTACH\s*'datatable(://[^':]+|\?[^':]*)?'\s*AS\s+([^ ;]+)"
).unwrap();
}
let cap = RE.captures(query)?;
let reference = match cap.get(1).map(|m| m.as_str()) {
Some(named) if named.starts_with("://") => named[3..].to_string(),
Some(query) => format!("main{query}"),
None => "main".to_string(),
};
let alias = cap.get(2).map(|m| m.as_str()).unwrap_or("");
Some(AttachedDatatable { reference, alias })
}
async fn transform_attach_datatable(
query: &str,
conn: &Connection,
hidden_passwords: &mut Arc<Mutex<Vec<String>>>,
w_id: &str,
job: &MiniPulledJob,
) -> Result<Option<Vec<String>>> {
lazy_static::lazy_static! {
static ref RE: regex::Regex = regex::Regex::new(r"(?i)ATTACH\s*'datatable(://[^':]+)?'\s*AS\s+([^ ;]+)").unwrap();
}
let Some(cap) = RE.captures(query) else {
let Some(attached) = parse_attach_datatable(query) else {
return Ok(None);
};
let name = cap.get(1).map(|m| &m.as_str()[3..]).unwrap_or("main");
let alias_name = cap.get(2).map(|m| m.as_str()).unwrap_or("");
// A query string that does not parse is refused rather than dropped: attaching under the
// default role when the statement asked for another one is the failure this guards.
let db_resource = match conn {
Connection::Http(client) => {
get_datatable_resource_from_agent_http(client, name, w_id).await?
let (name, role) =
windmill_common::workspaces::parse_datatable_ref(&attached.reference)?;
get_datatable_resource_from_agent_http(client, name, &job.workspace_id, role, &job.id)
.await?
}
Connection::Sql(db) => {
let (name, role) = windmill_common::workspaces::parse_datatable_ref_for(
db,
&job.workspace_id,
&attached.reference,
)
.await?;
get_datatable_resource_from_db(
db,
&job.workspace_id,
&name,
role.as_deref(),
DatatableAccess::PermissionedAs {
permissioned_as: &job.permissioned_as,
email: &job.permissioned_as_email,
},
)
.await?
}
Connection::Sql(db) => get_datatable_resource_from_db_unchecked(db, w_id, name).await?,
};
if let Some(pwd) = db_resource.get("password").and_then(|p| p.as_str()) {
hidden_passwords.lock().unwrap().push(pwd.to_string());
}
Ok(Some(pg_secret_attach_statements(db_resource, alias_name)?))
Ok(Some(pg_secret_attach_statements(
db_resource,
attached.alias,
)?))
}
// Secret names must be plain identifiers; the hash keeps two aliases distinct even
@@ -2680,6 +2718,11 @@ fn pg_secret_attach_statements(db_resource: Value, alias_name: &str) -> Result<V
esc(res.password.as_deref().unwrap_or("")),
),
format!("ATTACH 'sslmode={sslmode}' AS {alias_name} (TYPE postgres, SECRET {secret_name});"),
// The attachment keeps its own resolved connection string, so the secret is dead weight
// once attached — and a live one is a credential the script's own statements can name: an
// `ATTACH 'dbname=<other>' (TYPE postgres, SECRET …)` would reach a database nobody
// authorized this job for, as this role.
format!("DROP TEMPORARY SECRET {secret_name};"),
])
}
@@ -2753,6 +2796,45 @@ pub struct Arg {
mod tests {
use super::*;
#[test]
fn attach_datatable_parses_name_and_role() {
let reference_of = |q: &str| parse_attach_datatable(q).unwrap().reference;
let named =
parse_attach_datatable("ATTACH 'datatable://sales?role=analytics' AS dt").unwrap();
assert_eq!(
(named.reference.as_str(), named.alias),
("sales?role=analytics", "dt")
);
// A bare `datatable` is the default one, and still takes a role.
assert_eq!(
reference_of("ATTACH 'datatable?role=analytics' AS dt"),
"main?role=analytics"
);
assert_eq!(reference_of("ATTACH 'datatable://sales' AS dt"), "sales");
assert_eq!(reference_of("ATTACH 'datatable' AS dt"), "main");
assert!(parse_attach_datatable("SELECT 1").is_none());
// A stored name can contain `?`, so that is left to the workspace lookup to split.
assert_eq!(reference_of("ATTACH 'datatable://a?b' AS dt"), "a?b");
// The key matches case-insensitively, as the `-- role` annotation does, and a query string
// that does not parse is refused rather than attached under the default role.
let parse = |q: &str| {
windmill_common::workspaces::parse_datatable_ref(&reference_of(q))
.map(|(name, role)| (name.to_string(), role.map(str::to_string)))
};
assert_eq!(
parse("ATTACH 'datatable://sales?Role=analytics' AS dt").unwrap(),
("sales".to_string(), Some("analytics".to_string()))
);
for malformed in [
"ATTACH 'datatable://sales?role=' AS dt",
"ATTACH 'datatable://sales?role=an;alytics' AS dt",
"ATTACH 'datatable://sales?x=1&role=analytics' AS dt",
] {
assert!(parse(malformed).is_err(), "silently ignored: {malformed}");
}
}
#[test]
fn decode_ffi_error_unescapes_multiline_and_strips_quotes() {
// Mirror the FFI: JSON-encode the raw DuckDB message, prefix "ERROR ".
@@ -3868,6 +3950,8 @@ mod tests {
stmts[3],
format!("ATTACH 'sslmode=require' AS dt (TYPE postgres, SECRET {secret_name});")
);
assert_eq!(stmts[4], format!("DROP TEMPORARY SECRET {secret_name};"));
assert_eq!(stmts.len(), 5);
}
#[test]
backend/windmill-worker/src/pg_executor.rs
+30 -7
View File
@@ -26,9 +26,11 @@ use windmill_common::azure_workload_identity::WORKLOAD_IDENTITY_PASSWORD;
use windmill_common::error::to_anyhow;
use windmill_common::error::{self, Error};
use windmill_common::worker::{
to_raw_value, Connection, SqlResultCollectionStrategy, CLOUD_HOSTED,
to_raw_value, Connection, SqlAnnotations, SqlResultCollectionStrategy, CLOUD_HOSTED,
};
use windmill_common::workspaces::{
get_datatable_resource_from_db, parse_datatable_ref, parse_datatable_ref_for, DatatableAccess,
};
use windmill_common::workspaces::get_datatable_resource_from_db_unchecked;
use windmill_common::{PgDatabase, PrepareQueryColumnInfo, PrepareQueryResult, DB};
use windmill_parser::{Arg, Typ};
use windmill_parser_sql::{
@@ -680,15 +682,36 @@ pub async fn do_postgresql(
} else {
match pg_args.get("database").cloned() {
Some(Value::String(db_str)) if db_str.starts_with("datatable://") => {
let db_str = db_str.trim_start_matches("datatable://");
let reference = db_str.trim_start_matches("datatable://");
// The annotation wins: a generated query can carry a `?role=` in the reference it
// was handed, but only the script's author writes the leading comment block.
let annotated = SqlAnnotations::datatable_role(&query)?;
Some(match conn {
Connection::Http(client) => {
get_datatable_resource_from_agent_http(client, &db_str, &job.workspace_id)
.await?
let (name, uri_role) = parse_datatable_ref(reference)?;
get_datatable_resource_from_agent_http(
client,
name,
&job.workspace_id,
annotated.as_deref().or(uri_role),
&job.id,
)
.await?
}
Connection::Sql(db) => {
get_datatable_resource_from_db_unchecked(db, &job.workspace_id, &db_str)
.await?
let (name, uri_role) =
parse_datatable_ref_for(db, &job.workspace_id, reference).await?;
get_datatable_resource_from_db(
db,
&job.workspace_id,
&name,
annotated.as_deref().or(uri_role.as_deref()),
DatatableAccess::PermissionedAs {
permissioned_as: &job.permissioned_as,
email: &job.permissioned_as_email,
},
)
.await?
}
})
}
benchmarks/lib.ts
+1 -1
View File
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
export const VERSION = "v1.813.0";
export const VERSION = "v1.814.0";
export async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({
chat-sdk/README.md
+29 -3
View File
@@ -183,7 +183,7 @@ await chat.sendMessage('Hello')
| `workspace` | Detected inside a raw app. |
| `token` | A token, or a function returning one (called before every request, so it can fetch a short-lived token from your backend). Omit it inside a raw app. |
| `history` | `'server'`, `'local'` or `'none'`, see [History](#history). Defaults to `'server'` with a viewer session and `'local'` with an explicit `token`. |
| `inputs` | Extra flow inputs sent with every message. `sendMessage(text, { inputs })` adds per-message ones. |
| `inputs` | Extra flow inputs sent with every message. `sendMessage(text, { inputs })` adds per-message ones, and `{ attachments, attachmentsInput }` files, see [Attachments](#attachments). |
| `storageKey` | Namespace for `local` history, e.g. the signed-in user's id. Local history is per browser and per flow; without it, users sharing a browser share it. |
| `fetch`, `storage` | Replacements for the globals, for tests and unusual runtimes. |
| `pageSize` | Messages and conversations per page of server history. Default 50. |
@@ -226,8 +226,8 @@ A turn goes `submitted` (the flow is queued) → `streaming` (the answer is arri
answer, an `assistant` message with `success: false`. `status: 'error'` (with `error`
set) means the turn could not run or be followed at all, such as a refused request.
Methods: `sendMessage(text, { inputs? })`, `stop()`, `newConversation()`,
`selectConversation(id)`, `loadConversations({ page?, perPage?, kind? })`,
Methods: `sendMessage(text, { inputs?, attachments?, attachmentsInput? })`, `stop()`,
`newConversation()`, `selectConversation(id)`, `loadConversations({ page?, perPage?, kind? })`,
`deleteConversation(id)`, `renameConversation(id, title)`, `loadOlderMessages()`,
`destroy()`. `kind` lists the flow editor's test chats (`'test'`), the deployed flow's
own (`'deployed'`, the server's default) or both (`'all'`); each `Conversation` carries
@@ -235,6 +235,32 @@ own (`'deployed'`, the server's default) or both (`'all'`); each `Conversation`
stops following the current answer; the flow keeps running and, with server history,
its answer is there when you come back.
## Attachments
A flow whose AI agent step reads `user_attachments` from an `s3object[]` (or a single
`s3object`) flow input takes files with a message:
```ts
await chat.sendMessage('What does this contract say?', {
attachments: [{ name: file.name, data: file }], // a Blob/File, or a `data:` URL
attachmentsInput: { name: 'files', multiple: true }
})
```
Each file is uploaded to the workspace's object storage under
`windmill_uploads/chat/<turn>/<index>/<name>` and handed to the input as `{ s3, filename }`
objects (the object for a single-file input). Once the uploads return, the pending user
message lists them in `attachments`, as `{ input, s3, filename }` references. The name's
extension is corrected to the file's media type for PNG, JPEG and PDF, because the worker
reads the type off the key.
Files need message text to go with them. A failed upload rejects `sendMessage` before any
run starts, and `stop()` during the upload aborts it; both leave the transcript as it was.
The chat never deletes uploads, so files of a send that did not run stay in storage. The
workspace needs object storage set up. With Enterprise advanced storage permissions, the
user needs read and write on `windmill_uploads/*`, which the default rules grant. The upload goes through
`job_helpers`, so a restricted token needs `job_helpers:write`; a sandboxed raw app cannot
request that scope today, so attachments are not available there yet.
## History
Windmill stores every conversation of a chat-mode flow, and each Windmill user sees
chat-sdk/package-lock.json
Generated
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "windmill-chat",
"version": "1.813.0",
"version": "1.814.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "windmill-chat",
"version": "1.813.0",
"version": "1.814.0",
"license": "Apache-2.0",
"devDependencies": {
"@ai-sdk/react": "^4.0.102",
chat-sdk/package.json
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "windmill-chat",
"description": "Build chat interfaces on Windmill flows deployed in chat mode, from any frontend or raw app",
"version": "1.813.0",
"version": "1.814.0",
"author": "Ruben Fiszel",
"license": "Apache-2.0",
"homepage": "https://github.com/windmill-labs/windmill/tree/main/chat-sdk#readme",
chat-sdk/src/api.ts
+27 -1
View File
@@ -230,6 +230,27 @@ export class WindmillChatApi {
return (await res.json()) as FlowConversationMessage[]
}
/**
* Puts bytes in the workspace's object storage under `fileKey` and returns the key they were
* stored under (the server may rewrite it). Needs the workspace to have object storage set.
*/
async uploadFile(
fileKey: string,
body: Blob,
options: { contentType?: string; signal?: AbortSignal } = {}
): Promise<{ file_key: string }> {
const query: Record<string, string> = { file_key: fileKey }
if (options.contentType) query.content_type = options.contentType
const res = await this.#request('job_helpers/upload_s3_file', {
method: 'POST',
query,
raw: body,
contentType: options.contentType || 'application/octet-stream',
signal: options.signal
})
return (await res.json()) as { file_key: string }
}
async deleteConversation(conversationId: string): Promise<void> {
await this.#request(`flow_conversations/delete/${encodeURIComponent(conversationId)}`, {
method: 'DELETE'
@@ -241,7 +262,11 @@ export class WindmillChatApi {
init: {
method?: string
query?: Record<string, string>
/** JSON-encoded. */
body?: unknown
/** Sent as is, under `contentType`. */
raw?: Blob
contentType?: string
accept?: string
signal?: AbortSignal
} = {}
@@ -252,13 +277,14 @@ export class WindmillChatApi {
const headers: Record<string, string> = {}
if (init.accept) headers['Accept'] = init.accept
if (init.body !== undefined) headers['Content-Type'] = 'application/json'
else if (init.raw !== undefined) headers['Content-Type'] = init.contentType ?? 'application/octet-stream'
const token = typeof this.#token === 'function' ? await this.#token() : this.#token
if (token) headers['Authorization'] = `Bearer ${token}`
const res = await this.#fetch(url.toString(), {
method: init.method ?? 'GET',
headers,
body: init.body === undefined ? undefined : JSON.stringify(init.body),
body: init.body === undefined ? init.raw : JSON.stringify(init.body),
// A token must not be paired with ambient cookies; without one, the cookie is
// the credential and only rides same-origin requests.
credentials: token ? 'omit' : 'same-origin',
chat-sdk/src/attachments.ts
+116
View File
@@ -0,0 +1,116 @@
import type { WindmillChatApi } from './api'
import type { AttachmentUpload } from './types'
import { abortError, isAbortError } from './utils'
/**
* Where a chat's uploads live in the workspace's object storage. Under `windmill_uploads/`
* because the default Enterprise storage permissions grant every user write and read there
* and deny any other top-level prefix: a key outside it is refused for non-admins, both on
* upload and when the agent's job reads the file back.
*/
export const CHAT_UPLOADS_PREFIX = 'windmill_uploads/chat'
/** What an AI agent step reads out of `user_attachments`. */
export interface UploadedAttachment {
s3: string
filename: string
}
/**
* The extension each type must be stored under. The worker reads an attachment's media type
* from the key's extension only (`mime_guess` in `windmill-ai/src/image_handler.rs`, falling
* back to `image/png`), never from the stored content type, so the extension must be true.
*/
const EXTENSION_BY_MEDIA_TYPE: Record<string, string> = {
'image/png': 'png',
'image/jpeg': 'jpg',
'application/pdf': 'pdf'
}
/**
* The name an attachment is stored under: the picked name with the extension its media type
* needs, e.g. a `photo.webp` re-encoded to PNG becomes `photo.png`. Other types keep their name.
*/
export function storedAttachmentName(filename: string, mediaType: string): string {
const extension = EXTENSION_BY_MEDIA_TYPE[mediaType]
if (!extension) return filename
const stem = filename.replace(/\.[^./]+$/, '')
return `${stem || filename}.${extension}`
}
/** The bytes of an attachment as a Blob carrying its media type. */
export function attachmentBlob(attachment: AttachmentUpload): Blob {
const data =
typeof attachment.data === 'string'
? dataUrlToBlob(attachment.data, 'application/octet-stream')
: attachment.data
return attachment.mediaType && attachment.mediaType !== data.type
? new Blob([data], { type: attachment.mediaType })
: data
}
function dataUrlToBlob(dataUrl: string, fallbackType: string): Blob {
const comma = dataUrl.indexOf(',')
if (!dataUrl.startsWith('data:') || comma === -1) {
throw new Error('windmill-chat: an attachment given as a string must be a data: URL')
}
const header = dataUrl.slice(5, comma)
const isBase64 = header.endsWith(';base64')
const mediaType = (isBase64 ? header.slice(0, -';base64'.length) : header) || fallbackType
const payload = dataUrl.slice(comma + 1)
if (!isBase64) return new Blob([decodeURIComponent(payload)], { type: mediaType })
const binary = atob(payload)
const bytes = new Uint8Array(binary.length)
for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i)
return new Blob([bytes], { type: mediaType })
}
/**
* Put each attachment in the workspace's object storage and hand back what the agent reads.
* The key's turn prefix and per-file index keep two files with the same name, in this turn or
* an earlier one, from overwriting each other; the name stays the last segment.
*/
export async function uploadAttachments(
api: WindmillChatApi,
attachments: AttachmentUpload[],
turnId: string,
signal?: AbortSignal
): Promise<UploadedAttachment[]> {
const prefix = `${CHAT_UPLOADS_PREFIX}/${turnId}`
// One failed upload aborts the rest. Nothing already stored is deleted: the chat never
// removes objects from the workspace's storage, so a send that does not run leaves them.
if (signal?.aborted) throw abortError()
const batch = new AbortController()
const abortBatch = () => batch.abort()
signal?.addEventListener('abort', abortBatch, { once: true })
try {
const results = await Promise.allSettled(
attachments.map(async (attachment, index) => {
try {
const blob = attachmentBlob(attachment)
const filename = storedAttachmentName(
attachment.name || `attachment-${index + 1}`,
blob.type
)
const { file_key } = await api.uploadFile(`${prefix}/${index}/${filename}`, blob, {
contentType: blob.type,
signal: batch.signal
})
return { s3: file_key, filename }
} catch (e) {
batch.abort()
throw e
}
})
)
const reasons = results.flatMap((r) => (r.status === 'rejected' ? [r.reason] : []))
// A stop that lands once every upload has answered still withdraws the batch.
if (reasons.length === 0 && !signal?.aborted) {
return results.flatMap((r) => (r.status === 'fulfilled' ? [r.value] : []))
}
// The failure that started it, not the aborts it caused in the other uploads.
throw reasons.find((reason) => !isAbortError(reason)) ?? reasons[0] ?? abortError()
} finally {
signal?.removeEventListener('abort', abortBatch)
}
}
chat-sdk/src/chat.ts
+94 -8
View File
@@ -8,6 +8,7 @@ import {
import { resolveConfig, type ResolvedConfig } from './config'
import { followJob } from './follow'
import { createLocalHistory, type LocalHistory } from './history'
import { uploadAttachments } from './attachments'
import type { AgentStreamEvent } from './stream'
import type {
Chat,
@@ -15,6 +16,7 @@ import type {
ChatOptions,
ChatState,
Conversation,
SendMessageOptions,
ToolInvocation
} from './types'
import {
@@ -22,6 +24,7 @@ import {
truncateTitle,
errorResultMessage,
extractChatAnswer,
abortError,
isAbortError,
isErrorResult,
now,
@@ -41,6 +44,12 @@ interface Turn {
conversationId: string
/** Id of the turn's user message; the answer is whatever follows it. */
userMessageId: string
/** The turn opened the conversation; withdrawing it closes the conversation again. */
isNew: boolean
/** The run was asked for. Before that, a failure or a stop withdraws the turn instead of failing it. */
started: boolean
/** Both `stop()` and the send's own rejection withdraw; only the first may. */
withdrawn: boolean
jobId?: string
/** The flow job and its step jobs; a persisted answer carries one of them as `job_id`. */
jobIds?: Set<string>
@@ -101,21 +110,34 @@ class ChatImpl implements Chat {
}
}
sendMessage = async (
text: string,
options: { inputs?: Record<string, unknown> } = {}
): Promise<void> => {
sendMessage = async (text: string, options: SendMessageOptions = {}): Promise<void> => {
const content = text.trim()
if (!content) return
if (!content) {
// A run needs a message; files alone would otherwise be dropped without a word.
if (options.attachments?.length) throw new Error('windmill-chat: attachments need a message to go with them')
return
}
if (this.#turn) {
throw new Error('windmill-chat: a message is already being answered; call stop() first')
}
const attachments = options.attachments ?? []
const attachmentsInput = options.attachmentsInput
if (attachments.length > 0 && !attachmentsInput) {
throw new Error('windmill-chat: attachments need `attachmentsInput`, the flow input that takes them')
}
if (attachmentsInput && !attachmentsInput.multiple && attachments.length > 1) {
// Uploading all of them would run with the first and leave the rest stranded in storage.
throw new Error(`windmill-chat: \`${attachmentsInput.name}\` holds one file; got ${attachments.length}`)
}
const isNew = this.#state.conversationId === undefined
const conversationId = this.#state.conversationId ?? randomId()
const turn: Turn = {
controller: new AbortController(),
conversationId,
userMessageId: `pending-${randomId()}`,
isNew,
started: false,
withdrawn: false,
streamedText: false
}
this.#turn = turn
@@ -127,7 +149,6 @@ class ChatImpl implements Chat {
const touched = { ...conversation, updatedAt: timestamp }
this.#set({
conversationId,
conversations: [touched, ...this.#state.conversations.filter((c) => c.id !== conversationId)],
messages: [
...this.#state.messages,
{ id: turn.userMessageId, role: 'user', content, success: true, createdAt: timestamp, pending: true }
@@ -135,10 +156,32 @@ class ChatImpl implements Chat {
status: 'submitted',
error: undefined
})
this.#rememberConversation()
try {
const args = { ...this.#config.inputs, ...options.inputs, user_message: content }
const args: Record<string, unknown> = { ...this.#config.inputs, ...options.inputs, user_message: content }
if (attachmentsInput && attachments.length > 0) {
// Uploaded with the turn already shown as submitted: the message is in the transcript
// and `stop()` can abort the upload, while a second send is refused as usual.
const uploaded = await uploadAttachments(this.#api, attachments, randomId(), turn.controller.signal)
args[attachmentsInput.name] = attachmentsInput.multiple ? uploaded : uploaded[0]
// Shown on the pending message until its server row replaces it, carrying its own.
const carried = uploaded.map((u) => ({ input: attachmentsInput.name, s3: u.s3, filename: u.filename }))
if (this.#turnActive(turn)) {
this.#set({
messages: this.#state.messages.map((m) => (m.id === turn.userMessageId ? { ...m, attachments: carried } : m))
})
}
}
// Nothing may start once stop() or a conversation switch has withdrawn the turn, including
// a stop from a subscriber told of the attachments just above.
if (turn.controller.signal.aborted) throw abortError()
turn.started = true
// Listed only once the run is asked for: a send that never runs (an upload that failed
// or was stopped) then has no conversation entry to take back.
this.#set({
conversations: [touched, ...this.#state.conversations.filter((c) => c.id !== conversationId)]
})
this.#rememberConversation()
const context = { memoryId: conversationId, conversationId, signal: turn.controller.signal }
turn.jobId = this.#config.run
? await this.#config.run(args, context)
@@ -152,6 +195,13 @@ class ChatImpl implements Chat {
}
await this.#finishTurn(turn, result, isNew)
} catch (e) {
if (!turn.started) {
// Nothing ran: the message is withdrawn rather than shown as a failed turn, and the
// caller gets the reason (an upload that failed, or the AbortError of a stop()).
if (this.#turn === turn) this.#turn = undefined
this.#withdrawTurn(turn)
throw e
}
// stop() and a conversation switch abort the turn and settle the state themselves.
if (turn.controller.signal.aborted || isAbortError(e)) return
this.#failTurn(turn, e)
@@ -164,6 +214,12 @@ class ChatImpl implements Chat {
const turn = this.#turn
if (!turn) return
this.#detachTurn()
if (!turn.started) {
// Still uploading its attachments: there is no run to cancel, and the message the
// reader took back must not stay in the transcript as sent.
this.#withdrawTurn(turn)
return
}
if (this.#state.conversationId === turn.conversationId) {
this.#set({ messages: finalized(this.#state.messages), status: 'idle' })
this.#persistLocal()
@@ -578,6 +634,36 @@ class ChatImpl implements Chat {
}
}
/**
* Take back the user message of a turn that never ran. A conversation it would have opened
* was never listed (see `sendMessage`), so only the message goes, and, while it is the turn
* on screen, the busy status. A switch away mid-upload has already written the message to
* local history, so it is removed there too.
*/
#withdrawTurn(turn: Turn): void {
if (turn.withdrawn) return
turn.withdrawn = true
const id = turn.conversationId
const withoutTurn = (messages: ChatMessage[]) => messages.filter((m) => m.id !== turn.userMessageId)
if (this.#state.conversationId === id) {
const messages = withoutTurn(this.#state.messages)
// A turn started since, such as a resend right after Stop, owns the status.
const newerTurn = this.#turn !== undefined && this.#turn !== turn
if (newerTurn) {
this.#set({ messages })
} else {
const unopened = turn.isNew && messages.length === 0
this.#set({ messages, status: 'idle', error: undefined, ...(unopened ? { conversationId: undefined } : {}) })
}
this.#persistLocal()
}
if (this.#state.history === 'local' && this.#state.conversationId !== id) {
const stored = withoutTurn(this.#local.getMessages(id))
if (stored.length > 0) this.#local.saveMessages(id, stored)
else if (!this.#state.conversations.some((c) => c.id === id)) this.#local.deleteConversation(id)
}
}
#failTurn(turn: Turn, e: unknown): void {
if (!this.#turnActive(turn)) return
const error = toError(e)
chat-sdk/src/index.ts
+4
View File
@@ -14,7 +14,10 @@ export {
export { parseStreamEvents, createStreamEventParser, type AgentStreamEvent } from './stream'
export { followJob, type FollowEvent } from './follow'
export { extractChatAnswer, conversationIdFor } from './utils'
export { storedAttachmentName, uploadAttachments, CHAT_UPLOADS_PREFIX, type UploadedAttachment } from './attachments'
export type {
AttachmentsInput,
AttachmentUpload,
Chat,
ChatAttachment,
ChatMessage,
@@ -25,6 +28,7 @@ export type {
Conversation,
FetchLike,
HistoryMode,
SendMessageOptions,
StorageLike,
TokenSource,
ToolInvocation
chat-sdk/src/types.ts
+35 -2
View File
@@ -127,12 +127,45 @@ export interface ChatOptions {
onError?: (error: Error, turn: { conversationId: string; jobId?: string }) => void
}
/** A file sent with a message. It is uploaded to the workspace's object storage before the run starts. */
export interface AttachmentUpload {
/** Kept as the last segment of the stored key, its extension corrected to the media type for PNG, JPEG and PDF. */
name: string
/** The bytes: a Blob, or a `data:` URL of them. */
data: Blob | string
/** The file's media type. Defaults to the Blob's own type, or the data URL's. */
mediaType?: string
}
/** The flow input the uploaded attachments are handed to: an `s3object` (`multiple: false`) or an `s3object[]`. */
export interface AttachmentsInput {
name: string
multiple: boolean
}
export interface SendMessageOptions {
/** Extra flow inputs for this message, on top of `ChatOptions.inputs`. */
inputs?: Record<string, unknown>
/**
* Files to upload and hand to the flow as `{ s3, filename }` objects in `attachmentsInput`,
* the way an AI agent step reads `user_attachments`. A failed upload rejects `sendMessage`
* and the run never starts; `stop()` during the upload does the same with an `AbortError`.
*/
attachments?: AttachmentUpload[]
/** Required with `attachments`, which also need message text. With `multiple: false`, more than one attachment is refused before anything uploads. */
attachmentsInput?: AttachmentsInput
}
export interface Chat {
getState(): ChatState
/** Calls `listener` now and on every change; returns the unsubscribe function (Svelte store contract). */
subscribe(listener: (state: ChatState) => void): () => void
/** Sends a message in the current conversation, starting one when there is none. Resolves when the answer is complete. */
sendMessage(text: string, options?: { inputs?: Record<string, unknown> }): Promise<void>
/**
* Sends a message in the current conversation, starting one when there is none. Resolves
* when the answer is complete. Rejects when the message could not be sent at all — a turn
* already running, an attachment that failed to upload — without touching the transcript.
*/
sendMessage(text: string, options?: SendMessageOptions): Promise<void>
/** Stops following the answer and asks Windmill to cancel the run. */
stop(): Promise<void>
newConversation(): void
chat-sdk/test/attachments.test.ts
+444
View File
@@ -0,0 +1,444 @@
import { describe, expect, test } from 'bun:test'
import { WindmillChatApi } from '../src/api'
import { storedAttachmentName, uploadAttachments } from '../src/attachments'
import { createChat } from '../src/chat'
import type { ChatOptions } from '../src/types'
import { abortError } from '../src/utils'
import { fetchMock, json, memoryStorage, sse, text, type RecordedCall, type Route } from './support'
const BASE = 'http://wm.test'
const FLOW = 'f/chat/agent'
const UPLOAD_PATH = '/api/w/ws/job_helpers/upload_s3_file'
const run: Route = (c) =>
c.method === 'POST' && c.url.pathname === `/api/w/ws/jobs/run/f/${FLOW}`
? text('job-1')
: undefined
/** Stores under the key it was asked to, like the server with a `file_key`. */
const upload: Route = (c) =>
c.url.pathname === UPLOAD_PATH
? json({ file_key: c.url.searchParams.get('file_key') })
: undefined
const answer: Route = (c) =>
c.url.pathname === '/api/w/ws/jobs_u/getupdate_sse/job-1'
? sse([
{
type: 'update',
completed: true,
only_result: { output: 'ok', messages: [] }
}
])
: undefined
function options(fetch: ChatOptions['fetch']): ChatOptions {
return {
flowPath: FLOW,
baseUrl: BASE,
workspace: 'ws',
token: 'tok',
fetch,
storage: memoryStorage()
}
}
const uploads = (calls: RecordedCall[]) => calls.filter((c) => c.url.pathname === UPLOAD_PATH)
const runs = (calls: RecordedCall[]) =>
calls.filter((c) => c.url.pathname.startsWith('/api/w/ws/jobs/run/'))
const png = new Blob([new Uint8Array([0x89, 0x50, 0x4e, 0x47])], {
type: 'image/png'
})
const pdf = new Blob(['%PDF-1.7'], { type: 'application/pdf' })
describe('storedAttachmentName', () => {
// The worker reads the media type from the key's extension, so it has to match the bytes.
test('renames a re-encoded image and gives a bare name its extension', () => {
expect(storedAttachmentName('photo.webp', 'image/png')).toBe('photo.png')
expect(storedAttachmentName('holiday.png', 'image/jpeg')).toBe('holiday.jpg')
expect(storedAttachmentName('contract', 'application/pdf')).toBe('contract.pdf')
expect(storedAttachmentName('report.2026.final.webp', 'image/png')).toBe(
'report.2026.final.png'
)
})
test('leaves a type it does not know alone', () => {
expect(storedAttachmentName('notes.csv', 'text/csv')).toBe('notes.csv')
})
})
describe('sendMessage with attachments', () => {
test('uploads each file under the turn prefix and hands the list to the input', async () => {
const { fetch, calls } = fetchMock(upload, run, answer)
const chat = createChat(options(fetch))
await chat.sendMessage('read these', {
inputs: { locale: 'fr' },
attachments: [
{ name: 'photo.webp', data: png },
{ name: 'contract', data: pdf },
// A data URL is decoded to its bytes; the mediaType names what they are.
{
name: 'photo.webp',
data: `data:image/png;base64,${btoa('\x89PNG')}`
}
],
attachmentsInput: { name: 'files', multiple: true }
})
const keys = uploads(calls).map((c) => c.url.searchParams.get('file_key')!)
expect(keys).toHaveLength(3)
const prefix = keys[0].split('/').slice(0, 3).join('/')
expect(prefix).toMatch(/^windmill_uploads\/chat\/[0-9a-f-]{36}$/)
expect(keys).toEqual([
`${prefix}/0/photo.png`,
`${prefix}/1/contract.pdf`,
`${prefix}/2/photo.png`
])
expect(uploads(calls).map((c) => c.url.searchParams.get('content_type'))).toEqual([
'image/png',
'application/pdf',
'image/png'
])
expect(uploads(calls).map((c) => c.headers['content-type'])).toEqual([
'image/png',
'application/pdf',
'image/png'
])
expect(new Uint8Array(await uploads(calls)[2].raw!.arrayBuffer())).toEqual(
new Uint8Array([0x89, 0x50, 0x4e, 0x47])
)
expect(runs(calls)[0].body).toEqual({
locale: 'fr',
user_message: 'read these',
files: [
{ s3: `${prefix}/0/photo.png`, filename: 'photo.png' },
{ s3: `${prefix}/1/contract.pdf`, filename: 'contract.pdf' },
{ s3: `${prefix}/2/photo.png`, filename: 'photo.png' }
]
})
expect(chat.getState().status).toBe('idle')
})
test('hands a single object to an input that holds one file', async () => {
const { fetch, calls } = fetchMock(upload, run, answer)
const chat = createChat(options(fetch))
await chat.sendMessage('read this', {
attachments: [{ name: 'contract.pdf', data: pdf }],
attachmentsInput: { name: 'file', multiple: false }
})
const body = runs(calls)[0].body as Record<string, unknown>
expect(body.file).toEqual({
s3: expect.stringMatching(/\/0\/contract\.pdf$/),
filename: 'contract.pdf'
})
})
test('refuses several files for an input that holds one, before uploading any', async () => {
const { fetch, calls } = fetchMock(upload, run, answer)
const chat = createChat(options(fetch))
await expect(
chat.sendMessage('read these', {
attachments: [
{ name: 'a.pdf', data: pdf },
{ name: 'b.png', data: png }
],
attachmentsInput: { name: 'file', multiple: false }
})
).rejects.toThrow('holds one file')
expect(calls).toHaveLength(0)
expect(chat.getState().messages).toEqual([])
})
test('refuses attachments without message text, before uploading', async () => {
const { fetch, calls } = fetchMock(upload, run, answer)
const chat = createChat(options(fetch))
await expect(
chat.sendMessage(' ', {
attachments: [{ name: 'a.pdf', data: pdf }],
attachmentsInput: { name: 'files', multiple: true }
})
).rejects.toThrow('need a message')
expect(calls).toHaveLength(0)
})
test('refuses attachments without an input to put them in', async () => {
const { fetch, calls } = fetchMock(upload, run, answer)
const chat = createChat(options(fetch))
await expect(
chat.sendMessage('hi', { attachments: [{ name: 'a.pdf', data: pdf }] })
).rejects.toThrow('attachmentsInput')
expect(calls).toHaveLength(0)
})
test('a failed upload rejects without a run, and withdraws the message', async () => {
const { fetch, calls } = fetchMock(
(c) => (c.url.pathname === UPLOAD_PATH ? text('no object storage', 500) : undefined),
run,
answer
)
const chat = createChat(options(fetch))
const statuses: string[] = []
chat.subscribe((s) => statuses.push(s.status))
await expect(
chat.sendMessage('read this', {
attachments: [{ name: 'contract.pdf', data: pdf }],
attachmentsInput: { name: 'files', multiple: true }
})
).rejects.toThrow('no object storage')
expect(runs(calls)).toHaveLength(0)
// Shown as submitted while uploading, then withdrawn whole: no message, no conversation.
expect(statuses).toContain('submitted')
const state = chat.getState()
expect(state.status).toBe('idle')
expect(state.messages).toEqual([])
expect(state.conversationId).toBeUndefined()
expect(state.conversations).toEqual([])
// The chat is free for the next message.
await chat.sendMessage('plain')
expect(runs(calls)).toHaveLength(1)
})
test('stop() during the upload aborts it and withdraws the message', async () => {
const { fetch, calls } = fetchMock(
(c) =>
c.url.pathname === UPLOAD_PATH
? new Promise((_, reject) =>
c.signal!.addEventListener('abort', () => reject(abortError()))
)
: undefined,
run,
answer
)
const chat = createChat(options(fetch))
const sending = chat.sendMessage('read this', {
attachments: [{ name: 'contract.pdf', data: pdf }],
attachmentsInput: { name: 'files', multiple: true }
})
await new Promise((r) => setTimeout(r, 0))
expect(chat.getState().status).toBe('submitted')
await chat.stop()
await expect(sending).rejects.toMatchObject({ name: 'AbortError' })
expect(runs(calls)).toHaveLength(0)
expect(chat.getState()).toMatchObject({
status: 'idle',
messages: [],
conversationId: undefined
})
})
test('a switch away mid-upload leaves no conversation behind', async () => {
const storage = memoryStorage()
const { fetch, calls } = fetchMock(
(c) =>
c.url.pathname === UPLOAD_PATH
? new Promise((_, reject) =>
c.signal!.addEventListener('abort', () => reject(abortError()))
)
: undefined,
run,
answer
)
const chat = createChat({ ...options(fetch), storage })
const sending = chat.sendMessage('never runs', {
attachments: [{ name: 'contract.pdf', data: pdf }],
attachmentsInput: { name: 'files', multiple: true }
})
await new Promise((r) => setTimeout(r, 0))
const opened = chat.getState().conversationId!
chat.newConversation()
await expect(sending).rejects.toMatchObject({ name: 'AbortError' })
expect(runs(calls)).toHaveLength(0)
expect(chat.getState().conversations.map((c) => c.id)).not.toContain(opened)
const reloaded = createChat({ ...options(fetch), storage })
expect((await reloaded.loadConversations()).map((c) => c.id)).not.toContain(opened)
})
test('a failed upload aborts the rest of its batch and deletes nothing', async () => {
let first: (r: Response) => void = () => {}
const { fetch, calls } = fetchMock(
(c) => {
if (c.url.pathname !== UPLOAD_PATH) return undefined
const key = c.url.searchParams.get('file_key')!
// The first file lands after the second has already failed.
if (key.includes('/0/')) return new Promise<Response>((resolve) => (first = resolve))
setTimeout(() => first(json({ file_key: keys()[0] })), 5)
return text('quota exceeded', 507)
},
run,
answer
)
const keys = () => uploads(calls).map((c) => c.url.searchParams.get('file_key')!)
const chat = createChat(options(fetch))
await expect(
chat.sendMessage('read these', {
attachments: [
{ name: 'a.pdf', data: pdf },
{ name: 'b.png', data: png }
],
attachmentsInput: { name: 'files', multiple: true }
})
).rejects.toThrow('quota exceeded')
// The upload still in flight when the other failed was told to stop.
expect(uploads(calls)[0].signal?.aborted).toBe(true)
expect(calls.filter((c) => c.method === 'DELETE')).toEqual([])
expect(runs(calls)).toHaveLength(0)
})
test('a send made right after stop() is not reset by the stopped upload', async () => {
let releaseRun: (r: Response) => void = () => {}
const { fetch, calls } = fetchMock(
(c) =>
c.url.pathname === UPLOAD_PATH
? new Promise((_, reject) =>
c.signal!.addEventListener('abort', () => reject(abortError()))
)
: undefined,
(c) =>
c.method === 'POST' && c.url.pathname === `/api/w/ws/jobs/run/f/${FLOW}`
? new Promise<Response>((resolve) => (releaseRun = resolve))
: undefined,
answer
)
const chat = createChat(options(fetch))
// An existing conversation, so the stopped turn and the next one share it.
const first = chat.sendMessage('first')
await new Promise((r) => setTimeout(r, 0))
releaseRun(text('job-1'))
await first
const stopped = chat.sendMessage('with a file', {
attachments: [{ name: 'a.pdf', data: pdf }],
attachmentsInput: { name: 'files', multiple: true }
})
await new Promise((r) => setTimeout(r, 0))
void chat.stop()
const next = chat.sendMessage('right after')
await expect(stopped).rejects.toMatchObject({ name: 'AbortError' })
await new Promise((r) => setTimeout(r, 0))
expect(chat.getState().status).toBe('submitted')
expect(chat.getState().messages.map((m) => m.content)).toContain('right after')
expect(chat.getState().messages.map((m) => m.content)).not.toContain('with a file')
releaseRun(text('job-1'))
await next
expect(runs(calls)).toHaveLength(2)
})
test('a conversation is listed only once its run starts', async () => {
let failUpload: (r: Response) => void = () => {}
const { fetch } = fetchMock(
(c) =>
c.url.pathname === UPLOAD_PATH
? new Promise<Response>((resolve) => (failUpload = resolve))
: undefined,
run,
answer
)
const chat = createChat(options(fetch))
const sending = chat.sendMessage('read this', {
attachments: [{ name: 'a.pdf', data: pdf }],
attachmentsInput: { name: 'files', multiple: true }
})
await new Promise((r) => setTimeout(r, 0))
expect(chat.getState()).toMatchObject({ status: 'submitted', conversations: [] })
failUpload(text('boom', 500))
await expect(sending).rejects.toThrow('boom')
expect(chat.getState().conversations).toEqual([])
})
test('an already aborted signal uploads nothing', async () => {
const { fetch, calls } = fetchMock(upload)
const api = new WindmillChatApi({ baseUrl: BASE, workspace: 'ws', token: 'tok', fetch })
const controller = new AbortController()
controller.abort()
await expect(
uploadAttachments(api, [{ name: 'a.pdf', data: pdf }], 'turn', controller.signal)
).rejects.toMatchObject({ name: 'AbortError' })
expect(calls).toHaveLength(0)
})
test('the pending user message carries its uploaded files before the run returns', async () => {
let releaseRun: (r: Response) => void = () => {}
const { fetch, calls } = fetchMock(
upload,
(c) =>
c.method === 'POST' && c.url.pathname === `/api/w/ws/jobs/run/f/${FLOW}`
? new Promise<Response>((resolve) => (releaseRun = resolve))
: undefined,
answer
)
const chat = createChat(options(fetch))
const sending = chat.sendMessage('read these', {
attachments: [
{ name: 'photo.webp', data: png },
{ name: 'contract', data: pdf }
],
attachmentsInput: { name: 'files', multiple: true }
})
while (runs(calls).length === 0) await new Promise((r) => setTimeout(r, 1))
const keys = uploads(calls).map((c) => c.url.searchParams.get('file_key')!)
const pending = chat.getState().messages.find((m) => m.role === 'user')!
expect(pending.pending).toBe(true)
expect(pending.attachments).toEqual([
{ input: 'files', s3: keys[0], filename: 'photo.png' },
{ input: 'files', s3: keys[1], filename: 'contract.pdf' }
])
releaseRun(text('job-1'))
await sending
})
test('an explicit mediaType wins over the type a data URL declares', async () => {
const { fetch, calls } = fetchMock(upload, run, answer)
const chat = createChat(options(fetch))
await chat.sendMessage('read this', {
attachments: [
{
name: 'contract',
data: `data:application/octet-stream;base64,${btoa('%PDF')}`,
mediaType: 'application/pdf'
}
],
attachmentsInput: { name: 'files', multiple: true }
})
const call = uploads(calls)[0]
expect(call.url.searchParams.get('file_key')).toMatch(/\/0\/contract\.pdf$/)
expect(call.url.searchParams.get('content_type')).toBe('application/pdf')
})
test('stop() after the uploads land but before the run starts runs nothing', async () => {
const { fetch, calls } = fetchMock(upload, run, answer)
const chat = createChat(options(fetch))
const sending = chat.sendMessage('read this', {
attachments: [{ name: 'contract.pdf', data: pdf }],
attachmentsInput: { name: 'files', multiple: true }
})
// The upload responds at once; Stop lands before the send resumes after it.
while (uploads(calls).length === 0) await Promise.resolve()
await chat.stop()
await expect(sending).rejects.toMatchObject({ name: 'AbortError' })
expect(runs(calls)).toHaveLength(0)
expect(calls.filter((c) => c.method === 'DELETE')).toEqual([])
expect(chat.getState()).toMatchObject({ status: 'idle', messages: [], conversations: [] })
})
test('a subscriber stopping when the attachments appear runs nothing', async () => {
const { fetch, calls } = fetchMock(upload, run, answer)
const chat = createChat(options(fetch))
chat.subscribe((s) => {
if (s.messages.some((m) => m.attachments)) void chat.stop()
})
await expect(
chat.sendMessage('read this', {
attachments: [{ name: 'contract.pdf', data: pdf }],
attachmentsInput: { name: 'files', multiple: true }
})
).rejects.toMatchObject({ name: 'AbortError' })
expect(runs(calls)).toHaveLength(0)
expect(calls.filter((c) => c.method === 'DELETE')).toEqual([])
expect(chat.getState()).toMatchObject({ status: 'idle', messages: [], conversations: [] })
})
})
chat-sdk/test/support.ts
+6 -1
View File
@@ -5,6 +5,9 @@ export interface RecordedCall {
url: URL
headers: Record<string, string>
body: unknown
/** A body sent as is rather than as JSON (an upload). */
raw?: Blob
signal?: AbortSignal
}
export type Route = (call: RecordedCall) => Response | Promise<Response> | undefined
@@ -20,7 +23,9 @@ export function fetchMock(...routes: Route[]): { fetch: FetchLike; calls: Record
headers: Object.fromEntries(
Object.entries((init?.headers as Record<string, string>) ?? {}).map(([k, v]) => [k.toLowerCase(), v])
),
body: typeof init?.body === 'string' ? JSON.parse(init.body) : undefined
body: typeof init?.body === 'string' ? JSON.parse(init.body) : undefined,
raw: init?.body instanceof Blob ? init.body : undefined,
signal: init?.signal ?? undefined
}
calls.push(call)
for (const route of routes) {
cli/src/core/constants.ts
+1 -1
View File
@@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork";
// (e.g. utils.ts) can read it without importing main.ts and creating a circular
// dependency (main → workspace → utils → main) that triggers a TDZ.
// Re-exported from main.ts for backwards compatibility.
export const VERSION = "1.813.0";
export const VERSION = "1.814.0";
cli/src/core/settings.ts
+8 -1
View File
@@ -393,10 +393,17 @@ export async function pushWorkspaceSettings(
if (!deepEqual(localSettings.datatable, settings.datatable)) {
log.debug(`Updating datatable config...`);
await wmill.editDataTableConfig({
const { stranded_references } = await wmill.editDataTableConfig({
workspace,
requestBody: { settings: localSettings.datatable ?? { datatables: {} } },
});
if (stranded_references?.length) {
log.warn(
`Removed data tables governed data tables in other workspaces, which no longer resolve: ${stranded_references
.map((r) => `${r.workspace_id}/${r.datatable}`)
.join(", ")}. A superadmin can point them somewhere else.`,
);
}
}
if (localSettings.slack_command_script != settings.slack_command_script) {
cli/src/guidance/skills.gen.ts
Generated
+26 -8
View File
@@ -1110,6 +1110,8 @@ parseS3Object(s3Object: S3Object): S3ObjectRecord
/**
* Create a SQL template function for PostgreSQL/datatable queries
* @param name - Database/datatable name (default: "main")
* @param opts.role - Connect as this data table role instead of the data table's default one.
* Only meaningful on a data table under roles, and only for a role you are a tenant of.
* @returns SQL template function for building parameterized queries
* @example
* let sql = wmill.datatable()
@@ -1119,8 +1121,11 @@ parseS3Object(s3Object: S3Object): S3ObjectRecord
* SELECT * FROM friends
* WHERE name = \${name} AND age = \${age}::int
* \`.fetch()
* @example
* // Read through a restricted role
* let sql = wmill.datatable("main", { role: "analytics" })
*/
datatable(name: string = "main"): DatatableSqlTemplateFunction
datatable(name: string = "main", opts?: DatatableOptions): DatatableSqlTemplateFunction
/**
* Create a SQL template function for DuckDB/ducklake queries
@@ -1901,6 +1906,8 @@ parseS3Object(s3Object: S3Object): S3ObjectRecord
/**
* Create a SQL template function for PostgreSQL/datatable queries
* @param name - Database/datatable name (default: "main")
* @param opts.role - Connect as this data table role instead of the data table's default one.
* Only meaningful on a data table under roles, and only for a role you are a tenant of.
* @returns SQL template function for building parameterized queries
* @example
* let sql = wmill.datatable()
@@ -1910,8 +1917,11 @@ parseS3Object(s3Object: S3Object): S3ObjectRecord
* SELECT * FROM friends
* WHERE name = \${name} AND age = \${age}::int
* \`.fetch()
* @example
* // Read through a restricted role
* let sql = wmill.datatable("main", { role: "analytics" })
*/
datatable(name: string = "main"): DatatableSqlTemplateFunction
datatable(name: string = "main", opts?: DatatableOptions): DatatableSqlTemplateFunction
/**
* Create a SQL template function for DuckDB/ducklake queries
@@ -2786,6 +2796,8 @@ parseS3Object(s3Object: S3Object): S3ObjectRecord
/**
* Create a SQL template function for PostgreSQL/datatable queries
* @param name - Database/datatable name (default: "main")
* @param opts.role - Connect as this data table role instead of the data table's default one.
* Only meaningful on a data table under roles, and only for a role you are a tenant of.
* @returns SQL template function for building parameterized queries
* @example
* let sql = wmill.datatable()
@@ -2795,8 +2807,11 @@ parseS3Object(s3Object: S3Object): S3ObjectRecord
* SELECT * FROM friends
* WHERE name = \${name} AND age = \${age}::int
* \`.fetch()
* @example
* // Read through a restricted role
* let sql = wmill.datatable("main", { role: "analytics" })
*/
datatable(name: string = "main"): DatatableSqlTemplateFunction
datatable(name: string = "main", opts?: DatatableOptions): DatatableSqlTemplateFunction
/**
* Create a SQL template function for DuckDB/ducklake queries
@@ -4405,10 +4420,13 @@ def send_teams_message(conversation_id: str, text: str, success: bool = True, ca
#
# Args:
# name: Database name (default: "main")
# role: Connect as this data table role instead of the data table's default one.
# Only meaningful on a data table under roles, and only for a role you are a
# tenant of.
#
# Returns:
# DataTableClient instance
def datatable(name: str = 'main')
def datatable(name: str = 'main', *, role: Optional[str] = None)
# Get a DuckLake client for DuckDB queries.
#
@@ -4626,7 +4644,7 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]]
#
# @task(retry={"attempts": 3, "delay": 30, "multiplier": 2})
# async def call_api(payload: dict): ...
def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)
def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)
# Create a task that dispatches to a separate Windmill script.
#
@@ -4639,7 +4657,7 @@ def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, ti
# @workflow
# async def main():
# data = await extract(url="https://...")
def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)
def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)
# Create a task that dispatches to a separate Windmill flow.
#
@@ -4652,7 +4670,7 @@ def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = N
# @workflow
# async def main():
# result = await pipeline(input=data)
def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)
def task_flow(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None)
# Decorator marking an async function as a workflow-as-code entry point.
#
@@ -4717,7 +4735,7 @@ async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_
# ...
#
# results = await parallel(items, process, concurrency=5)
async def parallel(items, fn, concurrency: Optional[int] = None)
async def parallel(items, fn, *, concurrency: Optional[int] = None)
# Commit Kafka offsets for a trigger with auto_commit disabled.
#
flake.nix
+4 -1
View File
@@ -65,7 +65,10 @@
# Misc
libtool
postgresql
# Must not trail the server the dev database runs (postgres:18): pg_dump refuses a
# server newer than itself by a major version, which takes out every data table
# export, clone and fork-with-data.
postgresql_18
# Build tooling
pkg-config
frontend/package-lock.json
Generated
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "@windmill-labs/components",
"version": "1.813.0",
"version": "1.814.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@windmill-labs/components",
"version": "1.813.0",
"version": "1.814.0",
"hasInstallScript": true,
"license": "AGPL-3.0",
"dependencies": {
frontend/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@windmill-labs/components",
"version": "1.813.0",
"version": "1.814.0",
"scripts": {
"dev": "vite dev",
"dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev",
frontend/src/lib/components/FlowPreviewContent.svelte
+9 -1
View File
@@ -473,7 +473,9 @@
return jobId ?? ''
}}
conversationKind="test"
frame="boxed"
path={$pathStore}
identity={$initialPathStore || fakeInitialPath}
inputSchema={flowStore.val.schema}
flowModules={flowStore.val.value?.modules}
/>
@@ -560,7 +562,13 @@
</div>
{/if}
{/if}
<div class="pt-4 flex flex-col border-t relative">
<!-- The rule divides the inputs form from its results. Chat mode has no form: the
chat is its own panel, and a second line right under it reads as a stray edge. -->
<div
class="pt-4 flex flex-col relative {flowStore.val.value?.chat_input_enabled
? ''
: 'border-t'}"
>
{#if flowHasChanged()}
<div class="pb-2">
<div
frontend/src/lib/components/InputTransformSchemaForm.svelte
+2 -2
View File
@@ -7,7 +7,7 @@
import type { PickableProperties } from './flows/previousResults'
import InputTransformForm from './InputTransformForm.svelte'
import InputTransformPickers from './InputTransformPickers.svelte'
import { useS3StorageConfigured } from './inputTransformEnv.svelte'
import { useWorkspaceStorageConfigured } from './inputTransformEnv.svelte'
import type ItemPicker from './ItemPicker.svelte'
import type VariableEditor from './VariableEditor.svelte'
import ResizeTransitionWrapper from './common/ResizeTransitionWrapper.svelte'
@@ -88,7 +88,7 @@
let itemPicker: ItemPicker | undefined = $state(undefined)
let variableEditor: VariableEditor | undefined = $state(undefined)
const s3Storage = useS3StorageConfigured(() => ws)
const s3Storage = useWorkspaceStorageConfigured(() => ws)
let keys: string[] = $state([])
$effect(() => {
frontend/src/lib/components/InstanceSettings.svelte
+4 -2
View File
@@ -1093,7 +1093,8 @@
migrations are set up and used, how often an empty workspace home is seen, how often
the home page’s create menu and hub-project picker are opened and from which entry
point, the name of any public hub project imported from the home page and how far that
import got, and whether a pre-approved trial offer was opened, last 30 days)</li
import got, whether a pre-approved trial offer was opened, and whether data tables are
put under roles and whether callers name a role or take the default, last 30 days)</li
>
<li
>feature adoption (counts of which flow, script, trigger, worker and data table
@@ -1159,7 +1160,8 @@
migrations are set up and used, how often an empty workspace home is seen, how often
the home page’s create menu and hub-project picker are opened and from which entry
point, the name of any public hub project imported from the home page and how far that
import got, and whether a pre-approved trial offer was opened, last 30 days)</li
import got, whether a pre-approved trial offer was opened, and whether data tables are
put under roles and whether callers name a role or take the default, last 30 days)</li
>
<li
>feature adoption (counts of which flow, script, trigger, worker and data table
frontend/src/lib/components/Login.svelte
+2 -1
View File
@@ -396,7 +396,8 @@
if (loginsResult.status === 'fulfilled') {
logins = loginsResult.value.oauth.map((login) => ({
type: login.type,
displayName: login.display_name || login.type
displayName:
login.display_name || providers.find((p) => p.type === login.type)?.name || login.type
}))
saml = loginsResult.value.saml
autoLogin = loginsResult.value.auto_login
frontend/src/lib/components/ScrollFade.svelte
+66
View File
@@ -0,0 +1,66 @@
<script lang="ts">
/**
* A soft edge on a scroller, so content scrolling out of view fades instead of being
* cut against whatever borders it.
*
* Rendered as an overlay in the scroller's positioned ancestor rather than inside the
* scroller: `sticky` would resolve against the scroller's padding box and leave the
* first few pixels unfaded. It shows only when there is something hidden in that
* direction, so a transcript that fits shows no edge at all.
*/
import { twMerge } from 'tailwind-merge'
interface Props {
/** The scrolling element this masks. */
scroller: HTMLElement | undefined
edge?: 'top' | 'bottom'
/** Tailwind colour stop to fade from — the surface the scroller sits on. */
from?: string
/** Tailwind height of the fade band. */
height?: string
class?: string
}
let {
scroller,
edge = 'top',
from = 'from-surface',
height = 'h-4',
class: className = ''
}: Props = $props()
let hidden = $state(true)
$effect(() => {
const el = scroller
if (!el) return
const update = () => {
// A pixel of slack: fractional scroll offsets otherwise leave the bottom edge
// showing on a scroller that is already at its end.
hidden =
edge === 'top' ? el.scrollTop <= 1 : el.scrollTop + el.clientHeight >= el.scrollHeight - 1
}
update()
el.addEventListener('scroll', update, { passive: true })
// Content arriving or the pane resizing changes what is hidden without a scroll.
const observer = new ResizeObserver(update)
observer.observe(el)
if (el.firstElementChild) observer.observe(el.firstElementChild)
return () => {
el.removeEventListener('scroll', update)
observer.disconnect()
}
})
</script>
<div
class={twMerge(
'pointer-events-none absolute inset-x-0 transition-opacity duration-150',
edge === 'top' ? 'top-0 bg-gradient-to-b' : 'bottom-0 bg-gradient-to-t',
from,
'to-transparent',
height,
hidden ? 'opacity-0' : 'opacity-100',
className
)}
></div>
frontend/src/lib/components/SqlRepl.svelte
+15 -8
View File
@@ -105,14 +105,21 @@
// We don't always put the fix by default for row ordering concerns
let transformedCode = code
if (doPostgresRowToJsonFix) {
transformedCode = statements
.map((statement) => {
if (READ_OPS.some((op) => statement.trim().toUpperCase().startsWith(op))) {
return `SELECT row_to_json(__t__) FROM (${statement}) __t__`
}
return statement
})
.join(';')
// Rebuilt from the pruned statements, which drops the leading comment block — and
// with it the `-- role <name>` annotation that decides which login the query runs
// as. Carry it over, or the retry connects as the data table's default role and a
// query the first attempt was denied succeeds on the second.
const leadingAnnotations = code.match(/^(?:[^\S\n]*\n|[^\S\n]*--[^\n]*\n)*/)?.[0] ?? ''
transformedCode =
leadingAnnotations +
statements
.map((statement) => {
if (READ_OPS.some((op) => statement.trim().toUpperCase().startsWith(op))) {
return `SELECT row_to_json(__t__) FROM (${statement}) __t__`
}
return statement
})
.join(';')
}
const dbArg = getDatabaseArg(input)
frontend/src/lib/components/WorkspaceItemDrillPicker.svelte
+4 -3
View File
@@ -5,9 +5,10 @@ the workspace-specific public API (kinds, scope = `{ kind, dir? }`,
currentItem, leaf/branch icons) so callers (BreadcrumbSegment, EditorHeader)
don't need to know about the generic tree model underneath.
Surfaces AI-created localStorage drafts (via `listGlobalDrafts`) as extra
items alongside the backend-loaded list, so chat-scaffolded scripts/flows/
apps that haven't been deployed yet are still navigable. Gated on
Surfaces the session's drafts (via `listGlobalDrafts`: backend draft rows
overlaid with live editor cells) as extra items alongside the backend-loaded
list, so drafts the listing does not show yet (unsaved cells, a rename typed
in a live editor) are still navigable. Gated on
`isGlobalAiEnabled()` — without sessions, the only UserDrafts present are
standalone editor autosaves and surfacing those in the breadcrumb picker
would be surprising.
frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte
+70 -28
View File
@@ -35,6 +35,7 @@
import ChatQuickActions from './ChatQuickActions.svelte'
import ContextUsageIndicator from './ContextUsageIndicator.svelte'
import AIChatModelSettings from './AIChatModelSettings.svelte'
import ScrollFade from '$lib/components/ScrollFade.svelte'
import AssistantSettingsModal from './AssistantSettingsModal.svelte'
import { SkillsMenu } from './skills/skillsMenu.svelte'
import { McpMenu } from '$lib/components/mcp/mcpMenu.svelte'
@@ -361,7 +362,15 @@
chatHost.mode === AIMode.SCRIPT || chatHost.mode === AIMode.FLOW || chatHost.mode === AIMode.APP
)
const canAttachFiles = $derived(chatHost.supportsMessageAttachments && !disabled)
// Why attaching is off, when this chat takes attachments but cannot right now. The `+` is
// kept and disabled rather than dropped: the input is the composer's either way, so the
// reader has to be able to see here why nothing can be attached.
const attachmentsOffReason = $derived(
chatHost.supportsMessageAttachments ? chatHost.attachmentsUnavailableReason : undefined
)
const canAttachFiles = $derived(
chatHost.supportsMessageAttachments && !disabled && !attachmentsOffReason
)
// Folders are linked as session-wide assets, which only a host that reads files in
// the browser can do — a host running the turn server-side takes attachments only.
const canLinkFolders = $derived(chatHost.supportsLinkedFolders && !disabled)
@@ -430,24 +439,32 @@
return Array.from(e.dataTransfer?.types ?? []).includes('Files')
}
// A drop is claimed while attaching is off for a stated reason, too: the browser would
// otherwise navigate to the dropped file, and the reader is owed the reason instead.
const panelTakesDrops = $derived(canAttachFiles || attachmentsOffReason !== undefined)
function onPanelDragEnter(e: DragEvent) {
if (!canAttachFiles || !dragHasFiles(e)) return
if (!panelTakesDrops || !dragHasFiles(e)) return
e.preventDefault()
dragDepth++
}
function onPanelDragOver(e: DragEvent) {
if (!canAttachFiles || !dragHasFiles(e)) return
if (!panelTakesDrops || !dragHasFiles(e)) return
e.preventDefault()
if (e.dataTransfer) e.dataTransfer.dropEffect = 'copy'
}
function onPanelDragLeave(_e: DragEvent) {
if (!canAttachFiles) return
if (!panelTakesDrops) return
dragDepth = Math.max(0, dragDepth - 1)
}
async function onPanelDrop(e: DragEvent) {
dragDepth = 0
if (!canAttachFiles || !dragHasFiles(e)) return
if (!panelTakesDrops || !dragHasFiles(e)) return
e.preventDefault()
if (attachmentsOffReason) {
sendUserToast(attachmentsOffReason, true)
return
}
const dt = e.dataTransfer
if (!dt) return
// Images and loose text files attach to the message; folders link as session
@@ -484,9 +501,8 @@
handles.length === 0
? flatFiles
: await Promise.all(handles.filter(isFileHandle).map((h) => h.getFile()))
// Loose text files attach to the message, like images.
const textFiles = looseFiles.filter((f) => !isImageFile(f))
if (textFiles.length > 0) await aiChatInput?.addTextFiles(textFiles)
// Loose files attach to the message, like images.
await attachNonImageFiles(looseFiles.filter((f) => !isImageFile(f)))
// Folders link as a live handle.
const dirs = handles.filter(isDirectoryHandle)
if (dirs.length > 0 && !canLinkFolders) {
@@ -523,24 +539,31 @@
if (canLinkFolders) await handleAddFiles(folderEntries)
else sendUserToast('Folders cannot be attached in this chat — drop individual files.', true)
}
if (topLevelText.length > 0) await aiChatInput?.addTextFiles(topLevelText)
await attachNonImageFiles(topLevelText)
}
}
async function onFileInputChange(e: Event) {
const input = e.currentTarget as HTMLInputElement
if (input.files && input.files.length > 0) {
const picked = Array.from(input.files)
const imageFiles = picked.filter(isImageFile)
const textFiles = picked.filter((f) => !isImageFile(f))
// Reserved before the text work is awaited — see onPanelDrop.
const imageWork = imageFiles.length > 0 ? aiChatInput?.addImages(imageFiles) : undefined
if (textFiles.length > 0) await aiChatInput?.addTextFiles(textFiles)
await imageWork
await attachPickedFiles(Array.from(input.files))
}
input.value = '' // allow re-selecting the same file
}
async function attachNonImageFiles(files: File[]) {
await aiChatInput?.addNonImageFiles(files)
}
async function attachPickedFiles(picked: File[]) {
const imageFiles = picked.filter(isImageFile)
const others = picked.filter((f) => !isImageFile(f))
// Reserved before the other work is awaited — see onPanelDrop.
const imageWork = imageFiles.length > 0 ? aiChatInput?.addImages(imageFiles) : undefined
await attachNonImageFiles(others)
await imageWork
}
function onFolderInputChange(e: Event) {
const input = e.currentTarget as HTMLInputElement
// webkitdirectory files carry webkitRelativePath (`folder/sub/file`); addFiles groups
@@ -569,6 +592,15 @@
// The typing-dots indicator implies the AI is busy, which is misleading while
// the loop is parked on the user; surface a text pill instead so users know to
// act on the tool above.
// A step name hangs its icon in the column's left padding (see AssistantMessage), so a
// transcript carrying one widens the padding, on both sides to keep the column centred.
const agentGutter = $derived(messages.some((m) => m.role === 'assistant' && m.stepName))
const columnClass = $derived(
wideLayout
? `w-full max-w-3xl mx-auto ${agentGutter ? 'px-8' : 'px-7'}`
: `w-full max-w-2xl mx-auto ${agentGutter ? 'px-8' : 'px-3'}`
)
const waitingForUserAction = $derived(chatHost.loading && !!pendingUserAction(messages))
// Gated on `loading` because a card restored from history still looks parked:
@@ -622,6 +654,7 @@
const showFooterLeftControls = $derived(
!footerMessageShown &&
(canAttachFiles ||
attachmentsOffReason !== undefined ||
showContextPicker ||
showAutonomyModeSelector ||
(chatHost.mode === AIMode.SCRIPT && hasDiff))
@@ -800,12 +833,7 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. -->
bind:this={scrollElement}
onscroll={onScroll}
>
<div
class={wideLayout
? 'w-full max-w-3xl mx-auto px-7 flex flex-col pb-2'
: 'w-full max-w-2xl mx-auto px-3 flex flex-col pb-2'}
bind:clientHeight={height}
>
<div class="{columnClass} flex flex-col pb-2" bind:clientHeight={height}>
{#each messages as message, messageIndex (messageIndex)}
<AIChatMessage
{message}
@@ -844,6 +872,8 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. -->
{/if}
</div>
</div>
<!-- Sits below the scroll-to-latest button, which carries z-10. -->
<ScrollFade scroller={scrollElement} />
{#if showScrollToLatest}
<div
transition:fade={{ duration: 120 }}
@@ -869,11 +899,9 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. -->
</div>
{/if}
<div
class={wideLayout
? 'relative w-full max-w-3xl mx-auto px-6 pb-2'
: 'relative w-full max-w-2xl mx-auto px-2 pb-2'}
>
<!-- Same horizontal padding as the transcript above: the composer's edges line up with
the messages rather than sitting closer to the panel edge. -->
<div class="relative {columnClass} pb-2">
{#if showFlowPendingActionControls}
<div class="absolute -top-10 w-full flex flex-row justify-center gap-2">
<Button
@@ -991,7 +1019,21 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. -->
{/snippet}
</Popover>
{/if}
{#if canAttachFiles}
{#if attachmentsOffReason}
<Tooltip small placement="top">
<Button
nonCaptureEvent
unifiedSize="2xs"
variant="default"
iconOnly
disabled
startIcon={{ icon: Plus }}
/>
{#snippet text()}
<div class="max-w-64 text-xs">{attachmentsOffReason}</div>
{/snippet}
</Tooltip>
{:else if canAttachFiles}
<DropdownV2
items={async () => {
// Both submenus fetch on the menu's first open, so they start
frontend/src/lib/components/copilot/chat/AIChatInput.svelte
+202 -18
View File
@@ -42,6 +42,13 @@
textByteLength,
type AttachedTextFile
} from './textFileUtils'
import {
fileToAttachedBlob,
matchesAccept,
MAX_ATTACHED_BLOBS,
MAX_BLOB_BYTES,
type AttachedBlob
} from './blobUtils'
import { MessageDraft } from './messageDraft.svelte'
import ExpandableImage, {
isImageViewerOpen
@@ -213,6 +220,45 @@
: undefined
)
/**
* Free slots in one attachment lane, against both that lane's own cap and any limit the
* host's consumer imposes on the turn as a whole — a flow input holding a single file
* caps images and blobs together, not one each. In-flight decodes count: two drops that
* both read the staged count before either resolves would claim the same slots twice.
*/
function attachmentSlots(laneCap: number, laneStaged: number): number {
const laneRemaining = laneCap - laneStaged
const turnCap = chatHost.maxMessageAttachments
if (turnCap === undefined) return laneRemaining
const staged =
draft.images.length +
pendingImages +
draft.files.length +
pendingFiles +
draft.blobs.length +
pendingBlobs
// A queue counts too: what is held mid-run merges into one turn on flush, so a
// second file accepted now would be dropped there instead of refused here.
const queued =
chatHost.queuedImages.length + chatHost.queuedFiles.length + chatHost.queuedBlobs.length
return Math.min(laneRemaining, Math.max(0, turnCap - staged - queued))
}
/** What to say when the host's own limit is the one that bit. */
function turnCapMessage(): string {
const turnCap = chatHost.maxMessageAttachments
return turnCap === 1
? 'This chat sends one attachment per message.'
: `This chat sends up to ${turnCap} attachments per message.`
}
/** Why some of what was picked did not fit, naming whichever limit actually bit. */
function skippedMessage(laneCap: number, lane: 'images' | 'files', skipped: number): string {
return chatHost.maxMessageAttachments !== undefined
? `${turnCapMessage()} ${skipped} file(s) were not attached.`
: `You can attach up to ${laneCap} ${lane}; ${skipped} were skipped.`
}
// Images being decoded right now. Holds off sending so a message can never go
// out without an attachment the user already dropped, and reserves cap slots
// against a concurrent drop.
@@ -221,6 +267,14 @@
/** Attach dropped/pasted image files (downscaled + bounded). */
export async function addImages(files: (File | Blob)[]) {
if (!chatHost.supportsMessageAttachments) return
// Attaching can be off despite the chat taking attachments — no object storage to
// upload to, say. The `+` renders disabled with the reason; a drop and a paste reach
// here instead, and would otherwise become a chip that only fails once sent.
const unavailable = chatHost.attachmentsUnavailableReason
if (unavailable) {
sendUserToast(unavailable, true)
return
}
const imageFiles = files.filter(isImageFile)
if (imageFiles.length === 0) return
// The vision check is about the model this composer's own turn will hit, so it
@@ -240,9 +294,14 @@
// Count decodes already in flight: two drops that both read the image count
// before either resolves would each claim the same free slots and overshoot
// the cap.
const remaining = MAX_ATTACHED_IMAGES - draft.images.length - pendingImages
const remaining = attachmentSlots(MAX_ATTACHED_IMAGES, draft.images.length + pendingImages)
if (remaining <= 0) {
sendUserToast(`You can attach up to ${MAX_ATTACHED_IMAGES} images.`, true)
sendUserToast(
chatHost.maxMessageAttachments !== undefined
? turnCapMessage()
: `You can attach up to ${MAX_ATTACHED_IMAGES} images.`,
true
)
return
}
const oversized = imageFiles.filter((f) => f.size > MAX_IMAGE_BYTES)
@@ -255,7 +314,7 @@
const batch = usable.slice(0, remaining)
if (batch.length < usable.length) {
sendUserToast(
`You can attach up to ${MAX_ATTACHED_IMAGES} images; ${usable.length - batch.length} were skipped.`,
skippedMessage(MAX_ATTACHED_IMAGES, 'images', usable.length - batch.length),
true
)
}
@@ -326,9 +385,14 @@
export async function addTextFiles(candidates: File[]) {
if (!chatHost.supportsMessageAttachments) return
if (candidates.length === 0) return
const remaining = MAX_ATTACHED_FILES - draft.files.length - pendingFiles
const remaining = attachmentSlots(MAX_ATTACHED_FILES, draft.files.length + pendingFiles)
if (remaining <= 0) {
sendUserToast(`You can attach up to ${MAX_ATTACHED_FILES} files.`, true)
sendUserToast(
chatHost.maxMessageAttachments !== undefined
? turnCapMessage()
: `You can attach up to ${MAX_ATTACHED_FILES} files.`,
true
)
return
}
const oversized = candidates.filter((f) => f.size > MAX_TEXT_FILE_BYTES)
@@ -343,10 +407,7 @@
if (usable.length === 0) return
let batch = usable.slice(0, remaining)
if (batch.length < usable.length) {
sendUserToast(
`You can attach up to ${MAX_ATTACHED_FILES} files; ${usable.length - batch.length} were skipped.`,
true
)
sendUserToast(skippedMessage(MAX_ATTACHED_FILES, 'files', usable.length - batch.length), true)
}
// Conversation-level byte budget: transcript + queue + every live
// composer's stage (this one and, mid-edit, the other) + this composer's
@@ -420,6 +481,82 @@
draft.files = draft.files.filter((_, i) => i !== index)
}
// Blobs being read right now — same send-hold/slot-reservation role as pendingImages.
let pendingBlobs = $state(0)
/**
* Attach non-image files through the lane the host reads: text for a host that decodes
* them, blobs for one that forwards them verbatim. The picker, a drop and both pastes all
* route here, and `accept` is re-applied since drops and pastes bypass the picker's filter.
*/
export async function addNonImageFiles(files: File[]) {
if (files.length === 0) return
// Same reason as in addImages.
const unavailable = chatHost.attachmentsUnavailableReason
if (unavailable) {
sendUserToast(unavailable, true)
return
}
if (!chatHost.attachmentsAsBlobs) {
await addTextFiles(files)
return
}
const allowed = files.filter((f) => matchesAccept(f, chatHost.attachmentAccept))
if (allowed.length < files.length) {
sendUserToast(
`${files.length - allowed.length} file(s) skipped — this chat accepts ${chatHost.attachmentAccept}.`,
true
)
}
await addBlobs(allowed)
}
/** Attach files the host takes verbatim (a PDF, say). Kept out of addTextFiles:
* that one decodes to a string and drops anything the binary sniff rejects. */
export async function addBlobs(candidates: File[]) {
if (!chatHost.supportsMessageAttachments) return
if (candidates.length === 0) return
const oversized = candidates.filter((f) => f.size > MAX_BLOB_BYTES)
if (oversized.length > 0) {
const mb = Math.round(MAX_BLOB_BYTES / 1_000_000)
sendUserToast(`${oversized.length} file(s) over ${mb}MB were skipped.`, true)
}
const usable = candidates.filter((f) => f.size <= MAX_BLOB_BYTES)
if (usable.length === 0) return
const remaining = attachmentSlots(MAX_ATTACHED_BLOBS, draft.blobs.length + pendingBlobs)
if (remaining <= 0) {
sendUserToast(
chatHost.maxMessageAttachments !== undefined
? turnCapMessage()
: `You can attach up to ${MAX_ATTACHED_BLOBS} files.`,
true
)
return
}
const batch = usable.slice(0, remaining)
if (batch.length < usable.length) {
sendUserToast(skippedMessage(MAX_ATTACHED_BLOBS, 'files', usable.length - batch.length), true)
}
pendingBlobs += batch.length
try {
const added: AttachedBlob[] = []
for (const file of batch) {
try {
added.push(await fileToAttachedBlob(file))
} catch (e) {
sendUserToast(`Could not read ${file.name}`, true)
}
}
if (added.length > 0) draft.addBlobs(added)
} finally {
pendingBlobs -= batch.length
}
}
function removeBlob(index: number) {
draft.blobs = draft.blobs.filter((_, i) => i !== index)
}
// App mode @ mention state
let showAppContextTooltip = $state(false)
let appContextTooltipWord = $state('')
@@ -501,7 +638,8 @@
// Attachments still decoding/reading (or mid-drop-routing) count as
// occupancy too — they belong to a draft the user started even though
// their lane is still empty.
if (pendingImages > 0 || pendingFiles > 0 || ingestionHolds > 0) return false
if (pendingImages > 0 || pendingFiles > 0 || pendingBlobs > 0 || ingestionHolds > 0)
return false
if (
!draft.replaceIfEmpty({
text: value,
@@ -524,15 +662,17 @@
export function prependText(
text: string,
restoredImages: AttachedImage[] = [],
restoredFiles: AttachedTextFile[] = []
restoredFiles: AttachedTextFile[] = [],
restoredBlobs: AttachedBlob[] = []
): boolean {
// mergedIntoDraft: the restored text landed on top of a draft the user was
// already writing — both instructions now share one composer, so the caller
// must keep both their contexts rather than replacing one with the other.
const { mergedIntoDraft, droppedImages, droppedFiles } = draft.prepend({
const { mergedIntoDraft, droppedImages, droppedFiles, droppedBlobs } = draft.prepend({
text,
images: restoredImages,
files: restoredFiles
files: restoredFiles,
blobs: restoredBlobs
})
if (droppedImages > 0) {
sendUserToast(
@@ -546,6 +686,12 @@
true
)
}
if (droppedBlobs > 0) {
sendUserToast(
`You can attach up to ${MAX_ATTACHED_BLOBS} files; ${droppedBlobs} restored file(s) were dropped.`,
true
)
}
focusInput()
return mergedIntoDraft
}
@@ -717,7 +863,7 @@
function sendRequest() {
// The send button is disabled while decoding, but Enter reaches here directly.
// Sending now would drop the in-flight attachments onto the following message.
if (pendingImages > 0 || pendingFiles > 0 || ingestionHolds > 0) {
if (pendingImages > 0 || pendingFiles > 0 || pendingBlobs > 0 || ingestionHolds > 0) {
return
}
// A host whose consumer needs a message of its own refuses an attachment-only
@@ -761,7 +907,8 @@
expanded(chatDraft(sent.text, sent.pastes)),
sent.images,
[...selectedContext],
sent.files
sent.files,
sent.blobs
)
// Consumed at enqueue, not at flush: the entry above pinned them.
consumeMentionsIfGlobal()
@@ -789,11 +936,15 @@
// when given no override, and the consume below empties it.
const carried = chatHost.mode === AIMode.GLOBAL ? [...selectedContext] : undefined
consumeMentionsIfGlobal()
// A host that refuses the turn puts the draft back itself (see AIChatManager's
// restoreToInput and FlowChatViewHost's upload failure): restoring here too
// would double the text and every attachment.
chatHost.sendRequest({
instructions: sent.text,
pastes: sent.pastes,
images: sent.images,
files: sent.files,
blobs: sent.blobs,
contextOverride: carried,
contextOverrideOrigin: carried ? 'pinned' : undefined
})
@@ -1015,6 +1166,23 @@
updateAppTooltipPosition(appTooltipCurrentViewNumber)
}
})
/**
* Clipboard files on the plain composer, as ContextTextarea does for the rich one. Only
* when the clipboard has no text: a spreadsheet copy carries a bitmap next to the text,
* and pasting a cell range must paste the cells.
*/
function handlePlainPaste(e: ClipboardEvent) {
if (!chatHost.supportsMessageAttachments) return
if ((e.clipboardData?.getData('text/plain') ?? '').trim()) return
const pasted = Array.from(e.clipboardData?.files ?? [])
const images = pasted.filter((f) => f.type.startsWith('image/'))
const others = pasted.filter((f) => !f.type.startsWith('image/'))
if (images.length === 0 && others.length === 0) return
e.preventDefault()
if (images.length > 0) void addImages(images)
if (others.length > 0) void addNonImageFiles(others)
}
</script>
{#snippet sendStopButton()}
@@ -1035,6 +1203,7 @@
disabled ||
pendingImages > 0 ||
pendingFiles > 0 ||
pendingBlobs > 0 ||
ingestionHolds > 0 ||
needsText ||
(emptyDraft &&
@@ -1068,7 +1237,7 @@
thumbnails get their own row (different height). -->
{#snippet badgeRow()}
{@const contextChips = showContext ? selectedContext : domSelectorChips}
{#if contextChips.length > 0 || draft.files.length > 0 || pendingFiles > 0}
{#if contextChips.length > 0 || draft.files.length > 0 || pendingFiles > 0 || draft.blobs.length > 0 || pendingBlobs > 0}
<div class="flex flex-row flex-wrap items-center gap-1 px-2.5 pt-2">
{#each contextChips as element (contextKey(element))}
<ContextElementBadge
@@ -1087,7 +1256,19 @@
onDelete={() => removeFile(i)}
/>
{/each}
{#each { length: pendingFiles } as _, i (i)}
<!-- Blobs are shown by the same badge as text files. Their preview line stands
in for content the badge cannot render (a PDF has no text to show). -->
{#each draft.blobs as blob, i (i)}
<ContextElementBadge
contextElement={createAttachedFileContextElement(
blob.name,
`${blob.mediaType} · ${Math.max(1, Math.round(blob.size / 1024))} KB`
)}
deletable
onDelete={() => removeBlob(i)}
/>
{/each}
{#each { length: pendingFiles + pendingBlobs } as _, i (i)}
<div
class="h-6 w-24 rounded-md border bg-surface flex items-center justify-center"
title="Reading file..."
@@ -1152,6 +1333,7 @@
draft.isEmpty &&
pendingImages === 0 &&
pendingFiles === 0 &&
pendingBlobs === 0 &&
ingestionHolds === 0
) {
// Shell-style recall: ArrowUp in the empty main composer pulls the
@@ -1167,6 +1349,7 @@
chatHost.queuedMessage ||
chatHost.queuedImages.length > 0 ||
chatHost.queuedFiles.length > 0 ||
chatHost.queuedBlobs.length > 0 ||
(chatHost.queuedContext?.length ?? 0) > 0
) {
e.preventDefault()
@@ -1192,7 +1375,7 @@
? (pasted) => void addImages(pasted)
: undefined}
onTextFiles={chatHost.supportsMessageAttachments
? (pasted) => void addTextFiles(pasted)
? (pasted) => void addNonImageFiles(pasted)
: undefined}
{availableContext}
{selectedContext}
@@ -1299,6 +1482,7 @@
bind:this={instructionsTextareaComponent}
bind:value={draft.text}
use:autosize={{ maxHeight: '40vh' }}
onpaste={handlePlainPaste}
onkeydown={(e) => {
if (onKeyDown) {
onKeyDown(e)
frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts
+9 -6
View File
@@ -1,3 +1,4 @@
import type { AttachedBlob } from './blobUtils'
import type { ChatViewHost } from './chatViewHost'
import type { ScriptLang } from '$lib/gen/types.gen'
import { JobService, type CompletedJob } from '$lib/gen'
@@ -94,7 +95,7 @@ import { copilotInfo } from '$lib/aiStore'
import { copilotWorkspaceRequested, loadCopilot } from '$lib/components/copilot/loadCopilot'
import { askTools, prepareAskSystemMessage, prepareAskUserMessage } from './ask/core'
import { readDocsPageTool, searchDocsTool } from './docs/core'
import { TypewriterReveal } from './typewriterReveal'
import { prefersInstantReveal, TypewriterReveal } from './typewriterReveal'
import { chatState, DEFAULT_SIZE, triggerablesByAi } from './sharedChatState.svelte'
import {
createAppBackendRunnableContextElement,
@@ -154,11 +155,6 @@ import { appendAttachedFilesRoster } from './files/fileTools'
import { ENTER_PLAN_MODE_TOOL, EXIT_PLAN_MODE_TOOL } from './planMode'
import { PlanModeController, type PlanModeHost } from './planModeController.svelte'
// SSR and users who prefer reduced motion get no typewriter pacing.
function prefersInstantReveal(): boolean {
return !BROWSER || (window.matchMedia?.('(prefers-reduced-motion: reduce)').matches ?? false)
}
// Compaction of the stored history: once the projected request size
// (contextTokens — the provider's report when current, a fresh chars/4
// estimate otherwise — plus the new user message) reaches the trigger ratio of
@@ -463,6 +459,10 @@ export class AIChatManager implements ChatViewHost {
get supportsLinkedFolders() {
return this.mode === AIMode.GLOBAL
}
// The copilot reads attachments in the browser, so non-image files decode to text.
attachmentsAsBlobs = false
// The copilot decodes its attachments, so nothing ever lands in the blob lane.
queuedBlobs: AttachedBlob[] = []
// Steers the OS file picker toward text + image formats (a soft hint; both attach to
// the message — text files after a content sniff).
attachmentAccept =
@@ -3954,6 +3954,9 @@ export class AIChatManager implements ChatViewHost {
{
role: 'assistant',
content: this.currentReply,
// Stamped as it lands. A chat restored from history predates this and
// simply shows no time rather than a made-up one.
createdAt: new Date().toISOString(),
...(this.currentReasoning
? { reasoning: this.currentReasoning, reasoningDurationMs }
: {}),

Some files were not shown because too many files have changed in this diff Show More