diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 3a7939e0cc..103817a597 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.813.0" + ".": "1.814.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dd3b7f784..9a5ee31990 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,31 @@ # Changelog +## [1.814.0](https://github.com/windmill-labs/windmill/compare/v1.813.0...v1.814.0) (2026-09-17) + + +### Features + +* **ai-chat:** add list_workers and list_data_metrics global tools ([#11143](https://github.com/windmill-labs/windmill/issues/11143)) ([e954d33](https://github.com/windmill-labs/windmill/commit/e954d33613e4ff5027667eb8f646615d9bbd499d)) +* **ai-chat:** merge get_job_logs and get_flow_run_details into get_run ([#11172](https://github.com/windmill-labs/windmill/issues/11172)) ([5bb37ca](https://github.com/windmill-labs/windmill/commit/5bb37ca3388666fba72c55534e37f37bb3e9299e)) +* allow git sync auto-pull, promotion and PRs on Pro licenses ([#11173](https://github.com/windmill-labs/windmill/issues/11173)) ([02e47de](https://github.com/windmill-labs/windmill/commit/02e47de8b4c4f3f54753aabf8c67bc8e71ffb957)) +* badge chat-input flows on the home list ([#11164](https://github.com/windmill-labs/windmill/issues/11164)) ([3d08197](https://github.com/windmill-labs/windmill/commit/3d0819718221f885b61e73d02b43dcc853c7d02a)) +* collect flow conversations and agent memory once their last message goes ([#11178](https://github.com/windmill-labs/windmill/issues/11178)) ([23c24a9](https://github.com/windmill-labs/windmill/commit/23c24a9688d4c8c462f53221334d538280f16bca)) +* flow chat model picker on a shared model-settings component ([#11187](https://github.com/windmill-labs/windmill/issues/11187)) ([189793c](https://github.com/windmill-labs/windmill/commit/189793c2e4db7f1c853695ebcc895c1ec82ed19f)) +* keep flow inputs and seed the agent when chat mode is enabled ([#11177](https://github.com/windmill-labs/windmill/issues/11177)) ([68f2248](https://github.com/windmill-labs/windmill/commit/68f2248018fc218a090bf939e1eb22ff97d5bc22)) +* let plan mode search and read connected mcp servers ([#11205](https://github.com/windmill-labs/windmill/issues/11205)) ([5371519](https://github.com/windmill-labs/windmill/commit/5371519f0f5ce7750982dcdb374dca72115902e7)) +* let test_run_flow name the conversation of a chat-mode test run ([#11198](https://github.com/windmill-labs/windmill/issues/11198)) ([6e1ef93](https://github.com/windmill-labs/windmill/commit/6e1ef93f329cb396ffc3df3304d592e8fa0e0e71)) +* managed memory with an inherited or custom memory id per step ([#11118](https://github.com/windmill-labs/windmill/issues/11118)) ([c297ed0](https://github.com/windmill-labs/windmill/commit/c297ed0052d998fb8f063faa2a36c6eb03e327be)) +* render the flow chat through the shared session chat components ([#11175](https://github.com/windmill-labs/windmill/issues/11175)) ([a9ec0ae](https://github.com/windmill-labs/windmill/commit/a9ec0aec3ac0c6b0f7919d0eb2168816923826d7)) +* show flow step detail inside the graph tab on narrow detail layouts ([#11168](https://github.com/windmill-labs/windmill/issues/11168)) ([64dffe6](https://github.com/windmill-labs/windmill/commit/64dffe6106ad6a55b61a423c855a4b5b0cef533e)) +* store mcp tool call, result and reasoning on flow conversation rows ([#11176](https://github.com/windmill-labs/windmill/issues/11176)) ([a571117](https://github.com/windmill-labs/windmill/commit/a571117f3fd2cef14c920770645c60ee358fdfdd)) +* tell test flow conversations from deployed ones and rename a chat ([#11179](https://github.com/windmill-labs/windmill/issues/11179)) ([4eab995](https://github.com/windmill-labs/windmill/commit/4eab995cf7cf091a5e4640da4cb77e0921bb7fdf)) + + +### Bug Fixes + +* disable a schedule whose cron has no run left instead of panicking ([#11195](https://github.com/windmill-labs/windmill/issues/11195)) ([381d447](https://github.com/windmill-labs/windmill/commit/381d4470ef699ea82283742132e56556b95d2bd2)) +* skip expiry notifications for app embed and SDK tokens ([#11169](https://github.com/windmill-labs/windmill/issues/11169)) ([9d348f8](https://github.com/windmill-labs/windmill/commit/9d348f84c7830f36b6153472556fd70e3d84cd24)) + ## [1.813.0](https://github.com/windmill-labs/windmill/compare/v1.812.0...v1.813.0) (2026-09-16) diff --git a/backend/.sqlx/query-06abbf945bee93349ff88f64906b96ea1e853ef202510281427cfa9beeff81b3.json b/backend/.sqlx/query-06abbf945bee93349ff88f64906b96ea1e853ef202510281427cfa9beeff81b3.json new file mode 100644 index 0000000000..7adefa7eb8 --- /dev/null +++ b/backend/.sqlx/query-06abbf945bee93349ff88f64906b96ea1e853ef202510281427cfa9beeff81b3.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT 1 AS one FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "one", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "06abbf945bee93349ff88f64906b96ea1e853ef202510281427cfa9beeff81b3" +} diff --git a/backend/.sqlx/query-06ce02cd7ce2f5a57355153edb573c242f9ba758db66e9a5e16f30e3e1494201.json b/backend/.sqlx/query-06ce02cd7ce2f5a57355153edb573c242f9ba758db66e9a5e16f30e3e1494201.json new file mode 100644 index 0000000000..b543f00b44 --- /dev/null +++ b/backend/.sqlx/query-06ce02cd7ce2f5a57355153edb573c242f9ba758db66e9a5e16f30e3e1494201.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings ws\n SET datatable = (\n SELECT jsonb_set(ws.datatable, '{datatables}', jsonb_object_agg(\n dt.key,\n CASE WHEN dt.value->'reference'->>'workspace_id' = $1\n AND dt.value->'reference'->>'datatable' = $2\n THEN jsonb_set(dt.value, '{reference,datatable}', to_jsonb($3::text))\n ELSE dt.value END\n ))\n FROM jsonb_each(ws.datatable->'datatables') dt\n )\n WHERE EXISTS (\n SELECT 1 FROM jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) d\n WHERE d.value->'reference'->>'workspace_id' = $1\n AND d.value->'reference'->>'datatable' = $2\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "06ce02cd7ce2f5a57355153edb573c242f9ba758db66e9a5e16f30e3e1494201" +} diff --git a/backend/.sqlx/query-0d10e0fa5cf4033c7d93c9ed56be8209046007917f44da954eccf2188e5bff1f.json b/backend/.sqlx/query-0d10e0fa5cf4033c7d93c9ed56be8209046007917f44da954eccf2188e5bff1f.json new file mode 100644 index 0000000000..a99ffad770 --- /dev/null +++ b/backend/.sqlx/query-0d10e0fa5cf4033c7d93c9ed56be8209046007917f44da954eccf2188e5bff1f.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT permissioned_as, permissioned_as_email FROM v2_job\n WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "permissioned_as", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "permissioned_as_email", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "0d10e0fa5cf4033c7d93c9ed56be8209046007917f44da954eccf2188e5bff1f" +} diff --git a/backend/.sqlx/query-297c7a40dfce729d44aa37bc7c65560517bd25e40c0752a00467829191e2eb98.json b/backend/.sqlx/query-297c7a40dfce729d44aa37bc7c65560517bd25e40c0752a00467829191e2eb98.json new file mode 100644 index 0000000000..cda61b894a --- /dev/null +++ b/backend/.sqlx/query-297c7a40dfce729d44aa37bc7c65560517bd25e40c0752a00467829191e2eb98.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings ws\n SET datatable = (\n SELECT jsonb_set(ws.datatable, '{datatables}', jsonb_object_agg(\n dt.key,\n CASE WHEN dt.value->'reference'->>'workspace_id' = $2\n THEN jsonb_set(dt.value, '{reference,workspace_id}', to_jsonb($1::text))\n ELSE dt.value END\n ))\n FROM jsonb_each(ws.datatable->'datatables') dt\n )\n WHERE jsonb_typeof(ws.datatable->'datatables') = 'object'\n AND ws.datatable::text LIKE '%\"reference\"%'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "297c7a40dfce729d44aa37bc7c65560517bd25e40c0752a00467829191e2eb98" +} diff --git a/backend/.sqlx/query-2a391cc1bfcd2f75b46144a394c01237e09c3060da88170f1f6e06468309d213.json b/backend/.sqlx/query-2a391cc1bfcd2f75b46144a394c01237e09c3060da88170f1f6e06468309d213.json new file mode 100644 index 0000000000..a893edfbd7 --- /dev/null +++ b/backend/.sqlx/query-2a391cc1bfcd2f75b46144a394c01237e09c3060da88170f1f6e06468309d213.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE capture_config SET server_id = NULL, last_server_ping = NULL\n WHERE workspace_id = $1 AND trigger_kind = 'postgres'\n AND (trigger_config->>'postgres_resource_path' = $2\n OR trigger_config->>'postgres_resource_path' LIKE $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "2a391cc1bfcd2f75b46144a394c01237e09c3060da88170f1f6e06468309d213" +} diff --git a/backend/.sqlx/query-334dbcd48fb59c96c62c2705ab2d1ce716cd52417f487cc1a8dd376017b2db7d.json b/backend/.sqlx/query-334dbcd48fb59c96c62c2705ab2d1ce716cd52417f487cc1a8dd376017b2db7d.json new file mode 100644 index 0000000000..0c94309c58 --- /dev/null +++ b/backend/.sqlx/query-334dbcd48fb59c96c62c2705ab2d1ce716cd52417f487cc1a8dd376017b2db7d.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = $1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Name" + ] + }, + "nullable": [ + null + ] + }, + "hash": "334dbcd48fb59c96c62c2705ab2d1ce716cd52417f487cc1a8dd376017b2db7d" +} diff --git a/backend/.sqlx/query-4692810d2be817bbb5de9b476d68d695941bd4fb5ccef393e4da522ed479d601.json b/backend/.sqlx/query-4692810d2be817bbb5de9b476d68d695941bd4fb5ccef393e4da522ed479d601.json new file mode 100644 index 0000000000..b0ad1eaf86 --- /dev/null +++ b/backend/.sqlx/query-4692810d2be817bbb5de9b476d68d695941bd4fb5ccef393e4da522ed479d601.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE postgres_trigger SET server_id = NULL, last_server_ping = NULL\n WHERE workspace_id = $1\n AND (postgres_resource_path = $2 OR postgres_resource_path LIKE $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "4692810d2be817bbb5de9b476d68d695941bd4fb5ccef393e4da522ed479d601" +} diff --git a/backend/.sqlx/query-5048e21546f9710697100100e1255ab103979433bc386d7c89d0e30db12bfd57.json b/backend/.sqlx/query-5048e21546f9710697100100e1255ab103979433bc386d7c89d0e30db12bfd57.json new file mode 100644 index 0000000000..804e294296 --- /dev/null +++ b/backend/.sqlx/query-5048e21546f9710697100100e1255ab103979433bc386d7c89d0e30db12bfd57.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ws.workspace_id AS \"workspace_id!\", dt.key AS \"datatable!\"\n FROM workspace_settings ws\n CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt\n WHERE dt.value->'reference'->>'workspace_id' = $1\n AND dt.value->'reference'->>'datatable' = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "datatable!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "5048e21546f9710697100100e1255ab103979433bc386d7c89d0e30db12bfd57" +} diff --git a/backend/.sqlx/query-538dd1779874e4003932d7f17750239c625f85e25b3364bf2edf566f518c8ee2.json b/backend/.sqlx/query-538dd1779874e4003932d7f17750239c625f85e25b3364bf2edf566f518c8ee2.json new file mode 100644 index 0000000000..03613519ad --- /dev/null +++ b/backend/.sqlx/query-538dd1779874e4003932d7f17750239c625f85e25b3364bf2edf566f518c8ee2.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ws.workspace_id AS \"workspace_id!\", dt.key AS \"datatable!\"\n FROM workspace_settings ws\n CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt\n WHERE dt.value->'reference'->>'workspace_id' = $1\n AND dt.value->'reference'->>'datatable' = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "datatable!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "538dd1779874e4003932d7f17750239c625f85e25b3364bf2edf566f518c8ee2" +} diff --git a/backend/.sqlx/query-58e5cfe9eb87bda9f7de87c403861b6e7b9d35a41594681e2a92a87359e6a018.json b/backend/.sqlx/query-58e5cfe9eb87bda9f7de87c403861b6e7b9d35a41594681e2a92a87359e6a018.json new file mode 100644 index 0000000000..f4f7acd0b7 --- /dev/null +++ b/backend/.sqlx/query-58e5cfe9eb87bda9f7de87c403861b6e7b9d35a41594681e2a92a87359e6a018.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT username FROM usr WHERE workspace_id = $1 AND email = $2 AND disabled = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "username", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "58e5cfe9eb87bda9f7de87c403861b6e7b9d35a41594681e2a92a87359e6a018" +} diff --git a/backend/.sqlx/query-6f9fb5d72f486358fa25d6887bd69b93910e028f140c07048f2c1c8d63ee6909.json b/backend/.sqlx/query-6f9fb5d72f486358fa25d6887bd69b93910e028f140c07048f2c1c8d63ee6909.json new file mode 100644 index 0000000000..d87d3aeba5 --- /dev/null +++ b/backend/.sqlx/query-6f9fb5d72f486358fa25d6887bd69b93910e028f140c07048f2c1c8d63ee6909.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO global_settings (name, value) VALUES ($1, $2)\n ON CONFLICT (name) DO UPDATE SET value = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "6f9fb5d72f486358fa25d6887bd69b93910e028f140c07048f2c1c8d63ee6909" +} diff --git a/backend/.sqlx/query-71ee2cb6661cca1fa4d8874a7f6d368347c59f36fd87df6dc7996152ccb84af0.json b/backend/.sqlx/query-71ee2cb6661cca1fa4d8874a7f6d368347c59f36fd87df6dc7996152ccb84af0.json new file mode 100644 index 0000000000..a03fab7081 --- /dev/null +++ b/backend/.sqlx/query-71ee2cb6661cca1fa4d8874a7f6d368347c59f36fd87df6dc7996152ccb84af0.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, name, enabled, pwd FROM datatable_role", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "enabled", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "pwd", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false, + true + ] + }, + "hash": "71ee2cb6661cca1fa4d8874a7f6d368347c59f36fd87df6dc7996152ccb84af0" +} diff --git a/backend/.sqlx/query-79799b5a2e499df6c28e286c42b9ad2db940c2455ab19cc95e5198baf96d5629.json b/backend/.sqlx/query-79799b5a2e499df6c28e286c42b9ad2db940c2455ab19cc95e5198baf96d5629.json new file mode 100644 index 0000000000..c36b6fb361 --- /dev/null +++ b/backend/.sqlx/query-79799b5a2e499df6c28e286c42b9ad2db940c2455ab19cc95e5198baf96d5629.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT ws.workspace_id AS \"workspace_id!\", dt.key AS \"datatable!\"\n FROM workspace_settings ws\n JOIN workspace w ON w.id = ws.workspace_id AND w.deleted = false\n CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt\n WHERE ws.workspace_id <> $1\n AND dt.value->'database'->>'resource_type' = 'instance'\n AND dt.value->'database'->>'resource_path' = $2\n ORDER BY ws.workspace_id, dt.key\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "datatable!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "79799b5a2e499df6c28e286c42b9ad2db940c2455ab19cc95e5198baf96d5629" +} diff --git a/backend/.sqlx/query-86af9d51a158ea5cb6161461ecddf2a63695f8cbf8af648da5a0a77a5b9d02ba.json b/backend/.sqlx/query-86af9d51a158ea5cb6161461ecddf2a63695f8cbf8af648da5a0a77a5b9d02ba.json new file mode 100644 index 0000000000..7627d9828d --- /dev/null +++ b/backend/.sqlx/query-86af9d51a158ea5cb6161461ecddf2a63695f8cbf8af648da5a0a77a5b9d02ba.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO datatable_role (id, name, enabled, pwd) VALUES ($1, $2, $3, $4)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool", + "Text" + ] + }, + "nullable": [] + }, + "hash": "86af9d51a158ea5cb6161461ecddf2a63695f8cbf8af648da5a0a77a5b9d02ba" +} diff --git a/backend/.sqlx/query-975e404ac3a6258bb8220e122e3de094c7ab23330fdbc74d6e4ad472ddd3c820.json b/backend/.sqlx/query-975e404ac3a6258bb8220e122e3de094c7ab23330fdbc74d6e4ad472ddd3c820.json new file mode 100644 index 0000000000..87666986a3 --- /dev/null +++ b/backend/.sqlx/query-975e404ac3a6258bb8220e122e3de094c7ab23330fdbc74d6e4ad472ddd3c820.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO global_settings (name, value) VALUES ($1, $2)\n ON CONFLICT (name) DO UPDATE SET value = $2, updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "975e404ac3a6258bb8220e122e3de094c7ab23330fdbc74d6e4ad472ddd3c820" +} diff --git a/backend/.sqlx/query-9f663180166f53d117e794f3f3a5723a0a43db163ecca7d5a63d4e74ab1d3be1.json b/backend/.sqlx/query-9f663180166f53d117e794f3f3a5723a0a43db163ecca7d5a63d4e74ab1d3be1.json new file mode 100644 index 0000000000..4dd7ea9bd8 --- /dev/null +++ b/backend/.sqlx/query-9f663180166f53d117e794f3f3a5723a0a43db163ecca7d5a63d4e74ab1d3be1.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_xact_lock(hashtext('datatable_role_catalog'))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_xact_lock", + "type_info": "Void" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "9f663180166f53d117e794f3f3a5723a0a43db163ecca7d5a63d4e74ab1d3be1" +} diff --git a/backend/.sqlx/query-a57de2bb0442a5ee8a607cd63cfcf675de175796184f620cb4b09670c8b0b19f.json b/backend/.sqlx/query-a57de2bb0442a5ee8a607cd63cfcf675de175796184f620cb4b09670c8b0b19f.json new file mode 100644 index 0000000000..f3b2cb571e --- /dev/null +++ b/backend/.sqlx/query-a57de2bb0442a5ee8a607cd63cfcf675de175796184f620cb4b09670c8b0b19f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE capture_config SET server_id = NULL, last_server_ping = NULL\n WHERE workspace_id = $1 AND trigger_kind = 'postgres'\n AND (trigger_config->>'postgres_resource_path' = $2\n OR trigger_config->>'postgres_resource_path' LIKE $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "a57de2bb0442a5ee8a607cd63cfcf675de175796184f620cb4b09670c8b0b19f" +} diff --git a/backend/.sqlx/query-b42af37fb474bea4c5419b0a46d9eadfe384013ab970ccf9c5effd1c78321b7c.json b/backend/.sqlx/query-b42af37fb474bea4c5419b0a46d9eadfe384013ab970ccf9c5effd1c78321b7c.json new file mode 100644 index 0000000000..65c7f47b30 --- /dev/null +++ b/backend/.sqlx/query-b42af37fb474bea4c5419b0a46d9eadfe384013ab970ccf9c5effd1c78321b7c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ws.datatable->'datatables' FROM workspace_settings ws\n WHERE ws.workspace_id = $1 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "b42af37fb474bea4c5419b0a46d9eadfe384013ab970ccf9c5effd1c78321b7c" +} diff --git a/backend/.sqlx/query-b9842d2d8abf382bd82d8fa1de012373638be391f884f81dc387ffc465badac6.json b/backend/.sqlx/query-b9842d2d8abf382bd82d8fa1de012373638be391f884f81dc387ffc465badac6.json new file mode 100644 index 0000000000..4b3a2f33e9 --- /dev/null +++ b/backend/.sqlx/query-b9842d2d8abf382bd82d8fa1de012373638be391f884f81dc387ffc465badac6.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT jsonb_object_keys(value->'databases') FROM global_settings\n WHERE name = 'custom_instance_pg_databases'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "jsonb_object_keys", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "b9842d2d8abf382bd82d8fa1de012373638be391f884f81dc387ffc465badac6" +} diff --git a/backend/.sqlx/query-c1d026c886799dabc39ce73e1fe09ccb175c7271df75d67aa9c72ad6f825a992.json b/backend/.sqlx/query-c1d026c886799dabc39ce73e1fe09ccb175c7271df75d67aa9c72ad6f825a992.json new file mode 100644 index 0000000000..f5fde93bf7 --- /dev/null +++ b/backend/.sqlx/query-c1d026c886799dabc39ce73e1fe09ccb175c7271df75d67aa9c72ad6f825a992.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM usr WHERE email = $1 RETURNING username, workspace_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "username", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "workspace_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "c1d026c886799dabc39ce73e1fe09ccb175c7271df75d67aa9c72ad6f825a992" +} diff --git a/backend/.sqlx/query-c5451ea9d9fa5146af242d1ee8c19ebd65b80e7ed9f29b9fb2e03767c2aa94ba.json b/backend/.sqlx/query-c5451ea9d9fa5146af242d1ee8c19ebd65b80e7ed9f29b9fb2e03767c2aa94ba.json new file mode 100644 index 0000000000..5d8ca8dc00 --- /dev/null +++ b/backend/.sqlx/query-c5451ea9d9fa5146af242d1ee8c19ebd65b80e7ed9f29b9fb2e03767c2aa94ba.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ws.workspace_id AS \"workspace_id!\", dt.key AS \"datatable!\"\n FROM workspace_settings ws\n CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt\n WHERE dt.value->'reference'->>'workspace_id' = $1\n ORDER BY ws.workspace_id, dt.key", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "datatable!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "c5451ea9d9fa5146af242d1ee8c19ebd65b80e7ed9f29b9fb2e03767c2aa94ba" +} diff --git a/backend/.sqlx/query-c85d362fe2e652d4ac01a35bf470e80b993020a2ff5dcb5849dc570d52798587.json b/backend/.sqlx/query-c85d362fe2e652d4ac01a35bf470e80b993020a2ff5dcb5849dc570d52798587.json new file mode 100644 index 0000000000..7a7da4308d --- /dev/null +++ b/backend/.sqlx/query-c85d362fe2e652d4ac01a35bf470e80b993020a2ff5dcb5849dc570d52798587.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM datatable_role WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "c85d362fe2e652d4ac01a35bf470e80b993020a2ff5dcb5849dc570d52798587" +} diff --git a/backend/.sqlx/query-d48ca62c86b1af7a9dd2450c1c28dc45020a2a553d8874c49f9eafedea5a9d40.json b/backend/.sqlx/query-d48ca62c86b1af7a9dd2450c1c28dc45020a2a553d8874c49f9eafedea5a9d40.json new file mode 100644 index 0000000000..49875acd78 --- /dev/null +++ b/backend/.sqlx/query-d48ca62c86b1af7a9dd2450c1c28dc45020a2a553d8874c49f9eafedea5a9d40.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT dt.key AS \"datatable!\"\n FROM workspace_settings ws\n CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt\n WHERE ws.workspace_id = $1\n AND dt.key <> $2\n AND NOT dt.value ? 'permissions'\n AND dt.value->'database'->>'resource_type' = 'instance'\n AND dt.value->'database'->>'resource_path' = $3\n ORDER BY dt.key", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "datatable!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "d48ca62c86b1af7a9dd2450c1c28dc45020a2a553d8874c49f9eafedea5a9d40" +} diff --git a/backend/.sqlx/query-d5fb5dde6300862f978739a3d9249fc2b3e7697c0da7d3195398933d3d81aadf.json b/backend/.sqlx/query-d5fb5dde6300862f978739a3d9249fc2b3e7697c0da7d3195398933d3d81aadf.json new file mode 100644 index 0000000000..6297c4d366 --- /dev/null +++ b/backend/.sqlx/query-d5fb5dde6300862f978739a3d9249fc2b3e7697c0da7d3195398933d3d81aadf.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT datatable FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "datatable", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "d5fb5dde6300862f978739a3d9249fc2b3e7697c0da7d3195398933d3d81aadf" +} diff --git a/backend/.sqlx/query-da5057c86607327bafc2942c218025ca9181a0c396405984d87e422e129521c1.json b/backend/.sqlx/query-da5057c86607327bafc2942c218025ca9181a0c396405984d87e422e129521c1.json new file mode 100644 index 0000000000..4e63a2afa6 --- /dev/null +++ b/backend/.sqlx/query-da5057c86607327bafc2942c218025ca9181a0c396405984d87e422e129521c1.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings ws\n SET datatable = (\n SELECT jsonb_set(ws.datatable, '{datatables}', jsonb_object_agg(\n dt.key,\n CASE WHEN dt.value->'reference'->>'workspace_id' = $1\n AND dt.value->'reference'->>'datatable' = $2\n THEN jsonb_set(dt.value, '{reference,datatable}', to_jsonb($3::text))\n ELSE dt.value END\n ))\n FROM jsonb_each(ws.datatable->'datatables') dt\n )\n WHERE jsonb_typeof(ws.datatable->'datatables') = 'object'\n AND ws.datatable::text LIKE '%\"reference\"%'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "da5057c86607327bafc2942c218025ca9181a0c396405984d87e422e129521c1" +} diff --git a/backend/.sqlx/query-dc8dfc37559e9b6713bde48155f48b5a2c7b8199eace1508e102b60d1ff40c04.json b/backend/.sqlx/query-dc8dfc37559e9b6713bde48155f48b5a2c7b8199eace1508e102b60d1ff40c04.json new file mode 100644 index 0000000000..def9ee0edb --- /dev/null +++ b/backend/.sqlx/query-dc8dfc37559e9b6713bde48155f48b5a2c7b8199eace1508e102b60d1ff40c04.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value->'roles' FROM global_settings WHERE name = 'custom_instance_pg_databases'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "dc8dfc37559e9b6713bde48155f48b5a2c7b8199eace1508e102b60d1ff40c04" +} diff --git a/backend/.sqlx/query-e159b2ff15633f85e839ee4fe1ec2ecd11caf228ea8d0f52ad66def595644250.json b/backend/.sqlx/query-e159b2ff15633f85e839ee4fe1ec2ecd11caf228ea8d0f52ad66def595644250.json new file mode 100644 index 0000000000..3f7245f35f --- /dev/null +++ b/backend/.sqlx/query-e159b2ff15633f85e839ee4fe1ec2ecd11caf228ea8d0f52ad66def595644250.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE postgres_trigger SET server_id = NULL, last_server_ping = NULL\n WHERE workspace_id = $1\n AND (postgres_resource_path = $2 OR postgres_resource_path LIKE $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "e159b2ff15633f85e839ee4fe1ec2ecd11caf228ea8d0f52ad66def595644250" +} diff --git a/backend/.sqlx/query-e2061df65ffd4a72146c4ca316829265289c8d6f625ac272655c88e1ad0b1745.json b/backend/.sqlx/query-e2061df65ffd4a72146c4ca316829265289c8d6f625ac272655c88e1ad0b1745.json new file mode 100644 index 0000000000..0e3c0182a9 --- /dev/null +++ b/backend/.sqlx/query-e2061df65ffd4a72146c4ca316829265289c8d6f625ac272655c88e1ad0b1745.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT datatable FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "datatable", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "e2061df65ffd4a72146c4ca316829265289c8d6f625ac272655c88e1ad0b1745" +} diff --git a/backend/.sqlx/query-ebaf3ed3097621da59dd201b5a4b9d1f440692f183c7c378f59e4b73f1c6e241.json b/backend/.sqlx/query-ebaf3ed3097621da59dd201b5a4b9d1f440692f183c7c378f59e4b73f1c6e241.json new file mode 100644 index 0000000000..e5de67e73f --- /dev/null +++ b/backend/.sqlx/query-ebaf3ed3097621da59dd201b5a4b9d1f440692f183c7c378f59e4b73f1c6e241.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings\n SET datatable = jsonb_set(\n jsonb_set(\n datatable #- ARRAY['datatables', $2, 'reference'],\n ARRAY['datatables', $2, 'database'], $3::jsonb),\n ARRAY['datatables', $2, 'forked_from'], $4::jsonb\n )\n WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "ebaf3ed3097621da59dd201b5a4b9d1f440692f183c7c378f59e4b73f1c6e241" +} diff --git a/backend/.sqlx/query-f3ee09fb17955ca8d886f446d397063c4094546a7807343b570b823796372cef.json b/backend/.sqlx/query-f3ee09fb17955ca8d886f446d397063c4094546a7807343b570b823796372cef.json new file mode 100644 index 0000000000..64183afaad --- /dev/null +++ b/backend/.sqlx/query-f3ee09fb17955ca8d886f446d397063c4094546a7807343b570b823796372cef.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id FROM workspace_settings WHERE datatable::text LIKE $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "f3ee09fb17955ca8d886f446d397063c4094546a7807343b570b823796372cef" +} diff --git a/backend/.sqlx/query-f4adc9e26ebfebce18a29fb2c21bf06394cacb8a9699a608327b097e0ac1363e.json b/backend/.sqlx/query-f4adc9e26ebfebce18a29fb2c21bf06394cacb8a9699a608327b097e0ac1363e.json new file mode 100644 index 0000000000..b3dd09e2ba --- /dev/null +++ b/backend/.sqlx/query-f4adc9e26ebfebce18a29fb2c21bf06394cacb8a9699a608327b097e0ac1363e.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings\n SET datatable = CASE WHEN $3::jsonb = 'null'::jsonb\n THEN datatable #- ARRAY['datatables', $2, 'permissions']\n ELSE jsonb_set(datatable, ARRAY['datatables', $2, 'permissions'], $3::jsonb)\n END\n WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "f4adc9e26ebfebce18a29fb2c21bf06394cacb8a9699a608327b097e0ac1363e" +} diff --git a/backend/.sqlx/query-fcb34e643b888122766e115a01394ab31ac856252aaa76c75ea27a447009c363.json b/backend/.sqlx/query-fcb34e643b888122766e115a01394ab31ac856252aaa76c75ea27a447009c363.json new file mode 100644 index 0000000000..26920a2943 --- /dev/null +++ b/backend/.sqlx/query-fcb34e643b888122766e115a01394ab31ac856252aaa76c75ea27a447009c363.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE datatable_role SET name = $2, enabled = $3, pwd = $4 WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar", + "Bool", + "Text" + ] + }, + "nullable": [] + }, + "hash": "fcb34e643b888122766e115a01394ab31ac856252aaa76c75ea27a447009c363" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index b2b9d65c97..d39d7e9a94 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -728,7 +728,7 @@ dependencies = [ "futures-lite 2.6.1", "parking", "polling 3.11.0", - "rustix 1.1.4", + "rustix 1.1.5", "slab", "windows-sys 0.61.2", ] @@ -873,7 +873,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.6", ] [[package]] @@ -1976,7 +1976,7 @@ dependencies = [ "prettyplease 0.3.0", "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.6", ] [[package]] @@ -2000,7 +2000,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.6", ] [[package]] @@ -2144,7 +2144,7 @@ checksum = "6a1f896587b6f2c069c73d2f0913e2d590c3990285cd2f0b6aa02b786b4c679c" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.6", ] [[package]] @@ -2164,9 +2164,9 @@ dependencies = [ [[package]] name = "bytes-str" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "577d2bf5650f8554d5a372af5ac93535110a0fc75b3e702bb853369febf227c2" +checksum = "4dde6d05e75a31ec9610eb6446a6f0a10dd30ff5100d720fee4c7c7a9008b5ba" dependencies = [ "bytes", "serde", @@ -2338,9 +2338,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" [[package]] name = "cfg_aliases" @@ -2454,7 +2454,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.6", ] [[package]] @@ -3047,7 +3047,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 3.0.5", + "syn 3.0.6", ] [[package]] @@ -3102,7 +3102,7 @@ checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ "darling_core 0.24.1", "quote", - "syn 3.0.5", + "syn 3.0.6", ] [[package]] @@ -4580,7 +4580,7 @@ checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" dependencies = [ "libc", "option-ext", - "redox_users 0.5.2", + "redox_users 0.5.3", "windows-sys 0.61.2", ] @@ -4603,7 +4603,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.6", ] [[package]] @@ -4850,7 +4850,7 @@ checksum = "a65863d15a4ce2888bd2f0f543cc963d3879c3a022c8ee43f6141d479a3ac815" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.6", ] [[package]] @@ -5198,7 +5198,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" dependencies = [ - "rustix 1.1.4", + "rustix 1.1.5", "windows-sys 0.59.0", ] @@ -5319,7 +5319,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.6", ] [[package]] @@ -9438,7 +9438,7 @@ dependencies = [ "concurrent-queue", "hermit-abi 0.5.3", "pin-project-lite", - "rustix 1.1.4", + "rustix 1.1.5", "windows-sys 0.61.2", ] @@ -9574,7 +9574,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" dependencies = [ "proc-macro2", - "syn 3.0.5", + "syn 3.0.6", ] [[package]] @@ -10235,11 +10235,10 @@ dependencies = [ [[package]] name = "redox_users" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +checksum = "60dc65c0ff1a7ae1294b0c67b9f14baf70b644404010370171787bfac1038fc0" dependencies = [ - "getrandom 0.2.17", "libredox", "thiserror 2.0.20", ] @@ -10261,7 +10260,7 @@ checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.6", ] [[package]] @@ -10580,7 +10579,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 3.0.5", + "syn 3.0.6", ] [[package]] @@ -10798,9 +10797,9 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" dependencies = [ "bitflags 2.13.2", "errno", @@ -11217,7 +11216,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals 0.30.0", - "syn 3.0.5", + "syn 3.0.6", ] [[package]] @@ -11416,7 +11415,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.6", ] [[package]] @@ -11438,7 +11437,7 @@ checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.6", ] [[package]] @@ -11492,7 +11491,7 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.6", ] [[package]] @@ -11560,7 +11559,7 @@ dependencies = [ "darling 0.24.1", "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.6", ] [[package]] @@ -12708,9 +12707,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.5" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" dependencies = [ "proc-macro2", "quote", @@ -12745,7 +12744,7 @@ checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.6", ] [[package]] @@ -13036,7 +13035,7 @@ dependencies = [ "fastrand 2.5.0", "getrandom 0.4.3", "once_cell", - "rustix 1.1.4", + "rustix 1.1.5", "windows-sys 0.61.2", ] @@ -13055,7 +13054,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ - "rustix 1.1.4", + "rustix 1.1.5", "windows-sys 0.61.2", ] @@ -13115,7 +13114,7 @@ checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.6", ] [[package]] @@ -14075,7 +14074,7 @@ checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.6", ] [[package]] @@ -14179,9 +14178,9 @@ checksum = "81b79ad29b5e19de4260020f8919b443b2ef0277d242ce532ec7b7a2cc8b6007" [[package]] name = "unicode-ident" -version = "1.0.24" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" [[package]] name = "unicode-normalization" @@ -14546,7 +14545,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.6", "wasm-bindgen-shared", ] @@ -14589,7 +14588,7 @@ checksum = "8c89dcab8b516b6b603baca9d550b7282d68fcc7f367e3956cff7ebf406a3f12" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.6", ] [[package]] @@ -14794,7 +14793,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "async-nats", @@ -14882,7 +14881,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.813.0" +version = "1.814.0" dependencies = [ "async-stream", "async-trait", @@ -14916,7 +14915,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.813.0" +version = "1.814.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14929,7 +14928,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "argon2", @@ -15069,7 +15068,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.813.0" +version = "1.814.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15092,7 +15091,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.813.0" +version = "1.814.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15109,7 +15108,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15135,7 +15134,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.813.0" +version = "1.814.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15145,7 +15144,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.813.0" +version = "1.814.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15162,7 +15161,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.813.0" +version = "1.814.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -15184,7 +15183,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15207,7 +15206,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.813.0" +version = "1.814.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15223,7 +15222,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.813.0" +version = "1.814.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15245,7 +15244,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.813.0" +version = "1.814.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15267,7 +15266,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.813.0" +version = "1.814.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15281,7 +15280,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "async-nats", @@ -15316,7 +15315,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15341,7 +15340,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.813.0" +version = "1.814.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15369,7 +15368,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15391,7 +15390,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.813.0" +version = "1.814.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15411,7 +15410,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.813.0" +version = "1.814.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15449,7 +15448,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15471,13 +15470,14 @@ dependencies = [ "windmill-ai", "windmill-alerting", "windmill-api-auth", + "windmill-audit", "windmill-common", "windmill-object-store", ] [[package]] name = "windmill-api-sse" -version = "1.813.0" +version = "1.814.0" dependencies = [ "lazy_static", "serde", @@ -15489,7 +15489,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.813.0" +version = "1.814.0" dependencies = [ "argon2", "axum 0.8.9", @@ -15513,7 +15513,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.813.0" +version = "1.814.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15527,7 +15527,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.813.0" +version = "1.814.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15562,7 +15562,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.813.0" +version = "1.814.0" dependencies = [ "chrono", "lazy_static", @@ -15576,7 +15576,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15595,7 +15595,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.813.0" +version = "1.814.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15702,7 +15702,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.813.0" +version = "1.814.0" dependencies = [ "chrono", "futures", @@ -15722,7 +15722,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.813.0" +version = "1.814.0" dependencies = [ "regex", "serde", @@ -15739,7 +15739,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15766,7 +15766,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "futures", @@ -15783,7 +15783,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.813.0" +version = "1.814.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15799,7 +15799,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "async-trait", @@ -15820,7 +15820,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "async-trait", @@ -15851,7 +15851,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "arc-swap", @@ -15876,7 +15876,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "async-stream", @@ -15911,7 +15911,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "futures", @@ -15929,7 +15929,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.813.0" +version = "1.814.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15938,7 +15938,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "lazy_static", @@ -15950,7 +15950,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "serde_json", @@ -15962,7 +15962,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "gosyn", @@ -15974,7 +15974,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "lazy_static", @@ -15986,7 +15986,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "serde_json", @@ -15998,7 +15998,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "nu-parser", @@ -16009,7 +16009,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16020,7 +16020,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16032,7 +16032,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16043,7 +16043,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "async-recursion", @@ -16065,7 +16065,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "serde_json", @@ -16077,7 +16077,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "lazy_static", @@ -16091,7 +16091,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16108,7 +16108,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "lazy_static", @@ -16121,7 +16121,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "serde", @@ -16133,7 +16133,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "lazy_static", @@ -16151,7 +16151,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16167,7 +16167,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16183,7 +16183,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "lazy_static", @@ -16197,7 +16197,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "async-recursion", @@ -16236,7 +16236,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "const_format", @@ -16276,7 +16276,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.813.0" +version = "1.814.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16287,7 +16287,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "async-recursion", @@ -16322,7 +16322,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "async-trait", @@ -16346,7 +16346,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "async-trait", @@ -16379,7 +16379,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "async-trait", @@ -16406,7 +16406,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "async-trait", @@ -16439,7 +16439,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "async-trait", @@ -16459,7 +16459,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "async-trait", @@ -16493,7 +16493,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "async-trait", @@ -16529,7 +16529,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "async-trait", @@ -16552,7 +16552,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "async-trait", @@ -16576,7 +16576,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "async-nats", @@ -16600,7 +16600,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "async-trait", @@ -16635,7 +16635,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "async-trait", @@ -16663,7 +16663,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "async-trait", @@ -16688,7 +16688,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "bitflags 2.13.2", @@ -16707,7 +16707,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "async-once-cell", @@ -16825,7 +16825,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.813.0" +version = "1.814.0" dependencies = [ "bytes", "futures", @@ -17458,7 +17458,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" dependencies = [ "libc", - "rustix 1.1.4", + "rustix 1.1.5", ] [[package]] @@ -17519,7 +17519,7 @@ checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.6", "synstructure 0.14.0", ] @@ -17560,7 +17560,7 @@ checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.6", "synstructure 0.14.0", ] @@ -17616,7 +17616,7 @@ checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.6", ] [[package]] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index f4f529e0fb..395e49afdc 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.813.0" +version = "1.814.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.813.0" +version = "1.814.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 317ca12c5e..cc50737a1d 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -d252afcc80e77fcc4f9a2a346b80908c8605a6c0 +7e338e4dabf91689bfd7fb0333c6534040b17b59 diff --git a/backend/migrations/20260908142148_datatable_role_catalog.down.sql b/backend/migrations/20260908142148_datatable_role_catalog.down.sql new file mode 100644 index 0000000000..8d3f12bc4a --- /dev/null +++ b/backend/migrations/20260908142148_datatable_role_catalog.down.sql @@ -0,0 +1,18 @@ +-- Refuse while the catalog holds anything. Each row is a live Postgres login with a password +-- only this table carries, so dropping it would leave credentials on the cluster that Windmill can +-- no longer disable, delete or even name — and re-applying could not recreate them, because the +-- role names would already be taken. Cleaning them up here is not an option either: dropping a +-- role means reassigning what it owns in *every* instance database, and a migration runs in one. +-- +-- Delete the roles through instance settings first; that path does the cluster work. +LOCK TABLE datatable_role IN ACCESS EXCLUSIVE MODE; + +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM datatable_role) THEN + RAISE EXCEPTION 'Cannot roll back: % data table role(s) still exist as Postgres logins. Delete them in instance settings first, which drops them from the cluster.', + (SELECT count(*) FROM datatable_role); + END IF; +END $$; + +DROP TABLE IF EXISTS datatable_role; diff --git a/backend/migrations/20260908142148_datatable_role_catalog.up.sql b/backend/migrations/20260908142148_datatable_role_catalog.up.sql new file mode 100644 index 0000000000..59cce4cc08 --- /dev/null +++ b/backend/migrations/20260908142148_datatable_role_catalog.up.sql @@ -0,0 +1,21 @@ +-- The instance's data table role catalog: one row per Postgres login Windmill created for data +-- table access. +-- +-- A table rather than a `global_settings` key, because the value is a set of live cluster +-- credentials and that table has generic read, list, write and CLI round-trip paths that know +-- nothing about what they are carrying. Every one of them is a way to leak the passwords or to +-- overwrite the catalog with a copy that has none, and a row nothing generic touches has none of +-- those. One row per role also makes two concurrent creates two inserts rather than a +-- read-modify-write over one document. +CREATE TABLE datatable_role ( + id VARCHAR(50) PRIMARY KEY, + -- The Postgres role name, verbatim. Unique because it is the cluster's own key. + name VARCHAR(63) NOT NULL UNIQUE, + enabled BOOLEAN NOT NULL DEFAULT true, + -- Generated by Windmill, never entered by anyone, and never leaves the server. + pwd TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +GRANT ALL ON datatable_role TO windmill_user; +GRANT ALL ON datatable_role TO windmill_admin; diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 9ec3e74012..335b3f79cf 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.813.0" +version = "1.814.0" dependencies = [ "aho-corasick", "anyhow", @@ -6274,7 +6274,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.813.0" +version = "1.814.0" dependencies = [ "proc-macro2", "quote", @@ -6286,7 +6286,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.813.0" +version = "1.814.0" dependencies = [ "convert_case", "serde", @@ -6295,7 +6295,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "lazy_static", @@ -6307,7 +6307,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "serde_json", @@ -6319,7 +6319,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "gosyn", @@ -6331,7 +6331,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "lazy_static", @@ -6343,7 +6343,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "serde_json", @@ -6355,7 +6355,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "nu-parser", @@ -6366,7 +6366,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6377,7 +6377,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6389,7 +6389,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6400,7 +6400,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "async-recursion", @@ -6422,7 +6422,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "serde_json", @@ -6434,7 +6434,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "lazy_static", @@ -6448,7 +6448,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "convert_case", @@ -6465,7 +6465,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "lazy_static", @@ -6478,7 +6478,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "serde", @@ -6490,7 +6490,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "lazy_static", @@ -6508,7 +6508,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6524,7 +6524,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6540,7 +6540,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6572,7 +6572,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "lazy_static", @@ -6586,7 +6586,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.813.0" +version = "1.814.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index d5b9ed591b..1262be6a3b 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.813.0" +version = "1.814.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 30c6047255..70fe5aa28a 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -730,7 +730,12 @@ pub fn parse_asset_syntax( s: &str, enable_default_syntax: bool, ) -> Option<(AssetKind, Cow<'_, str>)> { - if enable_default_syntax && s == "datatable" { + // `datatable` and `datatable?role=analyst` both name the default data table: the role picks + // which Postgres login the connection is made as, not which data table is read. + if enable_default_syntax + && s.strip_prefix("datatable") + .is_some_and(|rest| rest.is_empty() || rest.starts_with('?')) + { return Some((AssetKind::DataTable, Cow::Borrowed("main"))); } else if enable_default_syntax && s == "ducklake" { return Some((AssetKind::Ducklake, Cow::Borrowed("main"))); @@ -741,6 +746,14 @@ pub fn parse_asset_syntax( if *kind == AssetKind::Dbt { return Some((*kind, Cow::Owned(canonicalize_table_asset_path(suffix)))); } + // Same reasoning as above, for the explicit form. Specific to data tables: a + // `Resource`'s `?table=` is part of what it names, and stripping it would merge two + // different assets. + if *kind == AssetKind::DataTable { + if let Some((path, _role)) = suffix.split_once('?') { + return Some((*kind, Cow::Borrowed(path))); + } + } // The suffix is kept verbatim. For S3 the path encodes the storage: // `s3:///`, with an EMPTY storage segment for the // workspace default — so `s3:///key` yields `/key` (leading slash @@ -1692,6 +1705,25 @@ fn parse_trigger_spec(s: &str) -> Option { mod pipeline_annotation_tests { use super::*; + #[test] + fn a_datatable_role_is_not_part_of_the_asset_it_names() { + // The role picks which Postgres login the connection is made as, so two references that + // differ only by role are the same asset and must land on one graph node. + assert_eq!( + parse_asset_syntax("datatable://sales?role=analytics", false), + Some((AssetKind::DataTable, Cow::Borrowed("sales"))) + ); + assert_eq!( + parse_asset_syntax("datatable?role=analytics", true), + Some((AssetKind::DataTable, Cow::Borrowed("main"))) + ); + // A resource's `?table=` is part of what it names, so it is kept. + assert_eq!( + parse_asset_syntax("$res:f/db/pg?table=users", false), + Some((AssetKind::Resource, Cow::Borrowed("f/db/pg?table=users"))) + ); + } + #[test] fn s3_path_keeps_storage_distinction() { // An S3 asset path is `/` with an empty storage segment diff --git a/backend/tests/list_jobs.rs b/backend/tests/list_jobs.rs index 82a9e197cc..29a49c1610 100644 --- a/backend/tests/list_jobs.rs +++ b/backend/tests/list_jobs.rs @@ -1199,3 +1199,63 @@ async fn test_wm_labels_from_result_merged_with_static_labels( Ok(()) } + +/// `tag` lives only on `v2_job`, which count_jobs joins only when `tags` is set. +#[sqlx::test(fixtures("base"))] +async fn test_count_completed_jobs_tags_filter(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let client = windmill_api_client::create_client( + &format!("http://localhost:{port}"), + "SECRET_TOKEN".to_string(), + ); + + for (ws, tag, status) in [ + ("test-workspace", "deno", "success"), + ("test-workspace", "deno", "failure"), + ("test-workspace", "python3", "success"), + ("other-workspace", "deno", "success"), + ] { + let id = uuid::Uuid::new_v4(); + sqlx::query("INSERT INTO v2_job (id, workspace_id, tag) VALUES ($1, $2, $3)") + .bind(id) + .bind(ws) + .bind(tag) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO v2_job_completed (id, workspace_id, status, duration_ms) VALUES ($1, $2, $3::job_status, 0)", + ) + .bind(id) + .bind(ws) + .bind(status) + .execute(&db) + .await?; + } + + for (query, expected) in [ + ("", 3), + ("tags=deno", 2), + ("tags=deno&success=true", 1), + ("tags=deno,python3&completed_after_s_ago=3600", 3), + ] { + let response = client + .client() + .get(format!( + "{}/w/test-workspace/jobs/completed/count_jobs?{query}", + client.baseurl() + )) + .send() + .await?; + assert!( + response.status().is_success(), + "{query}: {}", + response.text().await? + ); + assert_eq!(response.json::().await?, expected, "{query}"); + } + + Ok(()) +} diff --git a/backend/windmill-api-groups/src/folders.rs b/backend/windmill-api-groups/src/folders.rs index 646f764ec6..c52b8a6316 100644 --- a/backend/windmill-api-groups/src/folders.rs +++ b/backend/windmill-api-groups/src/folders.rs @@ -800,6 +800,14 @@ async fn delete_folder( not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?; + // See the same call in `delete_group`: a freed name must not stay in a tenant list. + windmill_common::workspaces::remove_datatable_tenant_in_workspace( + &mut tx, + &w_id, + &format!("f/{name}"), + ) + .await?; + let del = sqlx::query_scalar!( "DELETE FROM folder WHERE name = $1 AND workspace_id = $2 RETURNING 1", name, diff --git a/backend/windmill-api-groups/src/groups.rs b/backend/windmill-api-groups/src/groups.rs index c931cf2255..0209af8585 100644 --- a/backend/windmill-api-groups/src/groups.rs +++ b/backend/windmill-api-groups/src/groups.rs @@ -797,6 +797,15 @@ async fn delete_group( } not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?; + // A tenant list names a principal, so a freed name must not linger in one: a later group + // reusing it would silently inherit the data table access this one had. + windmill_common::workspaces::remove_datatable_tenant_in_workspace( + &mut tx, + &w_id, + &format!("g/{name}"), + ) + .await?; + sqlx::query!( "DELETE FROM usr_to_group WHERE group_ = $1 AND workspace_id = $2", name, diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs new file mode 100644 index 0000000000..57b45088bd --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -0,0 +1,1247 @@ +//! Who may connect to a data table as which role, across the two shapes an entry can take: one +//! that owns its database, and a fork's pointer at it. + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {token}")) +} + +/// The `analytics` role's tenant list as stored, so a cascade can be observed directly. +async fn tenants(db: &Pool, w_id: &str) -> Vec { + let value: Option = sqlx::query_scalar( + "SELECT datatable->'datatables'->'main'->'permissions'->'roles'->'role1'->'tenants' + FROM workspace_settings WHERE workspace_id = $1", + ) + .bind(w_id) + .fetch_one(db) + .await + .unwrap(); + serde_json::from_value(value.unwrap_or(json!([]))).unwrap() +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn freeing_a_principal_takes_its_datatable_tenant(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace"); + + assert_eq!( + tenants(&db, "test-workspace").await, + vec!["u/test-user-2", "g/analysts", "f/finance"] + ); + + let resp = authed( + client().delete(format!("{base}/groups/delete/analysts")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "delete group: {}", resp.text().await?); + + let resp = authed( + client().delete(format!("{base}/folders/delete/finance")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "delete folder: {}", resp.text().await?); + + let resp = authed( + client().delete(format!("{base}/users/delete/test-user-2")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "delete user: {}", resp.text().await?); + + // Leaving is the other way a membership ends, and there are two `/leave` routes — the one the + // UI and the generated client call is this one. A tenant left behind here comes back with the + // person on rejoin, or attaches to whoever takes the username next. + sqlx::query( + r#"UPDATE workspace_settings SET datatable = jsonb_set(datatable, + '{datatables,main,permissions,roles,role1,tenants}', '["u/test-user-3"]'::jsonb) + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + let resp = authed( + client().post(format!("{base}/workspaces/leave")), + "SECRET_TOKEN_3", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "leave: {}", resp.text().await?); + // Nothing left naming a principal that no longer exists: a later group or account reusing one + // of those names must not inherit the access this one had. + assert!( + tenants(&db, "test-workspace").await.is_empty(), + "leaving kept the tenant: {:?}", + tenants(&db, "test-workspace").await + ); + Ok(()) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_fork_uses_the_data_table_it_points_at_but_never_administers_it( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let fork = format!("http://localhost:{port}/api/w/wm-fork-dt/workspaces"); + + // `test-user-2` is an admin of the fork and a plain member of the parent. The roles they can + // use are the ones the parent's tenants give them there, not what their fork admin bit says. + let resp = authed( + client().get(format!("{fork}/datatable_usable_roles/main")), + "SECRET_TOKEN_2", + ) + .send() + .await?; + assert_eq!(resp.status(), 200); + let body: Value = resp.json().await?; + assert_eq!(body["roles"], json!(["analytics"]), "{body}"); + assert_eq!(body["default_role"], "analytics"); + + // The drawer names the workspace that decides, and refuses to let the fork edit it. + let resp = authed( + client().get(format!("{fork}/datatable_permissions/main")), + "SECRET_TOKEN_2", + ) + .send() + .await?; + let body: Value = resp.json().await?; + assert_eq!(body["governing_workspace_id"], "test-workspace"); + assert_eq!(body["editable"], false, "{body}"); + + let resp = authed( + client().post(format!("{fork}/datatable_permissions/main")), + "SECRET_TOKEN_2", + ) + .json(&json!({"permissioned": true, "default_role": "admin", + "roles": [{"id": "admin", "tenants": ["*"]}]})) + .send() + .await?; + assert_eq!( + resp.status(), + 401, + "a fork admin widened the parent's access" + ); + + // Nor by saving the settings form: the pointer is server-owned, so a payload naming the + // parent's database leaves the entry exactly as it was. + let resp = authed( + client().post(format!("{fork}/edit_datatable_config")), + "SECRET_TOKEN_2", + ) + .json(&json!({ + "settings": {"datatables": {"main": { + "database": {"resource_type": "instance", "resource_path": "dt_main"} + }}}, + "renames": [], "deleted_datatables": [] + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + let entry: Option = sqlx::query_scalar( + "SELECT datatable->'datatables'->'main' FROM workspace_settings WHERE workspace_id = $1", + ) + .bind("wm-fork-dt") + .fetch_one(&db) + .await?; + let entry = entry.unwrap(); + assert_eq!( + entry["reference"]["workspace_id"], "test-workspace", + "{entry}" + ); + assert!(entry["database"].is_null(), "{entry}"); + Ok(()) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_second_entry_on_the_same_database_is_reported_rather_than_governed( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + // A copy of the parent's entry, as a fork created before data table roles would hold. It keeps + // its own access, so the owner is told about it instead of being told it is covered. + sqlx::query( + r#"UPDATE workspace_settings SET datatable = '{"datatables": {"copy": { + "database": {"resource_type": "instance", "resource_path": "dt_main"}}}}'::jsonb + WHERE workspace_id = 'wm-fork-dt'"#, + ) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let resp = authed( + client().get(format!( + "http://localhost:{port}/api/w/test-workspace/workspaces/datatable_permissions/main" + )), + "SECRET_TOKEN", + ) + .send() + .await?; + let body: Value = resp.json().await?; + assert_eq!( + body["ungoverned_reachers"], + json!([{"workspace_id": "wm-fork-dt", "datatable": "copy"}]), + "{body}" + ); + Ok(()) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_resource_backed_data_table_cannot_be_put_under_roles( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + // A role is a login on Windmill's own cluster. A resource-backed data table dials a host the + // workspace admin chose, so accepting one here would hand that host a real cluster credential. + sqlx::query( + r#"UPDATE workspace_settings SET datatable = '{"datatables": {"byo": { + "database": {"resource_type": "postgresql", "resource_path": "u/test-user/pg"}}}}'::jsonb + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + + let resp = authed( + client().get(format!("{base}/datatable_permissions/byo")), + "SECRET_TOKEN", + ) + .send() + .await?; + let body: Value = resp.json().await?; + assert_eq!(body["supported"], false, "{body}"); + + let resp = authed( + client().post(format!("{base}/datatable_permissions/byo")), + "SECRET_TOKEN", + ) + .json(&json!({"permissioned": true, "default_role": "role1", + "roles": [{"id": "role1", "tenants": ["*"]}]})) + .send() + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_fork_renaming_its_own_entry_leaves_the_governing_bookkeeping_alone( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + // The parent's migration definitions. A rename or delete through the fork's settings form + // resolves through the pointer, so without a guard it would relabel or wipe these. + sqlx::query( + "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up) + VALUES ('test-workspace', 'main', 1, 'init', 'SELECT 1')", + ) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/w/wm-fork-dt/workspaces/edit_datatable_config" + )), + "SECRET_TOKEN_2", + ) + .json(&json!({ + "settings": {"datatables": {}}, + "renames": [], + "deleted_datatables": ["main"] + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + let left: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM datatable_migrations WHERE workspace_id = 'test-workspace'", + ) + .fetch_one(&db) + .await?; + assert_eq!(left, 1, "the fork's delete reached the parent's migrations"); + Ok(()) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_caller_who_is_not_a_member_of_the_governing_workspace_reaches_nothing( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + // A fork member who was never added to the parent. Their fork membership says nothing there, + // and the email lookup that would evaluate them as a member of it finds no row. + sqlx::query( + "INSERT INTO usr (workspace_id, email, username, is_admin, role) + VALUES ('wm-fork-dt', 'test3@windmill.dev', 'test-user-3', false, 'User')", + ) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let resp = authed( + client().get(format!( + "http://localhost:{port}/api/w/wm-fork-dt/workspaces/datatable_usable_roles/main" + )), + "SECRET_TOKEN_3", + ) + .send() + .await?; + assert_eq!(resp.status(), 200); + let body: Value = resp.json().await?; + assert_eq!(body["roles"], json!([]), "{body}"); + Ok(()) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_caller_with_no_identity_reaches_a_permissioned_data_table_not_at_all( + db: Pool, +) -> anyhow::Result<()> { + use windmill_common::workspaces::{get_datatable_resource_from_db, DatatableAccess}; + + initialize_tracing().await; + // The compatibility story for an agent worker that predates data table roles and sends no job + // id: it keeps resolving an unpermissioned data table, and is refused on a permissioned one + // rather than handed an unattributed admin connection. + let refused = get_datatable_resource_from_db( + &db, + "test-workspace", + "main", + None, + DatatableAccess::NoIdentity, + ) + .await; + assert!(refused.is_err(), "an unidentified caller was let in"); + + sqlx::query( + "UPDATE workspace_settings + SET datatable = datatable #- '{datatables,main,permissions}' + WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + let resolved = get_datatable_resource_from_db( + &db, + "test-workspace", + "main", + None, + DatatableAccess::NoIdentity, + ) + .await?; + assert_eq!(resolved["dbname"], "dt_main", "{resolved}"); + Ok(()) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn concurrent_role_creations_both_survive(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Postgres roles are cluster-wide and this cluster is shared with every other test database, + // so the names have to be unique to this run. + let suffix: String = uuid::Uuid::new_v4().simple().to_string()[..8].to_string(); + let names = [format!("wmtest_a_{suffix}"), format!("wmtest_b_{suffix}")]; + + // The cluster DDL is not visible to another transaction until commit, so without the lock both + // of these pass their `pg_roles` existence check and one loses — leaving a live cluster login + // the catalog never recorded. + let create = |name: String| async move { + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/settings/datatable_roles" + )), + "SECRET_TOKEN", + ) + .json(&json!({ "name": name })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + Ok::<_, anyhow::Error>((status, body)) + }; + let outcome = async { + let (a, b) = tokio::join!(create(names[0].clone()), create(names[1].clone())); + let (a, b) = (a?, b?); + assert_eq!(a.0, 200, "{}", a.1); + assert_eq!(b.0, 200, "{}", b.1); + + let catalog = windmill_common::datatable_roles::read_role_catalog(&db).await?; + let recorded: Vec<&str> = catalog.values().map(|r| r.name.as_str()).collect(); + for name in &names { + assert!( + recorded.contains(&name.as_str()), + "{name} is a live cluster login the catalog forgot: {recorded:?}" + ); + } + Ok::<_, anyhow::Error>(()) + } + .await; + + // Roles are cluster-wide, so they outlive this test's throwaway database. Dropped whatever + // happened above — a failing run is exactly the one that created them and did not record them. + for name in &names { + let _ = sqlx::query(&format!("DROP ROLE IF EXISTS \"{name}\"")) + .execute(&db) + .await; + } + outcome +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_role_delete_that_fails_part_way_leaves_the_role_disabled( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let suffix: String = uuid::Uuid::new_v4().simple().to_string()[..8].to_string(); + let name = format!("wmtest_del_{suffix}"); + + let outcome = async { + let created: Value = authed( + client().post(format!( + "http://localhost:{port}/api/settings/datatable_roles" + )), + "SECRET_TOKEN", + ) + .json(&json!({ "name": name })) + .send() + .await? + .error_for_status()? + .json() + .await?; + let id = created["id"].as_str().unwrap().to_string(); + + // Each database's pass commits on its own, so one that cannot be reached fails the delete + // after the others may already have stripped the role. + sqlx::query( + "UPDATE global_settings SET value = jsonb_set(value, '{databases,wm_unreachable}', '{}') + WHERE name = 'custom_instance_pg_databases'", + ) + .execute(&db) + .await?; + + let resp = authed( + client().delete(format!( + "http://localhost:{port}/api/settings/datatable_roles/{id}" + )), + "SECRET_TOKEN", + ) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, 400, "{body}"); + + let catalog = windmill_common::datatable_roles::read_role_catalog(&db).await?; + let role = catalog + .get(&id) + .expect("a failed delete keeps the entry to retry"); + assert!( + !role.enabled, + "a half-deleted role is still enabled in the catalog" + ); + let can_login: bool = + sqlx::query_scalar("SELECT rolcanlogin FROM pg_roles WHERE rolname = $1") + .bind(&name) + .fetch_one(&db) + .await?; + assert!(!can_login, "a half-deleted role can still log in"); + Ok::<_, anyhow::Error>(()) + } + .await; + + let _ = sqlx::query(&format!("DROP ROLE IF EXISTS \"{name}\"")) + .execute(&db) + .await; + outcome +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn renaming_a_governing_data_table_carries_its_forks( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // A pointer names the governing data table by name, so a rename that does not follow leaves + // every fork resolving to nothing — the data table vanishes from their pickers and their jobs + // stop, with nothing in the renaming workspace to suggest why. + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/w/test-workspace/workspaces/edit_datatable_config" + )), + "SECRET_TOKEN", + ) + .json(&json!({ + "settings": {"datatables": {"renamed": { + "database": {"resource_type": "instance", "resource_path": "dt_main"} + }}}, + "renames": [{"from": "main", "to": "renamed"}], + "deleted_datatables": [] + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + let entry: Option = sqlx::query_scalar( + "SELECT datatable->'datatables'->'main' FROM workspace_settings WHERE workspace_id = $1", + ) + .bind("wm-fork-dt") + .fetch_one(&db) + .await?; + let entry = entry.unwrap(); + assert_eq!(entry["reference"]["datatable"], "renamed", "{entry}"); + + // And it still resolves, which is the thing the fork actually cares about. + let resp = authed( + client().get(format!( + "http://localhost:{port}/api/w/wm-fork-dt/workspaces/datatable_usable_roles/main" + )), + "SECRET_TOKEN_2", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_rename_has_to_match_the_save_it_claims_to_describe( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let url = + format!("http://localhost:{port}/api/w/test-workspace/workspaces/edit_datatable_config"); + + let instance = + |path: &str| json!({"database": {"resource_type": "instance", "resource_path": path}}); + + // Fork pointers are rewritten from the rename list, so a rename nobody performed moves every + // fork of one data table onto another. `main` survives this save, so it was not renamed. + let resp = authed(client().post(&url), "SECRET_TOKEN") + .json(&json!({ + "settings": {"datatables": {"main": instance("dt_main"), "decoy": instance("dt_two")}}, + "renames": [{"from": "main", "to": "decoy"}], + "deleted_datatables": [] + })) + .send() + .await?; + assert_eq!(resp.status(), 400, "a forged rename was accepted"); + + let entry: Option = sqlx::query_scalar( + "SELECT datatable->'datatables'->'main'->'reference' FROM workspace_settings + WHERE workspace_id = $1", + ) + .bind("wm-fork-dt") + .fetch_one(&db) + .await?; + assert_eq!( + entry.unwrap()["datatable"], + "main", + "the fork was repointed anyway" + ); + + // A swap is two renames whose sources and targets cross. It cannot be done one at a time — + // `datatables` is keyed by name — so refusing it would be a regression, and applying the two + // in order without a temporary name would carry `main`'s pointers back to `main`. + let resp = authed(client().post(&url), "SECRET_TOKEN") + .json(&json!({ + "settings": {"datatables": {"main": instance("dt_two"), "other": instance("dt_main")}}, + "renames": [{"from": "main", "to": "other"}, {"from": "other", "to": "main"}], + "deleted_datatables": [] + })) + .send() + .await?; + // `other` does not exist yet, so this particular pair is still refused — the swap shape is + // covered by the pair below, which starts from two real data tables. + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + + sqlx::query( + r#"UPDATE workspace_settings SET datatable = jsonb_set(datatable, '{datatables,other}', + '{"database": {"resource_type": "instance", "resource_path": "dt_two"}}'::jsonb) + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + + let resp = authed(client().post(&url), "SECRET_TOKEN") + .json(&json!({ + "settings": {"datatables": {"main": instance("dt_two"), "other": instance("dt_main")}}, + "renames": [{"from": "main", "to": "other"}, {"from": "other", "to": "main"}], + "deleted_datatables": [] + })) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "a swap was refused: {}", + resp.text().await? + ); + + // The fork named `main`, which is now called `other`. + let entry: Option = sqlx::query_scalar( + "SELECT datatable->'datatables'->'main'->'reference' FROM workspace_settings + WHERE workspace_id = $1", + ) + .bind("wm-fork-dt") + .fetch_one(&db) + .await?; + assert_eq!( + entry.unwrap()["datatable"], + "other", + "the swap did not carry the pointer" + ); + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_data_table_under_roles_is_not_copied_into_a_fork( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + // `pg_dump` carries no roles and the restore drops ACLs, so a copy would arrive with the + // parent's tenants and none of the grants behind them: every role but admin denied by + // Postgres in a data table that reads as configured. Refuse the copy rather than ship that, + // and refuse it before any data moves. + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Both halves of the clone: the database the copy would land in, then the copy itself. The + // first has to refuse too, or a permissioned fork leaves an empty registered database that + // no data table entry names and nothing collects. + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/w/test-workspace/workspaces/create_pg_database" + )), + "SECRET_TOKEN", + ) + .json(&json!({"source": "datatable://main", "target_dbname": "wm_fork_dt_copy"})) + .send() + .await?; + assert_eq!(resp.status(), 400); + assert!( + resp.text().await?.contains("under roles"), + "the fork's database was created for a copy that cannot happen" + ); + + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/w/test-workspace/workspaces/import_pg_database" + )), + "SECRET_TOKEN", + ) + .json( + &json!({"source": "datatable://main", "target": "datatable://main", + "fork_behavior": "schema_only"}), + ) + .send() + .await?; + assert_eq!(resp.status(), 400); + assert!( + resp.text().await?.contains("under roles"), + "the copy was refused for some other reason" + ); + Ok(()) +} + +/// The fork's `forked_from` for one of its entries; `None` whether it is absent or `null`. +async fn forked_from_of(db: &Pool, name: &str) -> Option { + sqlx::query_scalar::<_, Option>( + "SELECT datatable->'datatables'->$1::text->'forked_from' + FROM workspace_settings WHERE workspace_id = 'wm-fork-dt'", + ) + .bind(name) + .fetch_one(db) + .await + .unwrap() + .filter(|v| !v.is_null()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_clone_stamp_is_carried_but_its_schema_baseline_advances( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + // Whether an entry is a clone is what marks its database droppable, so a save can neither + // stamp nor unstamp one. The schema baseline inside the stamp is what the fork's schema diff + // advances after applying a change; dropping it would offer that same change again. + sqlx::query( + r#"UPDATE workspace_settings SET datatable = '{"datatables": { + "clone": {"database": {"resource_type": "instance", "resource_path": "wm_fork_dt__clone"}, + "forked_from": {"schema": {}}}, + "plain": {"database": {"resource_type": "instance", "resource_path": "dt_plain"}}}}'::jsonb + WHERE workspace_id = 'wm-fork-dt'"#, + ) + .execute(&db) + .await?; + let server = ApiServer::start(db.clone()).await?; + let url = format!( + "http://localhost:{}/api/w/wm-fork-dt/workspaces/edit_datatable_config", + server.addr.port() + ); + let clone_db = json!({"resource_type": "instance", "resource_path": "wm_fork_dt__clone"}); + let plain_db = json!({"resource_type": "instance", "resource_path": "dt_plain"}); + let baseline = json!({"schema": {"public": {"orders": {"id": "int4"}}}}); + + let resp = authed(client().post(&url), "SECRET_TOKEN_2") + .json(&json!({"settings": {"datatables": { + "clone": {"database": clone_db, "forked_from": baseline}, + "plain": {"database": plain_db, "forked_from": {"schema": {}}} + }}})) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + assert_eq!( + forked_from_of(&db, "clone").await, + Some(baseline.clone()), + "the schema diff's baseline did not advance" + ); + assert_eq!( + forked_from_of(&db, "plain").await, + None, + "a save stamped a clone" + ); + + let resp = authed(client().post(&url), "SECRET_TOKEN_2") + .json(&json!({"settings": {"datatables": { + "clone": {"database": clone_db}, "plain": {"database": plain_db} + }}})) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + assert_eq!( + forked_from_of(&db, "clone").await, + Some(baseline), + "a save unstamped a clone" + ); + Ok(()) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn roles_cannot_be_turned_on_while_a_trigger_streams_the_data_table( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + // A replication stream reads every row whatever the roles grant, so a data table carries one + // or the other. An enabled trigger on it — here a fork's, through its pointer — keeps roles + // from being turned on, and disabling it is what lets them on. + sqlx::query( + "UPDATE workspace_settings SET datatable = datatable #- '{datatables,main,permissions}' + WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + sqlx::query( + r#"INSERT INTO postgres_trigger (path, script_path, is_flow, workspace_id, edited_by, + postgres_resource_path, replication_slot_name, publication_name, permissioned_as, mode) + VALUES ('u/test-user-2/fork_stream', 'u/test-user-2/s', false, 'wm-fork-dt', + 'test-user-2', 'datatable://main', 'slot_fork', 'pub_fork', 'u/test-user-2', + 'enabled')"#, + ) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let url = format!( + "http://localhost:{}/api/w/test-workspace/workspaces/datatable_permissions/main", + server.addr.port() + ); + let turn_on = json!({"permissioned": true, "default_role": "admin", + "roles": [{"id": "admin", "tenants": ["*"]}]}); + + let resp = authed(client().post(&url), "SECRET_TOKEN") + .json(&turn_on) + .send() + .await?; + assert_eq!(resp.status(), 400); + assert!( + resp.text() + .await? + .contains("wm-fork-dt/u/test-user-2/fork_stream"), + "the refusal does not name the trigger to disable" + ); + + // Disabled, but its listener pinged just now and stops only at its next heartbeat. + sqlx::query( + "UPDATE postgres_trigger SET mode = 'disabled', server_id = NULL, last_server_ping = now() + WHERE path = 'u/test-user-2/fork_stream'", + ) + .execute(&db) + .await?; + let resp = authed(client().post(&url), "SECRET_TOKEN") + .json(&turn_on) + .send() + .await?; + assert_eq!( + resp.status(), + 400, + "roles went on while a disabled trigger's listener was still attached: {}", + resp.text().await? + ); + + sqlx::query( + "UPDATE postgres_trigger SET last_server_ping = now() - interval '20 seconds' + WHERE path = 'u/test-user-2/fork_stream'", + ) + .execute(&db) + .await?; + let resp = authed(client().post(&url), "SECRET_TOKEN") + .json(&turn_on) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(()) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn roles_going_on_wait_for_a_trigger_being_enabled(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + sqlx::query( + "UPDATE workspace_settings SET datatable = datatable #- '{datatables,main,permissions}' + WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + + // A trigger enable in flight: it holds the stream lock and its row is not committed yet, so a + // roles save that looked for streams now would miss it and its listener would connect to a + // data table it is about to be refused. + let mut enabling = db.begin().await?; + windmill_common::datatable_roles::lock_datatable_streams(&mut *enabling, false).await?; + sqlx::query( + r#"INSERT INTO postgres_trigger (path, script_path, is_flow, workspace_id, edited_by, + postgres_resource_path, replication_slot_name, publication_name, permissioned_as, mode) + VALUES ('u/test-user-2/racing_stream', 'u/test-user-2/s', false, 'wm-fork-dt', + 'test-user-2', 'datatable://main', 'slot_race', 'pub_race', 'u/test-user-2', + 'enabled')"#, + ) + .execute(&mut *enabling) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let url = format!( + "http://localhost:{}/api/w/test-workspace/workspaces/datatable_permissions/main", + server.addr.port() + ); + let save = tokio::spawn( + authed(client().post(&url), "SECRET_TOKEN") + .json(&json!({"permissioned": true, "default_role": "admin", + "roles": [{"id": "admin", "tenants": ["*"]}]})) + .send(), + ); + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + assert!( + !save.is_finished(), + "roles went on while a trigger was being enabled" + ); + enabling.commit().await?; + + let resp = save.await??; + assert_eq!(resp.status(), 400); + assert!( + resp.text() + .await? + .contains("wm-fork-dt/u/test-user-2/racing_stream"), + "the roles save missed the trigger enabled while it waited" + ); + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_stored_name_containing_a_question_mark_resolves_as_itself( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + // Names could contain `?` before they were restricted, and such an entry is still stored. + sqlx::query( + "UPDATE workspace_settings + SET datatable = jsonb_set(datatable, '{datatables,legacy?dt}', datatable->'datatables'->'main') + WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + + let resolve = |reference: &'static str| { + let db = db.clone(); + async move { + windmill_common::workspaces::parse_datatable_ref_for(&db, "test-workspace", reference) + .await + } + }; + assert_eq!(resolve("legacy?dt").await?, ("legacy?dt".to_string(), None)); + assert_eq!( + resolve("main?role=analytics").await?, + ("main".to_string(), Some("analytics".to_string())) + ); + assert!( + resolve("main?dt").await.is_err(), + "an unknown parameter was ignored" + ); + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_settings_save_dropping_a_governing_entry_names_the_forks_it_strands( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + // The whole map and no `deleted_datatables`, as a settings sync sends it. + let resp = authed( + client().post(format!( + "http://localhost:{}/api/w/test-workspace/workspaces/edit_datatable_config", + server.addr.port() + )), + "SECRET_TOKEN", + ) + .json(&json!({ "settings": { "datatables": {} } })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, 200, "{body}"); + let result: Value = serde_json::from_str(&body)?; + assert!( + result["stranded_references"] + .as_array() + .is_some_and(|refs| refs.iter().any(|r| r["workspace_id"] == "wm-fork-dt")), + "the fork left pointing at nothing was not named: {body}" + ); + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn an_entry_without_roles_cannot_newly_reach_a_database_under_roles( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + sqlx::query( + r#"UPDATE workspace_settings SET datatable = jsonb_set(datatable, '{datatables,other}', + '{"database": {"resource_type": "instance", "resource_path": "dt_other"}}') + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Whole-map saves with no `renames`, as a settings sync sends them. + let dt_main = + json!({ "database": { "resource_type": "instance", "resource_path": "dt_main" } }); + let dt_other = + json!({ "database": { "resource_type": "instance", "resource_path": "dt_other" } }); + for (case, w_id, datatables) in [ + ( + "a rename to a new name", + "test-workspace", + json!({ "main_renamed": dt_main, "other": dt_other }), + ), + ( + "an existing name repointed", + "test-workspace", + json!({ "other": dt_main }), + ), + ( + "another workspace's entry", + "wm-fork-dt", + json!({ "direct": dt_main }), + ), + ] { + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/w/{w_id}/workspaces/edit_datatable_config" + )), + "SECRET_TOKEN", + ) + .json(&json!({ "settings": { "datatables": datatables } })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert!( + status == 400 && body.contains("which a data table under roles uses"), + "{case} reached the database under roles without them ({status}): {body}" + ); + } + + let still_governed: bool = sqlx::query_scalar( + "SELECT (datatable->'datatables'->'main') ? 'permissions' FROM workspace_settings + WHERE workspace_id = 'test-workspace'", + ) + .fetch_one(&db) + .await?; + assert!(still_governed, "the refused save still took effect"); + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn an_alias_saved_elsewhere_waits_for_roles_going_on_for_its_database( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + sqlx::query( + r#"UPDATE workspace_settings SET datatable = jsonb_set(datatable, '{datatables,other}', + '{"database": {"resource_type": "instance", "resource_path": "dt_other"}}') + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + + // Roles going on for `dt_other`, not committed yet: it holds only its own workspace's settings + // row, so an alias saved from another workspace that looked for roles now would miss them. + let enabling = { + let mut tx = db.begin().await?; + windmill_common::datatable_roles::lock_instance_databases_governance( + &mut *tx, + ["dt_other"], + ) + .await?; + sqlx::query( + r#"UPDATE workspace_settings SET datatable = jsonb_set(datatable, + '{datatables,other,permissions}', + '{"default_role": "admin", "roles": {"admin": {"tenants": ["*"]}}}') + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&mut *tx) + .await?; + tx + }; + + let server = ApiServer::start(db.clone()).await?; + let url = format!( + "http://localhost:{}/api/w/wm-fork-dt/workspaces/edit_datatable_config", + server.addr.port() + ); + let save = tokio::spawn( + authed(client().post(&url), "SECRET_TOKEN") + .json(&json!({ "settings": { "datatables": { + "direct": { "database": { "resource_type": "instance", "resource_path": "dt_other" } } + } } })) + .send(), + ); + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + assert!( + !save.is_finished(), + "an alias was saved while roles were going on for its database" + ); + enabling.commit().await?; + + let resp = save.await??; + let status = resp.status(); + let body = resp.text().await?; + assert!( + status == 400 && body.contains("which a data table under roles uses"), + "the alias reached the database whose roles went on while it waited ({status}): {body}" + ); + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_fork_waits_for_a_rename_of_the_data_table_it_keeps( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + // A rename in flight holds the parent's settings row and moves only the pointers it can see; a + // fork being created is invisible to it, so the fork has to read the name the rename commits. + let mut renaming = db.begin().await?; + sqlx::query( + "SELECT 1 FROM workspace_settings WHERE workspace_id = 'test-workspace' FOR UPDATE", + ) + .execute(&mut *renaming) + .await?; + sqlx::query( + "UPDATE workspace_settings SET datatable = jsonb_set(datatable #- '{datatables,main}', + '{datatables,renamed}', datatable->'datatables'->'main') + WHERE workspace_id = 'test-workspace'", + ) + .execute(&mut *renaming) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let url = format!( + "http://localhost:{}/api/w/test-workspace/workspaces/create_fork", + server.addr.port() + ); + let fork = tokio::spawn( + authed(client().post(&url), "SECRET_TOKEN") + .json(&json!({ "id": "wm-fork-race", "name": "race", "color": "#0000ff" })) + .send(), + ); + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + assert!( + !fork.is_finished(), + "the fork copied the parent's data tables while a rename held them" + ); + renaming.commit().await?; + + let resp = fork.await??; + assert!(resp.status().is_success(), "{}", resp.text().await?); + let datatables: Option = sqlx::query_scalar( + "SELECT datatable->'datatables' FROM workspace_settings WHERE workspace_id = 'wm-fork-race'", + ) + .fetch_one(&db) + .await?; + let datatables = datatables.unwrap(); + assert_eq!( + datatables["renamed"]["reference"], + json!({ "workspace_id": "test-workspace", "datatable": "renamed" }), + "{datatables}" + ); + Ok(()) +} + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +const ENTERPRISE_REFUSAL: &str = "Data table roles are a Windmill Enterprise Edition feature"; + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn every_roles_route_is_an_enterprise_feature(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let api = format!("http://localhost:{}/api", server.addr.port()); + let dt = format!("{api}/w/test-workspace/workspaces"); + + for (method, url, body) in [ + ( + reqwest::Method::GET, + format!("{api}/settings/datatable_roles"), + None, + ), + ( + reqwest::Method::POST, + format!("{api}/settings/datatable_roles"), + Some(json!({ "name": "wmtest_ce" })), + ), + ( + reqwest::Method::POST, + format!("{api}/settings/datatable_roles/role1"), + Some(json!({ "enabled": false })), + ), + ( + reqwest::Method::DELETE, + format!("{api}/settings/datatable_roles/role1"), + None, + ), + ( + reqwest::Method::GET, + format!("{dt}/datatable_permissions/main"), + None, + ), + ( + reqwest::Method::POST, + format!("{dt}/datatable_permissions/main"), + Some(json!({ "permissioned": false })), + ), + ( + reqwest::Method::GET, + format!("{dt}/datatable_usable_roles/main"), + None, + ), + ] { + let mut request = authed(client().request(method.clone(), &url), "SECRET_TOKEN"); + if let Some(body) = body { + request = request.json(&body); + } + let resp = request.send().await?; + let status = resp.status(); + let text = resp.text().await?; + assert!( + status == 400 && text.contains(ENTERPRISE_REFUSAL), + "{method} {url} answered {status}: {text}" + ); + } + + // Refused, not acted on: the catalog row and the data table's roles are where they were. + let untouched: (i64, bool) = sqlx::query_as( + "SELECT (SELECT count(*) FROM datatable_role), + (datatable->'datatables'->'main') ? 'permissions' + FROM workspace_settings WHERE workspace_id = 'test-workspace'", + ) + .fetch_one(&db) + .await?; + assert_eq!(untouched, (1, true)); + Ok(()) +} + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn without_the_enterprise_edition_a_data_table_under_roles_is_refused_a_connection( + db: Pool, +) -> anyhow::Result<()> { + use windmill_common::workspaces::{get_datatable_resource_from_db, DatatableAccess}; + initialize_tracing().await; + + // Saved under roles, as an enterprise build left it: refused whoever asks, never `admin`. + for access in [DatatableAccess::Unchecked, DatatableAccess::NoIdentity] { + let err = get_datatable_resource_from_db(&db, "test-workspace", "main", None, access) + .await + .expect_err("a data table under roles resolved"); + assert!(err.to_string().contains(ENTERPRISE_REFUSAL), "{err}"); + } + + // Not under roles, it resolves as it always has, including when `admin` is named — which every + // migration does; naming any other role on it is refused. + sqlx::query( + "UPDATE workspace_settings SET datatable = datatable #- '{datatables,main,permissions}' + WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + for role in [None, Some("admin")] { + let resolved = get_datatable_resource_from_db( + &db, + "test-workspace", + "main", + role, + DatatableAccess::NoIdentity, + ) + .await?; + assert_eq!(resolved["dbname"], "dt_main", "{resolved}"); + } + let err = get_datatable_resource_from_db( + &db, + "test-workspace", + "main", + Some("analytics"), + DatatableAccess::Unchecked, + ) + .await + .expect_err("a named role resolved"); + assert!(err.to_string().contains(ENTERPRISE_REFUSAL), "{err}"); + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/fixtures/datatable_roles.sql b/backend/windmill-api-integration-tests/tests/fixtures/datatable_roles.sql new file mode 100644 index 0000000000..55d50d212c --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/fixtures/datatable_roles.sql @@ -0,0 +1,45 @@ +-- A data table under roles in `test-workspace`, and a fork whose entry points at it rather than +-- carrying a copy. `test-user-2` is a non-admin of the parent and an admin of the fork: the shape +-- the pointer exists for. + +-- Empty registry: role provisioning grants CONNECT on every database named here, and the data +-- table's `dt_main` is a name in workspace settings, not a database that exists. +INSERT INTO global_settings (name, value) VALUES + ('custom_instance_pg_databases', '{"user_pwd": "pw", "databases": {}}'::jsonb) + ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value; + +INSERT INTO datatable_role (id, name, enabled, pwd) VALUES ('role1', 'analytics', true, 'pw'); + +UPDATE workspace_settings SET datatable = '{ + "datatables": { + "main": { + "database": {"resource_type": "instance", "resource_path": "dt_main"}, + "permissions": { + "default_role": "role1", + "roles": { + "admin": {"tenants": []}, + "role1": {"tenants": ["u/test-user-2", "g/analysts", "f/finance"]} + } + } + } + } +}'::jsonb WHERE workspace_id = 'test-workspace'; + +INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES + ('test-workspace', 'analysts', 'Analysts', '{}'); +INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) VALUES + ('test-workspace', 'finance', 'finance', '{}', '{}'); + +INSERT INTO workspace (id, name, owner, parent_workspace_id) VALUES + ('wm-fork-dt', 'fork of test-workspace', 'test2@windmill.dev', 'test-workspace'); +INSERT INTO workspace_key (workspace_id, kind, key) VALUES ('wm-fork-dt', 'cloud', 'test-key'); +INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES + ('wm-fork-dt', 'all', 'All users', '{}'); +INSERT INTO usr (workspace_id, email, username, is_admin, role) VALUES + ('wm-fork-dt', 'test2@windmill.dev', 'test-user-2', true, 'Admin'); + +INSERT INTO workspace_settings (workspace_id, datatable) VALUES ('wm-fork-dt', '{ + "datatables": { + "main": {"reference": {"workspace_id": "test-workspace", "datatable": "main"}} + } +}'::jsonb); diff --git a/backend/windmill-api-settings/Cargo.toml b/backend/windmill-api-settings/Cargo.toml index bc5078bce4..0ec99ef6d0 100644 --- a/backend/windmill-api-settings/Cargo.toml +++ b/backend/windmill-api-settings/Cargo.toml @@ -11,7 +11,7 @@ path = "src/lib.rs" [features] default = [] enterprise = ["license"] -private = ["windmill-common/private"] +private = ["windmill-common/private", "windmill-audit/private"] parquet = ["windmill-common/parquet", "windmill-object-store/parquet"] license = ["dep:rsa"] @@ -19,6 +19,7 @@ license = ["dep:rsa"] windmill-ai = { workspace = true, default-features = false } windmill-alerting.workspace = true windmill-api-auth.workspace = true +windmill-audit.workspace = true windmill-common = { workspace = true, default-features = false } axum.workspace = true anyhow.workspace = true diff --git a/backend/windmill-api-settings/src/datatable_roles_oss.rs b/backend/windmill-api-settings/src/datatable_roles_oss.rs new file mode 100644 index 0000000000..885d13ee9d --- /dev/null +++ b/backend/windmill-api-settings/src/datatable_roles_oss.rs @@ -0,0 +1,44 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Where the data table role catalog endpoints come from: the enterprise implementation, or a +//! refusal. Roles are an Enterprise Edition feature; see `windmill_common::datatable_roles_oss`. + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub(crate) use crate::datatable_roles_ee::{ + create_datatable_role, delete_datatable_role, list_datatable_roles, update_datatable_role, +}; + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub(crate) use ce::*; + +// The routes stay registered so the API has one shape; each answers after authentication, before +// anything is read. +#[cfg(not(all(feature = "private", feature = "enterprise")))] +mod ce { + use windmill_api_auth::ApiAuthed; + use windmill_common::{ + datatable_roles_oss::datatable_roles_unavailable as unavailable, error::Result, + }; + + pub(crate) async fn list_datatable_roles(_authed: ApiAuthed) -> Result { + Err(unavailable()) + } + + pub(crate) async fn create_datatable_role(_authed: ApiAuthed) -> Result { + Err(unavailable()) + } + + pub(crate) async fn update_datatable_role(_authed: ApiAuthed) -> Result { + Err(unavailable()) + } + + pub(crate) async fn delete_datatable_role(_authed: ApiAuthed) -> Result { + Err(unavailable()) + } +} diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index 2e2b3290fa..5d01baafbd 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -17,6 +17,9 @@ mod audit_logs_s3; mod audit_logs_s3_backfill; #[cfg(feature = "parquet")] mod background_task; +#[cfg(all(feature = "private", feature = "enterprise"))] +mod datatable_roles_ee; +mod datatable_roles_oss; #[cfg(feature = "private")] mod ee; pub mod ee_oss; @@ -151,6 +154,16 @@ pub fn global_service() -> Router { "/list_custom_instance_pg_databases", post(list_custom_instance_pg_databases), ) + .route( + "/datatable_roles", + get(datatable_roles_oss::list_datatable_roles) + .post(datatable_roles_oss::create_datatable_role), + ) + .route( + "/datatable_roles/{id}", + post(datatable_roles_oss::update_datatable_role) + .delete(datatable_roles_oss::delete_datatable_role), + ) .route( "/refresh_custom_instance_user_pwd", post(refresh_custom_instance_user_pwd), diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index b7d30eaa4b..2bfac244f5 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -1703,14 +1703,25 @@ async fn delete_user( .await?; windmill_common::user_drafts::delete_drafts_of_email(&mut *tx, &email_to_delete).await?; - let usernames = sqlx::query_scalar!( - "DELETE FROM usr WHERE email = $1 RETURNING username", + let memberships = sqlx::query!( + "DELETE FROM usr WHERE email = $1 RETURNING username, workspace_id", &email_to_delete ) .fetch_all(&mut *tx) .await?; - for username in usernames { + for row in memberships { + let username = row.username; + // A tenant list names a principal of its workspace, so the name has to be freed in every + // workspace this account belonged to: a later account taking the username would otherwise + // inherit the data table access it had. + windmill_common::workspaces::remove_datatable_tenant_in_workspace( + &mut tx, + &row.workspace_id, + &format!("u/{username}"), + ) + .await?; + sqlx::query!("DELETE FROM password WHERE email = $1", &email_to_delete) .execute(&mut *tx) .await?; @@ -2456,6 +2467,15 @@ pub async fn delete_workspace_user_internal( tx: &mut Transaction<'_, Postgres>, authed: Option<&ApiAuthed>, // None for system operations ) -> Result<()> { + // Same reasoning as the `extra_perms` sweep below: a freed username must not stay named + // anywhere that grants access, tenant lists included. + windmill_common::workspaces::remove_datatable_tenant_in_workspace( + tx, + w_id, + &format!("u/{username_to_delete}"), + ) + .await?; + // ---- Clean up extra_perms referencing this user ---- let extra_perms_tables = [ "script", @@ -3965,6 +3985,12 @@ async fn leave_workspace( ) -> Result { forbid_job_token_account_destruction(&authed)?; let mut tx = db.begin().await?; + windmill_common::workspaces::remove_datatable_tenant_in_workspace( + &mut tx, + &w_id, + &format!("u/{}", authed.username), + ) + .await?; sqlx::query!( "DELETE FROM usr WHERE workspace_id = $1 AND username = $2", &w_id, diff --git a/backend/windmill-api-workspaces/src/datatable_migrations.rs b/backend/windmill-api-workspaces/src/datatable_migrations.rs index ef5ea37e1d..fa433b2cbc 100644 --- a/backend/windmill-api-workspaces/src/datatable_migrations.rs +++ b/backend/windmill-api-workspaces/src/datatable_migrations.rs @@ -30,6 +30,7 @@ use windmill_api_auth::{require_super_admin, ApiAuthed}; use windmill_api_jobs::run_wait_result_internal; use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; +use windmill_common::datatable_roles::ADMIN_DATATABLE_ROLE; use windmill_common::db::UserDB; use windmill_common::error::{pg_error_message, Error, JsonResult, Result}; use windmill_common::jobs::{JobPayload, RawCode}; @@ -38,7 +39,11 @@ use windmill_common::runnable_settings::{ConcurrencySettingsWithCustom, Debounci use windmill_common::scripts::ScriptLang; use windmill_common::users::username_to_permissioned_as; use windmill_common::worker::to_raw_value; -use windmill_common::workspaces::get_datatable_resource_from_db_unchecked; +use windmill_common::worker::SqlAnnotations; +use windmill_common::workspaces::{ + ensure_can_use_datatable_role, ensure_datatable_admin_access, + get_datatable_resource_from_db_unchecked, resolve_governing_datatable, DatatableAccess, +}; use windmill_common::{PgDatabase, DB}; use windmill_git_sync::{ handle_deployment_metadata, handle_deployment_metadata_batch, DeployedObject, @@ -86,6 +91,42 @@ pub(crate) fn routes() -> Router { ) } +/// Refuse a migration whose role this caller may not use, before a job is pushed or a version +/// recorded. +/// +/// A migration that declares `-- role ` runs as that role, so the caller has to be one of its +/// tenants. One that declares none runs as `admin` and reaches every object in the database +/// whatever the roles grant, so it is for the admins of the workspace that governs the data table +/// — a fork can run a migration under a role it holds, never a migration under `admin`. +/// +/// The executor re-checks the role when it resolves the connection, so this is not the boundary. It +/// is what makes the refusal legible: which migration, and which role. +async fn ensure_migration_role_allowed( + db: &DB, + w_id: &str, + datatable_name: &str, + authed: &ApiAuthed, + sql: &str, + timestamp: i64, + name: &str, +) -> Result<()> { + let context = format!("Migration {timestamp} ({name})"); + let access = DatatableAccess::Authed(authed.to_authed_ref()); + match SqlAnnotations::datatable_role(sql)? { + Some(role) => { + ensure_can_use_datatable_role(db, w_id, datatable_name, Some(&role), &access, &context) + .await + } + None => ensure_datatable_admin_access(db, w_id, datatable_name, &access) + .await + .map_err(|e| { + Error::NotAuthorized(format!( + "{context} declares no role, so it would run as admin. {e}" + )) + }), + } +} + #[derive(Serialize)] struct AppliedMigration { version: i64, @@ -128,7 +169,18 @@ async fn datatable_database_arg( .await? .ok_or_else(|| Error::internal_err(format!("datatable {datatable_name} not found")))?; - Ok(to_raw_value(&format!("datatable://{datatable_name}"))) + // `?role=admin` rather than a bare reference, so a migration that declares no `-- role` runs + // as the connection that owns the schema instead of falling through to the data table's + // default role — which is what `ensure_migration_role_allowed` gated it as, and which is the + // only role a DDL statement can be expected to succeed under. A migration that does declare a + // role overrides this: the annotation wins over the reference. + // + // A legacy name containing `?` cannot be migrated through this reference: the appended query + // makes it neither an exact name nor a parseable one. Accepted on purpose, since such names can + // no longer be created and none are expected to carry migrations. + Ok(to_raw_value(&format!( + "datatable://{datatable_name}?role={ADMIN_DATATABLE_ROLE}" + ))) } /// Run a migration's SQL as a normal Windmill `postgresql` job, permissioned as @@ -384,6 +436,11 @@ async fn run_datatable_migrations( Path((w_id, datatable_name)): Path<(String, String)>, Query(query): Query, ) -> JsonResult { + // Before the admin connection is opened at all: the bookkeeping below is created and read + // through it, so a caller no role covers must be refused here rather than after the fact. + crate::datatable_permissions::ensure_reaches_datatable(&db, &w_id, &datatable_name, &authed) + .await?; + audit_log( &db, &authed, @@ -440,6 +497,16 @@ async fn run_datatable_migrations( if applied_versions.contains(&m.timestamp) { continue; } + ensure_migration_role_allowed( + &db, + &w_id, + &datatable_name, + &authed, + &m.code_up, + m.timestamp, + &m.name, + ) + .await?; run_datatable_migration_job(&db, &user_db, &authed, &w_id, &database_arg, &m.code_up) .await .map_err(|e| { @@ -506,6 +573,11 @@ async fn rollback_datatable_migrations( Path((w_id, datatable_name)): Path<(String, String)>, Query(query): Query, ) -> JsonResult { + // Before the admin connection is opened at all: the bookkeeping below is created and read + // through it, so a caller no role covers must be refused here rather than after the fact. + crate::datatable_permissions::ensure_reaches_datatable(&db, &w_id, &datatable_name, &authed) + .await?; + audit_log( &db, &authed, @@ -588,6 +660,17 @@ async fn rollback_datatable_migrations( )) })?; + ensure_migration_role_allowed( + &db, + &w_id, + &datatable_name, + &authed, + &code_down, + version, + &definition.name, + ) + .await?; + let database_arg = datatable_database_arg(&db, &w_id, &datatable_name).await?; run_datatable_migration_job(&db, &user_db, &authed, &w_id, &database_arg, &code_down) .await @@ -748,10 +831,15 @@ async fn read_applied_datatable_versions( /// List a data table's migrations annotated with whether each has been applied. async fn datatable_migrations_status( - _authed: ApiAuthed, + authed: ApiAuthed, Extension(db): Extension, Path((w_id, datatable_name)): Path<(String, String)>, ) -> JsonResult { + // Reads `_wm_migrations` through the data table's admin connection, so it answers to the same + // question as running one: may you reach this data table at all. + crate::datatable_permissions::ensure_reaches_datatable(&db, &w_id, &datatable_name, &authed) + .await?; + let enabled = datatable_migrations_enabled(&db, &w_id, &datatable_name).await?; if !enabled { return Ok(Json(DatatableMigrationsStatusResult { @@ -1431,6 +1519,15 @@ async fn generate_initial_datatable_migration( Extension(db): Extension, Path((w_id, datatable_name)): Path<(String, String)>, ) -> JsonResult { + // Returns a `pg_dump` of the whole schema and writes into the data table's own bookkeeping, so + // it answers to the workspace that governs it rather than to whoever is asking. + ensure_datatable_admin_access( + &db, + &w_id, + &datatable_name, + &DatatableAccess::Authed(authed.to_authed_ref()), + ) + .await?; validate_datatable_path_segment(&datatable_name)?; ensure_datatable_migrations_enabled(&db, &w_id, &datatable_name).await?; @@ -1601,9 +1698,21 @@ pub(crate) struct DatatableRename { pub(crate) to: String, } -async fn resolve_datatable_pg(db: &DB, w_id: &str, datatable: &str) -> Result { +/// The database whose `_wm_migrations` a rename or delete of `datatable` in `w_id` should touch — +/// `None` when that is somebody else's. +/// +/// A fork's entry points at the workspace that governs the data table, so renaming or removing it +/// changes what the fork calls the data table and nothing more. Following the pointer here would +/// let a fork admin relabel or wipe the *governing* workspace's migration bookkeeping through +/// their own settings form, and the parent would then re-run every migration from zero. +async fn resolve_datatable_pg(db: &DB, w_id: &str, datatable: &str) -> Result> { + let governing = resolve_governing_datatable(db, w_id, datatable).await?; + if governing.workspace_id != w_id { + return Ok(None); + } let db_resource = get_datatable_resource_from_db_unchecked(db, w_id, datatable).await?; serde_json::from_value(db_resource) + .map(Some) .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e))) } @@ -1621,7 +1730,9 @@ fn ignore_missing_wm_migrations(e: tokio_postgres::Error) -> Result<()> { /// Drop a data table's rows from its own database's `_wm_migrations`. async fn remote_forget_datatable_migrations(db: &DB, w_id: &str, datatable: &str) -> Result<()> { - let pg_db = resolve_datatable_pg(db, w_id, datatable).await?; + let Some(pg_db) = resolve_datatable_pg(db, w_id, datatable).await? else { + return Ok(()); + }; let (client, connection) = pg_db.connect(Some(db)).await?; tokio::spawn(async move { let _ = connection.await; @@ -1646,7 +1757,9 @@ async fn remote_rename_datatable_migrations( from: &str, to: &str, ) -> Result<()> { - let pg_db = resolve_datatable_pg(db, w_id, resolve_by).await?; + let Some(pg_db) = resolve_datatable_pg(db, w_id, resolve_by).await? else { + return Ok(()); + }; let (client, connection) = pg_db.connect(Some(db)).await?; tokio::spawn(async move { let _ = connection.await; diff --git a/backend/windmill-api-workspaces/src/datatable_permissions.rs b/backend/windmill-api-workspaces/src/datatable_permissions.rs new file mode 100644 index 0000000000..5cb1f3c1c3 --- /dev/null +++ b/backend/windmill-api-workspaces/src/datatable_permissions.rs @@ -0,0 +1,67 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Who may connect to a data table as which role. +//! +//! The decision lives on the data table entry of the workspace that governs it, which is not +//! necessarily the workspace asking: a fork's entry points at its parent's, and everything here +//! resolves through that pointer first. Nothing in this module runs SQL against the data table — +//! a save is tenant lists and a default, and the Postgres roles themselves are the instance +//! catalog's business. + +use axum::{routing::get, Router}; + +use windmill_api_auth::ApiAuthed; +use windmill_common::error::Result; +use windmill_common::workspaces::GoverningDatatable; +use windmill_common::DB; + +use crate::datatable_permissions_oss as roles; + +pub(crate) fn routes() -> Router { + Router::new() + .route( + "/datatable_permissions/{datatable_name}", + get(roles::get_datatable_permissions).post(roles::set_datatable_permissions), + ) + .route( + "/datatable_usable_roles/{datatable_name}", + get(roles::list_usable_datatable_roles), + ) +} + +/// Administering a data table — its permissions, its migrations that declare no role, its exports +/// — is for the admins of the workspace that governs it. A fork can use the data table; it never +/// administers it. +// The gate for whatever administers a data table under roles, which the routes of this module alone +// do not always reach. +#[allow(dead_code)] +pub(crate) async fn ensure_governs_datatable( + db: &DB, + authed: &ApiAuthed, + w_id: &str, + governing: &GoverningDatatable, +) -> Result<()> { + roles::ensure_governs_datatable(db, authed, w_id, governing).await +} + +/// Refuse a caller that no tenant of this data table covers. +/// +/// The bookkeeping endpoints below open the data table's `admin` connection to read or create +/// `_wm_migrations` before they know which migration will run — so without this, someone covered +/// by no role at all can still force admin-backed reads and writes on a database they may not +/// touch. It asks only "may you reach this data table as anything"; which role a given migration +/// runs as is still decided per migration, and by the executor after that. +pub(crate) async fn ensure_reaches_datatable( + db: &DB, + w_id: &str, + datatable_name: &str, + authed: &ApiAuthed, +) -> Result<()> { + roles::ensure_reaches_datatable(db, w_id, datatable_name, authed).await +} diff --git a/backend/windmill-api-workspaces/src/datatable_permissions_oss.rs b/backend/windmill-api-workspaces/src/datatable_permissions_oss.rs new file mode 100644 index 0000000000..f4d8c6a7ad --- /dev/null +++ b/backend/windmill-api-workspaces/src/datatable_permissions_oss.rs @@ -0,0 +1,73 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Where the data table permissions endpoints and their gates come from: the enterprise +//! implementation, or a refusal. Roles are an Enterprise Edition feature; see +//! `windmill_common::datatable_roles_oss`. + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub(crate) use crate::datatable_permissions_ee::{ + ensure_governs_datatable, ensure_reaches_datatable, get_datatable_permissions, + list_usable_datatable_roles, set_datatable_permissions, +}; + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub(crate) use ce::*; + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +mod ce { + use windmill_api_auth::ApiAuthed; + use windmill_common::{ + datatable_roles_oss::datatable_roles_unavailable as unavailable, + error::Result, + workspaces::{resolve_governing_datatable, GoverningDatatable}, + DB, + }; + + /// Nobody administers a data table's roles without them. + #[allow(dead_code)] + pub(crate) async fn ensure_governs_datatable( + _db: &DB, + _authed: &ApiAuthed, + _w_id: &str, + _governing: &GoverningDatatable, + ) -> Result<()> { + Err(unavailable()) + } + + /// A data table not under roles is reached as it was before roles existed. One under roles is + /// refused: no role of it can be connected as. + pub(crate) async fn ensure_reaches_datatable( + db: &DB, + w_id: &str, + datatable_name: &str, + _authed: &ApiAuthed, + ) -> Result<()> { + let governing = resolve_governing_datatable(db, w_id, datatable_name).await?; + if governing.datatable.permissions.is_none() { + Ok(()) + } else { + Err(unavailable()) + } + } + + // The routes stay registered so the API has one shape; each answers after authentication, + // before anything is read. + + pub(crate) async fn get_datatable_permissions(_authed: ApiAuthed) -> Result { + Err(unavailable()) + } + + pub(crate) async fn set_datatable_permissions(_authed: ApiAuthed) -> Result { + Err(unavailable()) + } + + pub(crate) async fn list_usable_datatable_roles(_authed: ApiAuthed) -> Result { + Err(unavailable()) + } +} diff --git a/backend/windmill-api-workspaces/src/lib.rs b/backend/windmill-api-workspaces/src/lib.rs index 22f2a2c5bb..c1eb6c1ffc 100644 --- a/backend/windmill-api-workspaces/src/lib.rs +++ b/backend/windmill-api-workspaces/src/lib.rs @@ -2,6 +2,8 @@ pub mod ai_session_backups; pub mod data_metrics; pub mod datatable_migrations; +pub mod datatable_permissions; +pub mod datatable_permissions_oss; pub mod deployment_requests; pub mod workspaces; pub mod workspaces_extra; @@ -9,3 +11,6 @@ pub mod workspaces_oss; #[cfg(feature = "private")] pub mod workspaces_ee; + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub mod datatable_permissions_ee; diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 2bef0d28ab..97ab07c0a5 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -45,10 +45,12 @@ use windmill_common::workspaces::GitRepositorySettings; #[cfg(feature = "enterprise")] use windmill_common::workspaces::WorkspaceDeploymentUISettings; use windmill_common::workspaces::{ - check_deploy_rules, check_user_against_rule, get_datatable_resource_from_db_unchecked, + check_deploy_rules, check_user_against_rule, get_datatable_resource_from_db, + get_datatable_resource_from_db_unchecked, parse_datatable_ref_for, resolve_governing_datatable, validate_dev_workspace_id, validate_fork_workspace_id, validate_workspace_name, DataTable, - DataTableCatalogResourceType, DataTableForkBehavior, ProtectionRuleKind, ProtectionRules, - ProtectionRuleset, RuleCheckResult, WorkspaceGitSyncSettings, DEV_WORKSPACE_LOCK_RULE_NAME, + DataTableCatalogResourceType, DataTableForkBehavior, DatatableAccess, GoverningDatatable, + ProtectionRuleKind, ProtectionRules, ProtectionRuleset, RuleCheckResult, + WorkspaceGitSyncSettings, DEV_WORKSPACE_LOCK_RULE_NAME, }; use windmill_common::workspaces::{Ducklake, DucklakeCatalogResourceType}; use windmill_common::PgDatabase; @@ -141,6 +143,7 @@ pub fn workspaced_service() -> Router { get(test_datatable_connection), ) .merge(crate::datatable_migrations::routes()) + .merge(crate::datatable_permissions::routes()) .route("/git_sync_enabled", get(get_git_sync_enabled)) .route("/git_sync_deploy_mode", get(get_git_sync_deploy_mode)) .route("/edit_git_sync_config", post(edit_git_sync_config)) @@ -1140,6 +1143,8 @@ async fn get_settings( if let Some(git_sync) = settings.git_sync.as_mut() { redact_git_sync_webhook_secrets(git_sync); } + settings.datatable = + windmill_common::workspaces::strip_datatable_permissions(settings.datatable.take()); Ok(Json(settings)) } @@ -1176,8 +1181,10 @@ async fn get_public_settings( .await .map_err(|e| Error::internal_err(format!("getting public settings: {e:#}")))?; - let settings = not_found_if_none(settings, "workspace settings", &w_id)?; + let mut settings = not_found_if_none(settings, "workspace settings", &w_id)?; tx.commit().await?; + settings.datatable = + windmill_common::workspaces::strip_datatable_permissions(settings.datatable.take()); Ok(Json(settings)) } @@ -2184,6 +2191,12 @@ struct DataTableListItem { name: String, resource_type: String, resource_path: String, + /// The workspace whose entry governs this one, when it is not this workspace — a fork pointing + /// at its parent. Its permissions apply here, and only its admins may edit them. + #[serde(skip_serializing_if = "Option::is_none")] + governing_workspace_id: Option, + /// Whether the governing entry is under roles. + permissioned: bool, } async fn list_datatables( @@ -2191,26 +2204,29 @@ async fn list_datatables( Extension(db): Extension, Path(w_id): Path, ) -> JsonResult> { - let config = sqlx::query_scalar!( - "SELECT datatable->'datatables' FROM workspace_settings WHERE workspace_id = $1", - &w_id - ) - .fetch_one(&db) - .await?; + // A pointer entry owns no database, so what it resolves to is the only truthful answer here. + // One that resolves to nothing — a pointer whose workspace was deleted — is dropped rather than + // listed with a database it does not have; what happened is named where it is actionable + // instead: by the delete that stranded it, and by any attempt to use it. + let resolved = + windmill_common::workspaces::resolve_workspace_governing_datatables(&db, &w_id).await?; - let items: Vec = match config { - Some(val) => { - let map: HashMap = serde_json::from_value(val).unwrap_or_default(); - map.into_iter() - .map(|(name, dt)| DataTableListItem { - name, - resource_type: dt.database.resource_type.as_ref().to_string(), - resource_path: dt.database.resource_path, - }) - .collect() - } - None => vec![], - }; + let mut items = Vec::with_capacity(resolved.len()); + for (name, governing) in resolved { + let database = governing + .datatable + .database + .as_ref() + .expect("a governing entry owns a database"); + items.push(DataTableListItem { + name, + resource_type: database.resource_type.as_ref().to_string(), + resource_path: database.resource_path.clone(), + governing_workspace_id: (governing.workspace_id != w_id) + .then(|| governing.workspace_id.clone()), + permissioned: governing.datatable.permissions.is_some(), + }); + } Ok(Json(items)) } @@ -2298,6 +2314,15 @@ async fn test_datatable_connection( Path((w_id, datatable_name)): Path<(String, String)>, ) -> JsonResult { require_admin(authed.is_admin, &authed.username)?; + // Reports what the admin connection can do, so it answers to the workspace that governs the + // data table rather than to whichever one is asking. + windmill_common::workspaces::ensure_datatable_admin_access( + &db, + &w_id, + &datatable_name, + &DatatableAccess::Authed(authed.to_authed_ref()), + ) + .await?; let db_resource = get_datatable_resource_from_db_unchecked(&db, &w_id, &datatable_name).await?; let pg_db: PgDatabase = serde_json::from_value(db_resource) @@ -2385,7 +2410,7 @@ async fn test_datatable_connection( } async fn list_datatable_schemas( - _authed: ApiAuthed, + authed: ApiAuthed, Extension(db): Extension, Path(w_id): Path, ) -> JsonResult> { @@ -2393,7 +2418,7 @@ async fn list_datatable_schemas( let mut results = Vec::new(); for datatable_name in datatable_names { - let schema = match get_datatable_schema(&db, &w_id, &datatable_name).await { + let schema = match get_datatable_schema(&db, &authed, &w_id, &datatable_name).await { Ok(schemas) => DataTableSchema { datatable_name, schemas, error: None }, Err(e) => DataTableSchema { datatable_name, @@ -2408,7 +2433,7 @@ async fn list_datatable_schemas( } async fn list_datatable_tables( - _authed: ApiAuthed, + authed: ApiAuthed, Extension(db): Extension, Path(w_id): Path, ) -> JsonResult> { @@ -2416,7 +2441,7 @@ async fn list_datatable_tables( let mut results = Vec::new(); for datatable_name in datatable_names { - let tables = match get_datatable_tables(&db, &w_id, &datatable_name).await { + let tables = match get_datatable_tables(&db, &authed, &w_id, &datatable_name).await { Ok(schemas) => DataTableTables { datatable_name, schemas, error: None }, Err(e) => DataTableTables { datatable_name, @@ -2431,13 +2456,14 @@ async fn list_datatable_tables( } async fn get_datatable_table_schema( - _authed: ApiAuthed, + authed: ApiAuthed, Extension(db): Extension, Path(w_id): Path, Query(query): Query, ) -> JsonResult { let columns = get_datatable_table_columns( &db, + &authed, &w_id, &query.datatable_name, &query.schema_name, @@ -2469,13 +2495,39 @@ async fn list_datatable_names(db: &DB, w_id: &str) -> Result> { .collect()) } -async fn get_datatable_schema(db: &DB, w_id: &str, datatable_name: &str) -> Result { - // Get the datatable resource (connection credentials) - let db_resource = get_datatable_resource_from_db_unchecked(db, w_id, datatable_name).await?; +/// Connect to a data table as the caller, not as `admin`: the role they named, or the data table's +/// default. A data table not under roles resolves as `admin`, exactly as it did before roles. +/// +/// Every schema-browsing query below then reports what this Postgres role can actually reach, +/// which is why they filter on `has_schema_privilege` — `pg_catalog` is world-readable, so an +/// unfiltered listing would name schemas the connection cannot even enter. +async fn resolve_datatable_pg_as_caller( + db: &DB, + authed: &ApiAuthed, + w_id: &str, + datatable_name: &str, +) -> Result { + let db_resource = get_datatable_resource_from_db( + db, + w_id, + datatable_name, + // The data table's default role. Browsing has no way to name another one yet; when the + // database manager grows a role picker it passes the pick through here. + None, + DatatableAccess::Authed(authed.to_authed_ref()), + ) + .await?; + serde_json::from_value(db_resource) + .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e))) +} - // Parse the resource as PgDatabase - let pg_db: PgDatabase = serde_json::from_value(db_resource) - .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?; +async fn get_datatable_schema( + db: &DB, + authed: &ApiAuthed, + w_id: &str, + datatable_name: &str, +) -> Result { + let pg_db = resolve_datatable_pg_as_caller(db, authed, w_id, datatable_name).await?; // Connect to the datatable database let (client, connection) = pg_db.connect(Some(db)).await?; @@ -2495,6 +2547,7 @@ async fn get_datatable_schema(db: &DB, w_id: &str, datatable_name: &str) -> Resu FROM pg_namespace WHERE nspname NOT IN ('information_schema', 'pg_toast', 'pg_catalog') AND nspname NOT LIKE 'pg_%' + AND has_schema_privilege(oid, 'USAGE') ORDER BY nspname "#, &[], @@ -2562,10 +2615,13 @@ async fn get_datatable_schema(db: &DB, w_id: &str, datatable_name: &str) -> Resu Ok(schema_map) } -async fn get_datatable_tables(db: &DB, w_id: &str, datatable_name: &str) -> Result { - let db_resource = get_datatable_resource_from_db_unchecked(db, w_id, datatable_name).await?; - let pg_db: PgDatabase = serde_json::from_value(db_resource) - .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?; +async fn get_datatable_tables( + db: &DB, + authed: &ApiAuthed, + w_id: &str, + datatable_name: &str, +) -> Result { + let pg_db = resolve_datatable_pg_as_caller(db, authed, w_id, datatable_name).await?; let (client, connection) = pg_db.connect(Some(db)).await?; tokio::spawn(async move { @@ -2581,6 +2637,7 @@ async fn get_datatable_tables(db: &DB, w_id: &str, datatable_name: &str) -> Resu FROM pg_namespace WHERE nspname NOT IN ('information_schema', 'pg_toast', 'pg_catalog') AND nspname NOT LIKE 'pg_%' + AND has_schema_privilege(oid, 'USAGE') ORDER BY nspname "#, &[], @@ -2629,6 +2686,7 @@ async fn get_datatable_tables(db: &DB, w_id: &str, datatable_name: &str) -> Resu async fn get_datatable_table_columns( db: &DB, + authed: &ApiAuthed, w_id: &str, datatable_name: &str, schema_name: &str, @@ -2641,9 +2699,7 @@ async fn get_datatable_table_columns( ))); } - let db_resource = get_datatable_resource_from_db_unchecked(db, w_id, datatable_name).await?; - let pg_db: PgDatabase = serde_json::from_value(db_resource) - .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?; + let pg_db = resolve_datatable_pg_as_caller(db, authed, w_id, datatable_name).await?; let (client, connection) = pg_db.connect(Some(db)).await?; tokio::spawn(async move { @@ -2853,7 +2909,10 @@ mod tests { } /// Resolve a source string to PgDatabase credentials with user-scoped permission checks. -/// For `datatable://name`: accessible to everyone (variables are resolved internally). +/// +/// For `datatable://name`: the **admin** connection, so it is gated on admin reach. Every caller +/// copies, dumps or drops a whole database, and a dump taken under a restricted role would be a +/// silently truncated copy rather than an error — which is worse than refusing. /// For `$res:path`: uses UserDB (row-level security) to verify the user can see the resource, /// then interpolates `$var:` references in the resource value. pub(crate) async fn resolve_pg_source_checked( @@ -2864,6 +2923,13 @@ pub(crate) async fn resolve_pg_source_checked( source: &str, ) -> Result { let db_resource = if let Some(name) = source.strip_prefix("datatable://") { + windmill_common::workspaces::ensure_datatable_admin_access( + db, + w_id, + name, + &DatatableAccess::Authed(authed.to_authed_ref()), + ) + .await?; get_datatable_resource_from_db_unchecked(db, w_id, name).await? } else if let Some(path) = source.strip_prefix("$res:") { let db_with_authed = windmill_common::db::DbWithOptAuthed::from_authed( @@ -2904,22 +2970,13 @@ pub(crate) async fn resolve_pg_source_checked( /// Whether the data table `name` is backed by the Windmill instance's own PostgreSQL /// rather than a user resource. pub(crate) async fn is_instance_datatable(db: &DB, w_id: &str, name: &str) -> Result { - let config = sqlx::query_scalar!( - "SELECT datatable->'datatables'->$2 FROM workspace_settings WHERE workspace_id = $1", - w_id, - name - ) - .fetch_optional(db) - .await? - .flatten(); - Ok(config - .and_then(|v| { - v.get("database") - .and_then(|d| d.get("resource_type")) - .and_then(|r| r.as_str()) - .map(|s| s == "instance") - }) - .unwrap_or(false)) + // Resolved rather than read: a pointer entry owns no database of its own, so only the entry it + // lands on can answer. A name that resolves to nothing keeps the historical `false`. + Ok(resolve_governing_datatable(db, w_id, name) + .await + .ok() + .and_then(|g| g.datatable.database) + .is_some_and(|d| d.resource_type == DataTableCatalogResourceType::Instance)) } /// Same, for the `datatable://` / `$res:` form the import endpoints take. @@ -3250,6 +3307,14 @@ async fn create_pg_database( ) -> Result { windmill_common::validate_dbname(&req.target_dbname)?; + // The copy this database is for is refused a call later, and nothing collects an instance + // database that no data table entry names. Refuse here too, so the clone stops before one + // exists rather than leaving an empty registered `wm_fork_…` behind. + if let Some(reference) = req.source.strip_prefix("datatable://") { + let (name, _) = parse_datatable_ref_for(&db, &w_id, reference).await?; + ensure_datatable_is_clonable(&db, &w_id, &name).await?; + } + // Non-superadmin: restrict dbname to wm_fork_ prefix if !windmill_api_auth::is_super_admin_authed(&db, &authed).await? { if !req.target_dbname.starts_with("wm_fork_") { @@ -3319,6 +3384,64 @@ struct ImportPgDatabaseRequest { fork_behavior: DataTableForkBehavior, } +/// Refuse to copy a data table that is under roles. +/// +/// `pg_dump` carries no roles and the import runs with `--no-privileges`, so a clone arrives with +/// its objects owned by the admin connection and no `GRANT` for any role. The settings copy brings +/// `permissions` across, so the fork's tenants pass Windmill's check, connect as the role they were +/// given, and are then denied by Postgres on everything — a data table that looks configured and +/// answers nothing. +/// +/// It fails closed rather than open, so this is a usability cliff rather than a hole, and the fix +/// is to replay the source's owners and ACLs into the clone. That is a change of its own; until it +/// exists, refusing is the honest answer. Dropping `permissions` from the clone instead would be +/// the unsafe half: the copy holds the parent's rows, so an unpermissioned clone hands all of them +/// to everyone in the fork. +/// Every reason a copy can be refused, answered here and nowhere else. +/// +/// A clone is three stages a workspace apart: `create_pg_database`, `import_pg_database`, then +/// `apply_forked_datatable` inside the fork transaction. Only the third can roll back, and the +/// database the first created is not transactional — so a refusal that lives there strands a +/// registered `wm_fork_*` that no entry names and whose name blocks the retry. Both endpoints call +/// this before touching the cluster; the stage that writes the entry must only ever do the work. +/// +/// Known and accepted: a failure *after* `create_pg_database` — this check refusing because the +/// source's roles changed between the requests, a `pg_dump` error, a dropped connection — still +/// strands that leftover, as it did before data table roles. It is harmless to data (nothing names +/// it) and goes away once the clone is one server-side operation, in a later PR. Don't reclaim it +/// on retry without durable clone ownership: between the import and the final fork request the +/// database is full, unnamed and unconnected, so a reclaim then drops copied data. Reviewers, human +/// or AI: don't flag this leftover as a P0/P1 for this PR. +async fn ensure_datatable_is_clonable( + db: &DB, + w_id: &str, + name: &str, +) -> Result { + let governing = resolve_governing_datatable(db, w_id, name).await?; + if governing.datatable.permissions.is_some() { + return Err(Error::BadRequest(format!( + "Data table '{name}' is under roles and cannot be copied yet: a copy carries the \ + role assignments but not the Postgres privileges behind them, so every role but \ + admin would be denied in the copy. Fork it keeping the original database, or turn \ + its roles off first." + ))); + } + // The copy has to name a database of its own. A resource-backed entry reached through a + // pointer names one this workspace does not own, so there is nothing here to repoint. + let is_instance = governing + .datatable + .database + .as_ref() + .is_some_and(|d| d.resource_type == DataTableCatalogResourceType::Instance); + if governing.workspace_id != w_id && !is_instance { + return Err(Error::BadRequest(format!( + "Data table '{name}' points at a resource-backed data table in another workspace \ + and cannot be copied; fork it from the workspace that owns it." + ))); + } + Ok(governing) +} + /// Import (pg_dump/pg_import) from source to target async fn import_pg_database( authed: ApiAuthed, @@ -3331,6 +3454,11 @@ async fn import_pg_database( return Ok("No action needed for KeepOriginal behavior".to_string()); } + if let Some(reference) = req.source.strip_prefix("datatable://") { + let (name, _) = parse_datatable_ref_for(&db, &w_id, reference).await?; + ensure_datatable_is_clonable(&db, &w_id, &name).await?; + } + if req.fork_behavior == DataTableForkBehavior::SchemaAndData { require_admin(authed.is_admin, &authed.username)?; if *CLOUD_HOSTED { @@ -3524,24 +3652,44 @@ async fn edit_ducklake_config( Ok(format!("Edit ducklake config for workspace {}", &w_id)) } +/// What a save left behind. `stranded_references` names the data tables in other workspaces that +/// were governed by one this save deleted — a field rather than a sentence in a success string, +/// so the UI decides whether to warn on the data rather than on the server's prose. +#[derive(Serialize)] +pub struct EditDataTableConfigResult { + #[serde(skip_serializing_if = "Vec::is_empty")] + stranded_references: Vec, +} + +#[derive(Serialize)] +pub struct StrandedReference { + workspace_id: String, + datatable: String, +} + async fn edit_datatable_config( authed: ApiAuthed, Extension(db): Extension, Path(w_id): Path, ApiAuthed { is_admin, username, .. }: ApiAuthed, Json(mut new_config): Json, -) -> Result { +) -> JsonResult { require_admin(is_admin, &username)?; let is_superadmin = require_super_admin(&db, &authed).await.is_ok(); let mut tx = db.begin().await?; + // Read under the row lock this transaction will write with. `permissions`, `reference` and + // `forked_from` are carried across from what this read returns, so a permissions save + // committing between the read and the whole-document write below would be silently rolled back + // by it. let old_datatables: HashMap = serde_json::from_value( sqlx::query_scalar!( - "SELECT ws.datatable->'datatables' FROM workspace_settings ws WHERE ws.workspace_id = $1", + "SELECT ws.datatable->'datatables' FROM workspace_settings ws + WHERE ws.workspace_id = $1 FOR UPDATE", &w_id ) - .fetch_one(&db) + .fetch_one(&mut *tx) .await? .unwrap_or(serde_json::Value::Null), ) @@ -3563,6 +3711,58 @@ async fn edit_datatable_config( crate::datatable_migrations::validate_datatable_path_segment(&r.from)?; crate::datatable_migrations::validate_new_datatable_name(&r.to)?; } + // A rename is a claim about what this save is doing, and other workspaces' pointers are + // rewritten from it — so the claim has to match the configuration it describes, or a caller + // can move every fork of one data table onto another by asserting a rename that did not + // happen. The shape below is what "these old keys became those new keys" actually means. + { + let old_keys = &old_datatables; + let new_keys = &new_config.settings.datatables; + let froms: std::collections::HashSet<&str> = + new_config.renames.iter().map(|r| r.from.as_str()).collect(); + let tos: std::collections::HashSet<&str> = + new_config.renames.iter().map(|r| r.to.as_str()).collect(); + if froms.len() != new_config.renames.len() { + return Err(Error::BadRequest( + "A data table is renamed twice in one save".to_string(), + )); + } + if tos.len() != new_config.renames.len() { + return Err(Error::BadRequest( + "Two data tables are renamed to the same name in one save".to_string(), + )); + } + for r in &new_config.renames { + if !old_keys.contains_key(&r.from) { + return Err(Error::BadRequest(format!( + "Cannot rename data table '{}': this workspace has no such data table", + r.from + ))); + } + if !new_keys.contains_key(&r.to) { + return Err(Error::BadRequest(format!( + "Cannot rename data table '{}' to '{}': the save does not contain '{}'", + r.from, r.to, r.to + ))); + } + // The source has to be gone, or gone-and-reoccupied by another rename — which is what + // a swap is. Without this, `main -> decoy` passes against a save that keeps both, and + // every fork of `main` silently follows onto a different data table. + if new_keys.contains_key(&r.from) && !tos.contains(r.from.as_str()) { + return Err(Error::BadRequest(format!( + "Data table '{}' is renamed to '{}' but the save still contains '{}'", + r.from, r.to, r.from + ))); + } + // And the target has to be free, or freed by another rename. + if old_keys.contains_key(&r.to) && !froms.contains(r.to.as_str()) { + return Err(Error::BadRequest(format!( + "Cannot rename data table '{}' to '{}': '{}' already exists", + r.from, r.to, r.to + ))); + } + } + } // Map new name -> old name so a renamed data table inherits the previous // flag instead of being treated as brand new. @@ -3583,18 +3783,52 @@ async fn edit_datatable_config( .get(name.as_str()) .copied() .unwrap_or(name.as_str()); - dt.migrations_enabled = match old_datatables.get(lookup) { + let old = old_datatables.get(lookup); + dt.migrations_enabled = match old { Some(old) => old.migrations_enabled, None => { // Keyed by how the substrate is serialized into `workspace_settings`, // so these line up with the `datatable_configured` adoption counts. - created_substrates.push(match dt.database.resource_type { - DataTableCatalogResourceType::Instance => "instance", - DataTableCatalogResourceType::Postgresql => "postgresql", + created_substrates.push(match dt.database.as_ref().map(|d| d.resource_type) { + Some(DataTableCatalogResourceType::Instance) => "instance", + Some(DataTableCatalogResourceType::Postgresql) => "postgresql", + None => "reference", }); Some(true) } }; + // Carried across from the stored entry rather than taken from the request. `permissions` + // is an access decision, edited through its own endpoint; `reference` is what makes a fork + // answer to the workspace that governs its data table, and letting a save clear it would + // hand the fork the database outright. `forked_from` is the clone stamp the fork flow + // writes: whether an entry has one is carried the same way, since it is what marks the + // database droppable, but the schema baseline inside it is the diff view's to advance. + dt.permissions = old.and_then(|old| old.permissions.clone()); + dt.reference = old.and_then(|old| old.reference.clone()); + dt.forked_from = match old.and_then(|old| old.forked_from.as_ref()) { + Some(stored) => Some(dt.forked_from.take().unwrap_or_else(|| stored.clone())), + None => None, + }; + // Carrying the block onto a resource-backed entry would produce a data table the chokepoint + // refuses on every job — a save that succeeds and breaks everything afterwards. Refuse it + // instead: turning roles off first is one step, and it keeps discarding an access decision + // something somebody chose rather than a side effect of moving a database. + if dt.permissions.is_some() + && dt + .database + .as_ref() + .is_some_and(|d| d.resource_type != DataTableCatalogResourceType::Instance) + { + return Err(Error::BadRequest(format!( + "Data table '{name}' is under roles, which only a data table on the instance \ + database can be. Turn its roles off before moving it to a PostgreSQL resource." + ))); + } + // A pointer names no database of its own, so the form's empty `database` is correct there. + if dt.reference.is_some() { + dt.database = None; + } + windmill_common::workspaces::validate_datatable_shape(name, dt)?; } let args_for_audit = format!("{:?}", new_config.settings); @@ -3609,16 +3843,23 @@ async fn edit_datatable_config( ) .await?; - // Check that non-superadmins are not abusing Instance databases + // Check that non-superadmins are not abusing Instance databases, which reach a database this + // workspace does not own. Pointing an entry at another workspace's data table is not checked + // here because it cannot be requested at all: `reference` is overwritten from the stored entry + // above, for every caller. if !is_superadmin { for (name, dt) in new_config.settings.datatables.iter() { - if dt.database.resource_type == DataTableCatalogResourceType::Instance { - let old_dt = old_datatables.get(name); - if old_dt.is_none() - || old_dt.unwrap().database.resource_type - != DataTableCatalogResourceType::Instance - || old_dt.unwrap().database.resource_path != dt.database.resource_path - { + let old_dt = old_datatables.get(name); + if dt + .database + .as_ref() + .is_some_and(|d| d.resource_type == DataTableCatalogResourceType::Instance) + { + let unchanged = old_dt.and_then(|o| o.database.as_ref()).is_some_and(|o| { + o.resource_type == DataTableCatalogResourceType::Instance + && Some(&o.resource_path) == dt.database.as_ref().map(|d| &d.resource_path) + }); + if !unchanged { return Err(Error::BadRequest( "Only superadmins can create or modify data tables with Instance databases" .to_string(), @@ -3628,6 +3869,84 @@ async fn edit_datatable_config( } } + // Worked out from the locked entries rather than taken from `deleted_datatables`: a settings + // sync sends the whole map without that list, and dropping a governing entry strands every + // fork pointing at it all the same. + let removed: Vec = old_datatables + .keys() + .filter(|name| { + !new_config.settings.datatables.contains_key(*name) + && !new_config.renames.iter().any(|r| &r.from == *name) + }) + .cloned() + .collect(); + + // A database under roles is reached only through an entry that carries them. Roles follow an + // entry through a declared rename alone, and a settings sync never declares one, so an entry + // without roles that newly points at such a database — a name added, or an existing one + // repointed — would answer everyone there as `admin`. That holds whichever workspace governs it. + let newly_pointed: Vec<(&String, &str)> = new_config + .settings + .datatables + .iter() + .filter(|(_, dt)| dt.permissions.is_none()) + .filter_map(|(name, dt)| { + let db = dt + .database + .as_ref() + .filter(|d| d.resource_type == DataTableCatalogResourceType::Instance)?; + let lookup = rename_src + .get(name.as_str()) + .copied() + .unwrap_or(name.as_str()); + let repointed = old_datatables + .get(lookup) + .and_then(|old| old.database.as_ref()) + .is_none_or(|old_db| { + old_db.resource_type != db.resource_type + || old_db.resource_path != db.resource_path + }); + repointed.then_some((name, db.resource_path.as_str())) + }) + .collect(); + // Another workspace turning roles on for the same database holds only its own settings row, so + // without this the scan below could read past its uncommitted write. + windmill_common::datatable_roles::lock_instance_databases_governance( + &mut *tx, + newly_pointed.iter().map(|(_, dbname)| *dbname), + ) + .await?; + let governed_elsewhere: Vec = if newly_pointed.is_empty() { + vec![] + } else { + sqlx::query_scalar( + "SELECT DISTINCT dt.value->'database'->>'resource_path' FROM workspace_settings ws + CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt + WHERE ws.workspace_id <> $1 AND dt.value ? 'permissions' + AND dt.value->'database'->>'resource_type' = 'instance'", + ) + .bind(&w_id) + .fetch_all(&mut *tx) + .await? + }; + for (name, dbname) in newly_pointed { + let governed_here = old_datatables.values().any(|old| { + old.permissions.is_some() + && old.database.as_ref().is_some_and(|d| { + d.resource_type == DataTableCatalogResourceType::Instance + && d.resource_path == dbname + }) + }); + if governed_here || governed_elsewhere.iter().any(|g| g == dbname) { + return Err(Error::BadRequest(format!( + "Data table '{name}' would point at database '{dbname}', which a data table under \ + roles uses, without carrying those roles: everyone reaching '{name}' would connect \ + there as `admin`. Rename the data table under roles from the data table settings, \ + which carries its roles, or turn its roles off first." + ))); + } + } + let config: serde_json::Value = serde_json::to_value(new_config.settings) .map_err(|err| Error::internal_err(err.to_string()))?; @@ -3649,6 +3968,44 @@ async fn edit_datatable_config( ) .await?; + // A fork points at a data table by name, so a rename here has to follow or every fork's entry + // resolves to nothing. In two passes through a temporary name, like the migration cascade one + // layer down: applied in order, `sa -> sb` then `sb -> sa` would move what pointed at `sa` all + // the way back to `sa`, and `A -> B`, `B -> C` would carry `A`'s pointers to `C`. Each pointer + // moves once, from what it named before this save. Inside the transaction: the rename and the + // pointers that name it are one change, and half of it is a fork whose jobs stop. + for (i, r) in new_config.renames.iter().enumerate() { + repoint_datatable_references(&mut tx, &w_id, &r.from, &format!("__wm_rename_tmp/{i}")) + .await?; + } + for (i, r) in new_config.renames.iter().enumerate() { + repoint_datatable_references(&mut tx, &w_id, &format!("__wm_rename_tmp/{i}"), &r.to) + .await?; + } + + // A deletion cannot be followed the same way — there is nothing to point at any more. Read who + // is left stranded so the caller is told, the way deleting a workspace does. + let mut stranded: Vec = Vec::new(); + for name in &removed { + let rows = sqlx::query!( + r#"SELECT ws.workspace_id AS "workspace_id!", dt.key AS "datatable!" + FROM workspace_settings ws + CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt + WHERE dt.value->'reference'->>'workspace_id' = $1 + AND dt.value->'reference'->>'datatable' = $2"#, + &w_id, + name, + ) + .fetch_all(&mut *tx) + .await?; + stranded.extend( + rows.into_iter().map(|r| StrandedReference { + workspace_id: r.workspace_id, + datatable: r.datatable, + }), + ); + } + tx.commit().await?; for substrate in created_substrates { @@ -3663,7 +4020,9 @@ async fn edit_datatable_config( ) .await?; - Ok(format!("Edit datatable config for workspace {}", &w_id)) + Ok(Json(EditDataTableConfigResult { + stranded_references: stranded, + })) } #[derive(Deserialize)] @@ -7545,13 +7904,144 @@ async fn snapshot_datatable_schema( .map_err(|e| Error::internal_err(format!("Failed to serialize schema: {}", e))) } +/// Turn every data table the fork chose to keep into a pointer at the parent's entry. +/// +/// `clone_workspace_data` copies `workspace_settings` wholesale, so a kept data table arrives as a +/// byte-identical copy naming the parent's database — including the parent's `permissions`, which +/// a fork admin could then edit to widen their own access to it. A pointer has nothing local to +/// edit: the parent's entry stays the only place the decision lives. +/// +/// The cloned data tables are skipped: they own a fresh database of their own, and they keep the +/// copied `permissions` as their starting point, which they then govern. +async fn point_kept_datatables_at_parent( + tx: &mut Transaction<'_, Postgres>, + parent_w_id: &str, + forked_w_id: &str, + cloned: &[ForkedDatatableInfo], +) -> Result<()> { + let settings: Option = sqlx::query_scalar!( + "SELECT datatable FROM workspace_settings WHERE workspace_id = $1", + forked_w_id + ) + .fetch_optional(&mut **tx) + .await? + .flatten(); + + let Some(mut settings) = settings else { + return Ok(()); + }; + let Some(datatables) = settings + .get_mut("datatables") + .and_then(|d| d.as_object_mut()) + else { + return Ok(()); + }; + + let mut changed = false; + for (name, entry) in datatables.iter_mut() { + if cloned.iter().any(|c| &c.name == name) { + continue; + } + let dt: DataTable = match serde_json::from_value(entry.clone()) { + Ok(dt) => dt, + Err(_) => continue, + }; + // Already a pointer: the parent was itself a fork, and its entry names the workspace that + // governs. Following it from here is the same answer, so leave it alone. + if dt.reference.is_some() { + continue; + } + // Only instance databases. A resource-backed data table names a resource, and the settings + // clone gave the fork its own copy of that resource in its own workspace — pointing at the + // parent's entry would silently move the fork onto the parent's resource instead. + if dt + .database + .as_ref() + .is_none_or(|d| d.resource_type != DataTableCatalogResourceType::Instance) + { + continue; + } + *entry = serde_json::to_value(DataTable { + database: None, + reference: Some(windmill_common::workspaces::DataTableReference { + workspace_id: parent_w_id.to_string(), + datatable: name.clone(), + }), + forked_from: None, + migrations_enabled: dt.migrations_enabled, + permissions: None, + }) + .map_err(|e| Error::internal_err(format!("serializing data table '{name}': {e}")))?; + changed = true; + } + + if changed { + sqlx::query!( + "UPDATE workspace_settings SET datatable = $1 WHERE workspace_id = $2", + settings, + forked_w_id + ) + .execute(&mut **tx) + .await?; + } + Ok(()) +} + +/// Move every pointer in any workspace that names `(w_id, from)` to `(w_id, to)`. +/// +/// `EXISTS` rather than a `LIKE` over the whole document: the update rewrites the row, so matching +/// every workspace that holds any pointer would rewrite rows to a byte-identical value and hold an +/// exclusive lock on them until commit. +async fn repoint_datatable_references( + tx: &mut Transaction<'_, Postgres>, + w_id: &str, + from: &str, + to: &str, +) -> Result<()> { + sqlx::query!( + r#"UPDATE workspace_settings ws + SET datatable = ( + SELECT jsonb_set(ws.datatable, '{datatables}', jsonb_object_agg( + dt.key, + CASE WHEN dt.value->'reference'->>'workspace_id' = $1 + AND dt.value->'reference'->>'datatable' = $2 + THEN jsonb_set(dt.value, '{reference,datatable}', to_jsonb($3::text)) + ELSE dt.value END + )) + FROM jsonb_each(ws.datatable->'datatables') dt + ) + WHERE EXISTS ( + SELECT 1 FROM jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) d + WHERE d.value->'reference'->>'workspace_id' = $1 + AND d.value->'reference'->>'datatable' = $2 + )"#, + w_id, + from, + to, + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + async fn apply_forked_datatable( db: &DB, tx: &mut Transaction<'_, Postgres>, + authed: &ApiAuthed, parent_w_id: &str, forked_w_id: &str, fdt: &ForkedDatatableInfo, ) -> Result<()> { + // Cloning reads the parent's whole schema as admin and hands the copy to the fork, so it is + // for the workspace that governs the data table — a fork can use one, never duplicate it. + windmill_common::workspaces::ensure_datatable_admin_access( + db, + parent_w_id, + &fdt.name, + &DatatableAccess::Authed(authed.to_authed_ref()), + ) + .await?; + let governing = ensure_datatable_is_clonable(db, parent_w_id, &fdt.name).await?; windmill_common::validate_dbname(&fdt.new_dbname)?; if !fdt.new_dbname.starts_with("wm_fork_") { return Err(Error::BadRequest(format!( @@ -7583,25 +8073,46 @@ async fn apply_forked_datatable( let dt: DataTable = serde_json::from_value(config_val) .map_err(|e| Error::internal_err(format!("Failed to parse datatable config: {}", e)))?; - if dt.database.resource_type == DataTableCatalogResourceType::Instance { - // Instance: update resource_path to the new dbname + // A clone owns its copy, so the fork's entry has to be terminal. When the parent was itself a + // fork the settings clone hands down a pointer instead, and what it points at is what the copy + // was taken from. `ensure_datatable_is_clonable` already settled that this shape can be + // cloned, so there is nothing left to refuse here — by now the database exists and is filled. + let database = match dt.database.clone() { + Some(database) => database, + None => governing.datatable.database.clone().ok_or_else(|| { + Error::internal_err(format!( + "Data table '{}' resolves to an entry that owns no database", + fdt.name + )) + })?, + }; + + if database.resource_type == DataTableCatalogResourceType::Instance { + // The whole `database` object, not just its `resource_path`: a pointer entry has none to + // patch. `reference` goes with it — exactly one of the two may be set. + let new_database = serde_json::json!({ + "resource_type": "instance", + "resource_path": &fdt.new_dbname, + }); sqlx::query!( r#"UPDATE workspace_settings SET datatable = jsonb_set( - jsonb_set(datatable, ARRAY['datatables', $2, 'database', 'resource_path'], to_jsonb($3::text)), + jsonb_set( + datatable #- ARRAY['datatables', $2, 'reference'], + ARRAY['datatables', $2, 'database'], $3::jsonb), ARRAY['datatables', $2, 'forked_from'], $4::jsonb ) WHERE workspace_id = $1"#, forked_w_id, &fdt.name, - &fdt.new_dbname, + new_database, forked_from, ) .execute(&mut **tx) .await?; } else { // Resource: update the resource's dbname and mark as ws_specific - let resource_path = &dt.database.resource_path; + let resource_path = &database.resource_path; sqlx::query!( r#"UPDATE resource SET value = jsonb_set(value, '{dbname}', to_jsonb($3::text)) @@ -8038,6 +8549,14 @@ async fn create_workspace_fork( .execute(&mut *tx) .await?; + // The pointers this fork writes to the parent's data tables stay invisible until it commits, so + // a rename of one of them cannot carry them. Holding the parent's settings row makes such a + // rename wait for this commit, and makes the copy below read one that committed first. + sqlx::query("SELECT 1 FROM workspace_settings WHERE workspace_id = $1 FOR SHARE") + .bind(&parent_workspace_id) + .execute(&mut *tx) + .await?; + // Clone all data from the parent workspace using Rust implementation if let Err(e) = clone_workspace_data(&mut tx, &db, &parent_workspace_id, &forked_id, &authed).await @@ -8075,9 +8594,18 @@ async fn create_workspace_fork( // Update forked datatable settings to point to new databases for fdt in &nw.forked_datatables { - apply_forked_datatable(&db, &mut tx, &parent_workspace_id, &forked_id, fdt).await?; + apply_forked_datatable(&db, &mut tx, &authed, &parent_workspace_id, &forked_id, fdt) + .await?; } + point_kept_datatables_at_parent( + &mut tx, + &parent_workspace_id, + &forked_id, + &nw.forked_datatables, + ) + .await?; + // The settings clone copies the source's ducklake config verbatim — including a parent // fork's own `fork_behavior` stamps. Sharing is a per-fork-creation choice, never // inherited: reset any cloned stamps first, then apply this fork's requested list. @@ -8880,6 +9408,14 @@ async fn leave_workspace( ) -> Result { windmill_api_auth::forbid_job_token_account_destruction(&authed)?; let mut tx = db.begin().await?; + // The membership is what made `u/` mean this person. Leaving it behind in a tenant + // list would hand their data table access back on rejoin, or to whoever takes the name next. + windmill_common::workspaces::remove_datatable_tenant_in_workspace( + &mut tx, + &w_id, + &format!("u/{}", authed.username), + ) + .await?; sqlx::query!( "DELETE FROM usr WHERE workspace_id = $1 AND email = $2", &w_id, diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index e75f0d1ef2..1cb5188e5b 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -492,6 +492,30 @@ pub(crate) async fn change_workspace_id( .fetch_all(&mut *tx) .await?; + // A fork's data table entry names the workspace that governs it by id, so the rename has to + // follow there too — anywhere, not just in the reparented children: a detached workspace can + // point at this one without being its fork. Left behind, the pointer resolves to the archived + // shell and every job through it stops. + info!("Re-pointing data table references to the new workspace id"); + sqlx::query!( + r#"UPDATE workspace_settings ws + SET datatable = ( + SELECT jsonb_set(ws.datatable, '{datatables}', jsonb_object_agg( + dt.key, + CASE WHEN dt.value->'reference'->>'workspace_id' = $2 + THEN jsonb_set(dt.value, '{reference,workspace_id}', to_jsonb($1::text)) + ELSE dt.value END + )) + FROM jsonb_each(ws.datatable->'datatables') dt + ) + WHERE jsonb_typeof(ws.datatable->'datatables') = 'object' + AND ws.datatable::text LIKE '%"reference"%'"#, + &rw.new_id, + &old_id, + ) + .execute(&mut *tx) + .await?; + info!("Updating workspace_protection_rule table"); sqlx::query!( "UPDATE workspace_protection_rule SET workspace_id = $1 WHERE workspace_id = $2", @@ -971,6 +995,22 @@ pub(crate) async fn delete_workspace( // but the destructive cleanup itself runs only after the commit below: a delete that // fails mid-way must never leave a live workspace with its fork data destroyed and no // registry row to retry from. Read-only: nothing is dropped here. + // Read before the delete: another workspace's data table entry can point at one of this + // workspace's, and deleting the workspace it names leaves that pointer resolving to nothing. + // Nothing sweeps them — turning them back into copies would hand each fork the database + // outright — so the deleter is told which data tables they just stranded. + let stranded_pointers = sqlx::query!( + r#"SELECT ws.workspace_id AS "workspace_id!", dt.key AS "datatable!" + FROM workspace_settings ws + CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt + WHERE dt.value->'reference'->>'workspace_id' = $1 + ORDER BY ws.workspace_id, dt.key"#, + &w_id, + ) + .fetch_all(&db) + .await + .unwrap_or_default(); + let fork_ducklake_cleanups = prepare_fork_ducklake_cleanups(&db, &w_id, None) .await .unwrap_or_else(|e| { @@ -1289,7 +1329,23 @@ pub(crate) async fn delete_workspace( tracing::warn!("failed to broadcast fork lineage change: {e:#}"); } - Ok(format!("Deleted workspace {}", &w_id)) + if stranded_pointers.is_empty() { + Ok(format!("Deleted workspace {}", &w_id)) + } else { + let stranded = stranded_pointers + .iter() + .map(|r| format!("{}/{}", r.workspace_id, r.datatable)) + .collect::>() + .join(", "); + Ok(format!( + concat!( + "Deleted workspace {}. These data tables were governed by it and no longer ", + "resolve: {}. Their databases still exist; a superadmin can point them at ", + "another workspace's data table." + ), + &w_id, stranded + )) + } } #[derive(Deserialize)] @@ -1343,15 +1399,20 @@ pub async fn drop_forked_datatable_databases( let mut errors: Vec = Vec::new(); for dt_name in &req.datatable_names { - let dt = match datatables.get(dt_name) { - Some(dt) if dt.forked_from.is_some() => dt, + // Only a clone is droppable, and a clone is terminal by construction: a kept data table is + // a pointer at the parent's database, which this fork does not own. + let database = match datatables.get(dt_name) { + Some(dt) if dt.forked_from.is_some() => match dt.database.as_ref() { + Some(database) => database, + None => continue, + }, _ => continue, }; - if dt.database.resource_type + if database.resource_type == windmill_common::workspaces::DataTableCatalogResourceType::Instance { - let db_to_drop = &dt.database.resource_path; + let db_to_drop = &database.resource_path; if !db_to_drop.starts_with("wm_fork_") { errors.push(format!( "Refusing to drop instance database '{}' for datatable://{}: name does not start with 'wm_fork_'", diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 8447666a94..39f769fe9a 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.813.0 + version: 1.814.0 title: Windmill API contact: @@ -1588,6 +1588,89 @@ paths: additionalProperties: $ref: "#/components/schemas/CustomInstanceDb" + /settings/datatable_roles: + get: + summary: list the instance's data table roles + operationId: listInstanceDatatableRoles + tags: + - setting + responses: + "200": + description: the instance role catalog + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/InstanceDatatableRole" + post: + summary: create a data table role on the instance's Postgres cluster + operationId: createInstanceDatatableRole + tags: + - setting + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [name] + properties: + name: + type: string + responses: + "200": + description: the created role + content: + application/json: + schema: + $ref: "#/components/schemas/InstanceDatatableRole" + + /settings/datatable_roles/{id}: + post: + summary: rename a data table role or turn its login on and off + operationId: updateInstanceDatatableRole + tags: + - setting + parameters: + - in: path + name: id + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + name: + type: string + enabled: + type: boolean + responses: + "200": + description: the updated role + content: + application/json: + schema: + $ref: "#/components/schemas/InstanceDatatableRole" + delete: + summary: drop a data table role from the cluster and from every workspace that named it + operationId: deleteInstanceDatatableRole + tags: + - setting + parameters: + - in: path + name: id + required: true + schema: + type: string + responses: + "200": + description: deleted + /settings/setup_custom_instance_pg_database/{name}: post: summary: Runs CREATE DATABASE on the Windmill Postgres and grants access to the custom_instance_user @@ -5141,7 +5224,7 @@ paths: type: array items: type: object - required: [name, resource_type, resource_path] + required: [name, resource_type, resource_path, permissioned] properties: name: type: string @@ -5150,6 +5233,97 @@ paths: enum: [postgres, instance] resource_path: type: string + governing_workspace_id: + type: string + permissioned: + type: boolean + + /w/{workspace}/workspaces/datatable_permissions/{datatable_name}: + get: + summary: get who may connect to a data table as which role + operationId: getDatatablePermissions + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + responses: + "200": + description: the data table's roles and their tenants + content: + application/json: + schema: + $ref: "#/components/schemas/DatatablePermissions" + post: + summary: set who may connect to a data table as which role + operationId: setDatatablePermissions + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [permissioned] + properties: + permissioned: + type: boolean + default_role: + type: string + roles: + type: array + items: + $ref: "#/components/schemas/DatatableRoleTenants" + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/datatable_usable_roles/{datatable_name}: + get: + summary: list the data table roles the caller may connect as + operationId: listUsableDatatableRoles + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + responses: + "200": + description: usable roles + content: + application/json: + schema: + type: object + required: [permissioned, roles, default_role] + properties: + permissioned: + type: boolean + roles: + type: array + items: + type: string + default_role: + type: string /w/{workspace}/workspaces/list_datatable_schemas: get: @@ -5331,7 +5505,22 @@ paths: description: status content: application/json: - schema: {} + schema: + type: object + properties: + stranded_references: + description: >- + Data tables in other workspaces that were governed by one this save deleted + and no longer resolve. + type: array + items: + type: object + required: [workspace_id, datatable] + properties: + workspace_id: + type: string + datatable: + type: string /w/{workspace}/workspaces/run_datatable_migrations/{datatable_name}: post: @@ -33400,6 +33589,66 @@ components: - ducklake - datatable + InstanceDatatableRole: + type: object + required: [id, name, enabled] + properties: + id: + type: string + name: + type: string + enabled: + type: boolean + + DatatableRoleTenants: + type: object + required: [id, tenants] + properties: + id: + type: string + name: + type: string + tenants: + type: array + items: + type: string + + DatatablePermissions: + type: object + required: [supported, permissioned, default_role, roles, editable, available_roles] + properties: + supported: + type: boolean + description: >- + Whether this data table can be put under roles at all. Only one backed by the + instance database can: a role is a login on that cluster. + permissioned: + type: boolean + default_role: + type: string + roles: + type: array + items: + $ref: "#/components/schemas/DatatableRoleTenants" + governing_workspace_id: + type: string + editable: + type: boolean + available_roles: + type: array + items: + $ref: "#/components/schemas/InstanceDatatableRole" + ungoverned_reachers: + type: array + items: + type: object + required: [workspace_id, datatable] + properties: + workspace_id: + type: string + datatable: + type: string + CustomInstanceDb: type: object required: @@ -35448,9 +35697,11 @@ components: type: object additionalProperties: type: object - required: [database] properties: database: + description: >- + Set on an entry that owns its database. Absent on a fork's entry, which points at + another workspace's data table instead. type: object properties: resource_type: @@ -35462,6 +35713,17 @@ components: type: string required: - resource_type + reference: + description: >- + The workspace and data table that govern this one. Server-owned: written by fork + creation, and carried across a settings save whatever the request says. + type: object + required: [workspace_id, datatable] + properties: + workspace_id: + type: string + datatable: + type: string migrations_enabled: type: boolean description: Whether the SQL migrations feature is opted in for this data table diff --git a/backend/windmill-api/src/capture.rs b/backend/windmill-api/src/capture.rs index 2e2058ea80..6255b5eb13 100644 --- a/backend/windmill-api/src/capture.rs +++ b/backend/windmill-api/src/capture.rs @@ -567,6 +567,9 @@ async fn set_config( }; let mut tx = user_db.begin(&authed).await?; + if matches!(nc.trigger_kind, TriggerKind::Postgres) { + windmill_common::datatable_roles::lock_datatable_streams(&mut *tx, false).await?; + } sqlx::query!( r#" @@ -614,6 +617,9 @@ async fn ping_config( )>, ) -> Result<()> { let mut tx = user_db.begin(&authed).await?; + if matches!(trigger_kind, TriggerKind::Postgres) { + windmill_common::datatable_roles::lock_datatable_streams(&mut *tx, false).await?; + } sqlx::query!( r#" diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 4149292200..74f24ca415 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -4369,12 +4369,12 @@ async fn count_completed_jobs_detail( Query(query): Query, ) -> error::JsonResult { let mut sqlb = SqlBuilder::select_from("v2_job_completed"); - //FOR RLS - sqlb.join("v2_job USING (id)"); sqlb.field("COUNT(*) as count"); + // Filtering on v2_job.workspace_id instead would keep the planner off + // ix_job_workspace_id_completed_at_all and scan the whole retention window. if !(w_id == "admins" && query.all_workspaces.unwrap_or(false)) { - sqlb.and_where_eq("v2_job.workspace_id", "?".bind(&w_id)); + sqlb.and_where_eq("v2_job_completed.workspace_id", "?".bind(&w_id)); } if let Some(after_s_ago) = query.completed_after_s_ago { @@ -4393,6 +4393,7 @@ async fn count_completed_jobs_detail( } if let Some(tags) = query.tags { + sqlb.join("v2_job USING (id)"); sqlb.and_where_in( "v2_job.tag", &tags.split(",").map(|t| quote(t)).collect::>(), @@ -4400,7 +4401,19 @@ async fn count_completed_jobs_detail( } let sql = sqlb.sql()?; - let stats = sqlx::query_scalar::<_, i64>(&sql).fetch_one(&db).await?; + let mut tx = db.begin().await?; + set_list_jobs_statement_timeout(&mut tx).await?; + let stats = sqlx::query_scalar::<_, i64>(&sql) + .fetch_one(&mut *tx) + .await + .map_err(|e| { + list_jobs_timeout_error( + e, + "Counting completed jobs", + "Lower completed_after_s_ago or narrow the filters.", + ) + })?; + tx.commit().await?; Ok(Json(stats)) } @@ -4429,6 +4442,33 @@ lazy_static::lazy_static! { .unwrap_or(30); } +/// A client that gives up does not cancel its query, so without this bound every retry of a +/// slow filter stacks another scan running until the connection-wide 5min timeout. +async fn set_list_jobs_statement_timeout(tx: &mut Transaction<'_, Postgres>) -> error::Result<()> { + let timeout_secs = *LIST_JOBS_STATEMENT_TIMEOUT_SECS; + if timeout_secs > 0 { + sqlx::query(&format!("SET LOCAL statement_timeout = '{timeout_secs}s'")) + .execute(&mut **tx) + .await?; + } + Ok(()) +} + +fn list_jobs_timeout_error(e: sqlx::Error, action: &str, hint: &str) -> Error { + let timeout_secs = *LIST_JOBS_STATEMENT_TIMEOUT_SECS; + match e { + sqlx::Error::Database(ref db_err) + if timeout_secs > 0 && db_err.code().as_deref() == Some("57014") => + { + Error::Generic( + StatusCode::BAD_REQUEST, + format!("{action} took more than {timeout_secs}s and was stopped. {hint}"), + ) + } + e => e.into(), + } +} + async fn list_jobs( authed: ApiAuthed, Extension(user_db): Extension, @@ -4545,32 +4585,14 @@ async fn list_jobs( }; // tracing::info!("sql: {}", &sql); let mut tx: Transaction<'_, Postgres> = user_db.begin(&authed).await?; - - // A client that gives up does not cancel its query, so without this bound every retry of a - // slow filter stacks another scan running until the connection-wide 5min timeout. - let timeout_secs = *LIST_JOBS_STATEMENT_TIMEOUT_SECS; - if timeout_secs > 0 { - sqlx::query(&format!("SET LOCAL statement_timeout = '{timeout_secs}s'")) - .execute(&mut *tx) - .await?; - } + set_list_jobs_statement_timeout(&mut tx).await?; let jobs: Vec = sqlx::query_as(&sql) .fetch_all(&mut *tx) .warn_after_seconds_with_sql(5, format!("list_jobs: {}", sql)) .await - .map_err(|e| match e { - sqlx::Error::Database(ref db_err) - if timeout_secs > 0 && db_err.code().as_deref() == Some("57014") => - { - Error::Generic( - StatusCode::BAD_REQUEST, - format!( - "Listing jobs took more than {timeout_secs}s and was stopped. Set a start date or narrow the filters." - ), - ) - } - e => e.into(), + .map_err(|e| { + list_jobs_timeout_error(e, "Listing jobs", "Set a start date or narrow the filters.") })?; tx.commit().await?; @@ -8343,8 +8365,9 @@ pub async fn run_wait_result_flow_by_version( /// job lives, in particular DuckDB, which runs in-process in the worker. /// /// What it does permit is any statement against the workspace's data tables, writes and DDL -/// included: the helper's body is an unrestricted SQL template and data tables carry no -/// per-user ACL. Narrowing that is a separate decision from this exemption. +/// included: the helper's body is an unrestricted SQL template. What that reaches is the +/// operator's own data table role — the preview job is permissioned as them, so the executor +/// resolves it under their tenancy like any other job. /// /// The database argument is only half the target: the executor honors a `-- database` /// directive in the SQL over it, and `-- s3` redirects the result set, so both are refused. @@ -8576,7 +8599,7 @@ async fn run_inline_preview_script( #[cfg(not(feature = "run_inline"))] async fn run_inline_preview_script() -> error::Result { Err(error::Error::InternalErr( - "inline preview requires the worker feature".to_string(), + "inline preview requires the run_inline feature on the worker".to_string(), )) } diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 4dd073b3af..a7f9aeda89 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -352,6 +352,17 @@ async fn update_username_in_workpsace<'c>( new_username: &str, w_id: &str, ) -> error::Result<()> { + // ---- data table tenants ---- + // Tenants name the user, so the rename has to follow here too; a list left naming the old + // username silently drops the access instead of moving it. + windmill_common::workspaces::rename_datatable_tenant_in_workspace( + tx, + w_id, + &format!("u/{old_username}"), + &format!("u/{new_username}"), + ) + .await?; + // ---- instance and workspace users ---- sqlx::query!( "UPDATE usr SET username = $1 WHERE email = $2", diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 69565ffcbd..4f6a6d9efb 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -1639,7 +1639,7 @@ pub(crate) async fn tarball_workspace( mute_critical_alerts: row.mute_critical_alerts, color: row.color.clone(), operator_settings: row.operator_settings.clone(), - datatable: row.datatable.clone(), + datatable: windmill_common::workspaces::strip_datatable_permissions(row.datatable.clone()), slack_team_id: row.slack_team_id.clone(), slack_name: row.slack_name.clone(), slack_command_script: row.slack_command_script.clone(), @@ -1703,7 +1703,7 @@ pub(crate) async fn tarball_workspace( mute_critical_alerts: row.mute_critical_alerts, color: row.color, operator_settings: row.operator_settings, - datatable: row.datatable, + datatable: windmill_common::workspaces::strip_datatable_permissions(row.datatable), slack_team_id: row.slack_team_id, slack_name: row.slack_name, slack_command_script: row.slack_command_script, diff --git a/backend/windmill-common/src/datatable_roles.rs b/backend/windmill-common/src/datatable_roles.rs new file mode 100644 index 0000000000..a9195055ac --- /dev/null +++ b/backend/windmill-common/src/datatable_roles.rs @@ -0,0 +1,362 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! The instance's data table role catalog. +//! +//! A data table role is a real Postgres login role on the Windmill cluster, named exactly as the +//! user named it, shared by every instance database. Windmill decides who may ask for a role (the +//! per-data-table tenant lists in [`crate::workspaces`]); Postgres decides what the role may then +//! touch. The catalog here is only the first half's vocabulary plus the cluster provisioning. +//! +//! Entries are keyed by a generated id so a rename moves nothing else: tenants name the id. + +use std::collections::BTreeMap; + +use crate::{ + error::{Error, Result}, + DB, +}; + +/// The connection every data table resolved to before roles existed (`custom_instance_user`). It +/// owns every pre-existing object, so it is a reserved name rather than a catalog entry: never +/// created, renamed or dropped. +pub const ADMIN_DATATABLE_ROLE: &str = "admin"; + +/// The login the admin connection uses, and the role every created role is granted to — that +/// membership is what later lets it `ALTER ... OWNER TO` a role and drop it. +pub const CUSTOM_INSTANCE_USER: &str = "custom_instance_user"; + +/// One catalog entry, as stored in `datatable_role`. The password is per role and instance-wide; +/// it belongs to the instance, not to any workspace's settings. +/// No `Serialize`/`Deserialize`: the catalog is rows now, and a derived `Serialize` would emit +/// `pwd` — the same way out for a credential that the hand-written `Debug` below closes on the log +/// side. +#[derive(Clone)] +pub struct InstanceDatatableRole { + /// The Postgres role name, verbatim. + pub name: String, + pub enabled: bool, + /// Absent only for a role whose provisioning did not finish; resolving as it then errors + /// rather than falling back to admin. + /// + /// A plain string rather than a `StringOrSecretRef` like the instance user's password: that + /// one is a secret ref because an operator supplies it and may want it to come from their own + /// backend, while this one is minted here and never entered by anyone, so there is nothing for + /// a ref to point at. Encrypting generated secrets at rest is a separate change that would + /// take the replication password with it. + pub pwd: Option, +} + +/// Hand-written so `{:?}` on a catalog cannot put a live Postgres password in a log line or an +/// audit record. Everything else about the entry is safe to print. +impl std::fmt::Debug for InstanceDatatableRole { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("InstanceDatatableRole") + .field("name", &self.name) + .field("enabled", &self.enabled) + .field("pwd", &self.pwd.as_ref().map(|_| "")) + .finish() + } +} + +pub type DatatableRoleCatalog = BTreeMap; + +/// Names Postgres or Windmill already owns. `admin` is excluded because it never reaches the +/// cluster as a role name at all — it resolves to `custom_instance_user`. +fn is_reserved_role_name(name: &str) -> bool { + let lower = name.to_ascii_lowercase(); + lower == ADMIN_DATATABLE_ROLE + || lower == "postgres" + || lower == "public" + || lower.starts_with("pg_") + || lower.starts_with("windmill_") + || lower.starts_with("custom_instance_") +} + +/// The charset is what makes every downstream interpolation safe: the name reaches Postgres as a +/// quoted identifier, a `-- role ` annotation, and a `?role=` query parameter. +pub fn validate_role_name(name: &str) -> Result<()> { + if name.is_empty() || name.len() > 63 { + return Err(Error::BadRequest(format!( + "Invalid data table role name '{name}': it must be between 1 and 63 characters" + ))); + } + if !name + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') + { + return Err(Error::BadRequest(format!( + "Invalid data table role name '{name}': only letters, digits, '_' and '-' are allowed" + ))); + } + if is_reserved_role_name(name) { + return Err(Error::BadRequest(format!( + "'{name}' is reserved and cannot be used as a data table role name" + ))); + } + Ok(()) +} + +/// A double-quoted Postgres identifier. Doubling `"` is Postgres's own escaping inside one, so this +/// quotes any name — schema, table or role. Role names are validated as well +/// ([`validate_role_name`]) because they also travel unquoted, in `-- role ` and `?role=`. +pub fn quote_ident(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} + +/// Serialize the mutations that are not already serialized by the row itself. +/// +/// A create is an insert and a delete is a delete, which Postgres orders for us — the unique index +/// on `name` is what makes two concurrent creates of the same name one winner and one error. What +/// still needs it is the window between the cluster DDL and the row: `CREATE ROLE` is not visible +/// to another transaction's `pg_roles` check until commit, so without this two creates of the same +/// name both pass their existence check and one fails on the index having already made the login. +/// Held for the transaction, so the DDL has to run on that same transaction to be covered. +pub async fn lock_role_catalog(tx: &mut sqlx::Transaction<'_, sqlx::Postgres>) -> Result<()> { + sqlx::query!("SELECT pg_advisory_xact_lock(hashtext('datatable_role_catalog'))") + .execute(&mut **tx) + .await?; + Ok(()) +} + +/// A replication stream reads every row whatever a data table's roles grant. Turning roles on looks +/// for streams holding this exclusive; whatever can start a Postgres trigger or capture streaming +/// holds it shared on the transaction that commits it. So either the look sees the stream, or the +/// stream's listener connects after roles are committed and refuses. Held for the transaction. +pub async fn lock_datatable_streams(conn: &mut sqlx::PgConnection, exclusive: bool) -> Result<()> { + let lock = if exclusive { + "pg_advisory_xact_lock" + } else { + "pg_advisory_xact_lock_shared" + }; + sqlx::query(&format!("SELECT {lock}(hashtext('datatable_streams'))")) + .execute(conn) + .await?; + Ok(()) +} + +/// Whether an instance database is reached only through entries under roles is decided by two +/// writes that lock different workspaces' settings rows: turning roles on for one entry, and a +/// settings save pointing an entry without roles at the database. Each holds this for every +/// database it decides on, so neither reads past the other's uncommitted write. Held for the +/// transaction; the names are locked in sorted order so two holders cannot deadlock. +pub async fn lock_instance_databases_governance<'a>( + conn: &mut sqlx::PgConnection, + dbnames: impl IntoIterator, +) -> Result<()> { + let dbnames: std::collections::BTreeSet<&str> = dbnames.into_iter().collect(); + for dbname in dbnames { + sqlx::query("SELECT pg_advisory_xact_lock(hashtext('datatable_instance_database:' || $1))") + .bind(dbname) + .execute(&mut *conn) + .await?; + } + Ok(()) +} + +/// Disclosure: returns every role's stored Postgres password in plaintext. Any server path that +/// has to resolve or name a role may call it — including handlers open to a workspace member, who +/// need the names — but callers MUST NOT let `pwd` reach a response, a log line, an audit record +/// or an export. Nothing about who may call it: the credential is the whole risk, and `Debug` is +/// hand-written to redact it for the same reason. +pub async fn read_role_catalog(db: &DB) -> Result { + crate::datatable_roles_oss::read_role_catalog(db).await +} + +/// As [`read_role_catalog`], reading inside the caller's transaction so the value is the one +/// [`lock_role_catalog`] is protecting. Same disclosure contract. +pub async fn read_role_catalog_tx( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, +) -> Result { + crate::datatable_roles_oss::read_role_catalog_tx(tx).await +} + +/// Record a role, in the caller's transaction so it commits with the `CREATE ROLE` it describes. +/// +/// Authorization: writes a generated Postgres credential. Callers MUST restrict this to superadmin +/// paths and MUST hold [`lock_role_catalog`] on `tx`. +pub async fn insert_role_catalog_entry( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + id: &str, + role: &InstanceDatatableRole, +) -> Result<()> { + crate::datatable_roles_oss::insert_role_catalog_entry(tx, id, role).await +} + +/// Update a role's recorded name, login flag and password. Same contract as +/// [`insert_role_catalog_entry`]. +pub async fn update_role_catalog_entry( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + id: &str, + role: &InstanceDatatableRole, +) -> Result<()> { + crate::datatable_roles_oss::update_role_catalog_entry(tx, id, role).await +} + +/// Forget a role. Same contract as [`insert_role_catalog_entry`]; run it in the transaction that +/// drops the cluster login, so the two cannot disagree. +pub async fn delete_role_catalog_entry( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + id: &str, +) -> Result<()> { + crate::datatable_roles_oss::delete_role_catalog_entry(tx, id).await +} + +/// Resolve the role a caller named to its catalog id. A disabled role is an error rather than a +/// silent fallback: the caller asked for something the instance deliberately turned off. +pub fn role_id_by_name<'a>(catalog: &'a DatatableRoleCatalog, name: &str) -> Result<&'a str> { + let entry = catalog + .iter() + .find(|(_, role)| role.name == name) + .ok_or_else(|| { + Error::NotFound(format!( + "'{name}' is not a data table role of this instance. Defined roles: {}.", + catalog + .values() + .map(|r| r.name.as_str()) + .collect::>() + .join(", ") + )) + })?; + if !entry.1.enabled { + return Err(Error::BadRequest(format!( + "Data table role '{name}' is disabled on this instance" + ))); + } + Ok(entry.0.as_str()) +} + +/// Every instance database the registry knows about. Role provisioning has to reach all of them: +/// a role that cannot `CONNECT` to a database is refused by Postgres before any grant matters. +/// +/// Authorization: checks nothing, and names every instance database across all workspaces. Callers +/// MUST be superadmin-gated or keep the names server-side; never return them to a workspace caller. +pub async fn registered_instance_databases(db: &DB) -> Result> { + crate::datatable_roles_oss::registered_instance_databases(db).await +} + +/// `CONNECT` on `dbname` for every enabled role, and none for `PUBLIC`. Run at role creation, at +/// database creation, and lazily whenever an instance data table is administered, so a database +/// provisioned before a role existed is repaired rather than left silently unreachable. +/// +/// Authorization: rewrites a database's ACL with the server's own credentials and checks nothing. +/// Callers MUST have authorized administration of `dbname` — superadmin, or an admin of the +/// workspace governing a data table on it. +pub async fn converge_connect_grants(db: &DB, dbname: &str) -> Result<()> { + crate::datatable_roles_oss::converge_connect_grants(db, dbname).await +} + +/// As [`converge_connect_grants`], with a catalog the caller already read. Same contract. +pub async fn converge_connect_grants_with( + db: &DB, + dbname: &str, + catalog: &DatatableRoleCatalog, +) -> Result<()> { + crate::datatable_roles_oss::converge_connect_grants_with(db, dbname, catalog).await +} + +/// `CREATE ROLE LOGIN PASSWORD ...; GRANT TO custom_instance_user`, and `CONNECT` on +/// every registered database. No privileges beyond that — an admin grants them through SQL or the +/// ACL editor. +/// +/// Authorization: creates a cluster-wide Postgres login. Callers MUST restrict this to superadmin +/// paths, and MUST hold [`lock_role_catalog`] on the same transaction. +pub async fn create_instance_role( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + name: &str, + password: &str, +) -> Result<()> { + crate::datatable_roles_oss::create_instance_role(tx, name, password).await +} + +/// Authorization: alters a cluster-wide Postgres login. Callers MUST restrict this to superadmin +/// paths, and MUST hold [`lock_role_catalog`] on the same transaction. +pub async fn set_instance_role_login( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + name: &str, + enabled: bool, +) -> Result<()> { + crate::datatable_roles_oss::set_instance_role_login(tx, name, enabled).await +} + +/// A rename discards an md5-hashed password, so the caller has to hand over a fresh one. +/// +/// Authorization: renames a cluster-wide Postgres login. Callers MUST restrict this to superadmin +/// paths, and MUST hold [`lock_role_catalog`] on the same transaction. +pub async fn rename_instance_role( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + from: &str, + to: &str, + password: &str, +) -> Result<()> { + crate::datatable_roles_oss::rename_instance_role(tx, from, to, password).await +} + +/// A role owning anything in any database blocks its own `DROP ROLE`, and both its objects and the +/// privileges granted to it are only visible from inside each database — hence the pass over the +/// registry. An unreachable database aborts the whole delete: dropping the role while one database +/// still holds objects owned by it leaves those objects owned by a numeric OID nobody can name. +/// +/// Each pass runs as the instance's own Postgres user rather than `custom_instance_user`, which +/// owns the databases and can therefore revoke a grant whoever made it. `custom_instance_user` +/// could only undo what it granted itself, so a privilege planted by an operator in psql — the +/// ordinary way privileges reach a role — would survive and block the drop. +/// +/// Authorization: drops a cluster-wide Postgres login and reassigns everything it owns. Callers +/// MUST restrict this to superadmin paths, and MUST hold [`lock_role_catalog`] on `tx`. +/// +/// The per-database passes open their own connections and cannot join `tx`; the lock is what keeps +/// a concurrent mutation out while they run. Only the final `DROP ROLE` is on `tx`, so it commits +/// or rolls back with the catalog write that forgets the role. Those passes commit as they go, so +/// callers MUST have disabled the role in an earlier committed transaction: a failure part-way +/// then leaves a disabled role to retry, not an enabled one already stripped in some databases. +pub async fn drop_instance_role( + db: &DB, + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + name: &str, +) -> Result<()> { + crate::datatable_roles_oss::drop_instance_role(db, tx, name).await +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn role_names_are_validated() { + assert!(validate_role_name("analytics").is_ok()); + assert!(validate_role_name("read-only_2").is_ok()); + assert!(validate_role_name("").is_err()); + assert!(validate_role_name(&"a".repeat(64)).is_err()); + assert!(validate_role_name("has space").is_err()); + assert!(validate_role_name("quote\"injection").is_err()); + // Reserved, case-insensitively. + assert!(validate_role_name("admin").is_err()); + assert!(validate_role_name("Postgres").is_err()); + assert!(validate_role_name("pg_read_all_data").is_err()); + assert!(validate_role_name("windmill_user").is_err()); + assert!(validate_role_name("custom_instance_user").is_err()); + } + + #[test] + fn a_disabled_role_is_an_error_not_a_fallback() { + let mut catalog = DatatableRoleCatalog::new(); + catalog.insert( + "id1".to_string(), + InstanceDatatableRole { + name: "analytics".to_string(), + enabled: false, + pwd: Some("x".to_string()), + }, + ); + assert!(role_id_by_name(&catalog, "analytics").is_err()); + assert!(role_id_by_name(&catalog, "nope").is_err()); + catalog.get_mut("id1").unwrap().enabled = true; + assert_eq!(role_id_by_name(&catalog, "analytics").unwrap(), "id1"); + } +} diff --git a/backend/windmill-common/src/datatable_roles_oss.rs b/backend/windmill-common/src/datatable_roles_oss.rs new file mode 100644 index 0000000000..057a739bd4 --- /dev/null +++ b/backend/windmill-common/src/datatable_roles_oss.rs @@ -0,0 +1,208 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Where data table roles come from: the enterprise implementation, or a refusal. +//! +//! Roles are an Enterprise Edition feature. An edition without them creates, grants and connects +//! as none, and a data table saved under roles — by an enterprise build, before a downgrade — is +//! refused rather than resolved as `admin`. A data table not under roles, asked for no role, +//! resolves as it always has. `private` alone is not that edition: community builds carry it. + +use crate::error::Error; + +/// What every roles path answers without the Enterprise Edition. +pub fn datatable_roles_unavailable() -> Error { + Error::BadRequest("Data table roles are a Windmill Enterprise Edition feature".to_string()) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub(crate) use crate::datatable_roles_ee::{ + can_use_datatable_role, can_use_datatable_role_in_governing_workspace, converge_connect_grants, + converge_connect_grants_with, create_instance_role, delete_role_catalog_entry, + drop_instance_role, ensure_can_use_datatable_role, ensure_datatable_admin_access, + ensure_instance_db_grant_options_unchecked, forget_datatable_role_everywhere, + insert_role_catalog_entry, read_role_catalog, read_role_catalog_tx, + registered_instance_databases, rename_instance_role, resolve_datatable_role_connection, + set_instance_role_login, update_role_catalog_entry, +}; + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub(crate) use ce::*; + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +mod ce { + use super::datatable_roles_unavailable as unavailable; + use crate::{ + datatable_roles::{DatatableRoleCatalog, InstanceDatatableRole}, + db::AuthedRef, + error::Result, + workspaces::{ + resolve_governing_datatable, DataTableRoleTenants, DatatableAccess, GoverningDatatable, + }, + DB, + }; + + type Tx<'a> = sqlx::Transaction<'a, sqlx::Postgres>; + + pub(crate) async fn read_role_catalog(_db: &DB) -> Result { + Err(unavailable()) + } + + pub(crate) async fn read_role_catalog_tx(_tx: &mut Tx<'_>) -> Result { + Err(unavailable()) + } + + pub(crate) async fn insert_role_catalog_entry( + _tx: &mut Tx<'_>, + _id: &str, + _role: &InstanceDatatableRole, + ) -> Result<()> { + Err(unavailable()) + } + + pub(crate) async fn update_role_catalog_entry( + _tx: &mut Tx<'_>, + _id: &str, + _role: &InstanceDatatableRole, + ) -> Result<()> { + Err(unavailable()) + } + + pub(crate) async fn delete_role_catalog_entry(_tx: &mut Tx<'_>, _id: &str) -> Result<()> { + Err(unavailable()) + } + + pub(crate) async fn registered_instance_databases(_db: &DB) -> Result> { + Err(unavailable()) + } + + /// Nothing to converge: with no roles to admit, an instance database keeps the `CONNECT` + /// grants it was created with, `PUBLIC`'s included, as it did before roles existed. + pub(crate) async fn converge_connect_grants(_db: &DB, _dbname: &str) -> Result<()> { + Ok(()) + } + + /// As [`converge_connect_grants`]. + pub(crate) async fn converge_connect_grants_with( + _db: &DB, + _dbname: &str, + _catalog: &DatatableRoleCatalog, + ) -> Result<()> { + Ok(()) + } + + pub(crate) async fn create_instance_role( + _tx: &mut Tx<'_>, + _name: &str, + _password: &str, + ) -> Result<()> { + Err(unavailable()) + } + + pub(crate) async fn set_instance_role_login( + _tx: &mut Tx<'_>, + _name: &str, + _enabled: bool, + ) -> Result<()> { + Err(unavailable()) + } + + pub(crate) async fn rename_instance_role( + _tx: &mut Tx<'_>, + _from: &str, + _to: &str, + _password: &str, + ) -> Result<()> { + Err(unavailable()) + } + + pub(crate) async fn drop_instance_role(_db: &DB, _tx: &mut Tx<'_>, _name: &str) -> Result<()> { + Err(unavailable()) + } + + pub(crate) async fn ensure_instance_db_grant_options_unchecked( + _db: &DB, + _dbname: &str, + ) -> Result<()> { + Err(unavailable()) + } + + /// No tenant list covers anyone: there is no role to connect as. + pub(crate) fn can_use_datatable_role( + _tenants: &DataTableRoleTenants, + _authed: &AuthedRef<'_>, + ) -> bool { + false + } + + pub(crate) async fn can_use_datatable_role_in_governing_workspace( + _db: &DB, + _governing_w_id: &str, + _w_id: &str, + _tenants: &DataTableRoleTenants, + _access: &DatatableAccess<'_>, + ) -> Result { + Err(unavailable()) + } + + /// Reached only for a data table under roles or a caller naming a role: both are refused. + pub(crate) async fn resolve_datatable_role_connection( + _db: &DB, + _w_id: &str, + _name: &str, + _governing: &GoverningDatatable, + _db_resource: serde_json::Value, + _role: Option<&str>, + _access: DatatableAccess<'_>, + ) -> Result { + Err(unavailable()) + } + + /// A data table not under roles, asked for no role or for `admin`, is not a role decision and + /// passes, as it did before roles existed. Anything else is refused. + pub(crate) async fn ensure_can_use_datatable_role( + db: &DB, + w_id: &str, + name: &str, + role: Option<&str>, + _access: &DatatableAccess<'_>, + _context: &str, + ) -> Result<()> { + let governing = resolve_governing_datatable(db, w_id, name).await?; + if governing.datatable.permissions.is_none() + && role.is_none_or(|r| r == crate::datatable_roles::ADMIN_DATATABLE_ROLE) + { + Ok(()) + } else { + Err(unavailable()) + } + } + + /// A data table not under roles is the `admin` connection for anyone who reaches it, as before + /// roles existed. One under roles is refused. + pub(crate) async fn ensure_datatable_admin_access( + db: &DB, + w_id: &str, + name: &str, + _access: &DatatableAccess<'_>, + ) -> Result<()> { + let governing = resolve_governing_datatable(db, w_id, name).await?; + if governing.datatable.permissions.is_none() { + Ok(()) + } else { + Err(unavailable()) + } + } + + pub(crate) async fn forget_datatable_role_everywhere( + _tx: &mut Tx<'_>, + _role_id: &str, + ) -> Result<()> { + Err(unavailable()) + } +} diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 8e17ec676e..52d1a85aec 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -37,6 +37,10 @@ pub mod bench; pub mod cache; pub mod client; pub mod data_metrics; +pub mod datatable_roles; +#[cfg(all(feature = "private", feature = "enterprise"))] +mod datatable_roles_ee; +pub mod datatable_roles_oss; pub mod db; #[cfg(all(feature = "enterprise", feature = "private"))] mod db_entra_ee; @@ -1514,6 +1518,41 @@ pub async fn drop_custom_instance_database(db: &DB, dbname: &str) -> error::Resu Ok(()) } +/// What `custom_instance_user` holds on an instance database. +/// +/// `WITH GRANT OPTION` throughout: this is the connection every data table resolves to as `admin`, +/// and it is the one that hands privileges to data table roles. Postgres refuses to let a role pass +/// on a privilege it does not itself hold with grant option, so without these an admin could own +/// the database and still be unable to grant `SELECT` on it to `analytics`. +pub(crate) fn instance_db_grants(dbname: &str) -> String { + format!( + "GRANT CONNECT ON DATABASE \"{dbname}\" TO custom_instance_user WITH GRANT OPTION; + GRANT CREATE ON DATABASE \"{dbname}\" TO custom_instance_user WITH GRANT OPTION; + DO $$ BEGIN + IF EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = 'public') THEN + GRANT USAGE ON SCHEMA public TO custom_instance_user WITH GRANT OPTION; + GRANT CREATE ON SCHEMA public TO custom_instance_user WITH GRANT OPTION; + ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO custom_instance_user; + END IF; + END $$;" + ) +} + +/// Re-apply [`instance_db_grants`] to an instance database provisioned before data table roles +/// existed, whose grants carry no grant option. Connects as the instance's own Postgres user — +/// the database and `public` schema owner — since only it can hand out an option it holds. +/// +/// Authorization: reaches an instance database with the server's own credentials and checks +/// nothing. Callers MUST have authorized administration of `dbname` — superadmin, or an admin of +/// the workspace governing a data table on it. +pub async fn ensure_instance_db_grant_options_unchecked( + db: &DB, + dbname: &str, +) -> error::Result<()> { + crate::datatable_roles_oss::ensure_instance_db_grant_options_unchecked(db, dbname).await +} + /// Create a custom instance database: CREATE DATABASE, grant permissions, register in global_settings. /// The `tag` is stored in global_settings metadata (e.g. "datatable" or "ducklake"). pub async fn create_custom_instance_database( @@ -1553,17 +1592,7 @@ pub async fn create_custom_instance_database( let (client, connection) = new_pg_creds.connect(Some(db)).await?; let join_handle = tokio::spawn(async move { connection.await }); - if let Err(e) = client - .batch_execute(&format!( - "GRANT CONNECT ON DATABASE \"{dbname}\" TO custom_instance_user; - GRANT USAGE ON SCHEMA public TO custom_instance_user; - GRANT CREATE ON SCHEMA public TO custom_instance_user; - GRANT CREATE ON DATABASE \"{dbname}\" TO custom_instance_user; - ALTER DEFAULT PRIVILEGES IN SCHEMA public - GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO custom_instance_user;" - )) - .await - { + if let Err(e) = client.batch_execute(&instance_db_grants(dbname)).await { tracing::warn!( "Failed to grant permissions on '{}': {}. Continuing.", dbname, @@ -1592,6 +1621,13 @@ pub async fn create_custom_instance_database( .execute(db) .await?; + // A data table role can only reach a database it may CONNECT to, and PUBLIC's default CONNECT + // would otherwise let every role in regardless of what this instance defines. Best-effort: a + // failure here leaves the database usable as `admin`, and the next role change repairs it. + if let Err(e) = crate::datatable_roles::converge_connect_grants(db, dbname).await { + tracing::warn!("Could not set CONNECT grants on instance database '{dbname}': {e}"); + } + tracing::info!("Created custom instance database '{}'", dbname); Ok(()) } diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index ee8a389bb7..c742a3efcb 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -1082,6 +1082,83 @@ pub struct SqlAnnotations { pub raw_output: bool, } +impl SqlAnnotations { + /// The data table role a query declares as `-- role `, if any. Only meaningful against a + /// `datatable://` database that is under roles; absent means the data table's default role. + /// + /// Hand-written rather than derived because the value matters, not just the presence, and + /// because the executor needs it before it knows the connection is a data table at all. Like + /// every annotation it lives in the leading comment block. + /// + /// A leading comment whose first word is `role` is an annotation *attempt*, and a malformed + /// one is an error. The alternative — ignoring what does not parse — resolves the query to the + /// data table's default role instead, so a typo silently runs it under a login the author did + /// not choose, which is the opposite of what naming a role is for. Only callers that already + /// know the target is a `datatable://` reference ever run this, so ordinary SQL keeps its + /// comments. + pub fn datatable_role(code: &str) -> error::Result> { + for line in code.lines() { + let line = line.trim(); + if line.is_empty() { + continue; + } + if !line.starts_with("--") { + break; + } + // The keyword may be followed by whitespace, `:` or `=` — `role x`, `role: x`, + // `role=x`, `Role = x` all open an attempt, while `rolexyz` does not. Each accepted + // separator is one spelling that would otherwise take the `continue` below and run the + // query as the data table's default role, which is the silence this exists to remove. + let body = line[2..].trim_start(); + let Some(after) = body + .get(..4) + .filter(|kw| kw.eq_ignore_ascii_case("role")) + .map(|_| &body[4..]) + else { + continue; + }; + if !after.is_empty() + && !after.starts_with(char::is_whitespace) + && !after.starts_with([':', '=']) + { + continue; + } + + // Past this point the line is an attempt to name a role, so a malformed one is an + // error rather than a miss. Falling through would run the query as the data table's + // default role — quietly, and under a login the author did not choose. + let after = after.trim_start(); + let after = after.strip_prefix([':', '=']).unwrap_or(after); + let mut tokens = after.split_whitespace(); + let role = tokens + .next() + .map(|role| role.strip_suffix(';').unwrap_or(role)); + let rest = tokens.next(); + match (role, rest) { + (Some(role), None) + if !role.is_empty() + && role.len() <= 63 + && role + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') => + { + return Ok(Some(role.to_string())); + } + _ => { + return Err(error::Error::BadRequest(format!( + "Malformed data table role annotation: `{line}`. Write it as \ + `-- role ` on a line of its own, where is letters, digits, \ + '_' or '-'. A comment in the leading block that starts with the word \ + 'role' is read as this annotation; move it below the first statement if \ + it is prose." + ))); + } + } + } + Ok(None) + } +} + #[annotations("#")] pub struct BashAnnotations { pub docker: bool, @@ -2653,6 +2730,56 @@ mod tests { use super::*; use std::collections::HashMap; + #[test] + fn datatable_role_is_read_from_the_leading_comment_block() { + let role = |code| SqlAnnotations::datatable_role(code); + assert_eq!( + role("-- role analytics\nSELECT 1").unwrap(), + Some("analytics".to_string()) + ); + // Blank lines and other annotations before it are fine. + assert_eq!( + role("\n-- prepare\n-- role read_only\nSELECT 1").unwrap(), + Some("read_only".to_string()) + ); + // Past the first statement it is an ordinary comment, not an annotation. + assert_eq!(role("SELECT 1;\n-- role analytics").unwrap(), None); + assert_eq!(role("SELECT 1").unwrap(), None); + + // Unambiguous intent is honoured: the keyword matches case-insensitively, a trailing + // semicolon is a habit carried over from SQL rather than a different role, and the colon + // spelling is the one most likely to be typed. + for accepted in [ + "-- Role operator\nSELECT 1", + "-- role operator;\nSELECT 1", + "-- role: operator\nSELECT 1", + "-- role:operator\nSELECT 1", + "-- role=operator\nSELECT 1", + "-- Role = operator\nSELECT 1", + ] { + assert_eq!( + role(accepted).unwrap(), + Some("operator".to_string()), + "not honoured: {accepted}" + ); + } + + // Anything else opening with the word is refused rather than resolved to the default role: + // the whole point of naming one is to not run as something else. + for near_miss in [ + "-- role operator -- why\nSELECT 1", + "-- role an;alytics\nSELECT 1", + "-- role\nSELECT 1", + "-- role:\nSELECT 1", + "-- role based access is handled below\nSELECT 1", + ] { + assert!(role(near_miss).is_err(), "silently ignored: {near_miss}"); + } + + // A word that merely starts with the keyword is not an attempt. + assert_eq!(role("-- rolebased notes\nSELECT 1").unwrap(), None); + } + fn matcher(id: &str) -> WorkspaceMatcher { WorkspaceMatcher { id: id.to_string(), include_forks: false } } diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 7cf910ef84..711b4dac29 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -6,6 +6,7 @@ use serde::{Deserialize, Serialize}; use strum::AsRefStr; use crate::{ + datatable_roles::{ADMIN_DATATABLE_ROLE, CUSTOM_INSTANCE_USER}, error::{self, to_anyhow, Error, Result}, get_database_url, secret_backend::{get_secret_value, is_external_stored_value}, @@ -1292,9 +1293,17 @@ impl Default for DataTableForkBehavior { } } -#[derive(Deserialize, Serialize, Debug)] +#[derive(Deserialize, Serialize, Debug, Clone)] pub struct DataTable { - pub database: DataTableDatabase, + /// Set on a *terminal* entry — one that owns its database. Mutually exclusive with + /// [`DataTable::reference`]; [`validate_datatable_shape`] is the one place that enforces it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub database: Option, + /// Set on a *pointer* entry — one that names another workspace's entry and owns nothing. + /// A keep-original fork gets one of these instead of a copy of the parent's entry, so there is + /// nothing local for a fork admin to widen. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reference: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub forked_from: Option, /// Whether the SQL-migrations feature is opted in for this data table. @@ -1302,22 +1311,85 @@ pub struct DataTable { /// when migrations already exist (see `datatable_migrations_enabled`). #[serde(default, skip_serializing_if = "Option::is_none")] pub migrations_enabled: Option, + /// Who may connect as which role. Absent = unpermissioned: every caller connects as `admin`, + /// which is how data tables behaved before roles existed. Only meaningful on a terminal entry; + /// a pointer is governed by what it points at. + /// + /// Never leaves the instance: stripped from the workspace export and ignored on import, since + /// tenants are workspace-scoped names and syncing them would make repo write access a second + /// door onto the access decision. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub permissions: Option, } -#[derive(Deserialize, Serialize, Debug)] +/// A pointer at another workspace's data table entry. +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] +pub struct DataTableReference { + pub workspace_id: String, + pub datatable: String, +} + +/// The access decision for one data table: which role a caller gets, and who may ask for each. +#[derive(Deserialize, Serialize, Debug, Clone, Default)] +pub struct DataTablePermissions { + /// A role id from the instance catalog, or `admin`. Absent = `admin`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub default_role: Option, + /// Keyed by instance role id, plus the reserved `admin` key. A role absent from this map + /// cannot be used on this data table at all, whatever the instance catalog says. + #[serde(default)] + pub roles: std::collections::BTreeMap, +} + +impl DataTablePermissions { + pub fn default_role(&self) -> &str { + self.default_role.as_deref().unwrap_or(ADMIN_DATATABLE_ROLE) + } +} + +#[derive(Deserialize, Serialize, Debug, Clone, Default)] +pub struct DataTableRoleTenants { + /// `u/`, `g/`, `f/`, or `*` for every member. + #[serde(default)] + pub tenants: Vec, +} + +/// Every member of the governing workspace. +pub const DATATABLE_TENANT_WILDCARD: &str = "*"; + +/// How deep a chain of pointer entries may go before it is called a loop. Data tables are not +/// expected to chain at all — a fork points at its parent — so this only has to be generous +/// enough to survive a fork of a fork. +const DATATABLE_REFERENCE_MAX_DEPTH: usize = 20; + +/// Exactly one of `database` and `reference` must be set. Called wherever an entry is persisted, +/// so nothing downstream has to handle an entry that is both or neither. +pub fn validate_datatable_shape(name: &str, dt: &DataTable) -> Result<()> { + match (&dt.database, &dt.reference) { + (Some(_), None) | (None, Some(_)) => Ok(()), + (Some(_), Some(_)) => Err(Error::BadRequest(format!( + "Data table '{name}' both owns a database and points at another one" + ))), + (None, None) => Err(Error::BadRequest(format!( + "Data table '{name}' names neither a database nor another data table" + ))), + } +} + +#[derive(Deserialize, Serialize, Debug, Clone)] pub struct DataTableForkedFrom { /// Schema snapshot at fork time #[serde(default, skip_serializing_if = "Option::is_none")] pub schema: Option, } -#[derive(Deserialize, Serialize, Debug)] +#[derive(Deserialize, Serialize, Debug, Clone)] pub struct DataTableDatabase { pub resource_type: DataTableCatalogResourceType, pub resource_path: String, } -#[derive(Deserialize, Serialize, Debug, PartialEq)] +#[derive(Deserialize, Serialize, Debug, PartialEq, Clone, Copy)] #[serde(rename_all = "lowercase")] #[derive(AsRefStr)] #[strum(serialize_all = "lowercase")] @@ -1353,37 +1425,13 @@ fn datatable_not_found_error(name: &str, datatables: Option<&serde_json::Value>) )) } -pub async fn get_datatable_resource_from_db_unchecked( - db: &DB, - w_id: &str, - name: &str, -) -> Result { - get_datatable_resource_inner(db, w_id, name, false).await -} - -/// Same as [`get_datatable_resource_from_db_unchecked`] but for postgres trigger -/// connections: custom-instance datatables resolve to -/// `custom_instance_replication_user` rather than `custom_instance_user`. BYO-postgres -/// datatables resolve to the user's own resource unchanged; configuring it for -/// replication there is the user's responsibility. +/// Read one workspace's data table entry, without following a pointer. /// -/// Authorization: like its `_unchecked` sibling, returns resolved connection -/// credentials and performs no authorization — callers MUST have already authorized -/// access to the datatable (e.g. the trigger's own create-time check). -pub async fn get_datatable_replication_resource_from_db_unchecked( - db: &DB, - w_id: &str, - name: &str, -) -> Result { - get_datatable_resource_inner(db, w_id, name, true).await -} - -async fn get_datatable_resource_inner( - db: &DB, - w_id: &str, - name: &str, - replication: bool, -) -> Result { +/// Disclosure: this is the primitive [`resolve_governing_datatable`] calls on every path, so it is +/// deliberately open to anything that has to resolve a data table, including for a workspace the +/// caller does not belong to. What it returns is not: callers MUST NOT put `permissions` into a +/// response, an export or a log — it names the governing workspace's users, groups and folders. +pub async fn read_datatable_entry(db: &DB, w_id: &str, name: &str) -> Result { let datatables = sqlx::query_scalar!( r#" SELECT ws.datatable->'datatables' AS datatables @@ -1401,37 +1449,590 @@ async fn get_datatable_resource_inner( .and_then(|d| d.get(name)) .filter(|v| !v.is_null()) .ok_or_else(|| datatable_not_found_error(name, datatables.as_ref()))?; - let datatable = serde_json::from_value::(datatable.clone())?; + Ok(serde_json::from_value::(datatable.clone())?) +} - let db_resource = if datatable.database.resource_type == DataTableCatalogResourceType::Instance - { +/// The terminal entry a reference chain lands on: the workspace that governs the data table, the +/// entry name there, and the entry itself. A terminal entry resolves to itself. +/// +/// Every decision downstream — which database to connect to, whose `permissions` apply, whose +/// members tenants are evaluated against, who may administer it — is taken on this, never on the +/// entry the caller named. +/// +/// Authorization: resolving deliberately crosses into the governing workspace, so it answers for a +/// workspace the caller may not belong to and checks nothing itself. It is the input to the +/// checks, not one of them: callers MUST pass what it returns to +/// [`can_use_datatable_role_in_governing_workspace`] or [`ensure_datatable_admin_access`] before +/// acting on it, and MUST NOT return its `permissions` or `workspace_id` to a caller from +/// elsewhere without gating on the answer. +pub struct GoverningDatatable { + pub workspace_id: String, + pub name: String, + pub datatable: DataTable, +} + +impl GoverningDatatable { + /// Backed by the Windmill instance's own Postgres, which is the only substrate data table + /// roles apply to. + pub fn is_instance(&self) -> bool { + self.datatable + .database + .as_ref() + .is_some_and(|d| d.resource_type == DataTableCatalogResourceType::Instance) + } +} + +pub async fn resolve_governing_datatable( + db: &DB, + w_id: &str, + name: &str, +) -> Result { + let mut workspace_id = w_id.to_string(); + let mut name = name.to_string(); + let mut hops = 0; + for _ in 0..DATATABLE_REFERENCE_MAX_DEPTH { + let datatable = read_datatable_entry(db, &workspace_id, &name) + .await + .map_err(|e| { + if hops == 0 { + e + } else { + // A pointer outlives the workspace it names: deleting one only nulls the fork + // lineage, it does not sweep the entries that pointed at it. Say which one is + // gone rather than reporting a data table this workspace never had. + Error::NotFound(format!( + "Data table '{name}' of workspace '{workspace_id}' governs this one and no \ + longer exists. A superadmin can point this data table somewhere else." + )) + } + })?; + hops += 1; + validate_datatable_shape(&name, &datatable)?; + match &datatable.reference { + None => return Ok(GoverningDatatable { workspace_id, name, datatable }), + Some(reference) => { + workspace_id = reference.workspace_id.clone(); + name = reference.datatable.clone(); + } + } + } + Err(Error::BadRequest(format!( + "Data table '{name}' points at another data table through more than \ + {DATATABLE_REFERENCE_MAX_DEPTH} hops; the chain is likely a loop" + ))) +} + +/// Every entry of a workspace resolved as [`resolve_governing_datatable`] resolves one, in stored +/// order, reading the settings rows one pointer hop at a time rather than once per entry. An entry +/// that does not resolve — malformed, a dangling pointer, a loop — is left out. Same authorization +/// contract as the single resolution: it checks nothing. +pub async fn resolve_workspace_governing_datatables( + db: &DB, + w_id: &str, +) -> Result> { + type Entries = + std::collections::HashMap>; + async fn load(db: &DB, workspaces: &[String], entries: &mut Entries) -> Result> { + let rows: Vec<(String, String, serde_json::Value)> = sqlx::query_as( + "SELECT ws.workspace_id, dt.key, dt.value FROM workspace_settings ws + CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt + WHERE ws.workspace_id = ANY($1)", + ) + .bind(workspaces) + .fetch_all(db) + .await?; + for ws in workspaces { + entries.entry(ws.clone()).or_default(); + } + let mut keys = Vec::with_capacity(rows.len()); + for (ws, key, value) in rows { + keys.push(key.clone()); + entries.entry(ws).or_default().insert(key, value); + } + Ok(keys) + } + + let mut entries = Entries::new(); + let listed = load(db, &[w_id.to_string()], &mut entries).await?; + // (index into `listed`, workspace, entry name) still to be followed. + let mut cursors: Vec<(usize, String, String)> = listed + .iter() + .enumerate() + .map(|(i, name)| (i, w_id.to_string(), name.clone())) + .collect(); + let mut resolved: Vec<(usize, GoverningDatatable)> = vec![]; + + for _ in 0..DATATABLE_REFERENCE_MAX_DEPTH { + let mut next = vec![]; + for (i, ws, name) in cursors.drain(..) { + let Some(value) = entries + .get(&ws) + .and_then(|m| m.get(&name)) + .filter(|v| !v.is_null()) + else { + continue; + }; + let Ok(datatable) = serde_json::from_value::(value.clone()) else { + continue; + }; + if validate_datatable_shape(&name, &datatable).is_err() { + continue; + } + match &datatable.reference { + None => { + resolved.push((i, GoverningDatatable { workspace_id: ws, name, datatable })) + } + Some(reference) => next.push(( + i, + reference.workspace_id.clone(), + reference.datatable.clone(), + )), + } + } + if next.is_empty() { + break; + } + let to_load: Vec = next + .iter() + .map(|(_, ws, _)| ws.clone()) + .filter(|ws| !entries.contains_key(ws)) + .collect::>() + .into_iter() + .collect(); + if !to_load.is_empty() { + load(db, &to_load, &mut entries).await?; + } + cursors = next; + } + + resolved.sort_by_key(|(i, _)| *i); + Ok(resolved + .into_iter() + .map(|(i, governing)| (listed[i].clone(), governing)) + .collect()) +} + +/// Build the `admin` connection for a governing entry: `custom_instance_user` for an instance +/// database, the user's own resource for a BYO-postgres one. +async fn resolve_datatable_connection_unchecked( + db: &DB, + governing: &GoverningDatatable, + replication: bool, +) -> Result { + let database = governing + .datatable + .database + .as_ref() + .expect("a governing entry owns a database"); + if database.resource_type == DataTableCatalogResourceType::Instance { let mut pg_creds = PgDatabase::parse_uri(&get_database_url().await?.as_str().await)?; - pg_creds.dbname = datatable.database.resource_path.clone(); + pg_creds.dbname = database.resource_path.clone(); if replication { pg_creds.user = Some("custom_instance_replication_user".to_string()); pg_creds.password = Some(get_custom_pg_instance_replication_password(&db).await?); } else { - pg_creds.user = Some("custom_instance_user".to_string()); + pg_creds.user = Some(CUSTOM_INSTANCE_USER.to_string()); pg_creds.password = Some(get_custom_pg_instance_password(&db).await?); } serde_json::to_value(&pg_creds) - .map_err(|e| Error::internal_err(format!("Error serializing pg creds: {}", e)))? + .map_err(|e| Error::internal_err(format!("Error serializing pg creds: {}", e))) } else { // Name the data table too: the caller asked for one by name, and a bare // "resource f/x/y does not exist" leaves them to work out which one points at it. transform_json_unchecked( - &serde_json::Value::String(format!("$res:{}", datatable.database.resource_path)), - w_id, + &serde_json::Value::String(format!("$res:{}", database.resource_path)), + &governing.workspace_id, db, ) .await .map_err(|e| match e { - Error::NotFound(m) => Error::NotFound(format!("data table {name}: {m}")), + Error::NotFound(m) => Error::NotFound(format!("data table {}: {m}", governing.name)), e => e, - })? + }) + } +} + +/// Resolve a data table to connection credentials **without authorizing anything**: always the +/// `admin` connection. +/// +/// Authorization: callers MUST have authorized access already. Anything that acts for a user or a +/// job wants [`get_datatable_resource_from_db`] instead. +pub async fn get_datatable_resource_from_db_unchecked( + db: &DB, + w_id: &str, + name: &str, +) -> Result { + let governing = resolve_governing_datatable(db, w_id, name).await?; + resolve_datatable_connection_unchecked(db, &governing, false).await +} + +/// Same as [`get_datatable_resource_from_db_unchecked`] but for postgres trigger +/// connections: custom-instance datatables resolve to +/// `custom_instance_replication_user` rather than `custom_instance_user`. BYO-postgres +/// datatables resolve to the user's own resource unchanged; configuring it for +/// replication there is the user's responsibility. +/// +/// Authorization: a replication connection reads every row whatever the roles grant, so no role or +/// admin check makes it safe. Callers MUST refuse a data table under roles outright — the Postgres +/// trigger crate's `ensure_not_under_roles` — and turning roles on is refused while one streams. +pub async fn get_datatable_replication_resource_from_db_unchecked( + db: &DB, + w_id: &str, + name: &str, +) -> Result { + let governing = resolve_governing_datatable(db, w_id, name).await?; + resolve_datatable_connection_unchecked(db, &governing, true).await +} + +/// The identity a resolution is made for. `Unchecked` is for callers that authorized already; +/// everything else is checked against the governing entry's tenants. +pub enum DatatableAccess<'a> { + /// Reaches every role. For callers that already authorized, or that have no user at all. + Unchecked, + Authed(crate::db::AuthedRef<'a>), + /// A job's owner, without reading the job row — only fetched if the data table turns out to + /// be permissioned. + PermissionedAs { + permissioned_as: &'a str, + email: &'a str, + }, + /// A job identified by id; its owner is read from `v2_job`. For agent workers and anything + /// else that authenticates as infrastructure rather than as the job's user. + Job(uuid::Uuid), + /// No identity established. Unpermissioned data tables resolve as before; permissioned ones + /// are refused, so a caller predating this feature fails closed. + NoIdentity, +} + +/// Does one tenant list cover this identity? Admins of the governing workspace pass everything — +/// they can edit the tenant lists anyway, so refusing them would only be theatre. +pub fn can_use_datatable_role( + tenants: &DataTableRoleTenants, + authed: &crate::db::AuthedRef<'_>, +) -> bool { + crate::datatable_roles_oss::can_use_datatable_role(tenants, authed) +} + +/// Evaluate a tenant list **as a member of the governing workspace**, whoever is calling. +/// +/// A caller reaching a data table through a pointer is a member of some other workspace, and being +/// its admin means nothing here — that is the whole point of the pointer. They are looked up in +/// the governing workspace by email and evaluated there, or refused when they are not a member. +/// A `g/` or `f/` permissioned-as from a foreign workspace is refused outright: those names are +/// defined per workspace and mean nothing outside the one that defined them. +pub async fn can_use_datatable_role_in_governing_workspace( + db: &DB, + governing_w_id: &str, + w_id: &str, + tenants: &DataTableRoleTenants, + access: &DatatableAccess<'_>, +) -> Result { + crate::datatable_roles_oss::can_use_datatable_role_in_governing_workspace( + db, + governing_w_id, + w_id, + tenants, + access, + ) + .await +} + +/// Resolve a data table to connection credentials for one identity. +/// +/// This is the chokepoint: everything that opens a connection to a data table on someone's behalf +/// goes through it. `role` is the role name the caller asked for — the `-- role` annotation, the +/// `?role=` on a `datatable://` reference, or `None` for the data table's default. +/// +/// The resolved role **logs in as itself**. Never `SET ROLE`: a script could `RESET ROLE` its way +/// back to admin. +pub async fn get_datatable_resource_from_db( + db: &DB, + w_id: &str, + name: &str, + role: Option<&str>, + access: DatatableAccess<'_>, +) -> Result { + let governing = resolve_governing_datatable(db, w_id, name).await?; + let db_resource = resolve_datatable_connection_unchecked(db, &governing, false).await?; + // Not under roles and asked for none, or for `admin` by name: the `admin` connection, as before + // roles existed, in every edition. Anything else is a role decision. Every migration names + // `admin` explicitly, so an edition without roles must not treat that as one. + if governing.datatable.permissions.is_none() && role.is_none_or(|r| r == ADMIN_DATATABLE_ROLE) { + return Ok(db_resource); + } + crate::datatable_roles_oss::resolve_datatable_role_connection( + db, + w_id, + name, + &governing, + db_resource, + role, + access, + ) + .await +} + +/// Would the chokepoint accept this identity connecting as this role? Answers without resolving +/// credentials, for callers that want to refuse early and say which thing was refused. +/// +/// Not the security boundary — [`get_datatable_resource_from_db`] re-checks when it actually opens +/// the connection. This is what turns "permission denied for table x" into a message naming the +/// migration and the role. +pub async fn ensure_can_use_datatable_role( + db: &DB, + w_id: &str, + name: &str, + role: Option<&str>, + access: &DatatableAccess<'_>, + context: &str, +) -> Result<()> { + crate::datatable_roles_oss::ensure_can_use_datatable_role(db, w_id, name, role, access, context) + .await +} + +/// Gate the operations that see the whole database whatever the roles grant: a migration that +/// declares no role, exports, and editing the permissions themselves. Not replication, which a +/// data table under roles refuses whoever asks (see `ensure_not_under_roles`). Passing +/// means the caller could have connected as `admin` anyway. +pub async fn ensure_datatable_admin_access( + db: &DB, + w_id: &str, + name: &str, + access: &DatatableAccess<'_>, +) -> Result<()> { + crate::datatable_roles_oss::ensure_datatable_admin_access(db, w_id, name, access).await +} + +/// Rewrite the `permissions` of every data table entry of one workspace, in the caller's +/// transaction. `change` reports whether it touched anything; the row is only written when +/// something did. +/// +/// Authorization: writes an access decision for any workspace named, with an arbitrary mutation, +/// and checks nothing. It exists for the cascades below — the transaction that frees or renames a +/// principal — so callers MUST be the operation that made the principal change, and MUST run in +/// its transaction. Anything editing a decision on purpose belongs in the permissions endpoint, +/// which is gated on the workspace that governs the data table. +/// +/// The tenant lists name principals of this workspace, so anything that frees or renames one has +/// to come through here in the same transaction that frees it — otherwise a `u/alice` reused by a +/// later account silently inherits her access. +pub async fn update_datatable_permissions_in_workspace( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + w_id: &str, + change: F, +) -> Result<()> +where + F: Fn(&mut DataTablePermissions) -> bool, +{ + let Some(mut settings) = sqlx::query_scalar!( + "SELECT datatable FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE", + w_id + ) + .fetch_optional(&mut **tx) + .await? + .flatten() else { + return Ok(()); }; - Ok(db_resource) + let Some(datatables) = settings + .get_mut("datatables") + .and_then(|d| d.as_object_mut()) + else { + return Ok(()); + }; + + let mut touched = false; + for entry in datatables.values_mut() { + let Some(permissions) = entry.get("permissions").filter(|p| !p.is_null()) else { + continue; + }; + let Ok(mut permissions) = + serde_json::from_value::(permissions.clone()) + else { + continue; + }; + if change(&mut permissions) { + entry["permissions"] = serde_json::to_value(&permissions) + .map_err(|e| Error::internal_err(format!("serializing permissions: {e}")))?; + touched = true; + } + } + + if touched { + sqlx::query!( + "UPDATE workspace_settings SET datatable = $1 WHERE workspace_id = $2", + settings, + w_id + ) + .execute(&mut **tx) + .await?; + } + Ok(()) +} + +/// Drop a freed principal (`u/alice`, `g/analysts`, `f/finance`) from every tenant list of one +/// workspace. Same contract as [`update_datatable_permissions_in_workspace`]: for the transaction +/// that frees the principal, not for editing a decision. +pub async fn remove_datatable_tenant_in_workspace( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + w_id: &str, + tenant: &str, +) -> Result<()> { + update_datatable_permissions_in_workspace(tx, w_id, |permissions| { + let mut touched = false; + for role in permissions.roles.values_mut() { + let before = role.tenants.len(); + role.tenants.retain(|t| t != tenant); + touched |= role.tenants.len() != before; + } + touched + }) + .await +} + +/// Follow a renamed principal through every tenant list of one workspace. Same contract as +/// [`update_datatable_permissions_in_workspace`]. +pub async fn rename_datatable_tenant_in_workspace( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + w_id: &str, + old: &str, + new: &str, +) -> Result<()> { + update_datatable_permissions_in_workspace(tx, w_id, |permissions| { + let mut touched = false; + for role in permissions.roles.values_mut() { + let mut role_touched = false; + for tenant in role.tenants.iter_mut() { + if tenant == old { + *tenant = new.to_string(); + role_touched = true; + } + } + if role_touched { + // The rename can collide with a name already in the list, and the two need not be + // adjacent — `Vec::dedup` only collapses neighbours, so it would leave the pair. + let mut seen = std::collections::HashSet::new(); + role.tenants.retain(|t| seen.insert(t.clone())); + touched = true; + } + } + touched + }) + .await +} + +/// Strip a deleted instance role from every workspace that had tenanted it, so nothing is left +/// naming a role that no longer exists. +/// +/// Authorization: reaches every workspace on the instance. Callers MUST be the superadmin path +/// dropping the role from the cluster — it exists to follow that, not to edit tenants. +/// +/// Takes that path's transaction rather than opening its own: run afterwards, a failure part-way +/// leaves the catalog row already gone, so the retry answers `NotFound` while some workspaces +/// still name a role nothing can connect as. In the transaction, the cluster drop, the catalog row +/// and every tenant list commit together or not at all. A data table whose default role was the deleted one falls +/// back to `admin` — the one role that is always present. +pub async fn forget_datatable_role_everywhere( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + role_id: &str, +) -> Result<()> { + crate::datatable_roles_oss::forget_datatable_role_everywhere(tx, role_id).await +} + +/// Drop the `permissions` block from a `workspace_settings.datatable` value before it leaves the +/// server. +/// +/// Who may connect as which role is an access decision, not configuration, and its tenants name +/// principals of one workspace — `g/analysts` in dev is a different group from `g/analysts` in +/// prod. Shipping it would both mean nothing at the far end and turn a settings push into a way to +/// widen access, so the decision stays where it was made. [`DataTable`] deserializes fine without +/// it, and the settings-editing endpoint carries the stored block across untouched. +pub fn strip_datatable_permissions( + datatable: Option, +) -> Option { + let mut datatable = datatable?; + if let Some(entries) = datatable + .get_mut("datatables") + .and_then(|d| d.as_object_mut()) + { + for entry in entries.values_mut() { + if let Some(entry) = entry.as_object_mut() { + entry.remove("permissions"); + } + } + } + Some(datatable) +} + +/// As [`parse_datatable_ref`], except that an entry whose stored name itself contains `?` — which +/// names could before they were restricted — resolves by that exact name, without a role. It is +/// looked up first, so `sales?role=x` never reaches a different entry than the one stored so. +/// +/// Authorization: checks nothing, and its answer reveals whether `w_id` stores that exact name. +/// Callers MUST already act for `w_id` — a job of it, or a caller authenticated into it — and +/// MUST still pass the name to [`get_datatable_resource_from_db`] or an admin-access check. +pub async fn parse_datatable_ref_for( + db: &DB, + w_id: &str, + reference: &str, +) -> Result<(String, Option)> { + if reference.contains('?') { + let exists = sqlx::query_scalar::<_, Option>( + "SELECT (datatable->'datatables') ? $2 FROM workspace_settings WHERE workspace_id = $1", + ) + .bind(w_id) + .bind(reference) + .fetch_optional(db) + .await? + .flatten() + .unwrap_or(false); + if exists { + return Ok((reference.to_string(), None)); + } + } + let (name, role) = parse_datatable_ref(reference)?; + Ok((name.to_string(), role.map(str::to_string))) +} + +/// Split a `datatable://` reference into its name and the role its query string names. +/// +/// A query string that does not parse is an error rather than an absent role. Falling back would +/// resolve the reference to the data table's default role, so `?Role=analytics` or a mistyped +/// `?role=` would quietly connect as something the caller did not ask for — the same trap as a +/// malformed `-- role` annotation, and `role` is the only parameter a reference takes. +pub fn parse_datatable_ref(reference: &str) -> Result<(&str, Option<&str>)> { + let (name, query) = reference.split_once('?').unwrap_or((reference, "")); + let mut role = None; + for param in query.split('&').filter(|p| !p.is_empty()) { + let (key, value) = param.split_once('=').unwrap_or((param, "")); + if !key.eq_ignore_ascii_case("role") { + return Err(Error::BadRequest(format!( + "Data table reference '{name}' carries an unknown parameter '{key}'. \ + The only one it takes is `?role=`." + ))); + } + if role.is_some() { + return Err(Error::BadRequest(format!( + "Data table reference '{name}' names a role more than once." + ))); + } + if value.is_empty() || !is_datatable_role_name(value) { + return Err(Error::BadRequest(format!( + "Data table reference '{name}' has a malformed role '{value}'. Write it as \ + `?role=`, where is letters, digits, '_' or '-'." + ))); + } + role = Some(value); + } + Ok((name, role)) +} + +fn is_datatable_role_name(role: &str) -> bool { + !role.is_empty() + && role.len() <= 63 + && role + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') } #[derive(Deserialize, Serialize, Debug)] @@ -2642,6 +3243,130 @@ async fn transform_json_unchecked( mod tests { use super::*; + fn tenants(list: &[&str]) -> DataTableRoleTenants { + DataTableRoleTenants { tenants: list.iter().map(|t| t.to_string()).collect() } + } + + #[cfg(all(feature = "private", feature = "enterprise"))] + #[test] + fn a_tenant_list_covers_users_groups_folders_and_the_wildcard() { + let groups = vec!["analysts".to_string()]; + let folders = vec![("finance".to_string(), true, false)]; + let scopes = None; + let token_prefix = None; + let is_admin = false; + let is_operator = false; + let authed = crate::db::AuthedRef { + email: "alice@windmill.dev", + username: "alice", + is_admin: &is_admin, + is_operator: &is_operator, + groups: &groups, + folders: &folders, + scopes: &scopes, + token_prefix: &token_prefix, + }; + + assert!(can_use_datatable_role(&tenants(&["u/alice"]), &authed)); + assert!(can_use_datatable_role(&tenants(&["g/analysts"]), &authed)); + assert!(can_use_datatable_role(&tenants(&["f/finance"]), &authed)); + assert!(can_use_datatable_role(&tenants(&["*"]), &authed)); + assert!(!can_use_datatable_role(&tenants(&[]), &authed)); + assert!(!can_use_datatable_role( + &tenants(&["u/bob", "g/ops"]), + &authed + )); + // A bare name is not a principal: only the three prefixes and the wildcard match. + assert!(!can_use_datatable_role(&tenants(&["alice"]), &authed)); + + // An admin of the governing workspace reaches every role: they can edit the lists anyway. + let is_admin = true; + let admin = crate::db::AuthedRef { is_admin: &is_admin, ..authed }; + assert!(can_use_datatable_role(&tenants(&[]), &admin)); + } + + #[cfg(not(all(feature = "private", feature = "enterprise")))] + #[test] + fn without_the_enterprise_edition_no_tenant_list_covers_anyone() { + let groups = vec![]; + let folders = vec![]; + let scopes = None; + let token_prefix = None; + let is_admin = true; + let is_operator = false; + let admin = crate::db::AuthedRef { + email: "alice@windmill.dev", + username: "alice", + is_admin: &is_admin, + is_operator: &is_operator, + groups: &groups, + folders: &folders, + scopes: &scopes, + token_prefix: &token_prefix, + }; + assert!(!can_use_datatable_role(&tenants(&["*"]), &admin)); + assert!(!can_use_datatable_role(&tenants(&["u/alice"]), &admin)); + } + + #[test] + fn a_datatable_ref_splits_off_its_role() { + assert_eq!(parse_datatable_ref("sales").unwrap(), ("sales", None)); + assert_eq!( + parse_datatable_ref("sales?role=analytics").unwrap(), + ("sales", Some("analytics")) + ); + // The key matches case-insensitively, the way the `-- role` annotation does. + assert_eq!( + parse_datatable_ref("sales?Role=analytics").unwrap(), + ("sales", Some("analytics")) + ); + + // A query string that does not parse is refused rather than read as "no role": resolving + // it to the data table's default would connect as a login the caller never asked for. + for malformed in [ + "sales?role=", + "sales?role=an;alytics", + "sales?x=1&role=analytics", + "sales?role=a&role=b", + ] { + assert!( + parse_datatable_ref(malformed).is_err(), + "silently ignored: {malformed}" + ); + } + } + + #[test] + fn an_entry_owns_a_database_or_points_at_one_but_never_both() { + let terminal = DataTable { + database: Some(DataTableDatabase { + resource_type: DataTableCatalogResourceType::Instance, + resource_path: "dt_main".to_string(), + }), + reference: None, + forked_from: None, + migrations_enabled: None, + permissions: None, + }; + assert!(validate_datatable_shape("main", &terminal).is_ok()); + + let pointer = DataTable { + database: None, + reference: Some(DataTableReference { + workspace_id: "prod".to_string(), + datatable: "main".to_string(), + }), + ..terminal.clone() + }; + assert!(validate_datatable_shape("main", &pointer).is_ok()); + + let both = DataTable { database: terminal.database.clone(), ..pointer.clone() }; + assert!(validate_datatable_shape("main", &both).is_err()); + + let neither = DataTable { database: None, reference: None, ..terminal.clone() }; + assert!(validate_datatable_shape("main", &neither).is_err()); + } + #[test] fn test_parse_fork_branch() { // Generated fork (`wm-fork-abc`) and dev workspace (`staging`) forms. diff --git a/backend/windmill-trigger-postgres/src/handler.rs b/backend/windmill-trigger-postgres/src/handler.rs index 92dd2e1e19..4294645b89 100644 --- a/backend/windmill-trigger-postgres/src/handler.rs +++ b/backend/windmill-trigger-postgres/src/handler.rs @@ -21,11 +21,12 @@ use windmill_common::{ use windmill_git_sync::DeployedObject; use windmill_api_auth::{check_scopes, ApiAuthed}; -use windmill_trigger::{Trigger, TriggerCrud, TriggerData}; +use windmill_trigger::{Trigger, TriggerCrud, TriggerData, TriggerMode}; use super::{ check_if_valid_publication_for_postgres_version, create_logical_replication_slot, - create_pg_publication, drop_publication, generate_random_string, get_default_pg_connection, + create_pg_publication, drop_publication, ensure_not_under_roles, generate_random_string, + get_default_pg_connection, mapper::{Mapper, MappingInfo}, PostgresConfig, PostgresConfigRequest, PostgresPublicationReplication, PostgresTrigger, PublicationData, Relations, Slot, SlotList, TableToTrack, TemplateScript, TestPostgresConfig, @@ -64,6 +65,29 @@ impl TriggerCrud for PostgresTrigger { DeployedObject::PostgresTrigger { path, parent_path } } + async fn validate_config( + &self, + db: &DB, + config: &Self::TriggerConfigRequest, + workspace_id: &str, + ) -> Result<()> { + ensure_not_under_roles(db, workspace_id, &config.postgres_resource_path).await + } + + async fn authorize_set_trigger_mode( + &self, + _authed: &ApiAuthed, + tx: &mut PgConnection, + _workspace_id: &str, + _path: &str, + mode: &TriggerMode, + ) -> Result<()> { + if *mode != TriggerMode::Disabled { + windmill_common::datatable_roles::lock_datatable_streams(tx, false).await?; + } + Ok(()) + } + async fn create_trigger( &self, db: &DB, @@ -72,6 +96,7 @@ impl TriggerCrud for PostgresTrigger { w_id: &str, trigger: TriggerData, ) -> Result<()> { + windmill_common::datatable_roles::lock_datatable_streams(&mut *tx, false).await?; let resolved_edited_by = trigger.base.resolve_edited_by(authed); let resolved_permissioned_as = trigger.base.resolve_permissioned_as(authed); let Self::TriggerConfigRequest { @@ -161,6 +186,7 @@ impl TriggerCrud for PostgresTrigger { path: &str, trigger: TriggerData, ) -> Result<()> { + windmill_common::datatable_roles::lock_datatable_streams(&mut *tx, false).await?; let resolved_edited_by = trigger.base.resolve_edited_by(authed); let resolved_permissioned_as = trigger.base.resolve_permissioned_as(authed); let Self::TriggerConfigRequest { diff --git a/backend/windmill-trigger-postgres/src/lib.rs b/backend/windmill-trigger-postgres/src/lib.rs index 571aac0a66..b92860662d 100644 --- a/backend/windmill-trigger-postgres/src/lib.rs +++ b/backend/windmill-trigger-postgres/src/lib.rs @@ -374,6 +374,35 @@ pub async fn get_raw_postgres_connection( Ok(client) } +/// A replication stream reads every row of every table whatever the data table's roles grant, so +/// the two don't mix: a data table under roles takes no triggers or captures, and roles cannot be +/// turned on while one is enabled on it. +/// +/// Authorization: checks nothing, and its refusal says whether `w_id`'s data table is under roles. +/// Callers MUST have established that the caller may manage triggers in `w_id` first. +pub(crate) async fn ensure_not_under_roles( + db: &DB, + w_id: &str, + postgres_resource_path: &str, +) -> Result<()> { + let Some(datatable_name) = postgres_resource_path.strip_prefix("datatable://") else { + return Ok(()); + }; + if windmill_common::workspaces::resolve_governing_datatable(db, w_id, datatable_name) + .await? + .datatable + .permissions + .is_some() + { + return Err(Error::BadRequest(format!( + "Data table '{datatable_name}' is under roles, and a Postgres trigger or capture \ + cannot read one: a replication stream sees every row whatever the roles grant. \ + Turn its roles off to stream it." + ))); + } + Ok(()) +} + pub async fn resolve_postgres_resource( authed: &ApiAuthed, user_db: Option, @@ -382,6 +411,7 @@ pub async fn resolve_postgres_resource( w_id: &str, ) -> Result { if let Some(datatable_name) = postgres_resource_path.strip_prefix("datatable://") { + ensure_not_under_roles(db, w_id, postgres_resource_path).await?; // Trigger connections (publication/slot management + logical replication) run // as the dedicated replication user on custom-instance databases. let resource_value = diff --git a/backend/windmill-trigger-postgres/src/listener.rs b/backend/windmill-trigger-postgres/src/listener.rs index 14d348b5f2..876b75661e 100644 --- a/backend/windmill-trigger-postgres/src/listener.rs +++ b/backend/windmill-trigger-postgres/src/listener.rs @@ -20,7 +20,8 @@ use windmill_common::{ use windmill_trigger::{listener::ListeningTrigger, trigger_helpers::TriggerJobArgs, Listener}; use super::{ - drop_publication, get_default_pg_connection, get_raw_postgres_connection, + drop_publication, ensure_not_under_roles, get_default_pg_connection, + get_raw_postgres_connection, handler::drop_logical_replication_slot, relation::RelationConverter, replication_message::{ @@ -135,8 +136,8 @@ impl PostgresSimpleClient { /// Resolves the Postgres resource, validates that the configured publication and /// replication slot still exist, and opens a fresh logical replication stream. /// -/// Returns `Error::BadConfig` when the publication or slot is missing (an -/// unrecoverable misconfiguration). Any other error is treated as transient +/// Returns `Error::BadConfig` when the publication or slot is missing, or the +/// data table is under roles (unrecoverable misconfigurations). Any other error is treated as transient /// (connection refused, network interruption, ...) and is retried by the caller. /// The resource is re-resolved on every call so credential rotations are picked /// up across reconnections. @@ -149,6 +150,14 @@ async fn connect_logical_replication_stream( let PostgresConfig { postgres_resource_path, publication_name, replication_slot_name, .. } = trigger_config; + // Retrying cannot lift roles, so this disables the trigger like a missing slot does. + ensure_not_under_roles(db, workspace_id, postgres_resource_path) + .await + .map_err(|e| match e { + Error::BadRequest(msg) => Error::BadConfig(msg), + e => e, + })?; + let database = resolve_postgres_resource( authed, Some(UserDB::new(db.clone())), diff --git a/backend/windmill-worker/src/agent_workers.rs b/backend/windmill-worker/src/agent_workers.rs index 8dccf691fa..e0971c08ce 100644 --- a/backend/windmill-worker/src/agent_workers.rs +++ b/backend/windmill-worker/src/agent_workers.rs @@ -64,16 +64,25 @@ pub async fn get_ducklake_from_agent_http( .await } +/// An agent worker authenticates as infrastructure, not as the job's user, so the job id travels +/// with the request: the server reads the job's owner from it and evaluates the data table's +/// tenants against them. A worker predating this sends neither, and the server fails it closed on +/// a data table under roles. #[allow(dead_code)] pub async fn get_datatable_resource_from_agent_http( client: &HttpClient, name: &str, w_id: &str, + role: Option<&str>, + job_id: &uuid::Uuid, ) -> anyhow::Result { + let role_query = role + .map(|r| format!("&role={}", urlencoding::encode(r))) + .unwrap_or_default(); client .get(&format!( - "/api/w/{}/agent_workers/get_datatable_resource/{}", - w_id, &name + "/api/w/{}/agent_workers/get_datatable_resource/{}?job_id={}{}", + w_id, &name, job_id, role_query )) .await } diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index b2e69d26d6..aecdb738b3 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -13,8 +13,8 @@ use windmill_common::error::{to_anyhow, Error, Result}; use windmill_common::utils::sanitize_string_from_password; use windmill_common::worker::{get_memory, to_raw_value, Connection, SqlResultCollectionStrategy}; use windmill_common::workspaces::{ - get_datatable_resource_from_db_unchecked, get_ducklake_from_db_unchecked, - strip_fork_reserved_attach_args, DucklakeCatalogResourceType, + get_datatable_resource_from_db, get_ducklake_from_db_unchecked, + strip_fork_reserved_attach_args, DatatableAccess, DucklakeCatalogResourceType, }; use windmill_common::PgDatabase; use windmill_object_store::S3_PROXY_LAST_ERRORS_CACHE; @@ -1494,13 +1494,9 @@ pub async fn do_duckdb( .await? { probe_blocks.extend(q); - } else if let Some(q) = transform_attach_datatable( - &query_block, - conn, - &mut hidden_passwords, - &job.workspace_id, - ) - .await? + } else if let Some(q) = + transform_attach_datatable(&query_block, conn, &mut hidden_passwords, job) + .await? { probe_blocks.extend(q); } else { @@ -1575,13 +1571,9 @@ pub async fn do_duckdb( .await? { v.extend(ducklake_query); - } else if let Some(datatable_query) = transform_attach_datatable( - &query_block, - conn, - &mut hidden_passwords, - &job.workspace_id, - ) - .await? + } else if let Some(datatable_query) = + transform_attach_datatable(&query_block, conn, &mut hidden_passwords, job) + .await? { v.extend(datatable_query); } else { @@ -2609,33 +2601,79 @@ fn fork_defer_statements( Ok(stmts) } +struct AttachedDatatable<'a> { + /// The data table reference, query string included; a bare `datatable` is `main`. + reference: String, + alias: &'a str, +} + +/// `ATTACH 'datatable[://][?role=]' AS `. A bare `datatable` names the default +/// data table, so the role query string has to be accepted with and without an explicit name. The +/// reference is split only once the workspace can be read, because a stored name may contain `?`. +fn parse_attach_datatable(query: &str) -> Option> { + lazy_static::lazy_static! { + static ref RE: regex::Regex = regex::Regex::new( + r"(?i)ATTACH\s*'datatable(://[^':]+|\?[^':]*)?'\s*AS\s+([^ ;]+)" + ).unwrap(); + } + let cap = RE.captures(query)?; + let reference = match cap.get(1).map(|m| m.as_str()) { + Some(named) if named.starts_with("://") => named[3..].to_string(), + Some(query) => format!("main{query}"), + None => "main".to_string(), + }; + let alias = cap.get(2).map(|m| m.as_str()).unwrap_or(""); + Some(AttachedDatatable { reference, alias }) +} + async fn transform_attach_datatable( query: &str, conn: &Connection, hidden_passwords: &mut Arc>>, - w_id: &str, + job: &MiniPulledJob, ) -> Result>> { - lazy_static::lazy_static! { - static ref RE: regex::Regex = regex::Regex::new(r"(?i)ATTACH\s*'datatable(://[^':]+)?'\s*AS\s+([^ ;]+)").unwrap(); - } - let Some(cap) = RE.captures(query) else { + let Some(attached) = parse_attach_datatable(query) else { return Ok(None); }; - let name = cap.get(1).map(|m| &m.as_str()[3..]).unwrap_or("main"); - let alias_name = cap.get(2).map(|m| m.as_str()).unwrap_or(""); + // A query string that does not parse is refused rather than dropped: attaching under the + // default role when the statement asked for another one is the failure this guards. let db_resource = match conn { Connection::Http(client) => { - get_datatable_resource_from_agent_http(client, name, w_id).await? + let (name, role) = + windmill_common::workspaces::parse_datatable_ref(&attached.reference)?; + get_datatable_resource_from_agent_http(client, name, &job.workspace_id, role, &job.id) + .await? + } + Connection::Sql(db) => { + let (name, role) = windmill_common::workspaces::parse_datatable_ref_for( + db, + &job.workspace_id, + &attached.reference, + ) + .await?; + get_datatable_resource_from_db( + db, + &job.workspace_id, + &name, + role.as_deref(), + DatatableAccess::PermissionedAs { + permissioned_as: &job.permissioned_as, + email: &job.permissioned_as_email, + }, + ) + .await? } - Connection::Sql(db) => get_datatable_resource_from_db_unchecked(db, w_id, name).await?, }; if let Some(pwd) = db_resource.get("password").and_then(|p| p.as_str()) { hidden_passwords.lock().unwrap().push(pwd.to_string()); } - Ok(Some(pg_secret_attach_statements(db_resource, alias_name)?)) + Ok(Some(pg_secret_attach_statements( + db_resource, + attached.alias, + )?)) } // Secret names must be plain identifiers; the hash keeps two aliases distinct even @@ -2680,6 +2718,11 @@ fn pg_secret_attach_statements(db_resource: Value, alias_name: &str) -> Result' (TYPE postgres, SECRET …)` would reach a database nobody + // authorized this job for, as this role. + format!("DROP TEMPORARY SECRET {secret_name};"), ]) } @@ -2753,6 +2796,45 @@ pub struct Arg { mod tests { use super::*; + #[test] + fn attach_datatable_parses_name_and_role() { + let reference_of = |q: &str| parse_attach_datatable(q).unwrap().reference; + let named = + parse_attach_datatable("ATTACH 'datatable://sales?role=analytics' AS dt").unwrap(); + assert_eq!( + (named.reference.as_str(), named.alias), + ("sales?role=analytics", "dt") + ); + // A bare `datatable` is the default one, and still takes a role. + assert_eq!( + reference_of("ATTACH 'datatable?role=analytics' AS dt"), + "main?role=analytics" + ); + assert_eq!(reference_of("ATTACH 'datatable://sales' AS dt"), "sales"); + assert_eq!(reference_of("ATTACH 'datatable' AS dt"), "main"); + assert!(parse_attach_datatable("SELECT 1").is_none()); + // A stored name can contain `?`, so that is left to the workspace lookup to split. + assert_eq!(reference_of("ATTACH 'datatable://a?b' AS dt"), "a?b"); + + // The key matches case-insensitively, as the `-- role` annotation does, and a query string + // that does not parse is refused rather than attached under the default role. + let parse = |q: &str| { + windmill_common::workspaces::parse_datatable_ref(&reference_of(q)) + .map(|(name, role)| (name.to_string(), role.map(str::to_string))) + }; + assert_eq!( + parse("ATTACH 'datatable://sales?Role=analytics' AS dt").unwrap(), + ("sales".to_string(), Some("analytics".to_string())) + ); + for malformed in [ + "ATTACH 'datatable://sales?role=' AS dt", + "ATTACH 'datatable://sales?role=an;alytics' AS dt", + "ATTACH 'datatable://sales?x=1&role=analytics' AS dt", + ] { + assert!(parse(malformed).is_err(), "silently ignored: {malformed}"); + } + } + #[test] fn decode_ffi_error_unescapes_multiline_and_strips_quotes() { // Mirror the FFI: JSON-encode the raw DuckDB message, prefix "ERROR ". @@ -3868,6 +3950,8 @@ mod tests { stmts[3], format!("ATTACH 'sslmode=require' AS dt (TYPE postgres, SECRET {secret_name});") ); + assert_eq!(stmts[4], format!("DROP TEMPORARY SECRET {secret_name};")); + assert_eq!(stmts.len(), 5); } #[test] diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index 1413924bf3..69aa3ee0fb 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -26,9 +26,11 @@ use windmill_common::azure_workload_identity::WORKLOAD_IDENTITY_PASSWORD; use windmill_common::error::to_anyhow; use windmill_common::error::{self, Error}; use windmill_common::worker::{ - to_raw_value, Connection, SqlResultCollectionStrategy, CLOUD_HOSTED, + to_raw_value, Connection, SqlAnnotations, SqlResultCollectionStrategy, CLOUD_HOSTED, +}; +use windmill_common::workspaces::{ + get_datatable_resource_from_db, parse_datatable_ref, parse_datatable_ref_for, DatatableAccess, }; -use windmill_common::workspaces::get_datatable_resource_from_db_unchecked; use windmill_common::{PgDatabase, PrepareQueryColumnInfo, PrepareQueryResult, DB}; use windmill_parser::{Arg, Typ}; use windmill_parser_sql::{ @@ -680,15 +682,36 @@ pub async fn do_postgresql( } else { match pg_args.get("database").cloned() { Some(Value::String(db_str)) if db_str.starts_with("datatable://") => { - let db_str = db_str.trim_start_matches("datatable://"); + let reference = db_str.trim_start_matches("datatable://"); + // The annotation wins: a generated query can carry a `?role=` in the reference it + // was handed, but only the script's author writes the leading comment block. + let annotated = SqlAnnotations::datatable_role(&query)?; Some(match conn { Connection::Http(client) => { - get_datatable_resource_from_agent_http(client, &db_str, &job.workspace_id) - .await? + let (name, uri_role) = parse_datatable_ref(reference)?; + get_datatable_resource_from_agent_http( + client, + name, + &job.workspace_id, + annotated.as_deref().or(uri_role), + &job.id, + ) + .await? } Connection::Sql(db) => { - get_datatable_resource_from_db_unchecked(db, &job.workspace_id, &db_str) - .await? + let (name, uri_role) = + parse_datatable_ref_for(db, &job.workspace_id, reference).await?; + get_datatable_resource_from_db( + db, + &job.workspace_id, + &name, + annotated.as_deref().or(uri_role.as_deref()), + DatatableAccess::PermissionedAs { + permissioned_as: &job.permissioned_as, + email: &job.permissioned_as_email, + }, + ) + .await? } }) } diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index cb408fb4e4..55735ed9fd 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.813.0"; +export const VERSION = "v1.814.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/chat-sdk/README.md b/chat-sdk/README.md index 0f1082de45..1d09b68265 100644 --- a/chat-sdk/README.md +++ b/chat-sdk/README.md @@ -183,7 +183,7 @@ await chat.sendMessage('Hello') | `workspace` | Detected inside a raw app. | | `token` | A token, or a function returning one (called before every request, so it can fetch a short-lived token from your backend). Omit it inside a raw app. | | `history` | `'server'`, `'local'` or `'none'`, see [History](#history). Defaults to `'server'` with a viewer session and `'local'` with an explicit `token`. | -| `inputs` | Extra flow inputs sent with every message. `sendMessage(text, { inputs })` adds per-message ones. | +| `inputs` | Extra flow inputs sent with every message. `sendMessage(text, { inputs })` adds per-message ones, and `{ attachments, attachmentsInput }` files, see [Attachments](#attachments). | | `storageKey` | Namespace for `local` history, e.g. the signed-in user's id. Local history is per browser and per flow; without it, users sharing a browser share it. | | `fetch`, `storage` | Replacements for the globals, for tests and unusual runtimes. | | `pageSize` | Messages and conversations per page of server history. Default 50. | @@ -226,8 +226,8 @@ A turn goes `submitted` (the flow is queued) → `streaming` (the answer is arri answer, an `assistant` message with `success: false`. `status: 'error'` (with `error` set) means the turn could not run or be followed at all, such as a refused request. -Methods: `sendMessage(text, { inputs? })`, `stop()`, `newConversation()`, -`selectConversation(id)`, `loadConversations({ page?, perPage?, kind? })`, +Methods: `sendMessage(text, { inputs?, attachments?, attachmentsInput? })`, `stop()`, +`newConversation()`, `selectConversation(id)`, `loadConversations({ page?, perPage?, kind? })`, `deleteConversation(id)`, `renameConversation(id, title)`, `loadOlderMessages()`, `destroy()`. `kind` lists the flow editor's test chats (`'test'`), the deployed flow's own (`'deployed'`, the server's default) or both (`'all'`); each `Conversation` carries @@ -235,6 +235,32 @@ own (`'deployed'`, the server's default) or both (`'all'`); each `Conversation` stops following the current answer; the flow keeps running and, with server history, its answer is there when you come back. +## Attachments + +A flow whose AI agent step reads `user_attachments` from an `s3object[]` (or a single +`s3object`) flow input takes files with a message: + +```ts +await chat.sendMessage('What does this contract say?', { + attachments: [{ name: file.name, data: file }], // a Blob/File, or a `data:` URL + attachmentsInput: { name: 'files', multiple: true } +}) +``` + +Each file is uploaded to the workspace's object storage under +`windmill_uploads/chat///` and handed to the input as `{ s3, filename }` +objects (the object for a single-file input). Once the uploads return, the pending user +message lists them in `attachments`, as `{ input, s3, filename }` references. The name's +extension is corrected to the file's media type for PNG, JPEG and PDF, because the worker +reads the type off the key. +Files need message text to go with them. A failed upload rejects `sendMessage` before any +run starts, and `stop()` during the upload aborts it; both leave the transcript as it was. +The chat never deletes uploads, so files of a send that did not run stay in storage. The +workspace needs object storage set up. With Enterprise advanced storage permissions, the +user needs read and write on `windmill_uploads/*`, which the default rules grant. The upload goes through +`job_helpers`, so a restricted token needs `job_helpers:write`; a sandboxed raw app cannot +request that scope today, so attachments are not available there yet. + ## History Windmill stores every conversation of a chat-mode flow, and each Windmill user sees diff --git a/chat-sdk/package-lock.json b/chat-sdk/package-lock.json index 5d72fb23b3..83475ba1c8 100644 --- a/chat-sdk/package-lock.json +++ b/chat-sdk/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-chat", - "version": "1.813.0", + "version": "1.814.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-chat", - "version": "1.813.0", + "version": "1.814.0", "license": "Apache-2.0", "devDependencies": { "@ai-sdk/react": "^4.0.102", diff --git a/chat-sdk/package.json b/chat-sdk/package.json index eb286df9b3..57df9050f6 100644 --- a/chat-sdk/package.json +++ b/chat-sdk/package.json @@ -1,7 +1,7 @@ { "name": "windmill-chat", "description": "Build chat interfaces on Windmill flows deployed in chat mode, from any frontend or raw app", - "version": "1.813.0", + "version": "1.814.0", "author": "Ruben Fiszel", "license": "Apache-2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/chat-sdk#readme", diff --git a/chat-sdk/src/api.ts b/chat-sdk/src/api.ts index 3ec5b0f4ff..a14e9800a1 100644 --- a/chat-sdk/src/api.ts +++ b/chat-sdk/src/api.ts @@ -230,6 +230,27 @@ export class WindmillChatApi { return (await res.json()) as FlowConversationMessage[] } + /** + * Puts bytes in the workspace's object storage under `fileKey` and returns the key they were + * stored under (the server may rewrite it). Needs the workspace to have object storage set. + */ + async uploadFile( + fileKey: string, + body: Blob, + options: { contentType?: string; signal?: AbortSignal } = {} + ): Promise<{ file_key: string }> { + const query: Record = { file_key: fileKey } + if (options.contentType) query.content_type = options.contentType + const res = await this.#request('job_helpers/upload_s3_file', { + method: 'POST', + query, + raw: body, + contentType: options.contentType || 'application/octet-stream', + signal: options.signal + }) + return (await res.json()) as { file_key: string } + } + async deleteConversation(conversationId: string): Promise { await this.#request(`flow_conversations/delete/${encodeURIComponent(conversationId)}`, { method: 'DELETE' @@ -241,7 +262,11 @@ export class WindmillChatApi { init: { method?: string query?: Record + /** JSON-encoded. */ body?: unknown + /** Sent as is, under `contentType`. */ + raw?: Blob + contentType?: string accept?: string signal?: AbortSignal } = {} @@ -252,13 +277,14 @@ export class WindmillChatApi { const headers: Record = {} if (init.accept) headers['Accept'] = init.accept if (init.body !== undefined) headers['Content-Type'] = 'application/json' + else if (init.raw !== undefined) headers['Content-Type'] = init.contentType ?? 'application/octet-stream' const token = typeof this.#token === 'function' ? await this.#token() : this.#token if (token) headers['Authorization'] = `Bearer ${token}` const res = await this.#fetch(url.toString(), { method: init.method ?? 'GET', headers, - body: init.body === undefined ? undefined : JSON.stringify(init.body), + body: init.body === undefined ? init.raw : JSON.stringify(init.body), // A token must not be paired with ambient cookies; without one, the cookie is // the credential and only rides same-origin requests. credentials: token ? 'omit' : 'same-origin', diff --git a/chat-sdk/src/attachments.ts b/chat-sdk/src/attachments.ts new file mode 100644 index 0000000000..ac5e71c7d3 --- /dev/null +++ b/chat-sdk/src/attachments.ts @@ -0,0 +1,116 @@ +import type { WindmillChatApi } from './api' +import type { AttachmentUpload } from './types' +import { abortError, isAbortError } from './utils' + +/** + * Where a chat's uploads live in the workspace's object storage. Under `windmill_uploads/` + * because the default Enterprise storage permissions grant every user write and read there + * and deny any other top-level prefix: a key outside it is refused for non-admins, both on + * upload and when the agent's job reads the file back. + */ +export const CHAT_UPLOADS_PREFIX = 'windmill_uploads/chat' + +/** What an AI agent step reads out of `user_attachments`. */ +export interface UploadedAttachment { + s3: string + filename: string +} + +/** + * The extension each type must be stored under. The worker reads an attachment's media type + * from the key's extension only (`mime_guess` in `windmill-ai/src/image_handler.rs`, falling + * back to `image/png`), never from the stored content type, so the extension must be true. + */ +const EXTENSION_BY_MEDIA_TYPE: Record = { + 'image/png': 'png', + 'image/jpeg': 'jpg', + 'application/pdf': 'pdf' +} + +/** + * The name an attachment is stored under: the picked name with the extension its media type + * needs, e.g. a `photo.webp` re-encoded to PNG becomes `photo.png`. Other types keep their name. + */ +export function storedAttachmentName(filename: string, mediaType: string): string { + const extension = EXTENSION_BY_MEDIA_TYPE[mediaType] + if (!extension) return filename + const stem = filename.replace(/\.[^./]+$/, '') + return `${stem || filename}.${extension}` +} + +/** The bytes of an attachment as a Blob carrying its media type. */ +export function attachmentBlob(attachment: AttachmentUpload): Blob { + const data = + typeof attachment.data === 'string' + ? dataUrlToBlob(attachment.data, 'application/octet-stream') + : attachment.data + return attachment.mediaType && attachment.mediaType !== data.type + ? new Blob([data], { type: attachment.mediaType }) + : data +} + +function dataUrlToBlob(dataUrl: string, fallbackType: string): Blob { + const comma = dataUrl.indexOf(',') + if (!dataUrl.startsWith('data:') || comma === -1) { + throw new Error('windmill-chat: an attachment given as a string must be a data: URL') + } + const header = dataUrl.slice(5, comma) + const isBase64 = header.endsWith(';base64') + const mediaType = (isBase64 ? header.slice(0, -';base64'.length) : header) || fallbackType + const payload = dataUrl.slice(comma + 1) + if (!isBase64) return new Blob([decodeURIComponent(payload)], { type: mediaType }) + const binary = atob(payload) + const bytes = new Uint8Array(binary.length) + for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i) + return new Blob([bytes], { type: mediaType }) +} + +/** + * Put each attachment in the workspace's object storage and hand back what the agent reads. + * The key's turn prefix and per-file index keep two files with the same name, in this turn or + * an earlier one, from overwriting each other; the name stays the last segment. + */ +export async function uploadAttachments( + api: WindmillChatApi, + attachments: AttachmentUpload[], + turnId: string, + signal?: AbortSignal +): Promise { + const prefix = `${CHAT_UPLOADS_PREFIX}/${turnId}` + // One failed upload aborts the rest. Nothing already stored is deleted: the chat never + // removes objects from the workspace's storage, so a send that does not run leaves them. + if (signal?.aborted) throw abortError() + const batch = new AbortController() + const abortBatch = () => batch.abort() + signal?.addEventListener('abort', abortBatch, { once: true }) + try { + const results = await Promise.allSettled( + attachments.map(async (attachment, index) => { + try { + const blob = attachmentBlob(attachment) + const filename = storedAttachmentName( + attachment.name || `attachment-${index + 1}`, + blob.type + ) + const { file_key } = await api.uploadFile(`${prefix}/${index}/${filename}`, blob, { + contentType: blob.type, + signal: batch.signal + }) + return { s3: file_key, filename } + } catch (e) { + batch.abort() + throw e + } + }) + ) + const reasons = results.flatMap((r) => (r.status === 'rejected' ? [r.reason] : [])) + // A stop that lands once every upload has answered still withdraws the batch. + if (reasons.length === 0 && !signal?.aborted) { + return results.flatMap((r) => (r.status === 'fulfilled' ? [r.value] : [])) + } + // The failure that started it, not the aborts it caused in the other uploads. + throw reasons.find((reason) => !isAbortError(reason)) ?? reasons[0] ?? abortError() + } finally { + signal?.removeEventListener('abort', abortBatch) + } +} diff --git a/chat-sdk/src/chat.ts b/chat-sdk/src/chat.ts index b13fa89173..2e2c52f3df 100644 --- a/chat-sdk/src/chat.ts +++ b/chat-sdk/src/chat.ts @@ -8,6 +8,7 @@ import { import { resolveConfig, type ResolvedConfig } from './config' import { followJob } from './follow' import { createLocalHistory, type LocalHistory } from './history' +import { uploadAttachments } from './attachments' import type { AgentStreamEvent } from './stream' import type { Chat, @@ -15,6 +16,7 @@ import type { ChatOptions, ChatState, Conversation, + SendMessageOptions, ToolInvocation } from './types' import { @@ -22,6 +24,7 @@ import { truncateTitle, errorResultMessage, extractChatAnswer, + abortError, isAbortError, isErrorResult, now, @@ -41,6 +44,12 @@ interface Turn { conversationId: string /** Id of the turn's user message; the answer is whatever follows it. */ userMessageId: string + /** The turn opened the conversation; withdrawing it closes the conversation again. */ + isNew: boolean + /** The run was asked for. Before that, a failure or a stop withdraws the turn instead of failing it. */ + started: boolean + /** Both `stop()` and the send's own rejection withdraw; only the first may. */ + withdrawn: boolean jobId?: string /** The flow job and its step jobs; a persisted answer carries one of them as `job_id`. */ jobIds?: Set @@ -101,21 +110,34 @@ class ChatImpl implements Chat { } } - sendMessage = async ( - text: string, - options: { inputs?: Record } = {} - ): Promise => { + sendMessage = async (text: string, options: SendMessageOptions = {}): Promise => { const content = text.trim() - if (!content) return + if (!content) { + // A run needs a message; files alone would otherwise be dropped without a word. + if (options.attachments?.length) throw new Error('windmill-chat: attachments need a message to go with them') + return + } if (this.#turn) { throw new Error('windmill-chat: a message is already being answered; call stop() first') } + const attachments = options.attachments ?? [] + const attachmentsInput = options.attachmentsInput + if (attachments.length > 0 && !attachmentsInput) { + throw new Error('windmill-chat: attachments need `attachmentsInput`, the flow input that takes them') + } + if (attachmentsInput && !attachmentsInput.multiple && attachments.length > 1) { + // Uploading all of them would run with the first and leave the rest stranded in storage. + throw new Error(`windmill-chat: \`${attachmentsInput.name}\` holds one file; got ${attachments.length}`) + } const isNew = this.#state.conversationId === undefined const conversationId = this.#state.conversationId ?? randomId() const turn: Turn = { controller: new AbortController(), conversationId, userMessageId: `pending-${randomId()}`, + isNew, + started: false, + withdrawn: false, streamedText: false } this.#turn = turn @@ -127,7 +149,6 @@ class ChatImpl implements Chat { const touched = { ...conversation, updatedAt: timestamp } this.#set({ conversationId, - conversations: [touched, ...this.#state.conversations.filter((c) => c.id !== conversationId)], messages: [ ...this.#state.messages, { id: turn.userMessageId, role: 'user', content, success: true, createdAt: timestamp, pending: true } @@ -135,10 +156,32 @@ class ChatImpl implements Chat { status: 'submitted', error: undefined }) - this.#rememberConversation() try { - const args = { ...this.#config.inputs, ...options.inputs, user_message: content } + const args: Record = { ...this.#config.inputs, ...options.inputs, user_message: content } + if (attachmentsInput && attachments.length > 0) { + // Uploaded with the turn already shown as submitted: the message is in the transcript + // and `stop()` can abort the upload, while a second send is refused as usual. + const uploaded = await uploadAttachments(this.#api, attachments, randomId(), turn.controller.signal) + args[attachmentsInput.name] = attachmentsInput.multiple ? uploaded : uploaded[0] + // Shown on the pending message until its server row replaces it, carrying its own. + const carried = uploaded.map((u) => ({ input: attachmentsInput.name, s3: u.s3, filename: u.filename })) + if (this.#turnActive(turn)) { + this.#set({ + messages: this.#state.messages.map((m) => (m.id === turn.userMessageId ? { ...m, attachments: carried } : m)) + }) + } + } + // Nothing may start once stop() or a conversation switch has withdrawn the turn, including + // a stop from a subscriber told of the attachments just above. + if (turn.controller.signal.aborted) throw abortError() + turn.started = true + // Listed only once the run is asked for: a send that never runs (an upload that failed + // or was stopped) then has no conversation entry to take back. + this.#set({ + conversations: [touched, ...this.#state.conversations.filter((c) => c.id !== conversationId)] + }) + this.#rememberConversation() const context = { memoryId: conversationId, conversationId, signal: turn.controller.signal } turn.jobId = this.#config.run ? await this.#config.run(args, context) @@ -152,6 +195,13 @@ class ChatImpl implements Chat { } await this.#finishTurn(turn, result, isNew) } catch (e) { + if (!turn.started) { + // Nothing ran: the message is withdrawn rather than shown as a failed turn, and the + // caller gets the reason (an upload that failed, or the AbortError of a stop()). + if (this.#turn === turn) this.#turn = undefined + this.#withdrawTurn(turn) + throw e + } // stop() and a conversation switch abort the turn and settle the state themselves. if (turn.controller.signal.aborted || isAbortError(e)) return this.#failTurn(turn, e) @@ -164,6 +214,12 @@ class ChatImpl implements Chat { const turn = this.#turn if (!turn) return this.#detachTurn() + if (!turn.started) { + // Still uploading its attachments: there is no run to cancel, and the message the + // reader took back must not stay in the transcript as sent. + this.#withdrawTurn(turn) + return + } if (this.#state.conversationId === turn.conversationId) { this.#set({ messages: finalized(this.#state.messages), status: 'idle' }) this.#persistLocal() @@ -578,6 +634,36 @@ class ChatImpl implements Chat { } } + /** + * Take back the user message of a turn that never ran. A conversation it would have opened + * was never listed (see `sendMessage`), so only the message goes, and, while it is the turn + * on screen, the busy status. A switch away mid-upload has already written the message to + * local history, so it is removed there too. + */ + #withdrawTurn(turn: Turn): void { + if (turn.withdrawn) return + turn.withdrawn = true + const id = turn.conversationId + const withoutTurn = (messages: ChatMessage[]) => messages.filter((m) => m.id !== turn.userMessageId) + if (this.#state.conversationId === id) { + const messages = withoutTurn(this.#state.messages) + // A turn started since, such as a resend right after Stop, owns the status. + const newerTurn = this.#turn !== undefined && this.#turn !== turn + if (newerTurn) { + this.#set({ messages }) + } else { + const unopened = turn.isNew && messages.length === 0 + this.#set({ messages, status: 'idle', error: undefined, ...(unopened ? { conversationId: undefined } : {}) }) + } + this.#persistLocal() + } + if (this.#state.history === 'local' && this.#state.conversationId !== id) { + const stored = withoutTurn(this.#local.getMessages(id)) + if (stored.length > 0) this.#local.saveMessages(id, stored) + else if (!this.#state.conversations.some((c) => c.id === id)) this.#local.deleteConversation(id) + } + } + #failTurn(turn: Turn, e: unknown): void { if (!this.#turnActive(turn)) return const error = toError(e) diff --git a/chat-sdk/src/index.ts b/chat-sdk/src/index.ts index aedaa8b19f..9ed2ed0b7e 100644 --- a/chat-sdk/src/index.ts +++ b/chat-sdk/src/index.ts @@ -14,7 +14,10 @@ export { export { parseStreamEvents, createStreamEventParser, type AgentStreamEvent } from './stream' export { followJob, type FollowEvent } from './follow' export { extractChatAnswer, conversationIdFor } from './utils' +export { storedAttachmentName, uploadAttachments, CHAT_UPLOADS_PREFIX, type UploadedAttachment } from './attachments' export type { + AttachmentsInput, + AttachmentUpload, Chat, ChatAttachment, ChatMessage, @@ -25,6 +28,7 @@ export type { Conversation, FetchLike, HistoryMode, + SendMessageOptions, StorageLike, TokenSource, ToolInvocation diff --git a/chat-sdk/src/types.ts b/chat-sdk/src/types.ts index 87fd5a7b0a..99e16fe304 100644 --- a/chat-sdk/src/types.ts +++ b/chat-sdk/src/types.ts @@ -127,12 +127,45 @@ export interface ChatOptions { onError?: (error: Error, turn: { conversationId: string; jobId?: string }) => void } +/** A file sent with a message. It is uploaded to the workspace's object storage before the run starts. */ +export interface AttachmentUpload { + /** Kept as the last segment of the stored key, its extension corrected to the media type for PNG, JPEG and PDF. */ + name: string + /** The bytes: a Blob, or a `data:` URL of them. */ + data: Blob | string + /** The file's media type. Defaults to the Blob's own type, or the data URL's. */ + mediaType?: string +} + +/** The flow input the uploaded attachments are handed to: an `s3object` (`multiple: false`) or an `s3object[]`. */ +export interface AttachmentsInput { + name: string + multiple: boolean +} + +export interface SendMessageOptions { + /** Extra flow inputs for this message, on top of `ChatOptions.inputs`. */ + inputs?: Record + /** + * Files to upload and hand to the flow as `{ s3, filename }` objects in `attachmentsInput`, + * the way an AI agent step reads `user_attachments`. A failed upload rejects `sendMessage` + * and the run never starts; `stop()` during the upload does the same with an `AbortError`. + */ + attachments?: AttachmentUpload[] + /** Required with `attachments`, which also need message text. With `multiple: false`, more than one attachment is refused before anything uploads. */ + attachmentsInput?: AttachmentsInput +} + export interface Chat { getState(): ChatState /** Calls `listener` now and on every change; returns the unsubscribe function (Svelte store contract). */ subscribe(listener: (state: ChatState) => void): () => void - /** Sends a message in the current conversation, starting one when there is none. Resolves when the answer is complete. */ - sendMessage(text: string, options?: { inputs?: Record }): Promise + /** + * Sends a message in the current conversation, starting one when there is none. Resolves + * when the answer is complete. Rejects when the message could not be sent at all — a turn + * already running, an attachment that failed to upload — without touching the transcript. + */ + sendMessage(text: string, options?: SendMessageOptions): Promise /** Stops following the answer and asks Windmill to cancel the run. */ stop(): Promise newConversation(): void diff --git a/chat-sdk/test/attachments.test.ts b/chat-sdk/test/attachments.test.ts new file mode 100644 index 0000000000..9c2f577567 --- /dev/null +++ b/chat-sdk/test/attachments.test.ts @@ -0,0 +1,444 @@ +import { describe, expect, test } from 'bun:test' +import { WindmillChatApi } from '../src/api' +import { storedAttachmentName, uploadAttachments } from '../src/attachments' +import { createChat } from '../src/chat' +import type { ChatOptions } from '../src/types' +import { abortError } from '../src/utils' +import { fetchMock, json, memoryStorage, sse, text, type RecordedCall, type Route } from './support' + +const BASE = 'http://wm.test' +const FLOW = 'f/chat/agent' +const UPLOAD_PATH = '/api/w/ws/job_helpers/upload_s3_file' + +const run: Route = (c) => + c.method === 'POST' && c.url.pathname === `/api/w/ws/jobs/run/f/${FLOW}` + ? text('job-1') + : undefined + +/** Stores under the key it was asked to, like the server with a `file_key`. */ +const upload: Route = (c) => + c.url.pathname === UPLOAD_PATH + ? json({ file_key: c.url.searchParams.get('file_key') }) + : undefined + +const answer: Route = (c) => + c.url.pathname === '/api/w/ws/jobs_u/getupdate_sse/job-1' + ? sse([ + { + type: 'update', + completed: true, + only_result: { output: 'ok', messages: [] } + } + ]) + : undefined + +function options(fetch: ChatOptions['fetch']): ChatOptions { + return { + flowPath: FLOW, + baseUrl: BASE, + workspace: 'ws', + token: 'tok', + fetch, + storage: memoryStorage() + } +} + +const uploads = (calls: RecordedCall[]) => calls.filter((c) => c.url.pathname === UPLOAD_PATH) +const runs = (calls: RecordedCall[]) => + calls.filter((c) => c.url.pathname.startsWith('/api/w/ws/jobs/run/')) + +const png = new Blob([new Uint8Array([0x89, 0x50, 0x4e, 0x47])], { + type: 'image/png' +}) +const pdf = new Blob(['%PDF-1.7'], { type: 'application/pdf' }) + +describe('storedAttachmentName', () => { + // The worker reads the media type from the key's extension, so it has to match the bytes. + test('renames a re-encoded image and gives a bare name its extension', () => { + expect(storedAttachmentName('photo.webp', 'image/png')).toBe('photo.png') + expect(storedAttachmentName('holiday.png', 'image/jpeg')).toBe('holiday.jpg') + expect(storedAttachmentName('contract', 'application/pdf')).toBe('contract.pdf') + expect(storedAttachmentName('report.2026.final.webp', 'image/png')).toBe( + 'report.2026.final.png' + ) + }) + + test('leaves a type it does not know alone', () => { + expect(storedAttachmentName('notes.csv', 'text/csv')).toBe('notes.csv') + }) +}) + +describe('sendMessage with attachments', () => { + test('uploads each file under the turn prefix and hands the list to the input', async () => { + const { fetch, calls } = fetchMock(upload, run, answer) + const chat = createChat(options(fetch)) + + await chat.sendMessage('read these', { + inputs: { locale: 'fr' }, + attachments: [ + { name: 'photo.webp', data: png }, + { name: 'contract', data: pdf }, + // A data URL is decoded to its bytes; the mediaType names what they are. + { + name: 'photo.webp', + data: `data:image/png;base64,${btoa('\x89PNG')}` + } + ], + attachmentsInput: { name: 'files', multiple: true } + }) + + const keys = uploads(calls).map((c) => c.url.searchParams.get('file_key')!) + expect(keys).toHaveLength(3) + const prefix = keys[0].split('/').slice(0, 3).join('/') + expect(prefix).toMatch(/^windmill_uploads\/chat\/[0-9a-f-]{36}$/) + expect(keys).toEqual([ + `${prefix}/0/photo.png`, + `${prefix}/1/contract.pdf`, + `${prefix}/2/photo.png` + ]) + expect(uploads(calls).map((c) => c.url.searchParams.get('content_type'))).toEqual([ + 'image/png', + 'application/pdf', + 'image/png' + ]) + expect(uploads(calls).map((c) => c.headers['content-type'])).toEqual([ + 'image/png', + 'application/pdf', + 'image/png' + ]) + expect(new Uint8Array(await uploads(calls)[2].raw!.arrayBuffer())).toEqual( + new Uint8Array([0x89, 0x50, 0x4e, 0x47]) + ) + + expect(runs(calls)[0].body).toEqual({ + locale: 'fr', + user_message: 'read these', + files: [ + { s3: `${prefix}/0/photo.png`, filename: 'photo.png' }, + { s3: `${prefix}/1/contract.pdf`, filename: 'contract.pdf' }, + { s3: `${prefix}/2/photo.png`, filename: 'photo.png' } + ] + }) + expect(chat.getState().status).toBe('idle') + }) + + test('hands a single object to an input that holds one file', async () => { + const { fetch, calls } = fetchMock(upload, run, answer) + const chat = createChat(options(fetch)) + await chat.sendMessage('read this', { + attachments: [{ name: 'contract.pdf', data: pdf }], + attachmentsInput: { name: 'file', multiple: false } + }) + const body = runs(calls)[0].body as Record + expect(body.file).toEqual({ + s3: expect.stringMatching(/\/0\/contract\.pdf$/), + filename: 'contract.pdf' + }) + }) + + test('refuses several files for an input that holds one, before uploading any', async () => { + const { fetch, calls } = fetchMock(upload, run, answer) + const chat = createChat(options(fetch)) + await expect( + chat.sendMessage('read these', { + attachments: [ + { name: 'a.pdf', data: pdf }, + { name: 'b.png', data: png } + ], + attachmentsInput: { name: 'file', multiple: false } + }) + ).rejects.toThrow('holds one file') + expect(calls).toHaveLength(0) + expect(chat.getState().messages).toEqual([]) + }) + + test('refuses attachments without message text, before uploading', async () => { + const { fetch, calls } = fetchMock(upload, run, answer) + const chat = createChat(options(fetch)) + await expect( + chat.sendMessage(' ', { + attachments: [{ name: 'a.pdf', data: pdf }], + attachmentsInput: { name: 'files', multiple: true } + }) + ).rejects.toThrow('need a message') + expect(calls).toHaveLength(0) + }) + + test('refuses attachments without an input to put them in', async () => { + const { fetch, calls } = fetchMock(upload, run, answer) + const chat = createChat(options(fetch)) + await expect( + chat.sendMessage('hi', { attachments: [{ name: 'a.pdf', data: pdf }] }) + ).rejects.toThrow('attachmentsInput') + expect(calls).toHaveLength(0) + }) + + test('a failed upload rejects without a run, and withdraws the message', async () => { + const { fetch, calls } = fetchMock( + (c) => (c.url.pathname === UPLOAD_PATH ? text('no object storage', 500) : undefined), + run, + answer + ) + const chat = createChat(options(fetch)) + const statuses: string[] = [] + chat.subscribe((s) => statuses.push(s.status)) + + await expect( + chat.sendMessage('read this', { + attachments: [{ name: 'contract.pdf', data: pdf }], + attachmentsInput: { name: 'files', multiple: true } + }) + ).rejects.toThrow('no object storage') + + expect(runs(calls)).toHaveLength(0) + // Shown as submitted while uploading, then withdrawn whole: no message, no conversation. + expect(statuses).toContain('submitted') + const state = chat.getState() + expect(state.status).toBe('idle') + expect(state.messages).toEqual([]) + expect(state.conversationId).toBeUndefined() + expect(state.conversations).toEqual([]) + // The chat is free for the next message. + await chat.sendMessage('plain') + expect(runs(calls)).toHaveLength(1) + }) + + test('stop() during the upload aborts it and withdraws the message', async () => { + const { fetch, calls } = fetchMock( + (c) => + c.url.pathname === UPLOAD_PATH + ? new Promise((_, reject) => + c.signal!.addEventListener('abort', () => reject(abortError())) + ) + : undefined, + run, + answer + ) + const chat = createChat(options(fetch)) + const sending = chat.sendMessage('read this', { + attachments: [{ name: 'contract.pdf', data: pdf }], + attachmentsInput: { name: 'files', multiple: true } + }) + await new Promise((r) => setTimeout(r, 0)) + expect(chat.getState().status).toBe('submitted') + await chat.stop() + await expect(sending).rejects.toMatchObject({ name: 'AbortError' }) + expect(runs(calls)).toHaveLength(0) + expect(chat.getState()).toMatchObject({ + status: 'idle', + messages: [], + conversationId: undefined + }) + }) + + test('a switch away mid-upload leaves no conversation behind', async () => { + const storage = memoryStorage() + const { fetch, calls } = fetchMock( + (c) => + c.url.pathname === UPLOAD_PATH + ? new Promise((_, reject) => + c.signal!.addEventListener('abort', () => reject(abortError())) + ) + : undefined, + run, + answer + ) + const chat = createChat({ ...options(fetch), storage }) + const sending = chat.sendMessage('never runs', { + attachments: [{ name: 'contract.pdf', data: pdf }], + attachmentsInput: { name: 'files', multiple: true } + }) + await new Promise((r) => setTimeout(r, 0)) + const opened = chat.getState().conversationId! + chat.newConversation() + await expect(sending).rejects.toMatchObject({ name: 'AbortError' }) + + expect(runs(calls)).toHaveLength(0) + expect(chat.getState().conversations.map((c) => c.id)).not.toContain(opened) + const reloaded = createChat({ ...options(fetch), storage }) + expect((await reloaded.loadConversations()).map((c) => c.id)).not.toContain(opened) + }) + + test('a failed upload aborts the rest of its batch and deletes nothing', async () => { + let first: (r: Response) => void = () => {} + const { fetch, calls } = fetchMock( + (c) => { + if (c.url.pathname !== UPLOAD_PATH) return undefined + const key = c.url.searchParams.get('file_key')! + // The first file lands after the second has already failed. + if (key.includes('/0/')) return new Promise((resolve) => (first = resolve)) + setTimeout(() => first(json({ file_key: keys()[0] })), 5) + return text('quota exceeded', 507) + }, + run, + answer + ) + const keys = () => uploads(calls).map((c) => c.url.searchParams.get('file_key')!) + const chat = createChat(options(fetch)) + await expect( + chat.sendMessage('read these', { + attachments: [ + { name: 'a.pdf', data: pdf }, + { name: 'b.png', data: png } + ], + attachmentsInput: { name: 'files', multiple: true } + }) + ).rejects.toThrow('quota exceeded') + // The upload still in flight when the other failed was told to stop. + expect(uploads(calls)[0].signal?.aborted).toBe(true) + expect(calls.filter((c) => c.method === 'DELETE')).toEqual([]) + expect(runs(calls)).toHaveLength(0) + }) + + test('a send made right after stop() is not reset by the stopped upload', async () => { + let releaseRun: (r: Response) => void = () => {} + const { fetch, calls } = fetchMock( + (c) => + c.url.pathname === UPLOAD_PATH + ? new Promise((_, reject) => + c.signal!.addEventListener('abort', () => reject(abortError())) + ) + : undefined, + (c) => + c.method === 'POST' && c.url.pathname === `/api/w/ws/jobs/run/f/${FLOW}` + ? new Promise((resolve) => (releaseRun = resolve)) + : undefined, + answer + ) + const chat = createChat(options(fetch)) + // An existing conversation, so the stopped turn and the next one share it. + const first = chat.sendMessage('first') + await new Promise((r) => setTimeout(r, 0)) + releaseRun(text('job-1')) + await first + const stopped = chat.sendMessage('with a file', { + attachments: [{ name: 'a.pdf', data: pdf }], + attachmentsInput: { name: 'files', multiple: true } + }) + await new Promise((r) => setTimeout(r, 0)) + void chat.stop() + const next = chat.sendMessage('right after') + await expect(stopped).rejects.toMatchObject({ name: 'AbortError' }) + await new Promise((r) => setTimeout(r, 0)) + expect(chat.getState().status).toBe('submitted') + expect(chat.getState().messages.map((m) => m.content)).toContain('right after') + expect(chat.getState().messages.map((m) => m.content)).not.toContain('with a file') + releaseRun(text('job-1')) + await next + expect(runs(calls)).toHaveLength(2) + }) + + test('a conversation is listed only once its run starts', async () => { + let failUpload: (r: Response) => void = () => {} + const { fetch } = fetchMock( + (c) => + c.url.pathname === UPLOAD_PATH + ? new Promise((resolve) => (failUpload = resolve)) + : undefined, + run, + answer + ) + const chat = createChat(options(fetch)) + const sending = chat.sendMessage('read this', { + attachments: [{ name: 'a.pdf', data: pdf }], + attachmentsInput: { name: 'files', multiple: true } + }) + await new Promise((r) => setTimeout(r, 0)) + expect(chat.getState()).toMatchObject({ status: 'submitted', conversations: [] }) + failUpload(text('boom', 500)) + await expect(sending).rejects.toThrow('boom') + expect(chat.getState().conversations).toEqual([]) + }) + + test('an already aborted signal uploads nothing', async () => { + const { fetch, calls } = fetchMock(upload) + const api = new WindmillChatApi({ baseUrl: BASE, workspace: 'ws', token: 'tok', fetch }) + const controller = new AbortController() + controller.abort() + await expect( + uploadAttachments(api, [{ name: 'a.pdf', data: pdf }], 'turn', controller.signal) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(calls).toHaveLength(0) + }) + + test('the pending user message carries its uploaded files before the run returns', async () => { + let releaseRun: (r: Response) => void = () => {} + const { fetch, calls } = fetchMock( + upload, + (c) => + c.method === 'POST' && c.url.pathname === `/api/w/ws/jobs/run/f/${FLOW}` + ? new Promise((resolve) => (releaseRun = resolve)) + : undefined, + answer + ) + const chat = createChat(options(fetch)) + const sending = chat.sendMessage('read these', { + attachments: [ + { name: 'photo.webp', data: png }, + { name: 'contract', data: pdf } + ], + attachmentsInput: { name: 'files', multiple: true } + }) + while (runs(calls).length === 0) await new Promise((r) => setTimeout(r, 1)) + const keys = uploads(calls).map((c) => c.url.searchParams.get('file_key')!) + const pending = chat.getState().messages.find((m) => m.role === 'user')! + expect(pending.pending).toBe(true) + expect(pending.attachments).toEqual([ + { input: 'files', s3: keys[0], filename: 'photo.png' }, + { input: 'files', s3: keys[1], filename: 'contract.pdf' } + ]) + releaseRun(text('job-1')) + await sending + }) + + test('an explicit mediaType wins over the type a data URL declares', async () => { + const { fetch, calls } = fetchMock(upload, run, answer) + const chat = createChat(options(fetch)) + await chat.sendMessage('read this', { + attachments: [ + { + name: 'contract', + data: `data:application/octet-stream;base64,${btoa('%PDF')}`, + mediaType: 'application/pdf' + } + ], + attachmentsInput: { name: 'files', multiple: true } + }) + const call = uploads(calls)[0] + expect(call.url.searchParams.get('file_key')).toMatch(/\/0\/contract\.pdf$/) + expect(call.url.searchParams.get('content_type')).toBe('application/pdf') + }) + + test('stop() after the uploads land but before the run starts runs nothing', async () => { + const { fetch, calls } = fetchMock(upload, run, answer) + const chat = createChat(options(fetch)) + const sending = chat.sendMessage('read this', { + attachments: [{ name: 'contract.pdf', data: pdf }], + attachmentsInput: { name: 'files', multiple: true } + }) + // The upload responds at once; Stop lands before the send resumes after it. + while (uploads(calls).length === 0) await Promise.resolve() + await chat.stop() + await expect(sending).rejects.toMatchObject({ name: 'AbortError' }) + expect(runs(calls)).toHaveLength(0) + expect(calls.filter((c) => c.method === 'DELETE')).toEqual([]) + expect(chat.getState()).toMatchObject({ status: 'idle', messages: [], conversations: [] }) + }) + + test('a subscriber stopping when the attachments appear runs nothing', async () => { + const { fetch, calls } = fetchMock(upload, run, answer) + const chat = createChat(options(fetch)) + chat.subscribe((s) => { + if (s.messages.some((m) => m.attachments)) void chat.stop() + }) + await expect( + chat.sendMessage('read this', { + attachments: [{ name: 'contract.pdf', data: pdf }], + attachmentsInput: { name: 'files', multiple: true } + }) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(runs(calls)).toHaveLength(0) + expect(calls.filter((c) => c.method === 'DELETE')).toEqual([]) + expect(chat.getState()).toMatchObject({ status: 'idle', messages: [], conversations: [] }) + }) +}) diff --git a/chat-sdk/test/support.ts b/chat-sdk/test/support.ts index c794e50f3d..e198f19355 100644 --- a/chat-sdk/test/support.ts +++ b/chat-sdk/test/support.ts @@ -5,6 +5,9 @@ export interface RecordedCall { url: URL headers: Record body: unknown + /** A body sent as is rather than as JSON (an upload). */ + raw?: Blob + signal?: AbortSignal } export type Route = (call: RecordedCall) => Response | Promise | undefined @@ -20,7 +23,9 @@ export function fetchMock(...routes: Route[]): { fetch: FetchLike; calls: Record headers: Object.fromEntries( Object.entries((init?.headers as Record) ?? {}).map(([k, v]) => [k.toLowerCase(), v]) ), - body: typeof init?.body === 'string' ? JSON.parse(init.body) : undefined + body: typeof init?.body === 'string' ? JSON.parse(init.body) : undefined, + raw: init?.body instanceof Blob ? init.body : undefined, + signal: init?.signal ?? undefined } calls.push(call) for (const route of routes) { diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 1ad44a8df9..a399ec1b85 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.813.0"; +export const VERSION = "1.814.0"; diff --git a/cli/src/core/settings.ts b/cli/src/core/settings.ts index 4c95090bcf..4174ec0358 100644 --- a/cli/src/core/settings.ts +++ b/cli/src/core/settings.ts @@ -393,10 +393,17 @@ export async function pushWorkspaceSettings( if (!deepEqual(localSettings.datatable, settings.datatable)) { log.debug(`Updating datatable config...`); - await wmill.editDataTableConfig({ + const { stranded_references } = await wmill.editDataTableConfig({ workspace, requestBody: { settings: localSettings.datatable ?? { datatables: {} } }, }); + if (stranded_references?.length) { + log.warn( + `Removed data tables governed data tables in other workspaces, which no longer resolve: ${stranded_references + .map((r) => `${r.workspace_id}/${r.datatable}`) + .join(", ")}. A superadmin can point them somewhere else.`, + ); + } } if (localSettings.slack_command_script != settings.slack_command_script) { diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 035c25a574..6da10b2504 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -1110,6 +1110,8 @@ parseS3Object(s3Object: S3Object): S3ObjectRecord /** * Create a SQL template function for PostgreSQL/datatable queries * @param name - Database/datatable name (default: "main") + * @param opts.role - Connect as this data table role instead of the data table's default one. + * Only meaningful on a data table under roles, and only for a role you are a tenant of. * @returns SQL template function for building parameterized queries * @example * let sql = wmill.datatable() @@ -1119,8 +1121,11 @@ parseS3Object(s3Object: S3Object): S3ObjectRecord * SELECT * FROM friends * WHERE name = \${name} AND age = \${age}::int * \`.fetch() + * @example + * // Read through a restricted role + * let sql = wmill.datatable("main", { role: "analytics" }) */ -datatable(name: string = "main"): DatatableSqlTemplateFunction +datatable(name: string = "main", opts?: DatatableOptions): DatatableSqlTemplateFunction /** * Create a SQL template function for DuckDB/ducklake queries @@ -1901,6 +1906,8 @@ parseS3Object(s3Object: S3Object): S3ObjectRecord /** * Create a SQL template function for PostgreSQL/datatable queries * @param name - Database/datatable name (default: "main") + * @param opts.role - Connect as this data table role instead of the data table's default one. + * Only meaningful on a data table under roles, and only for a role you are a tenant of. * @returns SQL template function for building parameterized queries * @example * let sql = wmill.datatable() @@ -1910,8 +1917,11 @@ parseS3Object(s3Object: S3Object): S3ObjectRecord * SELECT * FROM friends * WHERE name = \${name} AND age = \${age}::int * \`.fetch() + * @example + * // Read through a restricted role + * let sql = wmill.datatable("main", { role: "analytics" }) */ -datatable(name: string = "main"): DatatableSqlTemplateFunction +datatable(name: string = "main", opts?: DatatableOptions): DatatableSqlTemplateFunction /** * Create a SQL template function for DuckDB/ducklake queries @@ -2786,6 +2796,8 @@ parseS3Object(s3Object: S3Object): S3ObjectRecord /** * Create a SQL template function for PostgreSQL/datatable queries * @param name - Database/datatable name (default: "main") + * @param opts.role - Connect as this data table role instead of the data table's default one. + * Only meaningful on a data table under roles, and only for a role you are a tenant of. * @returns SQL template function for building parameterized queries * @example * let sql = wmill.datatable() @@ -2795,8 +2807,11 @@ parseS3Object(s3Object: S3Object): S3ObjectRecord * SELECT * FROM friends * WHERE name = \${name} AND age = \${age}::int * \`.fetch() + * @example + * // Read through a restricted role + * let sql = wmill.datatable("main", { role: "analytics" }) */ -datatable(name: string = "main"): DatatableSqlTemplateFunction +datatable(name: string = "main", opts?: DatatableOptions): DatatableSqlTemplateFunction /** * Create a SQL template function for DuckDB/ducklake queries @@ -4405,10 +4420,13 @@ def send_teams_message(conversation_id: str, text: str, success: bool = True, ca # # Args: # name: Database name (default: "main") +# role: Connect as this data table role instead of the data table's default one. +# Only meaningful on a data table under roles, and only for a role you are a +# tenant of. # # Returns: # DataTableClient instance -def datatable(name: str = 'main') +def datatable(name: str = 'main', *, role: Optional[str] = None) # Get a DuckLake client for DuckDB queries. # @@ -4626,7 +4644,7 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # # @task(retry={"attempts": 3, "delay": 30, "multiplier": 2}) # async def call_api(payload: dict): ... -def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) +def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill script. # @@ -4639,7 +4657,7 @@ def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, ti # @workflow # async def main(): # data = await extract(url="https://...") -def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) +def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill flow. # @@ -4652,7 +4670,7 @@ def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = N # @workflow # async def main(): # result = await pipeline(input=data) -def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) +def task_flow(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Decorator marking an async function as a workflow-as-code entry point. # @@ -4717,7 +4735,7 @@ async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_ # ... # # results = await parallel(items, process, concurrency=5) -async def parallel(items, fn, concurrency: Optional[int] = None) +async def parallel(items, fn, *, concurrency: Optional[int] = None) # Commit Kafka offsets for a trigger with auto_commit disabled. # diff --git a/flake.nix b/flake.nix index 9062fdb4fd..f005644e3d 100644 --- a/flake.nix +++ b/flake.nix @@ -65,7 +65,10 @@ # Misc libtool - postgresql + # Must not trail the server the dev database runs (postgres:18): pg_dump refuses a + # server newer than itself by a major version, which takes out every data table + # export, clone and fork-with-data. + postgresql_18 # Build tooling pkg-config diff --git a/frontend/package-lock.json b/frontend/package-lock.json index e95243b8d0..935a1aa475 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.813.0", + "version": "1.814.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.813.0", + "version": "1.814.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 93369b167e..eedca8f2c8 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.813.0", + "version": "1.814.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/frontend/src/lib/components/FlowPreviewContent.svelte b/frontend/src/lib/components/FlowPreviewContent.svelte index 79c990a063..9ccac1d8c6 100644 --- a/frontend/src/lib/components/FlowPreviewContent.svelte +++ b/frontend/src/lib/components/FlowPreviewContent.svelte @@ -473,7 +473,9 @@ return jobId ?? '' }} conversationKind="test" + frame="boxed" path={$pathStore} + identity={$initialPathStore || fakeInitialPath} inputSchema={flowStore.val.schema} flowModules={flowStore.val.value?.modules} /> @@ -560,7 +562,13 @@ {/if} {/if} -
+ +
{#if flowHasChanged()}
ws) + const s3Storage = useWorkspaceStorageConfigured(() => ws) let keys: string[] = $state([]) $effect(() => { diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index f38c0d3892..6ceab37872 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -1093,7 +1093,8 @@ migrations are set up and used, how often an empty workspace home is seen, how often the home page’s create menu and hub-project picker are opened and from which entry point, the name of any public hub project imported from the home page and how far that - import got, and whether a pre-approved trial offer was opened, last 30 days)
  • feature adoption (counts of which flow, script, trigger, worker and data table @@ -1159,7 +1160,8 @@ migrations are set up and used, how often an empty workspace home is seen, how often the home page’s create menu and hub-project picker are opened and from which entry point, the name of any public hub project imported from the home page and how far that - import got, and whether a pre-approved trial offer was opened, last 30 days)
  • feature adoption (counts of which flow, script, trigger, worker and data table diff --git a/frontend/src/lib/components/Login.svelte b/frontend/src/lib/components/Login.svelte index 44ba7e4e15..66c13c4ea3 100644 --- a/frontend/src/lib/components/Login.svelte +++ b/frontend/src/lib/components/Login.svelte @@ -396,7 +396,8 @@ if (loginsResult.status === 'fulfilled') { logins = loginsResult.value.oauth.map((login) => ({ type: login.type, - displayName: login.display_name || login.type + displayName: + login.display_name || providers.find((p) => p.type === login.type)?.name || login.type })) saml = loginsResult.value.saml autoLogin = loginsResult.value.auto_login diff --git a/frontend/src/lib/components/ScrollFade.svelte b/frontend/src/lib/components/ScrollFade.svelte new file mode 100644 index 0000000000..8726ea95ab --- /dev/null +++ b/frontend/src/lib/components/ScrollFade.svelte @@ -0,0 +1,66 @@ + + + diff --git a/frontend/src/lib/components/SqlRepl.svelte b/frontend/src/lib/components/SqlRepl.svelte index 98f79b39bc..d6c2d288e6 100644 --- a/frontend/src/lib/components/SqlRepl.svelte +++ b/frontend/src/lib/components/SqlRepl.svelte @@ -105,14 +105,21 @@ // We don't always put the fix by default for row ordering concerns let transformedCode = code if (doPostgresRowToJsonFix) { - transformedCode = statements - .map((statement) => { - if (READ_OPS.some((op) => statement.trim().toUpperCase().startsWith(op))) { - return `SELECT row_to_json(__t__) FROM (${statement}) __t__` - } - return statement - }) - .join(';') + // Rebuilt from the pruned statements, which drops the leading comment block — and + // with it the `-- role ` annotation that decides which login the query runs + // as. Carry it over, or the retry connects as the data table's default role and a + // query the first attempt was denied succeeds on the second. + const leadingAnnotations = code.match(/^(?:[^\S\n]*\n|[^\S\n]*--[^\n]*\n)*/)?.[0] ?? '' + transformedCode = + leadingAnnotations + + statements + .map((statement) => { + if (READ_OPS.some((op) => statement.trim().toUpperCase().startsWith(op))) { + return `SELECT row_to_json(__t__) FROM (${statement}) __t__` + } + return statement + }) + .join(';') } const dbArg = getDatabaseArg(input) diff --git a/frontend/src/lib/components/WorkspaceItemDrillPicker.svelte b/frontend/src/lib/components/WorkspaceItemDrillPicker.svelte index d80f282c02..ec285ce4dd 100644 --- a/frontend/src/lib/components/WorkspaceItemDrillPicker.svelte +++ b/frontend/src/lib/components/WorkspaceItemDrillPicker.svelte @@ -5,9 +5,10 @@ the workspace-specific public API (kinds, scope = `{ kind, dir? }`, currentItem, leaf/branch icons) so callers (BreadcrumbSegment, EditorHeader) don't need to know about the generic tree model underneath. -Surfaces AI-created localStorage drafts (via `listGlobalDrafts`) as extra -items alongside the backend-loaded list, so chat-scaffolded scripts/flows/ -apps that haven't been deployed yet are still navigable. Gated on +Surfaces the session's drafts (via `listGlobalDrafts`: backend draft rows +overlaid with live editor cells) as extra items alongside the backend-loaded +list, so drafts the listing does not show yet (unsaved cells, a rename typed +in a live editor) are still navigable. Gated on `isGlobalAiEnabled()` — without sessions, the only UserDrafts present are standalone editor autosaves and surfacing those in the breadcrumb picker would be surprising. diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index 82dcbc2dcd..b7f662b148 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -35,6 +35,7 @@ import ChatQuickActions from './ChatQuickActions.svelte' import ContextUsageIndicator from './ContextUsageIndicator.svelte' import AIChatModelSettings from './AIChatModelSettings.svelte' + import ScrollFade from '$lib/components/ScrollFade.svelte' import AssistantSettingsModal from './AssistantSettingsModal.svelte' import { SkillsMenu } from './skills/skillsMenu.svelte' import { McpMenu } from '$lib/components/mcp/mcpMenu.svelte' @@ -361,7 +362,15 @@ chatHost.mode === AIMode.SCRIPT || chatHost.mode === AIMode.FLOW || chatHost.mode === AIMode.APP ) - const canAttachFiles = $derived(chatHost.supportsMessageAttachments && !disabled) + // Why attaching is off, when this chat takes attachments but cannot right now. The `+` is + // kept and disabled rather than dropped: the input is the composer's either way, so the + // reader has to be able to see here why nothing can be attached. + const attachmentsOffReason = $derived( + chatHost.supportsMessageAttachments ? chatHost.attachmentsUnavailableReason : undefined + ) + const canAttachFiles = $derived( + chatHost.supportsMessageAttachments && !disabled && !attachmentsOffReason + ) // Folders are linked as session-wide assets, which only a host that reads files in // the browser can do — a host running the turn server-side takes attachments only. const canLinkFolders = $derived(chatHost.supportsLinkedFolders && !disabled) @@ -430,24 +439,32 @@ return Array.from(e.dataTransfer?.types ?? []).includes('Files') } + // A drop is claimed while attaching is off for a stated reason, too: the browser would + // otherwise navigate to the dropped file, and the reader is owed the reason instead. + const panelTakesDrops = $derived(canAttachFiles || attachmentsOffReason !== undefined) + function onPanelDragEnter(e: DragEvent) { - if (!canAttachFiles || !dragHasFiles(e)) return + if (!panelTakesDrops || !dragHasFiles(e)) return e.preventDefault() dragDepth++ } function onPanelDragOver(e: DragEvent) { - if (!canAttachFiles || !dragHasFiles(e)) return + if (!panelTakesDrops || !dragHasFiles(e)) return e.preventDefault() if (e.dataTransfer) e.dataTransfer.dropEffect = 'copy' } function onPanelDragLeave(_e: DragEvent) { - if (!canAttachFiles) return + if (!panelTakesDrops) return dragDepth = Math.max(0, dragDepth - 1) } async function onPanelDrop(e: DragEvent) { dragDepth = 0 - if (!canAttachFiles || !dragHasFiles(e)) return + if (!panelTakesDrops || !dragHasFiles(e)) return e.preventDefault() + if (attachmentsOffReason) { + sendUserToast(attachmentsOffReason, true) + return + } const dt = e.dataTransfer if (!dt) return // Images and loose text files attach to the message; folders link as session @@ -484,9 +501,8 @@ handles.length === 0 ? flatFiles : await Promise.all(handles.filter(isFileHandle).map((h) => h.getFile())) - // Loose text files attach to the message, like images. - const textFiles = looseFiles.filter((f) => !isImageFile(f)) - if (textFiles.length > 0) await aiChatInput?.addTextFiles(textFiles) + // Loose files attach to the message, like images. + await attachNonImageFiles(looseFiles.filter((f) => !isImageFile(f))) // Folders link as a live handle. const dirs = handles.filter(isDirectoryHandle) if (dirs.length > 0 && !canLinkFolders) { @@ -523,24 +539,31 @@ if (canLinkFolders) await handleAddFiles(folderEntries) else sendUserToast('Folders cannot be attached in this chat — drop individual files.', true) } - if (topLevelText.length > 0) await aiChatInput?.addTextFiles(topLevelText) + await attachNonImageFiles(topLevelText) } } async function onFileInputChange(e: Event) { const input = e.currentTarget as HTMLInputElement if (input.files && input.files.length > 0) { - const picked = Array.from(input.files) - const imageFiles = picked.filter(isImageFile) - const textFiles = picked.filter((f) => !isImageFile(f)) - // Reserved before the text work is awaited — see onPanelDrop. - const imageWork = imageFiles.length > 0 ? aiChatInput?.addImages(imageFiles) : undefined - if (textFiles.length > 0) await aiChatInput?.addTextFiles(textFiles) - await imageWork + await attachPickedFiles(Array.from(input.files)) } input.value = '' // allow re-selecting the same file } + async function attachNonImageFiles(files: File[]) { + await aiChatInput?.addNonImageFiles(files) + } + + async function attachPickedFiles(picked: File[]) { + const imageFiles = picked.filter(isImageFile) + const others = picked.filter((f) => !isImageFile(f)) + // Reserved before the other work is awaited — see onPanelDrop. + const imageWork = imageFiles.length > 0 ? aiChatInput?.addImages(imageFiles) : undefined + await attachNonImageFiles(others) + await imageWork + } + function onFolderInputChange(e: Event) { const input = e.currentTarget as HTMLInputElement // webkitdirectory files carry webkitRelativePath (`folder/sub/file`); addFiles groups @@ -569,6 +592,15 @@ // The typing-dots indicator implies the AI is busy, which is misleading while // the loop is parked on the user; surface a text pill instead so users know to // act on the tool above. + // A step name hangs its icon in the column's left padding (see AssistantMessage), so a + // transcript carrying one widens the padding, on both sides to keep the column centred. + const agentGutter = $derived(messages.some((m) => m.role === 'assistant' && m.stepName)) + const columnClass = $derived( + wideLayout + ? `w-full max-w-3xl mx-auto ${agentGutter ? 'px-8' : 'px-7'}` + : `w-full max-w-2xl mx-auto ${agentGutter ? 'px-8' : 'px-3'}` + ) + const waitingForUserAction = $derived(chatHost.loading && !!pendingUserAction(messages)) // Gated on `loading` because a card restored from history still looks parked: @@ -622,6 +654,7 @@ const showFooterLeftControls = $derived( !footerMessageShown && (canAttachFiles || + attachmentsOffReason !== undefined || showContextPicker || showAutonomyModeSelector || (chatHost.mode === AIMode.SCRIPT && hasDiff)) @@ -800,12 +833,7 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> bind:this={scrollElement} onscroll={onScroll} > -
    +
    {#each messages as message, messageIndex (messageIndex)} {/if}
    + + {#if showScrollToLatest}
    {/if} -
    + +
    {#if showFlowPendingActionControls}
    + {/if} +
    + {#if editable && row.id !== ADMIN_ROLE} + removeRole(row.id)} /> + {/if} +
    + +
    + {/each} +
    + + {#if editable && unusedRoles.length > 0} +
    + + dataTable.database.resource_type, - (resource_type) => { - dataTable.database = { - resource_type, - resource_path: - resource_type === 'instance' ? defaultInstanceDbName() : undefined + {#if dataTable.reference} +
    + Governed by + {dataTable.reference.workspace_id} + / + {dataTable.reference.datatable} + + This fork uses its parent's data table rather than a copy of it, so the database and + its roles are decided in that workspace. + +
    + {:else} +
    +
    + {#if dataTable.database.resource_type === 'instance'} + + Use Windmill's PostgreSQL instance + + {/if} +