From 64dffe6106ad6a55b61a423c855a4b5b0cef533e Mon Sep 17 00:00:00 2001 From: Guilhem Date: Wed, 16 Sep 2026 16:12:21 +0200 Subject: [PATCH 01/77] feat: show flow step detail inside the graph tab on narrow detail layouts (#11168) * feat: show flow step detail as a page inside the graph tab on narrow detail layouts * fix: pad the detail step tab, wrapped header row and raise the tabbed layout breakpoint * fix: keep the step header pinned and switch to triggers on every trigger node tap --- .../lib/components/FlowGraphViewerStep.svelte | 53 +++--------- .../FlowGraphViewerStepHeader.svelte | 80 +++++++++++++++++++ .../details/DetailPageHeader.svelte | 2 +- .../details/DetailPageLayout.svelte | 80 ++++++++++++++++--- .../(logged)/flows/get/[...path]/+page.svelte | 20 ++--- 5 files changed, 171 insertions(+), 64 deletions(-) create mode 100644 frontend/src/lib/components/FlowGraphViewerStepHeader.svelte diff --git a/frontend/src/lib/components/FlowGraphViewerStep.svelte b/frontend/src/lib/components/FlowGraphViewerStep.svelte index 67d7ab8c72..1e4525274f 100644 --- a/frontend/src/lib/components/FlowGraphViewerStep.svelte +++ b/frontend/src/lib/components/FlowGraphViewerStep.svelte @@ -19,6 +19,7 @@ import { Copy, Expand } from 'lucide-svelte' import HighlightTheme from './HighlightTheme.svelte' import LanguageIcon from './common/languageIcons/LanguageIcon.svelte' + import FlowGraphViewerStepHeader from './FlowGraphViewerStepHeader.svelte' interface Props { schema?: any | undefined @@ -28,6 +29,8 @@ // The workspace the viewed flow belongs to (differs from the nav workspace in fork/session // editors); used to qualify resource links. workspace?: string + /** Given, the step header starts with a back control that calls it. */ + onBack?: () => void } let { @@ -35,7 +38,8 @@ stepDetail = undefined, jobScriptHash = undefined, hideDefaultInputs = false, - workspace = undefined + workspace = undefined, + onBack = undefined }: Props = $props() let ws = $derived(workspace ?? $workspaceStore) let codeViewer: Drawer | undefined = $state() @@ -104,57 +108,20 @@ {/if} {:else if stepDetail == 'Input'} + {#if schema} {:else}

No input schema

{/if} {:else if stepDetail == 'Result'} +

End of the flow

{:else if typeof stepDetail != 'string' && stepDetail.value} + +
-
- {#if stepDetail.id && stepDetail.id != 'failure' && stepDetail.id != 'preprocessor'} - - {stepDetail.id} - - {/if} - - {#if stepDetail.summary} - {stepDetail.summary} - {:else if stepDetail.value.type == 'identity'} - Identity - {:else if stepDetail.value.type == 'forloopflow'} - For loop {#if stepDetail.value.parallel}(parallel){/if} - {#if stepDetail.value.skip_failures}(skip failures){/if} - {#if stepDetail.value.squash}(squash){/if} - {:else if stepDetail.value.type == 'branchall'} - Run all branches {#if stepDetail.value.parallel}(parallel){/if} - {:else if stepDetail.value.type == 'branchone'} - Run one branch - {:else if stepDetail.value.type == 'flow'} - Inner flow - {:else if stepDetail.value.type == 'whileloopflow'} - While loop {#if stepDetail.value.skip_failures}(skip failures){/if} - {#if stepDetail.value.squash}(squash){/if} - {:else if stepDetail.id === 'failure'} - Error handler - {:else if stepDetail.id === 'preprocessor'} - Preprocessor - {:else if stepDetail.value.type == 'rawscript'} - Inline {stepDetail.value.language} script - {:else if stepDetail.value.type == 'script'} - Workspace script - {:else if stepDetail.value.type == 'aiagent'} - AI Agent - {/if} - -
{#if stepDetail.value.type == 'script'}
+ import type { FlowModule } from '$lib/gen' + import { Badge, Button } from './common' + import { ArrowLeft } from 'lucide-svelte' + + interface Props { + /** A module, or the graph's pseudo-nodes by id (`Input`, `Result`). */ + stepDetail: FlowModule | string + /** Given, the row starts with a back control; the caller decides where back leads. */ + onBack?: () => void + } + + let { stepDetail, onBack = undefined }: Props = $props() + + const module = $derived(typeof stepDetail === 'string' ? undefined : stepDetail) + // The error handler and the preprocessor are named by their role, not by an id badge. + const showId = $derived( + module?.id !== undefined && module.id !== 'failure' && module.id !== 'preprocessor' + ) + + const title = $derived.by((): string => { + if (typeof stepDetail === 'string') { + if (stepDetail === 'Input') return 'Flow inputs' + if (stepDetail === 'Result') return 'Result' + return stepDetail + } + if (stepDetail.summary) return stepDetail.summary + if (stepDetail.id === 'failure') return 'Error handler' + if (stepDetail.id === 'preprocessor') return 'Preprocessor' + const v = stepDetail.value + switch (v?.type) { + case 'identity': + return 'Identity' + case 'forloopflow': + return ( + 'For loop' + + (v.parallel ? ' (parallel)' : '') + + (v.skip_failures ? ' (skip failures)' : '') + + (v.squash ? ' (squash)' : '') + ) + case 'whileloopflow': + return ( + 'While loop' + (v.skip_failures ? ' (skip failures)' : '') + (v.squash ? ' (squash)' : '') + ) + case 'branchall': + return 'Run all branches' + (v.parallel ? ' (parallel)' : '') + case 'branchone': + return 'Run one branch' + case 'flow': + return 'Inner flow' + case 'rawscript': + return `Inline ${v.language} script` + case 'script': + return 'Workspace script' + case 'aiagent': + return 'AI Agent' + default: + return stepDetail.id + } + }) + + + +
+ {#if onBack} +
diff --git a/frontend/src/lib/components/details/DetailPageHeader.svelte b/frontend/src/lib/components/details/DetailPageHeader.svelte index 77c802dcd2..4f91f1800f 100644 --- a/frontend/src/lib/components/details/DetailPageHeader.svelte +++ b/frontend/src/lib/components/details/DetailPageHeader.svelte @@ -137,7 +137,7 @@
diff --git a/frontend/src/lib/components/details/DetailPageLayout.svelte b/frontend/src/lib/components/details/DetailPageLayout.svelte index be6e1a9e79..a6a2aa7a8a 100644 --- a/frontend/src/lib/components/details/DetailPageLayout.svelte +++ b/frontend/src/lib/components/details/DetailPageLayout.svelte @@ -1,5 +1,6 @@
@@ -54,7 +74,7 @@
- {@render form?.()} + {@render form?.({ graphInline: true })} @@ -65,7 +85,11 @@ {@render save_inputs_render?.()} {/snippet} {#snippet flow_step()} - {@render flow_step_render?.()} + +
+ {@render flow_step_render?.({})} +
{/snippet} {#snippet triggers()} {@render triggers_render?.()} @@ -79,12 +103,15 @@
{@render header?.()}
- + + {#if !isChatMode} {/if} - {#if isChatMode && flow_json} + {#if flow_json} {/if} {#if !isOperator} @@ -99,7 +126,7 @@ {#snippet content()}
- {@render form?.()} + {@render form?.({ graphInline: false })} @@ -108,9 +135,9 @@ {@render triggers?.()} - {#if isChatMode && flow_json} + {#if flow_json} - {@render flow_graph_render?.()} + {@render pagedGraph()} {/if} {#if flow_json} @@ -128,3 +155,34 @@
{/if}
+ + +{#snippet pagedGraph()} + { + if (key === 'graph') selected = 'saved_inputs' + }} + pages={[ + { key: 'graph', content: graphPageContent }, + { key: 'step', content: stepPageContent } + ]} + /> +{/snippet} + +{#snippet graphPageContent()} +
+ {@render flow_graph_render?.()} +
+{/snippet} + +{#snippet stepPageContent()} + +
+ {@render flow_step_render?.({ onBack: () => (selected = 'saved_inputs') })} +
+{/snippet} diff --git a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte index 59ed08c8b8..a1384579af 100644 --- a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte @@ -491,6 +491,7 @@ let stepDetail: FlowModule | string | undefined = $state(undefined) let rightPaneSelected = $state('saved_inputs') let savedInputsV2: SavedInputsV2 | undefined = $state(undefined) + let detailLayout: DetailPageLayout | undefined = $state(undefined) let flowHistory: FlowHistory | undefined = $state(undefined) let path = $derived(page.params.path ?? '') @@ -546,6 +547,7 @@ {/if} { - rightPaneSelected = 'triggers' + detailLayout?.showTriggers() }} {mainButtons} menuItems={getMenuItems(flow, deployUiSettings)} @@ -596,7 +598,7 @@ isFlow selected={rightPaneSelected == 'triggers'} onSelect={async (triggerIndex: number) => { - rightPaneSelected = 'triggers' + detailLayout?.showTriggers() await tick() triggersState.selectedTriggerIndex = triggerIndex }} @@ -627,7 +629,7 @@ {/if} {/snippet} - {#snippet form()} + {#snippet form({ graphInline }: { graphInline: boolean })}
(showEditButtons = v)} />
@@ -771,7 +773,7 @@ {/if}
- {#if !chatInputEnabled} + {#if graphInline}
{ - rightPaneSelected = 'triggers' + detailLayout?.showTriggers() }} noBorder={true} /> @@ -817,10 +819,10 @@ /> {/snippet} - {#snippet flow_step()} + {#snippet flow_step({ onBack }: { onBack?: () => void })} {#if flow} {#if stepDetail} - + {/if} {/if} {/snippet} @@ -849,7 +851,7 @@ triggerNode={true} download {flow} - noSide={false} + noSide={true} noBorder minHeight={flowGraphHeight} on:select={(e) => { @@ -862,7 +864,7 @@ } }} on:triggerDetail={(e) => { - rightPaneSelected = 'triggers' + detailLayout?.showTriggers() }} />
From 4416283b17ba17ce8e9d8d707a7f1449838dec01 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Wed, 16 Sep 2026 16:32:36 +0200 Subject: [PATCH 02/77] fix: read the roles of a data table whose name contains '?' The generated client leaves a '?' in a path param unencoded, so the lookup 404'd and the raw-app picker blocked Start on such a data table. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/lib/components/datatableUsableRoles.ts | 14 ++++++++++++-- .../raw_apps/RawAppTemplatePicker.svelte | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/frontend/src/lib/components/datatableUsableRoles.ts b/frontend/src/lib/components/datatableUsableRoles.ts index 0f53a540af..84e52f29a8 100644 --- a/frontend/src/lib/components/datatableUsableRoles.ts +++ b/frontend/src/lib/components/datatableUsableRoles.ts @@ -1,4 +1,5 @@ -import { WorkspaceService, type ListUsableDatatableRolesResponse } from '$lib/gen' +import { OpenAPI, type ListUsableDatatableRolesResponse } from '$lib/gen' +import { request } from '$lib/gen/core/request' import { isCloudHosted } from '$lib/cloud' import { ADMIN_DATATABLE_ROLE } from './dbTypes' @@ -24,7 +25,16 @@ export async function listUsableDatatableRoles( ): Promise { if (isCloudHosted()) return NOT_UNDER_ROLES try { - return await WorkspaceService.listUsableDatatableRoles({ workspace, datatableName }) + // The generated client encodes path params with `encodeURI`, which leaves a '?' in a + // data table name created before names were restricted to cut the path short. + return await request( + { ...OpenAPI, ENCODE_PATH: encodeURIComponent }, + { + method: 'GET', + url: '/w/{workspace}/workspaces/datatable_usable_roles/{datatable_name}', + path: { workspace, datatable_name: datatableName } + } + ) } catch (e) { const body = (e as { body?: unknown })?.body const detail = `${typeof body === 'string' ? body : JSON.stringify(body ?? '')} ${(e as Error)?.message ?? e}` diff --git a/frontend/src/lib/components/raw_apps/RawAppTemplatePicker.svelte b/frontend/src/lib/components/raw_apps/RawAppTemplatePicker.svelte index 807ec8e9cf..1c889d7eba 100644 --- a/frontend/src/lib/components/raw_apps/RawAppTemplatePicker.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppTemplatePicker.svelte @@ -166,7 +166,7 @@ rolesSettled && selectedDatatable !== undefined && roles.current.permissioned && - loadedRoles.length === 0 + roles.current.roles.length === 0 ) const rolesUnknown = $derived(rolesSettled && roles.current.failed) const accessSettled = $derived( From 41e1101a2d7f6c01b26f344ec02cb16497b08e2c Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Wed, 16 Sep 2026 16:39:14 +0200 Subject: [PATCH 03/77] fix: take every pooled connection before the ACL apply locks Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb --- .../src/datatable_acl.rs | 70 ++++++++++++------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/backend/windmill-api-workspaces/src/datatable_acl.rs b/backend/windmill-api-workspaces/src/datatable_acl.rs index 39fa8fac5d..84bc949dd5 100644 --- a/backend/windmill-api-workspaces/src/datatable_acl.rs +++ b/backend/windmill-api-workspaces/src/datatable_acl.rs @@ -37,7 +37,8 @@ use windmill_common::datatable_roles::{ }; use windmill_common::error::{pg_error_message, Error, JsonResult, Result}; use windmill_common::workspaces::{ - get_datatable_resource_from_db_unchecked, resolve_governing_datatable, GoverningDatatable, + get_datatable_resource_from_db_unchecked, resolve_governing_datatable, DataTable, + GoverningDatatable, }; use windmill_common::{PgDatabase, DB}; @@ -1307,6 +1308,21 @@ async fn authorize_acl_change( Ok(governing) } +/// Whether the governing entry an apply was authorized on is still the one in the settings, read +/// under the lock: a save in between could have pointed it at another database or changed its roles. +fn entry_unchanged(governing: &GoverningDatatable, entry_now: Option) -> bool { + let Some(Ok(now)) = entry_now.map(serde_json::from_value::) else { + return false; + }; + match ( + serde_json::to_value(&now), + serde_json::to_value(&governing.datatable), + ) { + (Ok(now), Ok(authorized)) => now == authorized, + _ => false, + } +} + /// Plan one change against the catalog and the database as they are now. async fn build_plan( client: &tokio_postgres::Client, @@ -1502,33 +1518,11 @@ async fn apply_datatable_acl( .to_string(), ) })?; - // Refuses without taking a lock; everything is checked again once they are held. + // Everything that needs the pool happens before the locks: once `tx` holds them, a second pool + // connection could wait forever on a pool that concurrent applies, queued on the same locks, + // have exhausted. let governing = authorize_acl_change(&db, &authed, &w_id, &datatable_name).await?; - - // Held until the change is committed: a role renamed or dropped meanwhile would change what - // the plan names, and a settings save could move the entry onto another database. Taken in the - // same order as the permissions save, so the two cannot deadlock. - let mut tx = db.begin().await?; - lock_role_catalog(&mut tx).await?; - sqlx::query!( - "SELECT 1 AS one FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE", - &governing.workspace_id - ) - .fetch_optional(&mut *tx) - .await?; - let governing = authorize_acl_change(&db, &authed, &w_id, &datatable_name).await?; - let catalog = read_role_catalog_tx(&mut tx).await?; - let (mut client, mut notices, dbname) = connect_as_admin_unchecked(&db, &governing).await?; - let plan = build_plan(&client, &dbname, &catalog, &req.target, &req.change).await?; - if &plan.statements != confirmed { - return Err(Error::BadRequest( - "The data table or its roles changed since this was planned, so it would no longer \ - run what was confirmed. Plan it again." - .to_string(), - )); - } - // Postgres only lets a role pass on a privilege it holds with grant option, and an instance // database provisioned before data table roles holds none. Best-effort: a grant this fails to // enable is refused below rather than skipped. @@ -1537,6 +1531,30 @@ async fn apply_datatable_acl( tracing::warn!("Could not refresh grant options on '{dbname}': {e}"); } + // Held until the change is committed: a role renamed or dropped meanwhile would change what + // the plan names, and a settings save could move the entry onto another database. Taken in the + // same order as the permissions save, so the two cannot deadlock. + let mut tx = db.begin().await?; + lock_role_catalog(&mut tx).await?; + let entry_now = sqlx::query_scalar::<_, Option>( + "SELECT datatable->'datatables'->$2 FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE", + ) + .bind(&governing.workspace_id) + .bind(&governing.name) + .fetch_optional(&mut *tx) + .await? + .flatten(); + let catalog = read_role_catalog_tx(&mut tx).await?; + + let plan = build_plan(&client, &dbname, &catalog, &req.target, &req.change).await?; + if !entry_unchanged(&governing, entry_now) || &plan.statements != confirmed { + return Err(Error::BadRequest( + "The data table or its roles changed since this was planned, so it would no longer \ + run what was confirmed. Plan it again." + .to_string(), + )); + } + // One transaction: a half-applied ownership transfer leaves one schema's objects owned by two // different roles. let pg_tx = client.transaction().await.map_err(|e| { From d459b371d8ee7cf7226bb7d71ca51c37d54308f8 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Wed, 16 Sep 2026 16:53:13 +0200 Subject: [PATCH 04/77] fix: refresh grant options only after the ACL apply validates its plan Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb --- .../windmill-api-workspaces/src/datatable_acl.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/backend/windmill-api-workspaces/src/datatable_acl.rs b/backend/windmill-api-workspaces/src/datatable_acl.rs index 84bc949dd5..396a456e4e 100644 --- a/backend/windmill-api-workspaces/src/datatable_acl.rs +++ b/backend/windmill-api-workspaces/src/datatable_acl.rs @@ -1523,13 +1523,6 @@ async fn apply_datatable_acl( // have exhausted. let governing = authorize_acl_change(&db, &authed, &w_id, &datatable_name).await?; let (mut client, mut notices, dbname) = connect_as_admin_unchecked(&db, &governing).await?; - // Postgres only lets a role pass on a privilege it holds with grant option, and an instance - // database provisioned before data table roles holds none. Best-effort: a grant this fails to - // enable is refused below rather than skipped. - if let Err(e) = windmill_common::ensure_instance_db_grant_options_unchecked(&db, &dbname).await - { - tracing::warn!("Could not refresh grant options on '{dbname}': {e}"); - } // Held until the change is committed: a role renamed or dropped meanwhile would change what // the plan names, and a settings save could move the entry onto another database. Taken in the @@ -1555,6 +1548,15 @@ async fn apply_datatable_acl( )); } + // A role passes on only privileges it holds with grant option, which a database provisioned + // before data table roles lacks; a grant this fails to enable is refused below. After the plan + // check, since the planner reads these grants and would refuse what it just accepted; it + // connects as the server's own Postgres user, so it takes nothing from the pool. + if let Err(e) = windmill_common::ensure_instance_db_grant_options_unchecked(&db, &dbname).await + { + tracing::warn!("Could not refresh grant options on '{dbname}': {e}"); + } + // One transaction: a half-applied ownership transfer leaves one schema's objects owned by two // different roles. let pg_tx = client.transaction().await.map_err(|e| { From 10861bb52950ce64bd1e75101789d5507123fdc6 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Wed, 16 Sep 2026 17:03:32 +0200 Subject: [PATCH 05/77] fix(datatables): refuse a reference naming both a legacy data table and a role When a workspace stores both `sales` and a legacy `sales?role=analytics`, the reference resolved to the legacy entry without a role, so browsing `sales` as `analytics` reached another data table. Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/datatable_roles.rs | 12 ++++++++++ backend/windmill-common/src/workspaces.rs | 22 ++++++++++++++----- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs index d79dd5a3e3..5543947ce5 100644 --- a/backend/windmill-api-integration-tests/tests/datatable_roles.rs +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -912,6 +912,18 @@ async fn a_stored_name_containing_a_question_mark_resolves_as_itself( resolve("main?dt").await.is_err(), "an unknown parameter was ignored" ); + + sqlx::query( + "UPDATE workspace_settings + SET datatable = jsonb_set(datatable, '{datatables,main?role=analytics}', datatable->'datatables'->'main') + WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + assert!( + resolve("main?role=analytics").await.is_err(), + "a reference naming both a stored data table and a role on another resolved to one of them" + ); Ok(()) } diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index e8974da0e8..7ce41e78d3 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -1876,22 +1876,34 @@ pub fn strip_datatable_permissions( /// As [`parse_datatable_ref`], except that an entry whose stored name itself contains `?` — which /// names could before they were restricted — resolves by that exact name, without a role. It is /// looked up first, so `sales?role=x` never reaches a different entry than the one stored so. +/// When `sales` is stored too, the reference means either one, and is refused rather than +/// resolved to whichever is looked up first. pub async fn parse_datatable_ref_for( db: &DB, w_id: &str, reference: &str, ) -> Result<(String, Option)> { if reference.contains('?') { - let exists = sqlx::query_scalar::<_, Option>( - "SELECT (datatable->'datatables') ? $2 FROM workspace_settings WHERE workspace_id = $1", + let role_target = parse_datatable_ref(reference) + .ok() + .and_then(|(name, role)| role.map(|_| name)); + let (exists, target_exists) = sqlx::query_as::<_, (Option, Option)>( + "SELECT (datatable->'datatables') ? $2, (datatable->'datatables') ? $3 + FROM workspace_settings WHERE workspace_id = $1", ) .bind(w_id) .bind(reference) + .bind(role_target) .fetch_optional(db) .await? - .flatten() - .unwrap_or(false); - if exists { + .unwrap_or((None, None)); + if exists.unwrap_or(false) { + if let (Some(name), Some(true)) = (role_target, target_exists) { + return Err(Error::BadRequest(format!( + "Data table reference '{reference}' names both the data table '{reference}' \ + and a role on the data table '{name}'. Rename '{reference}' to use either." + ))); + } return Ok((reference.to_string(), None)); } } From dcc37bf6f2bc49cfd811b24468bf566e5dc13da9 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Wed, 16 Sep 2026 17:20:19 +0200 Subject: [PATCH 06/77] fix: declare the default role in migrations written for a data table whose name contains '?' Such a data table connects as its default role without naming it, so the migrations the manager wrote for it declared no role and ran as admin. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/lib/components/DBManagerDrawer.svelte | 16 ++++++++++++++-- frontend/src/lib/components/SqlRepl.svelte | 2 +- frontend/src/lib/components/dbOps.ts | 2 +- frontend/src/lib/components/dbTypes.ts | 13 +++++++++++++ .../raw_apps/RawAppDataTableDrawer.svelte | 18 ++++++++++++++---- .../raw_apps/RawAppTemplatePicker.svelte | 9 +++++++-- 6 files changed, 50 insertions(+), 10 deletions(-) diff --git a/frontend/src/lib/components/DBManagerDrawer.svelte b/frontend/src/lib/components/DBManagerDrawer.svelte index 330449eb7e..71dda5e1d3 100644 --- a/frontend/src/lib/components/DBManagerDrawer.svelte +++ b/frontend/src/lib/components/DBManagerDrawer.svelte @@ -17,6 +17,7 @@ import { ADMIN_DATATABLE_ROLE, datatableNameTakesRole, + defaultMigrationRole, type DatatableRowAction } from './dbTypes' import ResourcePicker from './ResourcePicker.svelte' @@ -103,6 +104,17 @@ if (effective) untrack(() => (uriState.selectedRole = effective)) }) + const contentInput = $derived.by(() => { + const input = uriState.effectiveInput + if (input?.type !== 'database' || selectedDatatable === undefined) return input + const migrationRole = defaultMigrationRole( + selectedDatatable, + rolesOfCurrent?.permissioned, + rolesOfCurrent?.default_role + ) + return migrationRole === undefined ? input : { ...input, migrationRole } + }) + // Every data table with its schemas and tables, in one call: this is what the // left pane's tree navigates, so it has to cover the data tables the user is // not currently on, not just the selected one. The privileges it reports are @@ -295,11 +307,11 @@ noPadding id="db-manager-drawer" > - {#if uriState.effectiveInput && ws && roleSettled} + {#if contentInput && ws && roleSettled} {#key `${selectedDatatable}~${selectedRole ?? ''}`} {/if} diff --git a/frontend/src/lib/components/dbOps.ts b/frontend/src/lib/components/dbOps.ts index 02c26f034f..e239a9e957 100644 --- a/frontend/src/lib/components/dbOps.ts +++ b/frontend/src/lib/components/dbOps.ts @@ -299,7 +299,7 @@ export function dbSchemaOpsWithPreviewScripts({ ? input.resourcePath.slice('datatable://'.length) : undefined // A migration declaring no role runs as admin, whatever role the manager connects as. - const migrationRole = input.type === 'database' ? input.role : undefined + const migrationRole = input.type === 'database' ? (input.role ?? input.migrationRole) : undefined function makeMarker(op: string, payload: Record): string { if (ducklake) payload.ducklake = ducklake diff --git a/frontend/src/lib/components/dbTypes.ts b/frontend/src/lib/components/dbTypes.ts index a9cb0a81e1..e6d3af12e8 100644 --- a/frontend/src/lib/components/dbTypes.ts +++ b/frontend/src/lib/components/dbTypes.ts @@ -6,6 +6,9 @@ export type DbInput = /** The data table role to connect as; the data table's default when unset. Only * meaningful for a `datatable://` path. */ role?: string + /** The role migrations written through this input declare when `role` is unset. A + * migration declaring none runs as admin, not as the role the manager connects as. */ + migrationRole?: string specificSchema?: string specificTable?: string } @@ -42,6 +45,16 @@ export function datatableNameTakesRole(name: string): boolean { return !name.includes('?') } +/** The `migrationRole` of a data table that cannot name a role in its reference: it connects as + * its default role, which its migrations must then declare. */ +export function defaultMigrationRole( + name: string, + permissioned: boolean | undefined, + defaultRole: string | undefined +): string | undefined { + return permissioned && !datatableNameTakesRole(name) ? defaultRole : undefined +} + /** `datatable://`, with `?role=` when a role is named. Throws rather than build a * reference the executor would refuse, or one that would silently mean another role. */ export function datatableReference(name: string, role: string | undefined): string { diff --git a/frontend/src/lib/components/raw_apps/RawAppDataTableDrawer.svelte b/frontend/src/lib/components/raw_apps/RawAppDataTableDrawer.svelte index a61bf0a0cb..f0ea7d1ce3 100644 --- a/frontend/src/lib/components/raw_apps/RawAppDataTableDrawer.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppDataTableDrawer.svelte @@ -11,7 +11,12 @@ import { resource } from 'runed' import { ArrowLeft, Expand, Minimize, Plus, RefreshCcw } from 'lucide-svelte' import DBManagerContent from '../DBManagerContent.svelte' - import { ADMIN_DATATABLE_ROLE, datatableNameTakesRole, type DbInput } from '../dbTypes' + import { + ADMIN_DATATABLE_ROLE, + datatableNameTakesRole, + defaultMigrationRole, + type DbInput + } from '../dbTypes' import type { PendingRowAction, SelectedTable } from '../DBManager.svelte' import { getRawAppOperatingWorkspace } from './rawAppWorkspace' import { useDbManagerTag } from '../dbManagerTag.svelte' @@ -107,12 +112,12 @@ // without one runs, and caches, as whatever the server defaults to. const roleSettled = $derived( selectedDatatable === undefined || - // Its reference cannot name a role, so it connects as the default one. - !datatableNameTakesRole(selectedDatatable) || (rolesOfCurrent !== undefined && (!rolesOfCurrent.permissioned || rolesOfCurrent.roles.length === 0 || - selectedRole !== undefined)) + selectedRole !== undefined || + // Its reference cannot name a role, so it connects as the default one. + !datatableNameTakesRole(selectedDatatable))) ) $effect(() => { @@ -307,6 +312,11 @@ resourceType: 'postgresql' as const, resourcePath: `datatable://${selectedDatatable}`, role: selectedRole, + migrationRole: defaultMigrationRole( + selectedDatatable, + rolesOfCurrent?.permissioned, + rolesOfCurrent?.default_role + ), specificSchema: openSchemaKey, specificTable: openTableKey } diff --git a/frontend/src/lib/components/raw_apps/RawAppTemplatePicker.svelte b/frontend/src/lib/components/raw_apps/RawAppTemplatePicker.svelte index 1c889d7eba..a48713a501 100644 --- a/frontend/src/lib/components/raw_apps/RawAppTemplatePicker.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppTemplatePicker.svelte @@ -34,7 +34,7 @@ toDatatableItems, toSchemaItems } from './datatableUtils.svelte' - import { datatableNameTakesRole } from '../dbTypes' + import { datatableNameTakesRole, defaultMigrationRole } from '../dbTypes' import RawAppDataTableList from './RawAppDataTableList.svelte' import RawAppDataTableDrawer from './RawAppDataTableDrawer.svelte' import FileEditorIcon from './FileEditorIcon.svelte' @@ -290,7 +290,12 @@ type: 'database', resourceType: 'postgresql', resourcePath: `datatable://${selectedDatatable}`, - role: effectiveRole + role: effectiveRole, + migrationRole: defaultMigrationRole( + selectedDatatable, + roles.current.permissioned, + roles.current.defaultRole + ) } }) await dbOps.onCreateSchema({ schema: newSchemaName }) From 707b0dc8eeddfd1c6196f120444833412971f600 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Wed, 16 Sep 2026 17:25:42 +0200 Subject: [PATCH 07/77] fix(datatables): let CE migrations connect as an explicitly named admin Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/datatable_roles.rs | 23 +++++++++++-------- .../src/datatable_roles_oss.rs | 8 ++++--- backend/windmill-common/src/workspaces.rs | 7 +++--- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs index a15439455f..08790386a4 100644 --- a/backend/windmill-api-integration-tests/tests/datatable_roles.rs +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -1098,22 +1098,25 @@ async fn without_the_enterprise_edition_a_data_table_under_roles_is_refused_a_co assert!(err.to_string().contains(ENTERPRISE_REFUSAL), "{err}"); } - // Not under roles, it resolves as it always has; naming a role on it is refused. + // Not under roles, it resolves as it always has, including when `admin` is named — which every + // migration does; naming any other role on it is refused. sqlx::query( "UPDATE workspace_settings SET datatable = datatable #- '{datatables,main,permissions}' WHERE workspace_id = 'test-workspace'", ) .execute(&db) .await?; - let resolved = get_datatable_resource_from_db( - &db, - "test-workspace", - "main", - None, - DatatableAccess::NoIdentity, - ) - .await?; - assert_eq!(resolved["dbname"], "dt_main", "{resolved}"); + for role in [None, Some("admin")] { + let resolved = get_datatable_resource_from_db( + &db, + "test-workspace", + "main", + role, + DatatableAccess::NoIdentity, + ) + .await?; + assert_eq!(resolved["dbname"], "dt_main", "{resolved}"); + } let err = get_datatable_resource_from_db( &db, "test-workspace", diff --git a/backend/windmill-common/src/datatable_roles_oss.rs b/backend/windmill-common/src/datatable_roles_oss.rs index 98fef4260f..057a739bd4 100644 --- a/backend/windmill-common/src/datatable_roles_oss.rs +++ b/backend/windmill-common/src/datatable_roles_oss.rs @@ -163,8 +163,8 @@ mod ce { Err(unavailable()) } - /// A data table not under roles, asked for no role, is not a role decision and passes, as it - /// did before roles existed. Anything else is refused. + /// A data table not under roles, asked for no role or for `admin`, is not a role decision and + /// passes, as it did before roles existed. Anything else is refused. pub(crate) async fn ensure_can_use_datatable_role( db: &DB, w_id: &str, @@ -174,7 +174,9 @@ mod ce { _context: &str, ) -> Result<()> { let governing = resolve_governing_datatable(db, w_id, name).await?; - if governing.datatable.permissions.is_none() && role.is_none() { + if governing.datatable.permissions.is_none() + && role.is_none_or(|r| r == crate::datatable_roles::ADMIN_DATATABLE_ROLE) + { Ok(()) } else { Err(unavailable()) diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index e8974da0e8..3e8dd0aae5 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -1664,9 +1664,10 @@ pub async fn get_datatable_resource_from_db( ) -> Result { let governing = resolve_governing_datatable(db, w_id, name).await?; let db_resource = resolve_datatable_connection_unchecked(db, &governing, false).await?; - // Not under roles and asked for none: the `admin` connection, as before roles existed, in - // every edition. Anything else is a role decision. - if governing.datatable.permissions.is_none() && role.is_none() { + // Not under roles and asked for none, or for `admin` by name: the `admin` connection, as before + // roles existed, in every edition. Anything else is a role decision. Every migration names + // `admin` explicitly, so an edition without roles must not treat that as one. + if governing.datatable.permissions.is_none() && role.is_none_or(|r| r == ADMIN_DATATABLE_ROLE) { return Ok(db_resource); } crate::datatable_roles_oss::resolve_datatable_role_connection( From 02e47de8b4c4f3f54753aabf8c67bc8e71ffb957 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Wed, 16 Sep 2026 17:28:40 +0200 Subject: [PATCH 08/77] feat: allow git sync auto-pull, promotion and PRs on Pro licenses (#11173) Co-authored-by: Claude Opus 5 --- backend/src/monitor.rs | 18 +--- .../windmill-api-workspaces/src/workspaces.rs | 92 +------------------ .../windmill-worker/src/result_processor.rs | 12 --- .../git_sync/GitSyncRepositoryCard.svelte | 5 +- 4 files changed, 10 insertions(+), 117 deletions(-) diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index dc3b9b6e8b..e250302209 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -4712,17 +4712,11 @@ const GIT_AUTO_PULL_LOCK_ID: i64 = 737_483_921; /// Poll every git-sync repository with auto-pull enabled and enqueue a pull when /// the tracked branch has new commits (repo → Windmill direction). /// -/// Runs on a single replica at a time (advisory lock) and only on -/// Enterprise-licensed instances. Detection is `git ls-remote`; GitHub-App -/// repositories are skipped here and sync via webhooks instead (phase 2). +/// Runs on a single replica at a time (advisory lock). Detection is +/// `git ls-remote`; GitHub-App repositories are skipped here and sync via +/// webhooks instead (phase 2). #[cfg(feature = "private")] pub async fn poll_git_auto_pull(db: &Pool) { - use windmill_common::ee_oss::{get_license_plan, LicensePlan}; - - if !matches!(get_license_plan().await, LicensePlan::Enterprise) { - return; - } - let mut lock_conn = match db.acquire().await { Ok(c) => c, Err(e) => { @@ -4792,12 +4786,6 @@ const GIT_CREDENTIAL_LOCK_ID: i64 = 737_483_923; /// sync down on its expiry date. #[cfg(all(feature = "enterprise", feature = "private"))] async fn maintain_git_credentials(db: &Pool) { - use windmill_common::ee_oss::{get_license_plan, LicensePlan}; - - if !matches!(get_license_plan().await, LicensePlan::Enterprise) { - return; - } - // Transaction-scoped advisory lock, as for the schedule reconcile above: a // session lock on a pooled connection would ride back into the pool still // held if the sweep died before unlocking, and wedge the pass on every diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 3e48e7baca..2bef0d28ab 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -1308,26 +1308,22 @@ async fn get_git_sync_deploy_mode( let configured = !settings.repositories.is_empty(); - // Auto-pull runs only on Enterprise-licensed instances (see poll_git_auto_pull); - // without a caller branch there is nothing to match. Either way deploy_on_push - // stays false and the caller falls back (git push via CI, or wmill sync push). + // Auto-pull runs only in builds that compile the poller (`private`); without a + // caller branch there is nothing to match. Either way deploy_on_push stays + // false and the caller falls back (git push via CI, or wmill sync push). let Some(branch) = q.branch.as_deref() else { return Ok(Json(GitSyncDeployMode { configured, deploy_on_push: false, })); }; - let licensed = matches!( - windmill_common::ee_oss::get_license_plan().await, - windmill_common::ee_oss::LicensePlan::Enterprise - ); // Count the auto-pull repos that would deploy this branch. We deliberately do // not check the caller's remote URL: with exactly one such repo the local // checkout is unambiguously it, and with several we can't tell which is the // caller's, so we report false and let the CLI ask the user. let mut matches = 0u32; - if licensed && !root_deleted { + if cfg!(feature = "private") && !root_deleted { for repo in &settings.repositories { let Some(auto_pull) = repo.auto_pull.as_ref() else { continue; @@ -3762,54 +3758,6 @@ fn cleanup_legacy_git_sync_settings_in_memory( #[cfg(not(feature = "enterprise"))] const CE_GIT_SYNC_MAX_USERS: i64 = 2; -/// Auto-pull is licensed per plan, not just per build: the poller only serves -/// Enterprise plans at runtime, so the save path must reject the setting too — -/// otherwise an EE binary without the plan could still register a webhook and -/// receive webhook-driven pulls. -#[cfg(feature = "enterprise")] -async fn check_git_sync_ee_license(feature: &str) -> Result<()> { - if !matches!( - windmill_common::ee_oss::get_license_plan().await, - windmill_common::ee_oss::LicensePlan::Enterprise - ) { - return Err(Error::BadRequest(format!( - "{feature} requires an Enterprise license" - ))); - } - Ok(()) -} - -#[cfg(feature = "enterprise")] -async fn check_auto_pull_license() -> Result<()> { - check_git_sync_ee_license("Automatic pull from git").await -} - -/// In-app PR creation (promotion/fork deploy branches) drives GitHub API calls -/// from the deploy completion hook; runtime-gate it like auto-pull. -#[cfg(feature = "enterprise")] -async fn check_open_prs_license<'a>( - mut repos: impl Iterator, -) -> Result<()> { - if repos.any(|r| r.promotion_open_prs || r.fork_open_prs) { - check_git_sync_ee_license("Opening pull requests from Windmill").await?; - } - Ok(()) -} - -/// Promotion mode (`use_individual_branch`: per-item `wm_deploy/**` deploy -/// branches) is an EE feature; runtime-gate it like auto-pull and PR creation -/// so an enterprise binary without an active plan can't enable it via either -/// git-sync edit endpoint. -#[cfg(feature = "enterprise")] -async fn check_promotion_license<'a>( - mut repos: impl Iterator, -) -> Result<()> { - if repos.any(|r| r.use_individual_branch.unwrap_or(false)) { - check_git_sync_ee_license("Promotion mode").await?; - } - Ok(()) -} - /// Promotion on a dev workspace needs the dev-aware sync script (hub >= 28796): /// an older pinned script bundles a CLI that force-disables per-item branches /// on every fork, so enabling promotion would silently keep deploying to the @@ -4061,18 +4009,6 @@ async fn edit_git_sync_config( )); } #[cfg(feature = "enterprise")] - if git_sync_settings - .repositories - .iter() - .any(|r| r.auto_pull.as_ref().is_some_and(|a| a.enabled)) - { - check_auto_pull_license().await?; - } - #[cfg(feature = "enterprise")] - check_open_prs_license(git_sync_settings.repositories.iter()).await?; - #[cfg(feature = "enterprise")] - check_promotion_license(git_sync_settings.repositories.iter()).await?; - #[cfg(feature = "enterprise")] check_dev_promotion_script_version(&db, &w_id, git_sync_settings.repositories.iter()) .await?; #[cfg(all(feature = "enterprise", feature = "private"))] @@ -4310,19 +4246,6 @@ async fn edit_git_sync_repository( )); } #[cfg(feature = "enterprise")] - if new_config - .repository - .auto_pull - .as_ref() - .is_some_and(|a| a.enabled) - { - check_auto_pull_license().await?; - } - #[cfg(feature = "enterprise")] - check_open_prs_license(std::iter::once(&new_config.repository)).await?; - #[cfg(feature = "enterprise")] - check_promotion_license(std::iter::once(&new_config.repository)).await?; - #[cfg(feature = "enterprise")] check_dev_promotion_script_version(&db, &w_id, std::iter::once(&new_config.repository)).await?; #[cfg(all(feature = "enterprise", feature = "private"))] check_dev_promotion_targets_parent_repo(&db, &w_id, std::iter::once(&new_config.repository)) @@ -4428,13 +4351,6 @@ async fn edit_git_sync_repository( } _ => {} } - // The request-side license gate above only saw the submitted config; the - // preservation can resurrect an enabled auto_pull (None arm), so re-check - // the effective state before it gets written and reconciled. - #[cfg(feature = "enterprise")] - if updated.auto_pull.as_ref().is_some_and(|a| a.enabled) { - check_auto_pull_license().await?; - } *existing_repo = updated; } else { // Repository doesn't exist, add it as a new repository diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 1529d474fa..0490ed1754 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -1254,18 +1254,6 @@ async fn maybe_open_git_sync_deploy_pr( if row.marker.is_none() { return; } - // Runtime Enterprise gate, like the poller: the toggles may have been set - // while a license was active (or written directly), and this hook drives - // GitHub API calls with the installation token. - if !matches!( - windmill_common::ee_oss::get_license_plan().await, - windmill_common::ee_oss::LicensePlan::Enterprise - ) { - tracing::warn!( - "git sync PR: skipping PR creation for {workspace_id}: requires an Enterprise license" - ); - return; - } let Some(repo_path) = row.repo_path else { return; }; diff --git a/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte b/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte index 32a510c54a..291c89c606 100644 --- a/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte +++ b/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte @@ -114,8 +114,9 @@ try { await gitSyncContext.saveRepository(idx) } catch (e) { - // The backend rejects promotion mode without an active EE plan; revert - // the optimistic toggle instead of leaving it stuck on until reload. + // The backend can reject promotion mode (non-EE build, outdated dev sync + // script, dev repo not matching its parent's); revert the optimistic + // toggle instead of leaving it stuck on until reload. if (repo) { repo.use_individual_branch = prevIndiv repo.group_by_folder = prevGbf From 9d348f84c7830f36b6153472556fd70e3d84cd24 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Wed, 16 Sep 2026 17:28:53 +0200 Subject: [PATCH 09/77] fix: skip expiry notifications for app embed and SDK tokens (#11169) * fix: skip expiry notifications for app embed and SDK tokens Co-Authored-By: Claude Opus 5 * refactor: share app token label prefixes between mint sites and the check Co-Authored-By: Claude Opus 5 * fix: skip expiry alerts for impersonation and test-connection tokens Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- AGENTS.md | 5 +-- ...f1d9f850e3e46c17130dd580426cda7a9f65.json} | 4 +-- backend/windmill-api-auth/src/lib.rs | 3 ++ .../tests/token_label_idor.rs | 4 ++- backend/windmill-api-users/src/users.rs | 7 ++-- backend/windmill-api/src/apps.rs | 12 +++++-- backend/windmill-common/src/auth.rs | 33 +++++++++++++++---- docs/auth-surface.md | 8 +++++ .../src/lib/components/TestConnection.svelte | 2 +- .../components/settings/TokensTable.svelte | 5 ++- 10 files changed, 64 insertions(+), 19 deletions(-) rename backend/.sqlx/{query-31ed2fb85c0c726e3cf6392be2a73c62bae004b2842a4828c6807232570f83a1.json => query-d631a26e5531589ff37e677f91a4f1d9f850e3e46c17130dd580426cda7a9f65.json} (63%) diff --git a/AGENTS.md b/AGENTS.md index 8a63c9828a..a315f1a67c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,8 +31,9 @@ Open-source platform for internal tools, workflows, API integrations, background `cargo run`; a normal build cannot start one at all. - **Enterprise**: `docs/enterprise.md` — EE file conventions and PR workflow - **Auth surface**: `docs/auth-surface.md` — credential precedence, session/cache invalidation - scope, how OAuth login matches `login_type`, and that every superadmin route refuses `$WM_TOKEN`. - Read before designing anything that creates users, tokens or sessions. + scope, which token labels email their owner at expiry, how OAuth login matches `login_type`, and + that every superadmin route refuses `$WM_TOKEN`. Read before designing anything that creates + users, tokens or sessions. - **Product telemetry**: `docs/feature-telemetry.md` — when to instrument a new feature with `feature_usage`, and the four-step recipe. An unregistered `(feature, kind)` pair is dropped silently, so frontend-only instrumentation records nothing. diff --git a/backend/.sqlx/query-31ed2fb85c0c726e3cf6392be2a73c62bae004b2842a4828c6807232570f83a1.json b/backend/.sqlx/query-d631a26e5531589ff37e677f91a4f1d9f850e3e46c17130dd580426cda7a9f65.json similarity index 63% rename from backend/.sqlx/query-31ed2fb85c0c726e3cf6392be2a73c62bae004b2842a4828c6807232570f83a1.json rename to backend/.sqlx/query-d631a26e5531589ff37e677f91a4f1d9f850e3e46c17130dd580426cda7a9f65.json index 556bd7c317..8cef9fc8aa 100644 --- a/backend/.sqlx/query-31ed2fb85c0c726e3cf6392be2a73c62bae004b2842a4828c6807232570f83a1.json +++ b/backend/.sqlx/query-d631a26e5531589ff37e677f91a4f1d9f850e3e46c17130dd580426cda7a9f65.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE token SET label = $1\n WHERE email = $2 AND token_prefix = $3\n AND (label IS NULL OR (\n label <> 'session'\n AND label <> 'guest_session'\n AND lower(label) NOT LIKE 'ephemeral%'\n AND label <> 'debugger-token'\n AND label NOT LIKE 'mcp-oauth-%'\n ))\n RETURNING token_prefix", + "query": "UPDATE token SET label = $1\n WHERE email = $2 AND token_prefix = $3\n AND (label IS NULL OR (\n label <> 'session'\n AND label <> 'guest_session'\n AND lower(label) NOT LIKE 'ephemeral%'\n AND label <> 'debugger-token'\n AND label NOT LIKE 'mcp-oauth-%'\n AND NOT starts_with(label, 'embed_app:')\n AND NOT starts_with(label, 'sdk_app:')\n AND NOT starts_with(label, 'impersonation:')\n ))\n RETURNING token_prefix", "describe": { "columns": [ { @@ -20,5 +20,5 @@ false ] }, - "hash": "31ed2fb85c0c726e3cf6392be2a73c62bae004b2842a4828c6807232570f83a1" + "hash": "d631a26e5531589ff37e677f91a4f1d9f850e3e46c17130dd580426cda7a9f65" } diff --git a/backend/windmill-api-auth/src/lib.rs b/backend/windmill-api-auth/src/lib.rs index 04513b4c05..0e26c2a8b8 100644 --- a/backend/windmill-api-auth/src/lib.rs +++ b/backend/windmill-api-auth/src/lib.rs @@ -1141,6 +1141,9 @@ impl NewToken { /// [`ensure_scopes_within_caller`] first (internal narrowing mints intentionally /// skip it, since their scopes derive from the action being authorized, not the /// caller's token). +/// +/// A token the system mints for itself with an `expiration` needs a label reserved in +/// `windmill_common::auth::is_user_token`, or its expiry alerts its owner (docs/auth-surface.md). pub async fn create_token_internal( tx: &mut sqlx::PgConnection, db: &DB, diff --git a/backend/windmill-api-integration-tests/tests/token_label_idor.rs b/backend/windmill-api-integration-tests/tests/token_label_idor.rs index ddec1a481a..e690bf673f 100644 --- a/backend/windmill-api-integration-tests/tests/token_label_idor.rs +++ b/backend/windmill-api-integration-tests/tests/token_label_idor.rs @@ -179,9 +179,11 @@ async fn test_trigger_token_labels_still_creatable(db: Pool) -> anyhow "http-test-user-2-cd34", "email-test-user-2-ef56", "my-ci-token", - // Minted client-side by the editor (every TypeScript editor load) and the debugger. + // Minted client-side by the editor (every TypeScript editor load), the debugger and + // the object-storage "Test from a worker" button. "Ephemeral lsp token", "debugger-token", + "ephemeral-test-connection: s3_bucket", ] { let resp = create_token_with_label(port, "SECRET_TOKEN_2", label).await; assert_eq!( diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index 476a900ef6..b7d30eaa4b 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -3882,8 +3882,8 @@ async fn update_token_label( Path(token_prefix): Path, Json(req): Json, ) -> Result { - // The new label must not collide with a system-token namespace (`session`, - // `ephemeral*`, `debugger-token`, `mcp-oauth-*`): those labels are + // The new label must not collide with a system-token namespace (see + // `windmill_common::auth::is_user_token`): those labels are // load-bearing, and a user-set collision would orphan the token — hidden // from the UI (`isUserToken`) and rejected by the editability guard below — // while it still authenticates. (`is_user_token(None)` is true, so clearing @@ -3922,6 +3922,9 @@ async fn update_token_label( AND lower(label) NOT LIKE 'ephemeral%' AND label <> 'debugger-token' AND label NOT LIKE 'mcp-oauth-%' + AND NOT starts_with(label, 'embed_app:') + AND NOT starts_with(label, 'sdk_app:') + AND NOT starts_with(label, 'impersonation:') )) RETURNING token_prefix", req.label.as_deref(), diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 6af6bbc37c..8b85c2919b 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -57,7 +57,7 @@ use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; use windmill_audit::ActionKind; use windmill_common::{ apps::{AppScriptId, ListAppQuery, APP_WORKSPACED_ROUTE}, - auth::TOKEN_PREFIX_LEN, + auth::{APP_EMBED_TOKEN_LABEL_PREFIX, RAW_APP_SDK_TOKEN_LABEL_PREFIX, TOKEN_PREFIX_LEN}, cache::{self, future::FutureCachedExt}, db::{DbWithOptAuthed, UserDB}, error::{to_anyhow, Error, JsonResult, Result}, @@ -1522,7 +1522,10 @@ async fn mint_raw_app_sdk_token( scopes.push(windmill_api_auth::scopes::GUEST_SENTINEL.to_string()); (label, exp) } - None => (format!("sdk_app:{app_path}"), requested_exp), + None => ( + format!("{RAW_APP_SDK_TOKEN_LABEL_PREFIX}{app_path}"), + requested_exp, + ), }; let token_config = NewToken::new( Some(label), @@ -1804,7 +1807,10 @@ pub async fn mint_app_embed_token( scopes.push(windmill_api_auth::scopes::GUEST_SENTINEL.to_string()); (label, exp) } - None => (format!("embed_app:{app_path}"), requested_exp), + None => ( + format!("{APP_EMBED_TOKEN_LABEL_PREFIX}{app_path}"), + requested_exp, + ), }; let token_config = NewToken::new( Some(label), diff --git a/backend/windmill-common/src/auth.rs b/backend/windmill-common/src/auth.rs index 73cdfba35d..3e8eb58a14 100644 --- a/backend/windmill-common/src/auth.rs +++ b/backend/windmill-common/src/auth.rs @@ -19,10 +19,11 @@ use crate::{ }; /// Whether `label` denotes a user-created token rather than a system token -/// (`session`, `guest_session`, `ephemeral*`, `debugger-token`, `mcp-oauth-*`). System-token -/// labels are load-bearing — session cleanup, super_admin propagation, expiry -/// notifications and username overrides all key off them — so they must not be -/// user-editable. `None` (no label) is treated as a user token. +/// (`session`, `guest_session`, `ephemeral*`, `debugger-token`, `mcp-oauth-*`, +/// `embed_app:*`, `sdk_app:*`, `impersonation:*`). System-token labels are load-bearing — +/// session cleanup, super_admin propagation, expiry notifications and username overrides +/// all key off them — so they must not be user-editable. `None` (no label) is treated as +/// a user token. /// /// This is the canonical copy. When updating it, also update its mirrors: /// - the `update_token_label` editability guard (SQL `WHERE`) in @@ -40,15 +41,30 @@ pub fn is_user_token(label: Option<&str>) -> bool { && !l.to_lowercase().starts_with("ephemeral") && l != "debugger-token" && !l.starts_with("mcp-oauth-") + // Short-lived tokens the server mints per app open or per service-account + // impersonation (EE `users_ee.rs`) and nobody manages, so an expiry warning + // for one is noise. + && !l.starts_with(APP_EMBED_TOKEN_LABEL_PREFIX) + && !l.starts_with(RAW_APP_SDK_TOKEN_LABEL_PREFIX) + && !l.starts_with("impersonation:") } } } +/// Label prefix, followed by the app path, of the token an app viewer's sandboxed iframe +/// runs with. Reserved in [`is_user_token`], whose SQL and frontend mirrors spell it out. +pub const APP_EMBED_TOKEN_LABEL_PREFIX: &str = "embed_app:"; + +/// Label prefix, followed by the app path, of the token a raw app's bundle uses for the +/// frontend SDK. Reserved in [`is_user_token`], whose SQL and frontend mirrors spell it out. +pub const RAW_APP_SDK_TOKEN_LABEL_PREFIX: &str = "sdk_app:"; + /// Whether `label` belongs to a namespace only the server mints, and which therefore must be /// rejected by `create_token`. Narrower than [`is_user_token`], which also drives label -/// editability and expiry notifications and can afford to reserve more: `Ephemeral lsp token` -/// and `debugger-token` are minted by the editor and the debugger through that same handler, -/// so reserving them would break those features. +/// editability and expiry notifications and can afford to reserve more: `Ephemeral lsp token`, +/// `debugger-token` and `ephemeral-test-connection: *` are minted by the editor, the debugger +/// and object-storage connection tests through that same handler, so reserving them would +/// break those features. /// /// `username_override_from_label` trusts a label to name the entity acting only if it is in /// here, so anything added must be unmintable by a member. @@ -961,6 +977,9 @@ mod tests { assert!(!is_user_token(Some("Ephemeral lsp token"))); assert!(!is_user_token(Some("debugger-token"))); assert!(!is_user_token(Some("mcp-oauth-client"))); + assert!(!is_user_token(Some("embed_app:f/team/dashboard"))); + assert!(!is_user_token(Some("sdk_app:u/admin/raw app"))); + assert!(!is_user_token(Some("impersonation:admin@windmill.dev"))); } #[test] diff --git a/docs/auth-surface.md b/docs/auth-surface.md index 0876a1184a..abeba1fe2b 100644 --- a/docs/auth-surface.md +++ b/docs/auth-surface.md @@ -12,6 +12,14 @@ Symbols, not line numbers, are cited: they drift less. by `create_session_token` (`windmill-api-users/src/users.rs`). `GET /api/users/refresh_token` mints one for any non-job token but returns plain text, no redirect. - **`tokens/impersonate`** (superadmin) returns a multi-use token and sets no cookie. +- **A token's label decides whether its expiry raises alerts.** When `delete_expired_items` + removes an expired `token` row, the monitor emails the owner and raises a critical alert (if + enabled); rows registered by `register_token_expiry_notification` also get an "expiring soon" + warning first. Neither happens when `is_user_token` (`windmill-common/src/auth.rs`) reserves + the label, so a token the system mints for itself, whether from the backend or from the frontend + through `tokens/create`, needs a reserved label. An `ephemeral-` prefix needs no other change + (keep it clear of `is_server_minted_label` if minted through `tokens/create`); a new prefix + also goes into the SQL and Svelte mirrors that function's doc lists. - **Every superadmin route refuses a job token**: `require_super_admin` (`windmill-api-auth/src/lib.rs`) errors on `authed.job_id.is_some()`. A script that needs `users/create`, `tokens/impersonate`, `set_login_type`, … must use a dedicated superadmin user diff --git a/frontend/src/lib/components/TestConnection.svelte b/frontend/src/lib/components/TestConnection.svelte index 6a58af8031..80ec59d206 100644 --- a/frontend/src/lib/components/TestConnection.svelte +++ b/frontend/src/lib/components/TestConnection.svelte @@ -174,7 +174,7 @@ export async function main(bucket: any, api_token: string) { async function mintApiToken(): Promise { return await UserService.createToken({ requestBody: { - label: `test connection: ${resourceType}`, + label: `ephemeral-test-connection: ${resourceType}`, expiration: new Date(Date.now() + API_TOKEN_TTL_MS).toISOString(), scopes: ['settings:write'] } diff --git a/frontend/src/lib/components/settings/TokensTable.svelte b/frontend/src/lib/components/settings/TokensTable.svelte index fe85d85f03..abfdd0f36d 100644 --- a/frontend/src/lib/components/settings/TokensTable.svelte +++ b/frontend/src/lib/components/settings/TokensTable.svelte @@ -59,7 +59,10 @@ label !== 'guest_session' && !label.toLowerCase().startsWith('ephemeral') && label !== 'debugger-token' && - !label.startsWith('mcp-oauth-') + !label.startsWith('mcp-oauth-') && + !label.startsWith('embed_app:') && + !label.startsWith('sdk_app:') && + !label.startsWith('impersonation:') ) } From 6a89ba0179c7772f400bec30caae7c48f30bf8e4 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Wed, 16 Sep 2026 17:31:06 +0200 Subject: [PATCH 10/77] fix: add only missing grant options before an ACL apply, never default privileges Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb --- .../src/datatable_acl.rs | 63 ++++++++++++++++--- 1 file changed, 54 insertions(+), 9 deletions(-) diff --git a/backend/windmill-api-workspaces/src/datatable_acl.rs b/backend/windmill-api-workspaces/src/datatable_acl.rs index 396a456e4e..6aa2f936b0 100644 --- a/backend/windmill-api-workspaces/src/datatable_acl.rs +++ b/backend/windmill-api-workspaces/src/datatable_acl.rs @@ -1308,6 +1308,59 @@ async fn authorize_acl_change( Ok(governing) } +/// A role passes on only privileges it holds with grant option, and an instance database +/// provisioned before data table roles gave `custom_instance_user` none. Adds that option to its +/// database and `public` privileges, and nothing else: default privileges are left alone, since a +/// schema's change of owner is planned against them. Best-effort, as a grant it fails to enable is +/// refused when it runs. +async fn ensure_grant_options(client: &tokio_postgres::Client, db: &DB, dbname: &str) { + let held = client + .query_one( + "SELECT has_database_privilege(current_database(), 'CONNECT WITH GRANT OPTION') + AND has_database_privilege(current_database(), 'CREATE WITH GRANT OPTION') + AND (to_regnamespace('public') IS NULL + OR (has_schema_privilege('public', 'USAGE WITH GRANT OPTION') + AND has_schema_privilege('public', 'CREATE WITH GRANT OPTION')))", + &[], + ) + .await + .is_ok_and(|row| row.get::<_, bool>(0)); + if held { + return; + } + if let Err(e) = grant_options_as_server(db, dbname).await { + tracing::warn!("Could not enable grant options on '{dbname}': {e}"); + } +} + +/// Only the database's owner, the server's own Postgres user, can hand out an option it holds. +async fn grant_options_as_server(db: &DB, dbname: &str) -> Result<()> { + let server = PgDatabase::parse_uri(&windmill_common::get_database_url().await?.as_str().await)?; + let creds = PgDatabase { dbname: dbname.to_string(), ..server }; + let (client, connection) = creds.connect(Some(db)).await?; + let join_handle = tokio::spawn(async move { connection.await }); + let role = quote_ident(CUSTOM_INSTANCE_USER); + let result = client + .batch_execute(&format!( + "GRANT CONNECT, CREATE ON DATABASE {} TO {role} WITH GRANT OPTION; + DO $$ BEGIN + IF to_regnamespace('public') IS NOT NULL THEN + GRANT USAGE, CREATE ON SCHEMA public TO {role} WITH GRANT OPTION; + END IF; + END $$;", + quote_ident(dbname) + )) + .await; + drop(client); + windmill_common::shutdown_pg_connection(join_handle).await?; + result.map_err(|e| { + Error::internal_err(format!( + "Failed to grant options on '{dbname}': {}", + pg_error_message(&e) + )) + }) +} + /// Whether the governing entry an apply was authorized on is still the one in the settings, read /// under the lock: a save in between could have pointed it at another database or changed its roles. fn entry_unchanged(governing: &GoverningDatatable, entry_now: Option) -> bool { @@ -1523,6 +1576,7 @@ async fn apply_datatable_acl( // have exhausted. let governing = authorize_acl_change(&db, &authed, &w_id, &datatable_name).await?; let (mut client, mut notices, dbname) = connect_as_admin_unchecked(&db, &governing).await?; + ensure_grant_options(&client, &db, &dbname).await; // Held until the change is committed: a role renamed or dropped meanwhile would change what // the plan names, and a settings save could move the entry onto another database. Taken in the @@ -1548,15 +1602,6 @@ async fn apply_datatable_acl( )); } - // A role passes on only privileges it holds with grant option, which a database provisioned - // before data table roles lacks; a grant this fails to enable is refused below. After the plan - // check, since the planner reads these grants and would refuse what it just accepted; it - // connects as the server's own Postgres user, so it takes nothing from the pool. - if let Err(e) = windmill_common::ensure_instance_db_grant_options_unchecked(&db, &dbname).await - { - tracing::warn!("Could not refresh grant options on '{dbname}': {e}"); - } - // One transaction: a half-applied ownership transfer leaves one schema's objects owned by two // different roles. let pg_tx = client.transaction().await.map_err(|e| { From a9ec0aec3ac0c6b0f7919d0eb2168816923826d7 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Wed, 16 Sep 2026 17:45:01 +0200 Subject: [PATCH 11/77] feat: render the flow chat through the shared session chat components (#11175) * feat: render the flow chat through the shared session chat components Co-Authored-By: Claude Fable 5.1 * fix: settle the flow chat queue on the turn's outcome and guard IME enter Co-Authored-By: Claude Fable 5.1 * fix: keep copilot free-tier banners and deploy-locked sends out of the flow chat Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 --- .../copilot/chat/AIChatDisplay.svelte | 235 +++++++------ .../copilot/chat/AIChatInput.svelte | 195 ++++++----- .../copilot/chat/AIChatManager.svelte.ts | 21 +- .../copilot/chat/AIChatMessage.svelte | 25 +- .../chat/AskUserQuestionDisplay.svelte | 10 +- .../copilot/chat/AssistantMessage.svelte | 25 +- .../copilot/chat/ContextTextarea.svelte | 23 +- .../copilot/chat/ContextUsageIndicator.svelte | 10 +- .../copilot/chat/QueuedMessageChip.svelte | 34 +- .../chat/ToolConfirmationFooter.svelte | 6 +- .../copilot/chat/ToolExecutionDisplay.svelte | 8 +- .../components/copilot/chat/chatViewHost.ts | 152 +++++++++ .../components/copilot/chat/composerBox.ts | 27 ++ .../src/lib/components/copilot/chat/shared.ts | 8 + .../flows/conversations/FlowChat.svelte | 28 +- .../conversations/FlowChatInterface.svelte | 254 +++++++------- .../conversations/flowChatViewHost.svelte.ts | 310 ++++++++++++++++++ .../conversations/flowChatViewHost.test.ts | 267 +++++++++++++++ .../(logged)/flows/get/[...path]/+page.svelte | 2 + 19 files changed, 1241 insertions(+), 399 deletions(-) create mode 100644 frontend/src/lib/components/copilot/chat/chatViewHost.ts create mode 100644 frontend/src/lib/components/copilot/chat/composerBox.ts create mode 100644 frontend/src/lib/components/flows/conversations/flowChatViewHost.svelte.ts create mode 100644 frontend/src/lib/components/flows/conversations/flowChatViewHost.test.ts diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index 7de7dd32dc..82dcbc2dcd 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -44,6 +44,7 @@ import Markdown from 'svelte-exmarkdown' import { twMerge } from 'tailwind-merge' import { AIAutonomyMode, AIMode } from './AIChatManager.svelte' + import { getChatViewHost } from './chatViewHost' import { getAiChatManager } from './aiChatManagerContext' import ChatTypingIndicator from './ChatTypingIndicator.svelte' import AIChatInput from './AIChatInput.svelte' @@ -68,14 +69,19 @@ import { base } from '$lib/base' const MAX_YOLO_TOOLTIP_TOOLS = 8 + const chatHost = getChatViewHost() + // The skill and MCP menus take an AIChatManager itself, which the seam deliberately + // doesn't carry. They render only under GLOBAL, which a non-copilot host never sets. const aiChatManager = getAiChatManager() + // The free grant pays for the copilot's own model, so its banners belong only to a host + // that sends to that model. A flow chat's turn runs on the flow's provider. + const freeTier = $derived(chatHost.supportsModelSettings ? $copilotInfo.freeTier : undefined) // The user spent their one-time free Windmill AI grant: there is no model left to send // to, so say so in the thread itself rather than only failing on send. - let freeTierExhausted = $derived($copilotInfo.freeTier?.exhausted === true) + let freeTierExhausted = $derived(freeTier?.exhausted === true) // Still on the free grant: keep how much is left in view right above the composer, so // running out isn't a surprise. Once spent, the exhausted banner replaces it. - let freeTier = $derived($copilotInfo.freeTier) let freeTierUsedPct = $derived(Math.min(100, Math.round((freeTier?.used_ratio ?? 0) * 100))) let showFreeTierUsage = $derived(!!freeTier && !freeTier.exhausted) @@ -174,8 +180,12 @@ wideLayout = false, emptyHint, inputPreface, + footerSettings, initialInstructions = undefined, - onDraftChange = undefined + onDraftChange = undefined, + placeholder = undefined, + scrollElement = $bindable(), + onTranscriptScroll = undefined }: { messages: DisplayMessage[] pastChats: { id: string; title: string }[] @@ -202,9 +212,18 @@ wideLayout?: boolean emptyHint?: Snippet inputPreface?: Snippet + /** The settings control at the footer's right edge, where the copilot puts its + * model picker. A host that configures its turn elsewhere replaces it here. */ + footerSettings?: Snippet // Seed / observe the main composer's draft text (see AIChatInput). initialInstructions?: string onDraftChange?: (text: string) => void + /** Composer placeholder. Falls back to the per-AI-mode wording. */ + placeholder?: string + /** The transcript's scroll container. A host that paginates older messages + * needs it to measure and restore the scroll position. */ + scrollElement?: HTMLDivElement | undefined + onTranscriptScroll?: () => void } = $props() let aiChatInput: AIChatInput | undefined = $state() @@ -223,7 +242,7 @@ let panelEl: HTMLDivElement | undefined = $state() $effect(() => { function onWindowKeydownCapture(e: KeyboardEvent) { - if (e.key !== 'Escape' || !aiChatManager.loading) return + if (e.key !== 'Escape' || !chatHost.loading) return const active = document.activeElement const focusOnChat = !active || active === document.body || (panelEl?.contains(active) ?? false) @@ -231,22 +250,21 @@ // row alone stops the turn — wherever it is mounted, since the preview panel holds the // form outside `panelEl`. Matched by call: two chats can be loading at once, and one's // row must not answer for the other. - if (aiChatManager.hasPendingRunForm) { + if (chatHost.hasPendingRunForm) { const row = active?.closest('[data-run-form-actions]') const toolCallId = row?.getAttribute('data-run-form-actions') - if (!toolCallId || !aiChatManager.isRunFormPending(toolCallId)) return + if (!toolCallId || !chatHost.isRunFormPending(toolCallId)) return } else if (!focusOnChat) return e.preventDefault() // Immediate form: other chat panels' identical listeners must not // also cancel on body focus, nor a drawer/modal close on this press. e.stopImmediatePropagation() - aiChatManager.cancel() + chatHost.cancel() } window.addEventListener('keydown', onWindowKeydownCapture, true) return () => window.removeEventListener('keydown', onWindowKeydownCapture, true) }) - let scrollEl: HTMLDivElement | undefined = $state() // Programmatic-scroll guard. `scrollDown()` triggers an async `scroll` // event; if a token-append between the scrollTo and the dispatch makes // scrollHeight grow, the gap can briefly exceed STICK_TO_BOTTOM_PX and @@ -259,22 +277,23 @@ // Instant scroll — smooth would animate every token append, racing with // the next scrollDown and confusing the onscroll bottom-detection below. function scrollDown() { - if (!scrollEl) return + if (!scrollElement) return programmaticScrollAt = Date.now() - scrollEl.scrollTo({ top: scrollEl.scrollHeight, behavior: 'auto' }) + scrollElement.scrollTo({ top: scrollElement.scrollHeight, behavior: 'auto' }) } let height = $state(0) $effect(() => { - if (aiChatManager.automaticScroll && height) { + if (chatHost.automaticScroll && height) { scrollDown() } // Recompute the scroll-to-latest visibility on every content-height // change. `onScroll` only fires for actual scroll events, so without // this the arrow can go stale when content grows past the threshold // while auto-scroll is disabled (user scrolled up mid-stream). - if (scrollEl && height) { - const distance = scrollEl.scrollHeight - scrollEl.scrollTop - scrollEl.clientHeight + if (scrollElement && height) { + const distance = + scrollElement.scrollHeight - scrollElement.scrollTop - scrollElement.clientHeight showScrollToLatest = distance > SCROLL_TO_LATEST_THRESHOLD_PX } }) @@ -289,8 +308,9 @@ const SCROLL_TO_LATEST_THRESHOLD_PX = 200 let showScrollToLatest = $state(false) function onScroll() { - if (!scrollEl) return - const distance = scrollEl.scrollHeight - scrollEl.scrollTop - scrollEl.clientHeight + if (!scrollElement) return + const distance = + scrollElement.scrollHeight - scrollElement.scrollTop - scrollElement.clientHeight // Always refresh the arrow visibility — even during the cooldown, // because clicking the arrow itself triggers a programmatic scroll // whose only event would otherwise be swallowed, leaving the arrow @@ -303,14 +323,15 @@ return } if (distance <= STICK_TO_BOTTOM_PX) { - aiChatManager.enableAutomaticScroll() + chatHost.enableAutomaticScroll() } else { - aiChatManager.disableAutomaticScroll() + chatHost.disableAutomaticScroll() } + onTranscriptScroll?.() } function submitSuggestion(suggestion: string) { - aiChatManager.sendRequest({ instructions: suggestion }) + chatHost.sendRequest({ instructions: suggestion }) } export function focusInput() { @@ -319,35 +340,31 @@ $effect(() => { if (aiChatInput) { - aiChatManager.setAiChatInput(aiChatInput) + chatHost.setAiChatInput(aiChatInput) } return () => { - aiChatManager.setAiChatInput(null) + chatHost.setAiChatInput(null) } }) // Also shown for a run held by another tab, labeled with where it is: the // dots say a turn is in flight even before the reader reaches the footer // note. Remote runs pause nothing and offer no Stop — this tab can't cancel. - const showTypingIndicator = $derived(aiChatManager.loading || aiChatManager.runHeldElsewhere) + const showTypingIndicator = $derived(chatHost.loading || chatHost.runHeldElsewhere) // The manual `@` context-picker button. Shown in SCRIPT/FLOW (workspace items + // code blocks) and APP (datatables, frontend files). Hidden in GLOBAL — there // `@`-context is still invoked inline by typing `@` in the input, so the button // is redundant. NAVIGATOR/ASK/API don't take @-context at all. const showContextPicker = $derived( - aiChatManager.mode === AIMode.SCRIPT || - aiChatManager.mode === AIMode.FLOW || - aiChatManager.mode === AIMode.APP + chatHost.mode === AIMode.SCRIPT || chatHost.mode === AIMode.FLOW || chatHost.mode === AIMode.APP ) - // File attachment is GLOBAL-mode only. - const canAttachFiles = $derived(aiChatManager.mode === AIMode.GLOBAL && !disabled) - // Steers the OS file picker toward text + image formats (soft hint; both attach - // to the message — text files after a content sniff). - const TEXT_FILE_ACCEPT = - 'image/*,text/*,.txt,.csv,.tsv,.json,.jsonl,.ndjson,.md,.markdown,.log,.yaml,.yml,.toml,.ini,.cfg,.conf,.env,.xml,.html,.htm,.css,.js,.mjs,.cjs,.ts,.tsx,.jsx,.py,.rb,.rs,.go,.java,.kt,.c,.h,.cpp,.cc,.cs,.php,.sh,.bash,.zsh,.sql,.svelte,.vue,.dockerfile' + const canAttachFiles = $derived(chatHost.supportsMessageAttachments && !disabled) + // Folders are linked as session-wide assets, which only a host that reads files in + // the browser can do — a host running the turn server-side takes attachments only. + const canLinkFolders = $derived(chatHost.supportsLinkedFolders && !disabled) let fileInputEl = $state(null) let folderInputEl = $state(null) let dragDepth = $state(0) @@ -373,12 +390,12 @@ } async function handleAddFiles(files: FileList | FileToAttach[]) { - const { added, rejected } = await aiChatManager.attachedFiles.addFiles(files) + const { added, rejected } = await chatHost.attachedFiles.addFiles(files) reportAddResult(added, rejected) } async function addDirHandle(dir: FileSystemDirectoryHandle) { - const { added, rejected } = await aiChatManager.attachedFiles.addFolder(dir) + const { added, rejected } = await chatHost.attachedFiles.addFolder(dir) reportAddResult(added, rejected) } @@ -471,8 +488,13 @@ const textFiles = looseFiles.filter((f) => !isImageFile(f)) if (textFiles.length > 0) await aiChatInput?.addTextFiles(textFiles) // Folders link as a live handle. - for (const h of handles.filter(isDirectoryHandle)) { - await addDirHandle(h) + const dirs = handles.filter(isDirectoryHandle) + if (dirs.length > 0 && !canLinkFolders) { + sendUserToast('Folders cannot be attached in this chat — drop individual files.', true) + } else { + for (const h of dirs) { + await addDirHandle(h) + } } } else { // Fallback (no File System Access API): snapshot dropped files AND folders by walking @@ -497,7 +519,10 @@ topLevelText.push(file) } } - if (folderEntries.length > 0) await handleAddFiles(folderEntries) + if (folderEntries.length > 0) { + if (canLinkFolders) await handleAddFiles(folderEntries) + else sendUserToast('Folders cannot be attached in this chat — drop individual files.', true) + } if (topLevelText.length > 0) await aiChatInput?.addTextFiles(topLevelText) } } @@ -524,9 +549,9 @@ input.value = '' } const autonomyAvailability = $derived({ - autoAcceptEditsAvailable: aiChatManager.autoAcceptEditsAvailable, - autoAcceptToolConfirmationsAvailable: aiChatManager.autoAcceptToolConfirmationsAvailable, - planModeAvailable: aiChatManager.planModeAvailable + autoAcceptEditsAvailable: chatHost.autoAcceptEditsAvailable, + autoAcceptToolConfirmationsAvailable: chatHost.autoAcceptToolConfirmationsAvailable, + planModeAvailable: chatHost.planModeAvailable }) const availableAutonomyModeOptions = $derived( autonomyModeOptions.filter((option) => option.isAvailable(autonomyAvailability)) @@ -534,8 +559,8 @@ // Fall back to ask-permission when the persisted mode isn't applicable in the // current AI mode (e.g. auto-accept edits while in a mode without edits). const effectiveAutonomyMode = $derived( - availableAutonomyModeOptions.some((option) => option.mode === aiChatManager.autonomyMode) - ? aiChatManager.autonomyMode + availableAutonomyModeOptions.some((option) => option.mode === chatHost.autonomyMode) + ? chatHost.autonomyMode : AIAutonomyMode.DEFAULT ) const showAutonomyModeSelector = $derived(!disabled && availableAutonomyModeOptions.length > 1) @@ -544,13 +569,13 @@ // The typing-dots indicator implies the AI is busy, which is misleading while // the loop is parked on the user; surface a text pill instead so users know to // act on the tool above. - const waitingForUserAction = $derived(aiChatManager.loading && !!pendingUserAction(messages)) + const waitingForUserAction = $derived(chatHost.loading && !!pendingUserAction(messages)) // Gated on `loading` because a card restored from history still looks parked: // its resolver left with the old page, so the composer must not advertise an // answer it cannot deliver. const pendingQuestionToolCallId = $derived.by(() => { - if (!aiChatManager.loading) { + if (!chatHost.loading) { return undefined } const pending = pendingUserActionDetail(messages) @@ -559,14 +584,14 @@ // Get app context for display when in APP mode const appContext = $derived.by((): SelectedContext | undefined => { - if (aiChatManager.mode !== AIMode.APP || !aiChatManager.appAiChatHelpers) { + if (chatHost.mode !== AIMode.APP || !chatHost.appAiChatHelpers) { return undefined } - return aiChatManager.appAiChatHelpers.getSelectedContext() + return chatHost.appAiChatHelpers.getSelectedContext() }) const yoloBypassedTools = $derived.by(() => { - return aiChatManager.tools + return chatHost.tools .filter((tool) => tool.requiresConfirmation === true || tool.bypassedByAutoAccept === true) .map((tool) => ({ name: tool.def.function.name, @@ -583,8 +608,7 @@ Math.max(0, yoloBypassedTools.length - visibleYoloBypassedTools.length) ) const showFlowPendingActionControls = $derived( - (aiChatManager.flowAiChatHelpers?.hasPendingChanges() ?? false) && - !aiChatManager.autoAcceptEditsActive + (chatHost.flowAiChatHelpers?.hasPendingChanges() ?? false) && !chatHost.autoAcceptEditsActive ) // A disabled state with no message (a remote hold, a spent free grant) keeps // the footer toolbar in place — swapping it for an empty strip would make @@ -592,11 +616,15 @@ // a real message (archived, AI off) still shows it, hold or not, matching // the precedence disabledMessage itself encodes. const footerMessageShown = $derived(disabled && disabledMessage !== '') + // `canAttachFiles` belongs in the group too: in GLOBAL mode the `+` always has the + // context picker or the autonomy selector beside it, but a host with attachments and + // nothing else would lose the group and the `+` with it. const showFooterLeftControls = $derived( !footerMessageShown && - (showContextPicker || + (canAttachFiles || + showContextPicker || showAutonomyModeSelector || - (aiChatManager.mode === AIMode.SCRIPT && hasDiff)) + (chatHost.mode === AIMode.SCRIPT && hasDiff)) ) @@ -694,12 +722,12 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> {#each pastChats as chat (chat.id)}
{/if}
- {:else if aiChatManager.mode === AIMode.APP} + {:else if chatHost.mode === AIMode.APP} {#if showContext} {@render badgeRow()} {/if} @@ -1264,30 +1289,38 @@ {/if} {:else} -
- - {#if !bottomRightSnippet} -
- {@render sendStopButton()} -
- {/if} + +
+ {@render badgeRow()} + {@render imageChipsRow()} +
+ + {#if !bottomRightSnippet} +
+ {@render sendStopButton()} +
+ {/if} +
{/if} {#if bottomRightSnippet} diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index aafb6a2ed1..34a6b5f3f1 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -1,3 +1,4 @@ +import type { ChatViewHost } from './chatViewHost' import type { ScriptLang } from '$lib/gen/types.gen' import { JobService, type CompletedJob } from '$lib/gen' import type { FlowOptions, ScriptOptions } from './ContextManager.svelte' @@ -445,9 +446,27 @@ function planModeHostFor(m: AIChatManager): PlanModeHost { } } -export class AIChatManager { +export class AIChatManager implements ChatViewHost { contextManager = new ContextManager() historyManager = new HistoryManager() + // The copilot owns its model choice and its own transcript, so both chat + // affordances apply here. See ChatViewHost for hosts where they don't. + supportsModelSettings = true + supportsMessageEditing = true + // The copilot turn is the attachments themselves when there is no text. + requiresMessageText = false + // Attachments and linked folders are GLOBAL-mode affordances. Declared as + // getters because `mode` changes under a mounted composer. + get supportsMessageAttachments() { + return this.mode === AIMode.GLOBAL + } + get supportsLinkedFolders() { + return this.mode === AIMode.GLOBAL + } + // Steers the OS file picker toward text + image formats (a soft hint; both attach to + // the message — text files after a content sniff). + attachmentAccept = + 'image/*,text/*,.txt,.csv,.tsv,.json,.jsonl,.ndjson,.md,.markdown,.log,.yaml,.yml,.toml,.ini,.cfg,.conf,.env,.xml,.html,.htm,.css,.js,.mjs,.cjs,.ts,.tsx,.jsx,.py,.rb,.rs,.go,.java,.kt,.c,.h,.cpp,.cc,.cs,.php,.sh,.bash,.zsh,.sql,.svelte,.vue,.dockerfile' /** Files the user attached to the current GLOBAL-mode conversation. */ attachedFiles = new AttachedFilesStore() /** Markdown artifacts the copilot created for the current session. */ diff --git a/frontend/src/lib/components/copilot/chat/AIChatMessage.svelte b/frontend/src/lib/components/copilot/chat/AIChatMessage.svelte index 12553be961..afcffbb400 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatMessage.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatMessage.svelte @@ -3,7 +3,7 @@ import type { DisplayMessage, ToolDisplayMessage } from './shared' import ContextElementBadge from './ContextElementBadge.svelte' import AssistantMessage from './AssistantMessage.svelte' - import { getAiChatManager } from './aiChatManagerContext' + import { getChatViewHost } from './chatViewHost' import { Button } from '$lib/components/common' import { RefreshCwIcon, Undo2Icon } from 'lucide-svelte' import AIChatInput from './AIChatInput.svelte' @@ -15,7 +15,7 @@ import ExpandableImage from '$lib/components/common/image/ExpandableImage.svelte' import { workspaceStore } from '$lib/stores' - const aiChatManager = getAiChatManager() + const chatHost = getChatViewHost() // Paths in a message name items the chat's tools reach, so they resolve against the // operating workspace, never `workspaceStore`: a fork session leaves the store on the @@ -25,7 +25,7 @@ // Registers the dependency that `operatingWorkspace`'s own untracked // `get(workspaceStore)` cannot. void $workspaceStore - return aiChatManager.operatingWorkspace + return chatHost.operatingWorkspace }) // Per-message expand/collapse state for paste chips shown in the bubble. @@ -62,7 +62,12 @@ let editContext = $state([]) function editMessage() { - if (message.role !== 'user' || editingMessageIndex !== null || aiChatManager.loading) { + if ( + !chatHost.supportsMessageEditing || + message.role !== 'user' || + editingMessageIndex !== null || + chatHost.loading + ) { return } editContext = [...(message.contextElements ?? [])] @@ -79,7 +84,9 @@ message.role === 'tool' && 'mb-1', message.role === 'user' && messageIndex > 0 && 'mt-4 mb-6', isLast && '!mb-12', - message.role !== 'user' ? 'cursor-default' : 'cursor-pointer' + message.role !== 'user' || !chatHost.supportsMessageEditing + ? 'cursor-default' + : 'cursor-pointer' )} role="button" tabindex="0" @@ -116,7 +123,7 @@ bind:selectedContext={editContext} initialInstructions={message.content} initialPastes={message.pastes} - initialImages={aiChatManager.storedImages(messageIndex)} + initialImages={chatHost.storedImages(messageIndex)} initialFiles={message.files} {editingMessageIndex} onClickOutside={() => (editingMessageIndex = null)} @@ -185,9 +192,9 @@ on:click={() => { if (message.snapshot) { if (message.snapshot.type === 'flow') { - aiChatManager.flowAiChatHelpers?.revertToSnapshot(message.snapshot.value) + chatHost.flowAiChatHelpers?.revertToSnapshot(message.snapshot.value) } else if (message.snapshot.type === 'app') { - aiChatManager.appAiChatHelpers?.revertToSnapshot(message.snapshot.value) + chatHost.appAiChatHelpers?.revertToSnapshot(message.snapshot.value) } } }} @@ -206,7 +213,7 @@ variant="default" title="Retry generation" startIcon={{ icon: RefreshCwIcon }} - onclick={() => aiChatManager.retryRequest(messageIndex)} + onclick={() => chatHost.retryRequest(messageIndex)} > Retry diff --git a/frontend/src/lib/components/copilot/chat/AskUserQuestionDisplay.svelte b/frontend/src/lib/components/copilot/chat/AskUserQuestionDisplay.svelte index 403b42864d..17cc2bb4c1 100644 --- a/frontend/src/lib/components/copilot/chat/AskUserQuestionDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AskUserQuestionDisplay.svelte @@ -4,7 +4,7 @@ import { CircleHelp, ArrowUp, Plus, Square, SquareCheck } from 'lucide-svelte' import Button from '$lib/components/common/button/Button.svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' - import { getAiChatManager } from './aiChatManagerContext' + import { getChatViewHost } from './chatViewHost' import type { UserQuestionDisplay } from './shared' // Sessions inject a per-pane `AIChatManager` via context; outside of @@ -12,7 +12,7 @@ // this, answers clicked inside a session would dispatch to the singleton's // pending callbacks map (which doesn't have the session manager's question // callback), and the AI loop would stall. - const aiChatManager = getAiChatManager() + const chatHost = getChatViewHost() interface Props { toolCallId: string @@ -93,14 +93,14 @@ } return } - aiChatManager.handleUserQuestionAnswer(toolCallId, [choice]) + chatHost.handleUserQuestionAnswer(toolCallId, [choice]) } function submitPicked() { if (!multiSelect || picked.size === 0) { return } - aiChatManager.handleUserQuestionAnswer(toolCallId, [...picked]) + chatHost.handleUserQuestionAnswer(toolCallId, [...picked]) } function submitCustomAnswer() { @@ -119,7 +119,7 @@ return } - aiChatManager.handleUserQuestionAnswer(toolCallId, [answer]) + chatHost.handleUserQuestionAnswer(toolCallId, [answer]) } function handleChoiceKeydown(event: KeyboardEvent, choice: string, index: number) { diff --git a/frontend/src/lib/components/copilot/chat/AssistantMessage.svelte b/frontend/src/lib/components/copilot/chat/AssistantMessage.svelte index f06941c2a4..44cedf323c 100644 --- a/frontend/src/lib/components/copilot/chat/AssistantMessage.svelte +++ b/frontend/src/lib/components/copilot/chat/AssistantMessage.svelte @@ -12,6 +12,7 @@ workspaceItemRegistry } from './workspaceItems.svelte' import { markdownProse } from '$lib/components/markdownProse' + import DisplayResult from '$lib/components/DisplayResult.svelte' interface Props { message: DisplayMessage @@ -60,6 +61,20 @@ return rest === 0 ? `${minutes}m` : `${minutes}m ${rest}s` } + const stepName = $derived(message.role === 'assistant' ? message.stepName : undefined) + + // A flow step can return a file rather than text; the raw JSON would be + // unreadable, so hand it to the result viewer instead of the markdown renderer. + const s3Object = $derived.by(() => { + if (!message.content.startsWith('{')) return undefined + try { + const parsed = JSON.parse(message.content) + return parsed?.type === 'windmill_s3_object' && parsed?.s3 ? parsed : undefined + } catch { + return undefined + } + }) + const candidatePaths = $derived(extractCandidatePaths(message.content)) const rendererPlugin = { renderer: { @@ -98,6 +113,12 @@ }) +{#if stepName} +
+ {stepName} +
+{/if} + {#if reasoning} {/if} -{#if message.content} +{#if s3Object} + +{:else if message.content}
diff --git a/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte b/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte index 8ae3b51b56..976b969b3f 100644 --- a/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte +++ b/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte @@ -1,4 +1,5 @@ - - -
+
{@render leading?.()} @@ -830,11 +817,7 @@ {placeholder} class={twMerge( 'textarea-input resize-none caret-black dark:caret-white overflow-clip', - // The box (border/ring) lives on the wrapper; kill the textarea's own - // @tailwindcss/forms border, focus ring, and background so only the - // wrapper reads as the field. - '!border-transparent !bg-transparent !shadow-none focus:!border-transparent focus:!ring-0', - 'disabled:cursor-not-allowed disabled:placeholder:text-disabled', + COMPOSER_FIELD_RESET, CHAT_INPUT_PADDING, className )} diff --git a/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte b/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte index 96b452120b..cd6c58b2a3 100644 --- a/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte +++ b/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte @@ -1,16 +1,16 @@ -{#if aiChatManager.queuedMessage || aiChatManager.queuedImages.length > 0 || aiChatManager.queuedFiles.length > 0 || (aiChatManager.queuedContext?.length ?? 0) > 0} +{#if chatHost.queuedMessage || chatHost.queuedImages.length > 0 || chatHost.queuedFiles.length > 0 || (chatHost.queuedContext?.length ?? 0) > 0}
- {#if aiChatManager.queuedMessage || aiChatManager.queuedImages.length > 0 || aiChatManager.queuedFiles.length > 0} + {#if chatHost.queuedMessage || chatHost.queuedImages.length > 0 || chatHost.queuedFiles.length > 0} - {:else if aiChatManager.queuedContext?.length} + {:else if chatHost.queuedContext?.length}
- {#each aiChatManager.queuedContext as element (contextElementKey(element))} + {#each chatHost.queuedContext as element (contextElementKey(element))} {/each}
@@ -82,7 +82,7 @@ iconOnly title="Remove queued message and put it back in the input" startIcon={{ icon: X }} - on:click={() => aiChatManager.dequeueMessage()} + on:click={() => chatHost.dequeueMessage()} />
{/if} diff --git a/frontend/src/lib/components/copilot/chat/ToolConfirmationFooter.svelte b/frontend/src/lib/components/copilot/chat/ToolConfirmationFooter.svelte index 699b79a292..fcf943cb34 100644 --- a/frontend/src/lib/components/copilot/chat/ToolConfirmationFooter.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolConfirmationFooter.svelte @@ -1,7 +1,7 @@ diff --git a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte index 620357ff87..a0d16043a4 100644 --- a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte @@ -21,9 +21,9 @@ } from './planMode' import { Button } from '$lib/components/common' import { markdownProse } from '$lib/components/markdownProse' - import { getAiChatManager } from './aiChatManagerContext' + import { getChatViewHost } from './chatViewHost' - const aiChatManager = getAiChatManager() + const chatHost = getChatViewHost() import { isActiveUserQuestion, type ToolDisplayMessage } from './shared' import ChatCollapsibleCard from './ChatCollapsibleCard.svelte' import { twMerge } from 'tailwind-merge' @@ -69,7 +69,7 @@ const planLabel = $derived((planState && planCopy?.[planState]) ?? '') const planDoc = $derived( message.planArtifactId - ? aiChatManager.artifacts.artifacts.find((a) => a.id === message.planArtifactId) + ? chatHost.artifacts.artifacts.find((a) => a.id === message.planArtifactId) : undefined ) // The version this card wrote, not the document's current one, since later proposals move it on. @@ -201,7 +201,7 @@ title="Open this plan in the side panel: {planDoc.name}" startIcon={{ icon: FileText, classes: PLAN_MODE_TEXT_COLOR }} endIcon={{ icon: PanelRight }} - on:click={() => aiChatManager.openArtifact?.(planDoc.id, planDoc.name, planCardVersion)} + on:click={() => chatHost.openArtifact?.(planDoc.id, planDoc.name, planCardVersion)} > Plan diff --git a/frontend/src/lib/components/copilot/chat/chatViewHost.ts b/frontend/src/lib/components/copilot/chat/chatViewHost.ts new file mode 100644 index 0000000000..9e3bdb44ac --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/chatViewHost.ts @@ -0,0 +1,152 @@ +import { getContext, setContext } from 'svelte' +import type { AIMode, AIAutonomyMode } from './AIChatManager.svelte' +import { getAiChatManager } from './aiChatManagerContext' +import type { DisplayMessage, Tool } from './shared' +import type { ContextElement } from './context' +import type { AttachedImage } from './imageUtils' +import type { AttachedTextFile } from './textFileUtils' +import type { PasteAttachment } from './pasteTokens' +import type { AttachedFilesStore } from './files/attachedFiles.svelte' +import type { SessionArtifactsStore } from './artifacts/artifactsState.svelte' +import type { ArtifactVersionTarget } from '$lib/components/sessions/previewRouter' +import type { FlowAIChatHelpers } from './flow/core' +import type { AppAIChatHelpers } from './app/core' +import type AIChatInput from './AIChatInput.svelte' + +export type ChatSendRequestOptions = { + instructions?: string + pastes?: PasteAttachment[] + images?: AttachedImage[] + files?: AttachedTextFile[] + /** Selected-context snapshot for this turn, in place of the live selection. Set + * whenever a send settles its context ahead of the turn. A host with no context + * of its own ignores it. */ + contextOverride?: ContextElement[] + /** Where `contextOverride` came from. 'pinned': chips picked for THIS message, so + * they are consumed from the live selection on send. 'replay': an edit or retry + * resending an older message's context, already consumed long ago. */ + contextOverrideOrigin?: 'pinned' | 'replay' +} + +/** + * What the chat view components (AIChatDisplay and everything it renders) need + * from whatever is driving the conversation. AIChatManager implements it for the + * copilot's own LLM loop; FlowChatViewHost implements it over a flow run's + * conversation so both chats render through the same components. + * + * Each affordance is gated by the field that answers for it — attachments by + * `supportsMessageAttachments`, the model button by `supportsModelSettings`, and so on — + * so a host turns on exactly what it can serve, and a new one is a matter of answering + * these fields rather than of being a copilot. `mode` is the exception, still read + * directly for chrome that only the copilot has. + */ +export interface ChatViewHost { + // Transcript + displayMessages: DisplayMessage[] + /** API-level messages. Only the count is read (context usage visibility). */ + messages: readonly unknown[] + contextTokens: number + /** The workspace a message's paths and jobs resolve against, which a fork session + * pins away from the navigated one. */ + readonly operatingWorkspace: string | undefined + loading: boolean + /** A turn this tab can neither follow nor stop, held by another tab on the same chat. */ + readonly runHeldElsewhere: boolean + loadingLabel: string | undefined + compacting: boolean + currentReply: string + currentReasoning: string + currentReasoningActive: boolean + readonly reasoningHiddenIndicatorLabel: string | undefined + readonly automaticScroll: boolean + enableAutomaticScroll: () => void + disableAutomaticScroll: () => void + + // Composer + instructions: string + readonly sendInFlight: boolean + /** Resolves to whether the draft was consumed as a turn. */ + sendRequest: (options?: ChatSendRequestOptions) => Promise + cancel: (reason?: string) => void + setAiChatInput: (aiChatInput: AIChatInput | null) => void + readonly queuedMessage: string + queuedContext: ContextElement[] | undefined + readonly queuedImages: AttachedImage[] + readonly queuedFiles: AttachedTextFile[] + queueMessage: ( + text: string, + images?: AttachedImage[], + context?: ContextElement[], + files?: AttachedTextFile[] + ) => void + dequeueMessage: () => void + setComposerStaged: (key: string, editingIndex: number | null, bytes: number) => void + clearComposerStaged: (key: string) => void + attachmentBytesExcluding: (selfKey: string) => number + + // Per-message actions + storedImages: (displayMessageIndex: number) => AttachedImage[] | undefined + retryRequest: (messageIndex: number) => void + restartGeneration: ( + displayMessageIndex: number, + newContent?: string, + pastes?: PasteAttachment[], + images?: AttachedImage[], + editedContext?: ContextElement[], + files?: AttachedTextFile[] + ) => void | Promise + handleUserQuestionAnswer: (toolId: string, choices: string[]) => boolean + handleToolConfirmation: (toolId: string, confirmed: boolean) => void + /** A tool is waiting on a run form the user is filling in. Escape belongs to that form + * then, not to the turn — see AIChatDisplay's window handler. */ + readonly hasPendingRunForm: boolean + isRunFormPending: (toolCallId: string) => boolean + + // Copilot-only surfaces. Left undefined/false by hosts that have no LLM loop + // of their own; the chrome they drive hides itself. + mode?: AIMode + isSessionChat: boolean + /** Model + reasoning picker. Off where the model is configured elsewhere. */ + supportsModelSettings: boolean + /** Click a user message to edit and resend it. Needs a host that can rewind + * its own transcript, which a host replaying a server-side run cannot. */ + supportsMessageEditing: boolean + /** The `+` menu's file entry and drag-and-drop onto the panel. */ + supportsMessageAttachments: boolean + /** The turn needs text: attachments alone cannot be sent. True where the consumer + * requires a message of its own — an AI agent step refuses a run with neither a + * `user_message` nor manual memory. */ + requiresMessageText: boolean + /** The `+` menu's folder entries, backed by `attachedFiles`. A linked folder is a + * live handle on the user's disk, so only a host reading files in the browser has one. */ + supportsLinkedFolders: boolean + /** `accept` for the file picker. */ + attachmentAccept: string + tools: Tool[] + autonomyMode: AIAutonomyMode + setAutonomyMode: (mode: AIAutonomyMode) => void + readonly autoAcceptEditsActive: boolean + readonly autoAcceptEditsAvailable: boolean + readonly autoAcceptToolConfirmationsAvailable: boolean + readonly planModeAvailable: boolean + attachedFiles: AttachedFilesStore + artifacts: SessionArtifactsStore + openArtifact?: (artifactId: string, name: string, version?: ArtifactVersionTarget) => void + flowAiChatHelpers?: FlowAIChatHelpers + appAiChatHelpers?: AppAIChatHelpers +} + +const CHAT_VIEW_HOST_CONTEXT_KEY = 'chatViewHost' + +export function setChatViewHost(host: ChatViewHost) { + setContext(CHAT_VIEW_HOST_CONTEXT_KEY, host) +} + +/** + * Resolve the host driving the chat in this subtree. Falls back to the + * AIChatManager (scoped instance or app-wide singleton) so every existing + * copilot chat keeps working without setting anything. + */ +export function getChatViewHost(): ChatViewHost { + return getContext(CHAT_VIEW_HOST_CONTEXT_KEY) ?? getAiChatManager() +} diff --git a/frontend/src/lib/components/copilot/chat/composerBox.ts b/frontend/src/lib/components/copilot/chat/composerBox.ts new file mode 100644 index 0000000000..f85783d988 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/composerBox.ts @@ -0,0 +1,27 @@ +/** + * The composer's box, shared by both of AIChatInput's branches — the rich + * ContextTextarea and the plain textarea a host without @-context gets. + * + * Border and rounding live on the WRAPPER, never on the field, so the chip rows + * (context, files, images) sit inside the box above the text. The field's own + * @tailwindcss/forms border, ring and background are neutralised so only the + * wrapper reads as the input. + * + * The disabled treatment is on the wrapper for the same reason: `disabled` on the + * field alone leaves it looking exactly like a usable one, so the only cue that + * typing is refused is placeholder text the eye reads as an invitation. + */ + +const BOX_BASE = 'w-full scroll-pb-2 rounded-md border border-border-light transition-colors' + +export function composerBoxClass(disabled: boolean = false): string { + return `${BOX_BASE} ${ + disabled + ? 'bg-surface-disabled cursor-not-allowed' + : 'bg-surface-input focus-within:border-border-selected' + }` +} + +/** Applied to the field inside the box; without it the field draws a second border. */ +export const COMPOSER_FIELD_RESET = + '!border-transparent !bg-transparent !shadow-none focus:!border-transparent focus:!ring-0 disabled:cursor-not-allowed disabled:placeholder:text-disabled' diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index 73efd2e37e..bbcb76b30c 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -671,6 +671,14 @@ export type AssistantDisplayMessage = BaseDisplayMessage & { * would look like it is still streaming forever. */ streaming?: boolean + /** Flow step that produced this message, when the conversation is a flow run + * rather than a copilot turn. Rendered as a label above the content. */ + stepName?: string + /** The run behind this answer. Flow chats only: a copilot turn happens in the + * browser and has no job. */ + jobId?: string + /** When the message was stored, as the server reports it. */ + createdAt?: string } /** diff --git a/frontend/src/lib/components/flows/conversations/FlowChat.svelte b/frontend/src/lib/components/flows/conversations/FlowChat.svelte index 160af2bf27..dc4eb92b57 100644 --- a/frontend/src/lib/components/flows/conversations/FlowChat.svelte +++ b/frontend/src/lib/components/flows/conversations/FlowChat.svelte @@ -22,6 +22,9 @@ path: string hideSidebar?: boolean inputSchema?: Record + /** The flow's description, shown under the empty transcript's prompt. */ + description?: string + wideLayout?: boolean } let { @@ -29,7 +32,9 @@ deploymentInProgress = false, path, hideSidebar = false, - inputSchema = undefined + inputSchema = undefined, + description = undefined, + wideLayout = false }: Props = $props() const flowEditorContext = getContext('FlowEditorContext') @@ -90,13 +95,18 @@ {#if !hideSidebar} {/if} - + + {#key chat} + + {/key} {/if}
diff --git a/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte b/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte index 22fe037728..535be3ca83 100644 --- a/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte +++ b/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte @@ -1,79 +1,37 @@ @@ -164,82 +141,71 @@ {/if} -
- -
- {#if deploymentInProgress} - - {/if} - {#if chatState.loadingMessages && chatState.messages.length === 0} -
- -
- {:else if chatState.messages.length === 0} -
- -

Start a conversation

-

Send a message to run the flow and see the results

-
+{#snippet emptyHint()} +
+ {#if chatHost.state.loadingMessages} + {:else} -
- {#each chatState.messages as message (message.id)} - - {/each} - {#if busy} -
- - Processing... -
- {/if} -
- {/if} -
- - -
- {#if additionalInputsSchema} -
-
- - {#if hasMissingRequired} - - {/if} + +

Start a conversation

+

Send a message to run the flow and see the results

+ {#if !emptyString(description)} +
+
-
+ {/if} {/if} -
- { - if (e.key === 'Enter' && !e.shiftKey && !e.isComposing) { - e.preventDefault() - handleSendMessage() - } - }} - showCancelButton={busy} - onCancel={() => chat.stop()} - sendTitle={deploymentInProgress ? 'Deployment in progress' : 'Send message (Enter)'} - /> -
+{/snippet} + +{#snippet footerSettings()} + {#if additionalInputsSchema} +
+ + {#if hasMissingRequired} + + {/if} +
+ {/if} +{/snippet} + + +
0} + class:min-h-64={chatHost.displayMessages.length === 0} +> + {}} + deletePastChat={() => {}} + saveAndClear={() => {}} + />
diff --git a/frontend/src/lib/components/flows/conversations/flowChatViewHost.svelte.ts b/frontend/src/lib/components/flows/conversations/flowChatViewHost.svelte.ts new file mode 100644 index 0000000000..78118546ac --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/flowChatViewHost.svelte.ts @@ -0,0 +1,310 @@ +import type { Chat, ChatMessage, ChatState } from 'windmill-chat' +import type { + ChatSendRequestOptions, + ChatViewHost +} from '$lib/components/copilot/chat/chatViewHost' +import type { DisplayMessage } from '$lib/components/copilot/chat/shared' +import type { AIAutonomyMode } from '$lib/components/copilot/chat/AIChatManager.svelte' +import { isPlanCardTool } from '$lib/components/copilot/chat/planMode' +import { AttachedFilesStore } from '$lib/components/copilot/chat/files/attachedFiles.svelte' +import { SessionArtifactsStore } from '$lib/components/copilot/chat/artifacts/artifactsState.svelte' +import type { AttachedImage } from '$lib/components/copilot/chat/imageUtils' +import type { AttachedTextFile } from '$lib/components/copilot/chat/textFileUtils' + +export type FlowChatViewHostOptions = { + /** The flow inputs sent next to `user_message` with every turn. */ + additionalInputs?: () => Record | undefined + /** The workspace the transcript's paths resolve against. */ + workspace?: () => string | undefined + /** Whether sending is refused right now (a deployment in progress, say). The composer + * is disabled on the same condition; this covers the sends the composer does not + * make itself: a queued message going out, a retry. */ + sendDisabled?: () => boolean +} + +function isBusy(status: ChatState['status']): boolean { + return status === 'submitted' || status === 'streaming' +} + +/** A tool's arguments or result as the card shows them: parsed where the string is JSON. */ +function parseToolPayload(raw: string | undefined): unknown { + if (raw === undefined || raw === '') return undefined + try { + return JSON.parse(raw) + } catch { + return raw + } +} + +/** + * Whether the turn the user message at `index` started failed: its last row before the + * next user message reports `success: false`. The last row, not any row: a tool call can + * fail and the agent still answer, and that turn completed. + */ +export function turnFailed(messages: readonly ChatMessage[], index: number): boolean { + let last: ChatMessage | undefined + for (let i = index + 1; i < messages.length; i++) { + const message = messages[i] + if (message.role === 'user') break + last = message + } + return last?.success === false +} + +/** Whether the latest turn failed, per `turnFailed`. False before any turn. */ +function lastTurnFailed(messages: readonly ChatMessage[]): boolean { + for (let i = messages.length - 1; i >= 0; i--) { + if (messages[i].role === 'user') return turnFailed(messages, i) + } + return false +} + +export function toDisplayMessages(messages: readonly ChatMessage[]): DisplayMessage[] { + let userIndex = 0 + return messages.map((message, i): DisplayMessage => { + switch (message.role) { + case 'user': + return { + role: 'user', + index: userIndex++, + content: message.content, + // Drives the shared Retry button. + error: turnFailed(messages, i) || undefined + } + case 'tool': { + const parameters = parseToolPayload(message.tool?.arguments) + const result = parseToolPayload(message.tool?.result) + const failed = message.success === false + return { + role: 'tool', + tool_call_id: message.id, + // The card's header is the row's text, which the server only words once the + // tool has returned; until then the row says what is running. + content: message.content || (message.tool ? `Running ${message.tool.name}` : ''), + // Withheld for the copilot's two plan-mode names: `toolName` is what makes + // ToolExecutionDisplay render a plan card, and an agent tool that happened to + // share one would silently become one. + toolName: isPlanCardTool(message.tool?.name) ? undefined : message.tool?.name, + parameters, + result, + showDetails: parameters !== undefined || result !== undefined, + error: failed ? message.content : undefined, + isLoading: message.pending && message.tool?.status === 'running' + } + } + default: + return { + role: 'assistant', + content: message.content, + // Only the message a turn is still writing: a finalized reasoning-only + // message must not look in progress. + streaming: message.pending || undefined, + reasoning: message.reasoning, + stepName: message.stepName, + jobId: message.jobId, + createdAt: message.createdAt + } + } + }) +} + +/** + * Renders a flow run's conversation, as the `windmill-chat` SDK keeps it, through the + * copilot's chat components. The turn is a flow job rather than an LLM call this host + * makes, so what it can offer is what the SDK's `Chat` can: a message in, an answer + * streamed back, Stop. Every copilot-only field is answered with "no" (see ChatViewHost). + * + * The host owns its subscription to the chat, so a panel swapping chats mounts a new one. + */ +export class FlowChatViewHost implements ChatViewHost { + #chat: Chat + #options: FlowChatViewHostOptions + #state = $state.raw() as ChatState + #unsubscribe: () => void + + constructor(chat: Chat, options: FlowChatViewHostOptions = {}) { + this.#chat = chat + this.#options = options + this.#state = chat.getState() + this.#unsubscribe = chat.subscribe((state) => this.#onState(state)) + } + + #disposed = false + /** Stops following the chat, and drops what was queued: a flush still waiting on the + * turn's release would otherwise start a run from a panel that is gone. The chat itself + * is the caller's to destroy. */ + dispose() { + this.#disposed = true + this.#queued = '' + this.#unsubscribe() + } + + /** The latest `ChatState`, for what the interface reads beyond the seam (paging, loading). */ + get state(): ChatState { + return this.#state + } + + #onState(state: ChatState) { + const previous = this.#state + this.#state = state + if (previous.conversationId !== state.conversationId) { + // A conversation opens at its end, whatever the reader was doing in the last one. + this.#automaticScroll = true + // The queue was typed into the conversation that just went away; a message sent + // after the switch would ride out of the wrong one, so it goes back to the composer. + this.dequeueMessage() + return + } + if (isBusy(previous.status) && !isBusy(state.status)) { + // The turn settled. What was typed during it goes out once the turn is released, + // not now: the chat publishes `idle` from inside its own `sendMessage`, which still + // counts the turn as open until it returns, and a send made before that would be + // refused as a second turn. After a failure it goes back to the composer instead, + // where the reader would rather look at the error than pile on. A failed flow + // settles as `idle` too, with its error as the answer, so the messages decide. + const succeeded = state.status === 'idle' && !lastTurnFailed(state.messages) + if (succeeded) void this.#turnDone.then(this.flushQueuedMessage) + else this.dequeueMessage() + } + } + + // Transcript + displayMessages = $derived.by(() => toDisplayMessages(this.#state.messages)) + get messages(): readonly unknown[] { + return this.#state.messages + } + contextTokens = 0 + get operatingWorkspace(): string | undefined { + return this.#options.workspace?.() + } + get loading(): boolean { + return isBusy(this.#state.status) + } + runHeldElsewhere = false + loadingLabel = undefined + compacting = false + // The answer streams into the message list itself, so the live lanes stay empty. + currentReply = '' + currentReasoning = '' + currentReasoningActive = false + reasoningHiddenIndicatorLabel = undefined + #automaticScroll = $state(true) + get automaticScroll(): boolean { + return this.#automaticScroll + } + enableAutomaticScroll = () => { + this.#automaticScroll = true + } + disableAutomaticScroll = () => { + this.#automaticScroll = false + } + + // Composer + instructions = '' + // The user message lands in the transcript before `sendMessage` awaits anything. + sendInFlight = false + sendRequest = async (options: ChatSendRequestOptions = {}): Promise => { + const text = options.instructions?.trim() ?? '' + if (!text) return false + if (this.loading) { + this.queueMessage(text) + return true + } + if (this.#options.sendDisabled?.()) { + // Refused, not dropped: the text waits in the composer for sending to reopen. + this.#aiChatInput?.prependText(text) + return false + } + this.#automaticScroll = true + // A run that fails is reported through the chat's `onError` and as a failed message; + // the promise itself only rejects when the chat refuses the turn outright, and the + // text is then handed back rather than dropped. + const turn = this.#chat + .sendMessage(text, { inputs: this.#options.additionalInputs?.() }) + .catch(() => this.#aiChatInput?.prependText(text)) + this.#turnDone = turn + await turn + return true + } + /** Settles when the chat has released the last turn this host started. */ + #turnDone: Promise = Promise.resolve() + cancel = () => { + // Stop means stop: what was typed during the run goes back to the composer rather + // than waiting there to go out after some later turn settles. + this.dequeueMessage() + void this.#chat.stop() + } + // Typed off the interface: a Svelte component's own type resolves differently + // across import specifiers, and the two would then not be assignable. + #aiChatInput: Parameters[0] = null + setAiChatInput: ChatViewHost['setAiChatInput'] = (aiChatInput) => { + this.#aiChatInput = aiChatInput + } + + // One message typed while the turn runs, sent whole once it settles. Enter again + // appends a line rather than replacing what waits. + #queued = $state('') + get queuedMessage(): string { + return this.#queued + } + queuedContext = undefined + queuedImages: AttachedImage[] = [] + queuedFiles: AttachedTextFile[] = [] + queueMessage = (text: string) => { + const trimmed = text.trim() + if (!trimmed) return + this.#queued = this.#queued ? `${this.#queued}\n${trimmed}` : trimmed + } + /** Put the queued draft back in the composer. */ + dequeueMessage = () => { + const text = this.#queued + if (!text) return + this.#queued = '' + this.#aiChatInput?.prependText(text) + } + flushQueuedMessage = () => { + const text = this.#queued + if (!text || this.#disposed) return + this.#queued = '' + void this.sendRequest({ instructions: text }) + } + setComposerStaged = () => {} + clearComposerStaged = () => {} + attachmentBytesExcluding = () => 0 + + // Per-message actions + storedImages = () => undefined + /** Send the user message at this transcript position again. */ + retryRequest = (messageIndex: number) => { + const message = this.#state.messages[messageIndex] + if (!message || message.role !== 'user' || this.loading) return + void this.sendRequest({ instructions: message.content }) + } + restartGeneration = () => {} + handleUserQuestionAnswer = () => false + handleToolConfirmation = () => {} + hasPendingRunForm = false + isRunFormPending = () => false + + // Copilot-only surfaces + mode = undefined + isSessionChat = false + supportsModelSettings = false + supportsMessageEditing = false + supportsMessageAttachments = false + // An AI agent step refuses a run with no `user_message`. + requiresMessageText = true + supportsLinkedFolders = false + attachmentAccept = '' + tools = [] + // The enum's value, written out so this module never imports the copilot manager at + // runtime: its unit test would otherwise load the manager and the editor it pulls in. + autonomyMode = 'default' as AIAutonomyMode.DEFAULT + setAutonomyMode = () => {} + autoAcceptEditsActive = false + autoAcceptEditsAvailable = false + autoAcceptToolConfirmationsAvailable = false + planModeAvailable = false + attachedFiles = new AttachedFilesStore() + artifacts = new SessionArtifactsStore() +} diff --git a/frontend/src/lib/components/flows/conversations/flowChatViewHost.test.ts b/frontend/src/lib/components/flows/conversations/flowChatViewHost.test.ts new file mode 100644 index 0000000000..a7e953ecf5 --- /dev/null +++ b/frontend/src/lib/components/flows/conversations/flowChatViewHost.test.ts @@ -0,0 +1,267 @@ +import { describe, expect, it, vi } from 'vitest' +import type { Chat, ChatMessage, ChatState } from 'windmill-chat' +import { FlowChatViewHost, toDisplayMessages } from './flowChatViewHost.svelte' + +function message(partial: Partial & Pick): ChatMessage { + return { + id: partial.id ?? `${partial.role}-${Math.random()}`, + content: '', + success: true, + createdAt: '2026-09-16T10:00:00Z', + pending: false, + ...partial + } +} + +function idleState(partial: Partial = {}): ChatState { + return { + conversationId: 'c1', + messages: [], + status: 'idle', + error: undefined, + conversations: [], + history: 'server', + loadingMessages: false, + hasMoreMessages: false, + ...partial + } +} + +/** A `Chat` whose state the test drives by hand. */ +function fakeChat(initial: ChatState = idleState()) { + let state = initial + const listeners = new Set<(s: ChatState) => void>() + const chat = { + getState: () => state, + subscribe: (listener: (s: ChatState) => void) => { + listeners.add(listener) + listener(state) + return () => listeners.delete(listener) + }, + sendMessage: vi.fn(async () => {}), + stop: vi.fn(async () => {}), + newConversation: vi.fn(), + selectConversation: vi.fn(async () => {}), + loadConversations: vi.fn(async () => []), + deleteConversation: vi.fn(async () => {}), + loadOlderMessages: vi.fn(async () => {}), + destroy: vi.fn() + } satisfies Chat + const set = (patch: Partial) => { + state = { ...state, ...patch } + for (const listener of listeners) listener(state) + } + return { chat, set } +} + +describe('toDisplayMessages', () => { + it('maps user, assistant and tool rows, marking the user message of a failed turn', () => { + const rows = [ + message({ role: 'user', content: 'hi' }), + message({ + role: 'assistant', + content: 'hello', + reasoning: 'thinking', + stepName: 'agent', + jobId: 'job-1', + createdAt: '2026-09-16T10:00:01Z' + }), + message({ role: 'user', content: 'again' }), + // A tool still running has no text yet: the server words the row on its return. + message({ + role: 'tool', + id: 'tool-row', + pending: true, + tool: { name: 'search', status: 'running', arguments: '{"q":"x"}' } + }), + message({ role: 'assistant', content: 'boom', success: false }) + ] + const display = toDisplayMessages(rows) + expect(display[0]).toEqual({ role: 'user', index: 0, content: 'hi', error: undefined }) + expect(display[1]).toMatchObject({ + role: 'assistant', + content: 'hello', + reasoning: 'thinking', + stepName: 'agent', + jobId: 'job-1', + createdAt: '2026-09-16T10:00:01Z', + streaming: undefined + }) + expect(display[2]).toEqual({ role: 'user', index: 1, content: 'again', error: true }) + expect(display[3]).toMatchObject({ + role: 'tool', + tool_call_id: 'tool-row', + content: 'Running search', + toolName: 'search', + parameters: { q: 'x' }, + showDetails: true, + isLoading: true, + error: undefined + }) + }) + + it('does not flag a turn whose tool failed but whose agent still answered', () => { + const display = toDisplayMessages([ + message({ role: 'user', content: 'try' }), + message({ + role: 'tool', + content: 'Error executing search', + success: false, + tool: { name: 'search', status: 'error' } + }), + message({ role: 'assistant', content: 'search is down, here is what I know' }) + ]) + expect(display[0]).toMatchObject({ role: 'user', error: undefined }) + }) + + it('flags the streaming assistant message and a failed tool', () => { + const display = toDisplayMessages([ + message({ role: 'assistant', content: 'partial', pending: true }), + message({ + role: 'tool', + content: 'Error executing search', + success: false, + tool: { name: 'search', status: 'error' } + }) + ]) + expect(display[0]).toMatchObject({ role: 'assistant', streaming: true }) + expect(display[1]).toMatchObject({ + role: 'tool', + error: 'Error executing search', + showDetails: false, + isLoading: false + }) + }) +}) + +describe('FlowChatViewHost', () => { + it('sends the text with the additional inputs and reports loading from the status', async () => { + const { chat, set } = fakeChat() + const host = new FlowChatViewHost(chat, { additionalInputs: () => ({ tone: 'brief' }) }) + expect(host.loading).toBe(false) + expect(await host.sendRequest({ instructions: ' hello ' })).toBe(true) + expect(chat.sendMessage).toHaveBeenCalledWith('hello', { inputs: { tone: 'brief' } }) + expect(await host.sendRequest({ instructions: ' ' })).toBe(false) + set({ status: 'streaming' }) + expect(host.loading).toBe(true) + set({ status: 'idle' }) + expect(host.loading).toBe(false) + host.dispose() + }) + + it('queues a message typed during a turn and sends it once the turn is released', async () => { + const { chat, set } = fakeChat() + // The chat publishes `idle` from inside `sendMessage`, before that call returns and + // releases the turn; a flush in between is refused as a second turn. + let releaseTurn = () => {} + chat.sendMessage.mockImplementationOnce( + () => new Promise((resolve) => (releaseTurn = resolve)) + ) + const host = new FlowChatViewHost(chat) + void host.sendRequest({ instructions: 'start' }) + set({ status: 'streaming' }) + host.queueMessage('first') + host.queueMessage('second') + expect(host.queuedMessage).toBe('first\nsecond') + set({ status: 'idle' }) + await new Promise((resolve) => setTimeout(resolve, 0)) + expect(chat.sendMessage).toHaveBeenCalledTimes(1) + releaseTurn() + await new Promise((resolve) => setTimeout(resolve, 0)) + expect(host.queuedMessage).toBe('') + expect(chat.sendMessage).toHaveBeenLastCalledWith('first\nsecond', { inputs: undefined }) + host.dispose() + }) + + it('hands the queue back when a failed flow settles as idle', () => { + const { chat, set } = fakeChat( + idleState({ status: 'streaming', messages: [message({ role: 'user', content: 'go' })] }) + ) + const host = new FlowChatViewHost(chat) + const prependText = vi.fn() + host.setAiChatInput({ prependText } as any) + host.queueMessage('later') + // A failed flow is answered with its error and the status still returns to idle. + set({ + status: 'idle', + messages: [ + message({ role: 'user', content: 'go' }), + message({ role: 'assistant', content: 'boom', success: false }) + ] + }) + expect(prependText).toHaveBeenCalledWith('later') + expect(chat.sendMessage).not.toHaveBeenCalled() + host.dispose() + }) + + it('hands the queue back instead of sending while sending is disabled', async () => { + const { chat, set } = fakeChat(idleState({ status: 'streaming' })) + let deploying = false + const host = new FlowChatViewHost(chat, { sendDisabled: () => deploying }) + const prependText = vi.fn() + host.setAiChatInput({ prependText } as any) + host.queueMessage('after deploy') + // A deployment starts while the turn is still running; the composer is disabled. + deploying = true + set({ status: 'idle' }) + await new Promise((resolve) => setTimeout(resolve, 0)) + expect(chat.sendMessage).not.toHaveBeenCalled() + expect(prependText).toHaveBeenCalledWith('after deploy') + expect(host.queuedMessage).toBe('') + host.dispose() + }) + + it('drops a flush still waiting on the turn once disposed', async () => { + const { chat, set } = fakeChat() + let releaseTurn = () => {} + chat.sendMessage.mockImplementationOnce( + () => new Promise((resolve) => (releaseTurn = resolve)) + ) + const host = new FlowChatViewHost(chat) + void host.sendRequest({ instructions: 'start' }) + set({ status: 'streaming' }) + host.queueMessage('never') + set({ status: 'idle' }) + host.dispose() + releaseTurn() + await new Promise((resolve) => setTimeout(resolve, 0)) + expect(chat.sendMessage).toHaveBeenCalledTimes(1) + }) + + it('hands the text back when the chat refuses the turn', async () => { + const { chat } = fakeChat() + chat.sendMessage.mockRejectedValueOnce(new Error('a message is already being answered')) + const host = new FlowChatViewHost(chat) + const prependText = vi.fn() + host.setAiChatInput({ prependText } as any) + await host.sendRequest({ instructions: 'kept' }) + expect(prependText).toHaveBeenCalledWith('kept') + host.dispose() + }) + + it('hands the queue back to the composer on Stop and on a failed turn', async () => { + const { chat, set } = fakeChat(idleState({ status: 'streaming' })) + const host = new FlowChatViewHost(chat) + const prependText = vi.fn() + host.setAiChatInput({ prependText } as any) + host.queueMessage('later') + host.cancel() + expect(chat.stop).toHaveBeenCalled() + expect(prependText).toHaveBeenCalledWith('later') + expect(host.queuedMessage).toBe('') + + host.queueMessage('after error') + set({ status: 'error' }) + expect(prependText).toHaveBeenLastCalledWith('after error') + expect(chat.sendMessage).not.toHaveBeenCalled() + host.dispose() + }) + + it('stops following the chat once disposed', () => { + const { chat, set } = fakeChat() + const host = new FlowChatViewHost(chat) + host.dispose() + set({ status: 'streaming' }) + expect(host.loading).toBe(false) + }) +}) diff --git a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte index a1384579af..5cd5b653da 100644 --- a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte @@ -703,7 +703,9 @@ onRunFlow={runFlowForChat} {deploymentInProgress} path={flow?.path ?? ''} + description={flow?.description} inputSchema={flow?.schema} + wideLayout /> {:else} {@const hasSchema = From e5ec5217d41885b4f9f098202e3a7a7d124af7b8 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Wed, 16 Sep 2026 18:09:07 +0200 Subject: [PATCH 12/77] fix(datatables): serialize roles going on with aliases saved from other workspaces Co-Authored-By: Claude Opus 5 (1M context) --- backend/ee-repo-ref.txt | 2 +- .../tests/datatable_roles.rs | 62 ++++++++++++++ .../windmill-api-workspaces/src/workspaces.rs | 84 +++++++++++-------- .../windmill-common/src/datatable_roles.rs | 19 +++++ 4 files changed, 131 insertions(+), 36 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 1d1f24af71..9717f3f96f 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -53614778ef0eec12ef22c6001c8905cd46f5ee17 +38d6fcf2aeb39cfdac21814bbdbbcc02911e566a diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs index 08790386a4..f12daf1953 100644 --- a/backend/windmill-api-integration-tests/tests/datatable_roles.rs +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -1009,6 +1009,68 @@ async fn an_entry_without_roles_cannot_newly_reach_a_database_under_roles( Ok(()) } +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn an_alias_saved_elsewhere_waits_for_roles_going_on_for_its_database( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + sqlx::query( + r#"UPDATE workspace_settings SET datatable = jsonb_set(datatable, '{datatables,other}', + '{"database": {"resource_type": "instance", "resource_path": "dt_other"}}') + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + + // Roles going on for `dt_other`, not committed yet: it holds only its own workspace's settings + // row, so an alias saved from another workspace that looked for roles now would miss them. + let enabling = { + let mut tx = db.begin().await?; + windmill_common::datatable_roles::lock_instance_databases_governance( + &mut *tx, + ["dt_other"], + ) + .await?; + sqlx::query( + r#"UPDATE workspace_settings SET datatable = jsonb_set(datatable, + '{datatables,other,permissions}', + '{"default_role": "admin", "roles": {"admin": {"tenants": ["*"]}}}') + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&mut *tx) + .await?; + tx + }; + + let server = ApiServer::start(db.clone()).await?; + let url = format!( + "http://localhost:{}/api/w/wm-fork-dt/workspaces/edit_datatable_config", + server.addr.port() + ); + let save = tokio::spawn( + authed(client().post(&url), "SECRET_TOKEN") + .json(&json!({ "settings": { "datatables": { + "direct": { "database": { "resource_type": "instance", "resource_path": "dt_other" } } + } } })) + .send(), + ); + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + assert!( + !save.is_finished(), + "an alias was saved while roles were going on for its database" + ); + enabling.commit().await?; + + let resp = save.await??; + let status = resp.status(); + let body = resp.text().await?; + assert!( + status == 400 && body.contains("which a data table under roles uses"), + "the alias reached the database whose roles went on while it waited ({status}): {body}" + ); + Ok(()) +} + #[cfg(not(all(feature = "private", feature = "enterprise")))] const ENTERPRISE_REFUSAL: &str = "Data table roles are a Windmill Enterprise Edition feature"; diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 38b3797518..ed01dede99 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -3891,50 +3891,64 @@ async fn edit_datatable_config( // entry through a declared rename alone, and a settings sync never declares one, so an entry // without roles that newly points at such a database — a name added, or an existing one // repointed — would answer everyone there as `admin`. That holds whichever workspace governs it. - let governed_elsewhere: Vec = sqlx::query_scalar( - "SELECT DISTINCT dt.value->'database'->>'resource_path' FROM workspace_settings ws - CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt - WHERE ws.workspace_id <> $1 AND dt.value ? 'permissions' - AND dt.value->'database'->>'resource_type' = 'instance'", + let newly_pointed: Vec<(&String, &str)> = new_config + .settings + .datatables + .iter() + .filter(|(_, dt)| dt.permissions.is_none()) + .filter_map(|(name, dt)| { + let db = dt + .database + .as_ref() + .filter(|d| d.resource_type == DataTableCatalogResourceType::Instance)?; + let lookup = rename_src + .get(name.as_str()) + .copied() + .unwrap_or(name.as_str()); + let repointed = old_datatables + .get(lookup) + .and_then(|old| old.database.as_ref()) + .is_none_or(|old_db| { + old_db.resource_type != db.resource_type + || old_db.resource_path != db.resource_path + }); + repointed.then_some((name, db.resource_path.as_str())) + }) + .collect(); + // Another workspace turning roles on for the same database holds only its own settings row, so + // without this the scan below could read past its uncommitted write. + windmill_common::datatable_roles::lock_instance_databases_governance( + &mut *tx, + newly_pointed.iter().map(|(_, dbname)| *dbname), ) - .bind(&w_id) - .fetch_all(&mut *tx) .await?; - for (name, dt) in new_config.settings.datatables.iter() { - if dt.permissions.is_some() { - continue; - } - let Some(db) = dt - .database - .as_ref() - .filter(|d| d.resource_type == DataTableCatalogResourceType::Instance) - else { - continue; - }; - let lookup = rename_src - .get(name.as_str()) - .copied() - .unwrap_or(name.as_str()); - let repointed = old_datatables - .get(lookup) - .and_then(|old| old.database.as_ref()) - .is_none_or(|old_db| { - old_db.resource_type != db.resource_type || old_db.resource_path != db.resource_path - }); + let governed_elsewhere: Vec = if newly_pointed.is_empty() { + vec![] + } else { + sqlx::query_scalar( + "SELECT DISTINCT dt.value->'database'->>'resource_path' FROM workspace_settings ws + CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt + WHERE ws.workspace_id <> $1 AND dt.value ? 'permissions' + AND dt.value->'database'->>'resource_type' = 'instance'", + ) + .bind(&w_id) + .fetch_all(&mut *tx) + .await? + }; + for (name, dbname) in newly_pointed { let governed_here = old_datatables.values().any(|old| { old.permissions.is_some() && old.database.as_ref().is_some_and(|d| { d.resource_type == DataTableCatalogResourceType::Instance - && d.resource_path == db.resource_path + && d.resource_path == dbname }) }); - if repointed && (governed_here || governed_elsewhere.contains(&db.resource_path)) { + if governed_here || governed_elsewhere.iter().any(|g| g == dbname) { return Err(Error::BadRequest(format!( - "Data table '{name}' would point at database '{}', which a data table under roles \ - uses, without carrying those roles: everyone reaching '{name}' would connect there \ - as `admin`. Rename the data table under roles from the data table settings, which \ - carries its roles, or turn its roles off first.", - db.resource_path + "Data table '{name}' would point at database '{dbname}', which a data table under \ + roles uses, without carrying those roles: everyone reaching '{name}' would connect \ + there as `admin`. Rename the data table under roles from the data table settings, \ + which carries its roles, or turn its roles off first." ))); } } diff --git a/backend/windmill-common/src/datatable_roles.rs b/backend/windmill-common/src/datatable_roles.rs index 3c269d1ada..4dd9b06fde 100644 --- a/backend/windmill-common/src/datatable_roles.rs +++ b/backend/windmill-common/src/datatable_roles.rs @@ -140,6 +140,25 @@ pub async fn lock_datatable_streams(conn: &mut sqlx::PgConnection, exclusive: bo Ok(()) } +/// Whether an instance database is reached only through entries under roles is decided by two +/// writes that lock different workspaces' settings rows: turning roles on for one entry, and a +/// settings save pointing an entry without roles at the database. Each holds this for every +/// database it decides on, so neither reads past the other's uncommitted write. Held for the +/// transaction; the names are locked in sorted order so two holders cannot deadlock. +pub async fn lock_instance_databases_governance<'a>( + conn: &mut sqlx::PgConnection, + dbnames: impl IntoIterator, +) -> Result<()> { + let dbnames: std::collections::BTreeSet<&str> = dbnames.into_iter().collect(); + for dbname in dbnames { + sqlx::query("SELECT pg_advisory_xact_lock(hashtext('datatable_instance_database:' || $1))") + .bind(dbname) + .execute(&mut *conn) + .await?; + } + Ok(()) +} + /// Disclosure: returns every role's stored Postgres password in plaintext. Any server path that /// has to resolve or name a role may call it — including handlers open to a workspace member, who /// need the names — but callers MUST NOT let `pwd` reach a response, a log line, an audit record From 110876d0c9e30cfd2484572b7f5bca1d2ac7ed9a Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Wed, 16 Sep 2026 22:25:27 +0200 Subject: [PATCH 13/77] docs(datatables): note that legacy names with ? cannot be migrated Co-Authored-By: Claude Opus 5 (1M context) --- backend/windmill-api-workspaces/src/datatable_migrations.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/windmill-api-workspaces/src/datatable_migrations.rs b/backend/windmill-api-workspaces/src/datatable_migrations.rs index 8900e708e4..fa433b2cbc 100644 --- a/backend/windmill-api-workspaces/src/datatable_migrations.rs +++ b/backend/windmill-api-workspaces/src/datatable_migrations.rs @@ -174,6 +174,10 @@ async fn datatable_database_arg( // default role — which is what `ensure_migration_role_allowed` gated it as, and which is the // only role a DDL statement can be expected to succeed under. A migration that does declare a // role overrides this: the annotation wins over the reference. + // + // A legacy name containing `?` cannot be migrated through this reference: the appended query + // makes it neither an exact name nor a parseable one. Accepted on purpose, since such names can + // no longer be created and none are expected to carry migrations. Ok(to_raw_value(&format!( "datatable://{datatable_name}?role={ADMIN_DATATABLE_ROLE}" ))) From 8d0d51e99dd14d326d27534ea215d52ebc9072cf Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Wed, 16 Sep 2026 22:25:51 +0200 Subject: [PATCH 14/77] fix: run one data table ACL apply at a time per server before it connects Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb --- backend/windmill-api-workspaces/src/datatable_acl.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/backend/windmill-api-workspaces/src/datatable_acl.rs b/backend/windmill-api-workspaces/src/datatable_acl.rs index 6aa2f936b0..8ec966d475 100644 --- a/backend/windmill-api-workspaces/src/datatable_acl.rs +++ b/backend/windmill-api-workspaces/src/datatable_acl.rs @@ -1308,6 +1308,8 @@ async fn authorize_acl_change( Ok(governing) } +static APPLY_SLOT: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(1); + /// A role passes on only privileges it holds with grant option, and an instance database /// provisioned before data table roles gave `custom_instance_user` none. Adds that option to its /// database and `public` privileges, and nothing else: default privileges are left alone, since a @@ -1575,6 +1577,13 @@ async fn apply_datatable_acl( // connection could wait forever on a pool that concurrent applies, queued on the same locks, // have exhausted. let governing = authorize_acl_change(&db, &authed, &w_id, &datatable_name).await?; + // Applies queue on an instance-wide lock while each holds a direct connection to the instance's + // Postgres; unbounded, the queue alone could exhaust its connection limit. One at a time per + // server, and the ones waiting hold no connection at all. + let _slot = APPLY_SLOT + .acquire() + .await + .map_err(|e| Error::internal_err(format!("ACL apply slot closed: {e}")))?; let (mut client, mut notices, dbname) = connect_as_admin_unchecked(&db, &governing).await?; ensure_grant_options(&client, &db, &dbname).await; From 0a2ebfb3db73932e966db0662708aadcadcc1e4b Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Wed, 16 Sep 2026 23:55:43 +0200 Subject: [PATCH 15/77] fix: hold the ACL connection to the database that was authorized Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb --- .../windmill-api-workspaces/src/datatable_acl.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/backend/windmill-api-workspaces/src/datatable_acl.rs b/backend/windmill-api-workspaces/src/datatable_acl.rs index 8ec966d475..ce63555267 100644 --- a/backend/windmill-api-workspaces/src/datatable_acl.rs +++ b/backend/windmill-api-workspaces/src/datatable_acl.rs @@ -328,6 +328,22 @@ async fn connect_as_admin_unchecked( .await?; let pg: PgDatabase = serde_json::from_value(resource) .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {e}")))?; + // Resolving reads the settings again, and a save since `governing` was authorized can point + // the entry elsewhere and back. An instance entry's database is its `resource_path`, so the + // connection is held to the database that was authorized, and a later check of the entry + // cannot pass while this talks to another one. + if governing + .datatable + .database + .as_ref() + .map(|d| d.resource_path.as_str()) + != Some(&pg.dbname) + { + return Err(Error::BadRequest(format!( + "Data table '{}' was pointed at another database while this ran; try again", + governing.name + ))); + } let dbname = pg.dbname.clone(); let (client, mut connection) = pg.connect(Some(db)).await?; // Unbounded: the driver must never wait on the receiver, which only drains once the statement From b7623f12ebafbbc6bbc296119bfec356093a327e Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Thu, 17 Sep 2026 00:09:29 +0200 Subject: [PATCH 16/77] fix: build the ACL connection from the authorized data table entry Resolving the settings again could land on a resource with the same database name on another server, which the later entry checks never see. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb --- .../src/datatable_acl.rs | 34 +++++++------------ 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/backend/windmill-api-workspaces/src/datatable_acl.rs b/backend/windmill-api-workspaces/src/datatable_acl.rs index ce63555267..1dac05beb0 100644 --- a/backend/windmill-api-workspaces/src/datatable_acl.rs +++ b/backend/windmill-api-workspaces/src/datatable_acl.rs @@ -36,10 +36,7 @@ use windmill_common::datatable_roles::{ ADMIN_DATATABLE_ROLE, CUSTOM_INSTANCE_USER, }; use windmill_common::error::{pg_error_message, Error, JsonResult, Result}; -use windmill_common::workspaces::{ - get_datatable_resource_from_db_unchecked, resolve_governing_datatable, DataTable, - GoverningDatatable, -}; +use windmill_common::workspaces::{resolve_governing_datatable, DataTable, GoverningDatatable}; use windmill_common::{PgDatabase, DB}; use crate::datatable_permissions::{ensure_governs_datatable, ensure_reaches_datatable}; @@ -323,27 +320,20 @@ async fn connect_as_admin_unchecked( mpsc::UnboundedReceiver, String, )> { - let resource = - get_datatable_resource_from_db_unchecked(db, &governing.workspace_id, &governing.name) - .await?; - let pg: PgDatabase = serde_json::from_value(resource) - .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {e}")))?; - // Resolving reads the settings again, and a save since `governing` was authorized can point - // the entry elsewhere and back. An instance entry's database is its `resource_path`, so the - // connection is held to the database that was authorized, and a later check of the entry - // cannot pass while this talks to another one. - if governing + ensure_instance(governing)?; + // Built from the authorized entry, never by resolving the settings again: a save in between + // could point the entry at a resource on another server and back, and this connection would + // then alter a database the later checks of the entry never see. + let mut pg = PgDatabase::parse_uri(&windmill_common::get_database_url().await?.as_str().await)?; + pg.dbname = governing .datatable .database .as_ref() - .map(|d| d.resource_path.as_str()) - != Some(&pg.dbname) - { - return Err(Error::BadRequest(format!( - "Data table '{}' was pointed at another database while this ran; try again", - governing.name - ))); - } + .expect("a governing entry owns a database") + .resource_path + .clone(); + pg.user = Some(CUSTOM_INSTANCE_USER.to_string()); + pg.password = Some(windmill_common::utils::get_custom_pg_instance_password(db).await?); let dbname = pg.dbname.clone(); let (client, mut connection) = pg.connect(Some(db)).await?; // Unbounded: the driver must never wait on the receiver, which only drains once the statement From e79bc14903e03938897aa7f5e4320c8b9655b6d0 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Thu, 17 Sep 2026 09:00:04 +0200 Subject: [PATCH 17/77] fix: check ACL read reach against the entry it connects from Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb --- backend/ee-repo-ref.txt | 2 +- .../src/datatable_acl.rs | 4 ++-- .../src/datatable_permissions.rs | 12 ++++++++++++ .../src/datatable_permissions_oss.rs | 18 ++++++++++++++++-- 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 1ba2346e6c..8cab265586 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -85e15fc93b238159fc90acfc8427d29e219b88ca +0edd40979cf36bfba59323f3f6a0811ae1369cf5 diff --git a/backend/windmill-api-workspaces/src/datatable_acl.rs b/backend/windmill-api-workspaces/src/datatable_acl.rs index 1dac05beb0..6bd14316fc 100644 --- a/backend/windmill-api-workspaces/src/datatable_acl.rs +++ b/backend/windmill-api-workspaces/src/datatable_acl.rs @@ -39,7 +39,7 @@ use windmill_common::error::{pg_error_message, Error, JsonResult, Result}; use windmill_common::workspaces::{resolve_governing_datatable, DataTable, GoverningDatatable}; use windmill_common::{PgDatabase, DB}; -use crate::datatable_permissions::{ensure_governs_datatable, ensure_reaches_datatable}; +use crate::datatable_permissions::{ensure_governs_datatable, ensure_reaches_governing_datatable}; pub(crate) fn routes() -> Router { Router::new() @@ -1018,8 +1018,8 @@ async fn get_datatable_acl( ) -> JsonResult { crate::datatable_acl_oss::ensure_datatable_acl_available()?; let target: AclTarget = query.try_into()?; - ensure_reaches_datatable(&db, &w_id, &datatable_name, &authed).await?; let governing = resolve_governing_datatable(&db, &w_id, &datatable_name).await?; + ensure_reaches_governing_datatable(&db, &w_id, &datatable_name, &governing, &authed).await?; ensure_instance(&governing)?; let editable = ensure_governs_datatable(&db, &authed, &w_id, &governing) .await diff --git a/backend/windmill-api-workspaces/src/datatable_permissions.rs b/backend/windmill-api-workspaces/src/datatable_permissions.rs index 5cb1f3c1c3..69e04e328d 100644 --- a/backend/windmill-api-workspaces/src/datatable_permissions.rs +++ b/backend/windmill-api-workspaces/src/datatable_permissions.rs @@ -65,3 +65,15 @@ pub(crate) async fn ensure_reaches_datatable( ) -> Result<()> { roles::ensure_reaches_datatable(db, w_id, datatable_name, authed).await } + +/// [`ensure_reaches_datatable`] against an entry already resolved, for a caller that goes on to +/// connect from that same entry. +pub(crate) async fn ensure_reaches_governing_datatable( + db: &DB, + w_id: &str, + datatable_name: &str, + governing: &GoverningDatatable, + authed: &ApiAuthed, +) -> Result<()> { + roles::ensure_reaches_governing_datatable(db, w_id, datatable_name, governing, authed).await +} diff --git a/backend/windmill-api-workspaces/src/datatable_permissions_oss.rs b/backend/windmill-api-workspaces/src/datatable_permissions_oss.rs index f4d8c6a7ad..c7c7314885 100644 --- a/backend/windmill-api-workspaces/src/datatable_permissions_oss.rs +++ b/backend/windmill-api-workspaces/src/datatable_permissions_oss.rs @@ -12,8 +12,8 @@ #[cfg(all(feature = "private", feature = "enterprise"))] pub(crate) use crate::datatable_permissions_ee::{ - ensure_governs_datatable, ensure_reaches_datatable, get_datatable_permissions, - list_usable_datatable_roles, set_datatable_permissions, + ensure_governs_datatable, ensure_reaches_datatable, ensure_reaches_governing_datatable, + get_datatable_permissions, list_usable_datatable_roles, set_datatable_permissions, }; #[cfg(not(all(feature = "private", feature = "enterprise")))] @@ -56,6 +56,20 @@ mod ce { } } + pub(crate) async fn ensure_reaches_governing_datatable( + _db: &DB, + _w_id: &str, + _datatable_name: &str, + governing: &GoverningDatatable, + _authed: &ApiAuthed, + ) -> Result<()> { + if governing.datatable.permissions.is_none() { + Ok(()) + } else { + Err(unavailable()) + } + } + // The routes stay registered so the API has one shape; each answers after authentication, // before anything is read. From 23c24a9688d4c8c462f53221334d538280f16bca Mon Sep 17 00:00:00 2001 From: Guilhem Date: Thu, 17 Sep 2026 09:27:12 +0200 Subject: [PATCH 18/77] feat: collect flow conversations and agent memory once their last message goes (#11178) * feat: collect flow conversations and agent memory when their last message goes * fix: lock the conversation lookup so a new turn orders against its cleanup * fix: let concurrent turns recreate a collected conversation without conflicting --- ...9a1fabeaa167818206a25abfe31d5582f942a.json | 15 ++ ...7cb3b31ee6b69abae79a214dddba0dee4425c.json | 15 -- ...0054b753cfbeb5dad1e8120fd5e8a672d50ef.json | 23 ++ ...3659915d9fcb3d58f16532aaffe06c44ec976.json | 23 ++ ...3ca33b3f82ca38289dfa4a69aa95041027d57.json | 59 +++++ ...9672f8c98d2579fc72c07752361cc5fd683dc.json | 15 ++ ...8e44eb1c0062a8241dc8fe0ed3dd95b65f89a.json | 14 + ...df90dad5246681bde6caaa633b85a5e8b2352.json | 22 ++ ...741a51ca0ce66bf1c674097705cf2e1b72eff.json | 14 + ...2893147b4e5ff078531de36147d908132d636.json | 14 - ...fee4a74f61b061e5025ecb75882208b3ba8f.json} | 4 +- ...7a30d71a5689efe72daf58bd1e8f6cf90c410.json | 22 ++ backend/tests/v2_job_delete_orphans.rs | 248 +++++++++++++++++- backend/windmill-api-jobs/src/execution.rs | 14 +- backend/windmill-api-jobs/src/jobs_export.rs | 60 ++++- backend/windmill-api/src/jobs.rs | 1 + .../windmill-common/src/flow_conversations.rs | 55 ++-- backend/windmill-common/src/jobs.rs | 55 +++- 18 files changed, 614 insertions(+), 59 deletions(-) create mode 100644 backend/.sqlx/query-07a005f0f9e80a156cd2a5a0ae39a1fabeaa167818206a25abfe31d5582f942a.json delete mode 100644 backend/.sqlx/query-462d2b2822b185a6f51fafcfa957cb3b31ee6b69abae79a214dddba0dee4425c.json create mode 100644 backend/.sqlx/query-4f52bf546579f26a1d22c239b8b0054b753cfbeb5dad1e8120fd5e8a672d50ef.json create mode 100644 backend/.sqlx/query-69bfbe9b39414b724488532cc3b3659915d9fcb3d58f16532aaffe06c44ec976.json create mode 100644 backend/.sqlx/query-6f32c1feed096ff706ae359ad6a3ca33b3f82ca38289dfa4a69aa95041027d57.json create mode 100644 backend/.sqlx/query-89ea81b765550cf665e30533efc9672f8c98d2579fc72c07752361cc5fd683dc.json create mode 100644 backend/.sqlx/query-90b910da8d00a7c7bcf29c167e38e44eb1c0062a8241dc8fe0ed3dd95b65f89a.json create mode 100644 backend/.sqlx/query-967f52005f4a044b3a2e9f02ceadf90dad5246681bde6caaa633b85a5e8b2352.json create mode 100644 backend/.sqlx/query-a1b23f3e62c6433d95cdac58215741a51ca0ce66bf1c674097705cf2e1b72eff.json delete mode 100644 backend/.sqlx/query-bfdd60b42e32bd81e2d20b327462893147b4e5ff078531de36147d908132d636.json rename backend/.sqlx/{query-6bd23a98838e3eec309e6b696edc776bd56fc9dae1238b3272557d1562400dbe.json => query-c1e3ed3ecc3bcb98f60ba8196d33fee4a74f61b061e5025ecb75882208b3ba8f.json} (84%) create mode 100644 backend/.sqlx/query-ec295b3890a0018475ec0a3774c7a30d71a5689efe72daf58bd1e8f6cf90c410.json diff --git a/backend/.sqlx/query-07a005f0f9e80a156cd2a5a0ae39a1fabeaa167818206a25abfe31d5582f942a.json b/backend/.sqlx/query-07a005f0f9e80a156cd2a5a0ae39a1fabeaa167818206a25abfe31d5582f942a.json new file mode 100644 index 0000000000..635ec2b94d --- /dev/null +++ b/backend/.sqlx/query-07a005f0f9e80a156cd2a5a0ae39a1fabeaa167818206a25abfe31d5582f942a.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM flow_conversation c\n WHERE c.id = ANY($1)\n AND c.workspace_id = $2\n AND NOT EXISTS (\n SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray", + "Text" + ] + }, + "nullable": [] + }, + "hash": "07a005f0f9e80a156cd2a5a0ae39a1fabeaa167818206a25abfe31d5582f942a" +} diff --git a/backend/.sqlx/query-462d2b2822b185a6f51fafcfa957cb3b31ee6b69abae79a214dddba0dee4425c.json b/backend/.sqlx/query-462d2b2822b185a6f51fafcfa957cb3b31ee6b69abae79a214dddba0dee4425c.json deleted file mode 100644 index 0c4d90b073..0000000000 --- a/backend/.sqlx/query-462d2b2822b185a6f51fafcfa957cb3b31ee6b69abae79a214dddba0dee4425c.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM flow_conversation_message m\n USING flow_conversation c\n WHERE m.conversation_id = c.id AND c.workspace_id = $1 AND m.job_id = ANY($2)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "UuidArray" - ] - }, - "nullable": [] - }, - "hash": "462d2b2822b185a6f51fafcfa957cb3b31ee6b69abae79a214dddba0dee4425c" -} diff --git a/backend/.sqlx/query-4f52bf546579f26a1d22c239b8b0054b753cfbeb5dad1e8120fd5e8a672d50ef.json b/backend/.sqlx/query-4f52bf546579f26a1d22c239b8b0054b753cfbeb5dad1e8120fd5e8a672d50ef.json new file mode 100644 index 0000000000..a5695f75fd --- /dev/null +++ b/backend/.sqlx/query-4f52bf546579f26a1d22c239b8b0054b753cfbeb5dad1e8120fd5e8a672d50ef.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM flow_conversation WHERE id = ANY($1) AND workspace_id = $2 ORDER BY id FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "4f52bf546579f26a1d22c239b8b0054b753cfbeb5dad1e8120fd5e8a672d50ef" +} diff --git a/backend/.sqlx/query-69bfbe9b39414b724488532cc3b3659915d9fcb3d58f16532aaffe06c44ec976.json b/backend/.sqlx/query-69bfbe9b39414b724488532cc3b3659915d9fcb3d58f16532aaffe06c44ec976.json new file mode 100644 index 0000000000..52105ca605 --- /dev/null +++ b/backend/.sqlx/query-69bfbe9b39414b724488532cc3b3659915d9fcb3d58f16532aaffe06c44ec976.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM flow_conversation_message m\n USING flow_conversation c\n WHERE m.conversation_id = c.id AND c.workspace_id = $1 AND m.job_id = ANY($2)\n RETURNING m.conversation_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "conversation_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "UuidArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "69bfbe9b39414b724488532cc3b3659915d9fcb3d58f16532aaffe06c44ec976" +} diff --git a/backend/.sqlx/query-6f32c1feed096ff706ae359ad6a3ca33b3f82ca38289dfa4a69aa95041027d57.json b/backend/.sqlx/query-6f32c1feed096ff706ae359ad6a3ca33b3f82ca38289dfa4a69aa95041027d57.json new file mode 100644 index 0000000000..0e92e3aa99 --- /dev/null +++ b/backend/.sqlx/query-6f32c1feed096ff706ae359ad6a3ca33b3f82ca38289dfa4a69aa95041027d57.json @@ -0,0 +1,59 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, workspace_id, flow_path, title, created_at, updated_at, created_by\n FROM flow_conversation\n WHERE id = $1 AND workspace_id = $2\n FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "flow_path", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "title", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 5, + "name": "updated_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "created_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true, + false, + false, + false + ] + }, + "hash": "6f32c1feed096ff706ae359ad6a3ca33b3f82ca38289dfa4a69aa95041027d57" +} diff --git a/backend/.sqlx/query-89ea81b765550cf665e30533efc9672f8c98d2579fc72c07752361cc5fd683dc.json b/backend/.sqlx/query-89ea81b765550cf665e30533efc9672f8c98d2579fc72c07752361cc5fd683dc.json new file mode 100644 index 0000000000..7a7110030c --- /dev/null +++ b/backend/.sqlx/query-89ea81b765550cf665e30533efc9672f8c98d2579fc72c07752361cc5fd683dc.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM ai_agent_memory a\n USING flow_conversation c\n WHERE c.id = ANY($1)\n AND c.workspace_id = $2\n AND a.conversation_id = c.id\n AND a.workspace_id = c.workspace_id\n AND NOT EXISTS (\n SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray", + "Text" + ] + }, + "nullable": [] + }, + "hash": "89ea81b765550cf665e30533efc9672f8c98d2579fc72c07752361cc5fd683dc" +} diff --git a/backend/.sqlx/query-90b910da8d00a7c7bcf29c167e38e44eb1c0062a8241dc8fe0ed3dd95b65f89a.json b/backend/.sqlx/query-90b910da8d00a7c7bcf29c167e38e44eb1c0062a8241dc8fe0ed3dd95b65f89a.json new file mode 100644 index 0000000000..1555f3683d --- /dev/null +++ b/backend/.sqlx/query-90b910da8d00a7c7bcf29c167e38e44eb1c0062a8241dc8fe0ed3dd95b65f89a.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM flow_conversation c\n WHERE c.id = ANY($1)\n AND NOT EXISTS (\n SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "90b910da8d00a7c7bcf29c167e38e44eb1c0062a8241dc8fe0ed3dd95b65f89a" +} diff --git a/backend/.sqlx/query-967f52005f4a044b3a2e9f02ceadf90dad5246681bde6caaa633b85a5e8b2352.json b/backend/.sqlx/query-967f52005f4a044b3a2e9f02ceadf90dad5246681bde6caaa633b85a5e8b2352.json new file mode 100644 index 0000000000..5161f716ff --- /dev/null +++ b/backend/.sqlx/query-967f52005f4a044b3a2e9f02ceadf90dad5246681bde6caaa633b85a5e8b2352.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM flow_conversation_message WHERE job_id = ANY($1) RETURNING conversation_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "conversation_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "967f52005f4a044b3a2e9f02ceadf90dad5246681bde6caaa633b85a5e8b2352" +} diff --git a/backend/.sqlx/query-a1b23f3e62c6433d95cdac58215741a51ca0ce66bf1c674097705cf2e1b72eff.json b/backend/.sqlx/query-a1b23f3e62c6433d95cdac58215741a51ca0ce66bf1c674097705cf2e1b72eff.json new file mode 100644 index 0000000000..d8fdd5140b --- /dev/null +++ b/backend/.sqlx/query-a1b23f3e62c6433d95cdac58215741a51ca0ce66bf1c674097705cf2e1b72eff.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM ai_agent_memory a\n USING flow_conversation c\n WHERE c.id = ANY($1)\n AND a.conversation_id = c.id\n AND a.workspace_id = c.workspace_id\n AND NOT EXISTS (\n SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "a1b23f3e62c6433d95cdac58215741a51ca0ce66bf1c674097705cf2e1b72eff" +} diff --git a/backend/.sqlx/query-bfdd60b42e32bd81e2d20b327462893147b4e5ff078531de36147d908132d636.json b/backend/.sqlx/query-bfdd60b42e32bd81e2d20b327462893147b4e5ff078531de36147d908132d636.json deleted file mode 100644 index f9fbc7a58b..0000000000 --- a/backend/.sqlx/query-bfdd60b42e32bd81e2d20b327462893147b4e5ff078531de36147d908132d636.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM flow_conversation_message WHERE job_id = ANY($1)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "UuidArray" - ] - }, - "nullable": [] - }, - "hash": "bfdd60b42e32bd81e2d20b327462893147b4e5ff078531de36147d908132d636" -} diff --git a/backend/.sqlx/query-6bd23a98838e3eec309e6b696edc776bd56fc9dae1238b3272557d1562400dbe.json b/backend/.sqlx/query-c1e3ed3ecc3bcb98f60ba8196d33fee4a74f61b061e5025ecb75882208b3ba8f.json similarity index 84% rename from backend/.sqlx/query-6bd23a98838e3eec309e6b696edc776bd56fc9dae1238b3272557d1562400dbe.json rename to backend/.sqlx/query-c1e3ed3ecc3bcb98f60ba8196d33fee4a74f61b061e5025ecb75882208b3ba8f.json index d7f5fc45d4..50ba9d2897 100644 --- a/backend/.sqlx/query-6bd23a98838e3eec309e6b696edc776bd56fc9dae1238b3272557d1562400dbe.json +++ b/backend/.sqlx/query-c1e3ed3ecc3bcb98f60ba8196d33fee4a74f61b061e5025ecb75882208b3ba8f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by, title)\n VALUES ($1, $2, $3, $4, $5)\n RETURNING id, workspace_id, flow_path, title, created_at, updated_at, created_by", + "query": "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by, title)\n VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (id) DO NOTHING\n RETURNING id, workspace_id, flow_path, title, created_at, updated_at, created_by", "describe": { "columns": [ { @@ -58,5 +58,5 @@ false ] }, - "hash": "6bd23a98838e3eec309e6b696edc776bd56fc9dae1238b3272557d1562400dbe" + "hash": "c1e3ed3ecc3bcb98f60ba8196d33fee4a74f61b061e5025ecb75882208b3ba8f" } diff --git a/backend/.sqlx/query-ec295b3890a0018475ec0a3774c7a30d71a5689efe72daf58bd1e8f6cf90c410.json b/backend/.sqlx/query-ec295b3890a0018475ec0a3774c7a30d71a5689efe72daf58bd1e8f6cf90c410.json new file mode 100644 index 0000000000..1e35bd15ac --- /dev/null +++ b/backend/.sqlx/query-ec295b3890a0018475ec0a3774c7a30d71a5689efe72daf58bd1e8f6cf90c410.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM flow_conversation WHERE id = ANY($1) ORDER BY id FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "ec295b3890a0018475ec0a3774c7a30d71a5689efe72daf58bd1e8f6cf90c410" +} diff --git a/backend/tests/v2_job_delete_orphans.rs b/backend/tests/v2_job_delete_orphans.rs index 95cd1673b6..ded760b8ff 100644 --- a/backend/tests/v2_job_delete_orphans.rs +++ b/backend/tests/v2_job_delete_orphans.rs @@ -37,7 +37,7 @@ async fn seed_side_rows(db: &Pool, ws: &str, job_id: Uuid) -> anyhow:: .bind(ws) .execute(db) .await?; - // created_seq is assigned by a trigger; inserting a value is rejected. + // created_seq is an identity column; supplying a value is rejected. sqlx::query( "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id) VALUES ($1, 'assistant', 'hi', $2)", @@ -121,6 +121,184 @@ async fn test_delete_jobs_removes_side_rows(db: Pool) -> anyhow::Resul Ok(()) } +/// (conversation rows, agent-memory rows) for one conversation. +async fn conversation_and_memory_counts( + db: &Pool, + conversation_id: Uuid, +) -> anyhow::Result<(i64, i64)> { + Ok(( + count( + db, + "SELECT count(*) FROM flow_conversation WHERE id = $1", + conversation_id, + ) + .await?, + count( + db, + "SELECT count(*) FROM ai_agent_memory WHERE conversation_id = $1", + conversation_id, + ) + .await?, + )) +} + +/// A conversation outlives the jobs behind its messages until the last one goes: only then +/// are the row and the agent's memory for it left with nothing, and only then are they +/// deleted. Both halves matter — the surviving half is what a single data-modifying CTE +/// would break, since its emptiness check would read the snapshot from before the delete. +#[sqlx::test(fixtures("base"))] +async fn test_delete_jobs_removes_a_conversation_once_its_last_message_goes( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let first_job = Uuid::new_v4(); + let second_job = Uuid::new_v4(); + insert_job(&db, WS, first_job).await?; + insert_job(&db, WS, second_job).await?; + + let conv_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by) + VALUES ($1, $2, 'f/flow', 'test-user')", + ) + .bind(conv_id) + .bind(WS) + .execute(&db) + .await?; + for job_id in [first_job, second_job] { + sqlx::query( + "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id) + VALUES ($1, 'assistant', 'hi', $2)", + ) + .bind(conv_id) + .bind(job_id) + .execute(&db) + .await?; + } + sqlx::query( + "INSERT INTO ai_agent_memory (workspace_id, conversation_id, step_id, messages) + VALUES ($1, $2, 'a', '[]'::jsonb)", + ) + .bind(WS) + .bind(conv_id) + .execute(&db) + .await?; + + let mut conn = db.acquire().await?; + windmill_common::jobs::delete_jobs(&mut conn, &[first_job]).await?; + drop(conn); + assert_eq!( + conversation_and_memory_counts(&db, conv_id).await?, + (1, 1), + "a conversation with a message left must survive, memory included" + ); + + let mut conn = db.acquire().await?; + windmill_common::jobs::delete_jobs(&mut conn, &[second_job]).await?; + drop(conn); + assert_eq!( + conversation_and_memory_counts(&db, conv_id).await?, + (0, 0), + "the last message going should take the conversation and its memory" + ); + Ok(()) +} + +/// Turns that start while retention is collecting their conversation must land, not fail: +/// the conversation lookup locks the row, so each turn waits for the collector's commit, +/// finds the conversation gone, and creates it again — the first insert wins and the other +/// reads its row. Without the lock a turn's message insert is what waits, on the parent +/// row's key lock, and fails its FK check afterwards. +#[sqlx::test(fixtures("base"))] +async fn test_new_turns_wait_for_conversation_cleanup_and_recreate( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let old_job = Uuid::new_v4(); + let new_jobs = [Uuid::new_v4(), Uuid::new_v4()]; + insert_job(&db, WS, old_job).await?; + for job in new_jobs { + insert_job(&db, WS, job).await?; + } + + let conv_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by) + VALUES ($1, $2, 'f/flow', 'test-user')", + ) + .bind(conv_id) + .bind(WS) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id) + VALUES ($1, 'user', 'hi', $2)", + ) + .bind(conv_id) + .bind(old_job) + .execute(&db) + .await?; + + // The collector holds the conversation row locked and deleted, uncommitted. + let mut cleanup = db.begin().await?; + windmill_common::jobs::delete_jobs(&mut *cleanup, &[old_job]).await?; + + let turns: Vec<_> = new_jobs + .into_iter() + .map(|new_job| { + let db = db.clone(); + tokio::spawn(async move { + let mut tx = db.begin().await?; + windmill_common::flow_conversations::get_or_create_conversation_with_id( + &mut tx, + WS, + "f/flow", + "test-user", + "hi again", + conv_id, + ) + .await?; + windmill_common::flow_conversations::add_message_to_conversation_tx( + &mut tx, + conv_id, + Some(new_job), + "hi again", + windmill_common::flow_conversations::MessageType::User, + None, + true, + ) + .await?; + tx.commit().await?; + anyhow::Ok(()) + }) + }) + .collect(); + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + cleanup.commit().await?; + for turn in turns { + turn.await??; + } + + assert_eq!( + conversation_and_memory_counts(&db, conv_id).await?.0, + 1, + "the turns must have created the conversation again, once" + ); + assert_eq!( + count( + &db, + "SELECT count(*) FROM flow_conversation_message WHERE conversation_id = $1", + conv_id, + ) + .await?, + 2, + "both turns' messages should be there" + ); + Ok(()) +} + #[sqlx::test(fixtures("base"))] async fn test_clear_schedule_removes_side_rows(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; @@ -192,6 +370,74 @@ async fn test_workspace_delete_removes_side_rows(db: Pool) -> anyhow:: Ok(()) } +/// The purge endpoint carries its own copy of the emptied-conversation rule, so it gets the +/// same guard: the conversation and its memory go with the last message, and not before. +#[sqlx::test(fixtures("base"))] +async fn test_jobs_export_delete_removes_a_conversation_once_its_last_message_goes( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let first_job = Uuid::new_v4(); + let second_job = Uuid::new_v4(); + insert_job(&db, WS, first_job).await?; + insert_job(&db, WS, second_job).await?; + + let conv_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by) + VALUES ($1, $2, 'f/flow', 'test-user')", + ) + .bind(conv_id) + .bind(WS) + .execute(&db) + .await?; + for job_id in [first_job, second_job] { + sqlx::query( + "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id) + VALUES ($1, 'assistant', 'hi', $2)", + ) + .bind(conv_id) + .bind(job_id) + .execute(&db) + .await?; + } + sqlx::query( + "INSERT INTO ai_agent_memory (workspace_id, conversation_id, step_id, messages) + VALUES ($1, $2, 'a', '[]'::jsonb)", + ) + .bind(WS) + .bind(conv_id) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let purge = |job_id: Uuid| async move { + reqwest::Client::new() + .post(format!("http://localhost:{port}/api/w/{WS}/jobs/delete")) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&[job_id]) + .send() + .await + }; + + assert!(purge(first_job).await?.status().is_success()); + assert_eq!( + conversation_and_memory_counts(&db, conv_id).await?, + (1, 1), + "a conversation with a message left must survive the purge endpoint too" + ); + + assert!(purge(second_job).await?.status().is_success()); + assert_eq!( + conversation_and_memory_counts(&db, conv_id).await?, + (0, 0), + "the last message going should take the conversation and its memory" + ); + Ok(()) +} + /// The `/jobs/delete` purge endpoint must scope every side-table delete to the path /// workspace. A `test-workspace` admin passing a job id from another workspace must not be /// able to delete that workspace's job or side rows (the side tables no longer cascade, so diff --git a/backend/windmill-api-jobs/src/execution.rs b/backend/windmill-api-jobs/src/execution.rs index 310e44ae1f..c15068583d 100644 --- a/backend/windmill-api-jobs/src/execution.rs +++ b/backend/windmill-api-jobs/src/execution.rs @@ -668,10 +668,16 @@ pub async fn handle_chat_conversation_messages( flow_path: &str, run_query: &RunJobQuery, user_message_raw: Option<&Box>, + job_id: Uuid, ) -> error::Result<()> { + // Names the query parameter rather than the field: it is not a flow argument, and + // supplying it as one is the first thing tried on reading `memory_id is required`. let memory_id = run_query.memory_id.ok_or_else(|| { windmill_common::error::Error::BadRequest( - "memory_id is required for chat-enabled flows".to_string(), + "memory_id is required for chat-enabled flows. Pass it as the `memory_id` query \ + parameter, not as a flow argument: it names the conversation the turn belongs to, \ + so a fresh UUID starts one and reusing a UUID continues it." + .to_string(), ) })?; @@ -698,10 +704,13 @@ pub async fn handle_chat_conversation_messages( ) .await?; + // The run this message started. Its args are the only record of what the message + // carried besides its text — attachments and every other flow input — and nothing + // written later points at them: an assistant row holds the AI agent step's job. add_message_to_conversation_tx( tx, memory_id, - None, + Some(job_id), &user_message, MessageType::User, None, @@ -826,6 +835,7 @@ pub async fn run_flow<'c>( &flow_path.to_string(), &run_query, args.args.get("user_message"), + uuid, ) .await?; } diff --git a/backend/windmill-api-jobs/src/jobs_export.rs b/backend/windmill-api-jobs/src/jobs_export.rs index 05f5a51bbc..6389134031 100644 --- a/backend/windmill-api-jobs/src/jobs_export.rs +++ b/backend/windmill-api-jobs/src/jobs_export.rs @@ -692,16 +692,64 @@ pub async fn delete_jobs( .await? .rows_affected(); - let conversation_message_deleted = sqlx::query!( + // One row per message deleted, so the conversation of a chat losing several appears + // several times: the count is taken before the dedup below. + let mut conversation_ids: Vec = sqlx::query_scalar!( "DELETE FROM flow_conversation_message m USING flow_conversation c - WHERE m.conversation_id = c.id AND c.workspace_id = $1 AND m.job_id = ANY($2)", + WHERE m.conversation_id = c.id AND c.workspace_id = $1 AND m.job_id = ANY($2) + RETURNING m.conversation_id", &w_id, &job_ids ) - .execute(&mut *tx) - .await? - .rows_affected(); + .fetch_all(&mut *tx) + .await?; + let conversation_message_deleted = conversation_ids.len() as u64; + + // Same rule, lock and statement order as retention (windmill_common::jobs::delete_jobs, + // which says why): a conversation with no messages left goes, and its memory with it. + conversation_ids.sort_unstable(); + conversation_ids.dedup(); + let mut memory_deleted = 0; + let mut conversation_deleted = 0; + if !conversation_ids.is_empty() { + sqlx::query_scalar!( + "SELECT id FROM flow_conversation WHERE id = ANY($1) AND workspace_id = $2 ORDER BY id FOR UPDATE", + &conversation_ids, + &w_id + ) + .fetch_all(&mut *tx) + .await?; + memory_deleted = sqlx::query!( + "DELETE FROM ai_agent_memory a + USING flow_conversation c + WHERE c.id = ANY($1) + AND c.workspace_id = $2 + AND a.conversation_id = c.id + AND a.workspace_id = c.workspace_id + AND NOT EXISTS ( + SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id + )", + &conversation_ids, + &w_id + ) + .execute(&mut *tx) + .await? + .rows_affected(); + conversation_deleted = sqlx::query!( + "DELETE FROM flow_conversation c + WHERE c.id = ANY($1) + AND c.workspace_id = $2 + AND NOT EXISTS ( + SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id + )", + &conversation_ids, + &w_id + ) + .execute(&mut *tx) + .await? + .rows_affected(); + } // Resolutions are not exported, so a delete-then-reimport of the same UUID would // otherwise resurrect the old annotation on a job that never carried one. @@ -737,6 +785,8 @@ pub async fn delete_jobs( + zombie_deleted + dispatch_event_deleted + conversation_message_deleted + + memory_deleted + + conversation_deleted + resolution_deleted + jobs_deleted; diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 84968ebbcc..bcc2fed94f 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -9553,6 +9553,7 @@ async fn run_preview_flow_job( &flow_path, &run_query, user_message.as_ref(), + uuid, ) .await?; } diff --git a/backend/windmill-common/src/flow_conversations.rs b/backend/windmill-common/src/flow_conversations.rs index 21b1f56389..b62f768bbc 100644 --- a/backend/windmill-common/src/flow_conversations.rs +++ b/backend/windmill-common/src/flow_conversations.rs @@ -36,30 +36,20 @@ pub async fn get_or_create_conversation_with_id( title: &str, conversation_id: Uuid, ) -> Result { - // Check if conversation already exists - let existing_conversation = sqlx::query_as!( - FlowConversation, - "SELECT id, workspace_id, flow_path, title, created_at, updated_at, created_by - FROM flow_conversation - WHERE id = $1 AND workspace_id = $2", - conversation_id, - w_id - ) - .fetch_optional(&mut **tx) - .await?; - - if let Some(existing) = existing_conversation { + if let Some(existing) = lock_conversation(tx, w_id, conversation_id).await? { return Ok(existing); } // Truncate title to 25 characters max let title = truncate_with_ellipsis(title, 25); - // Create new conversation with provided ID - let conversation = sqlx::query_as!( + // Every turn released by the same collector's commit finds no row: the first insert + // wins, the others wait on it, do nothing, and read the row it created. + let created = sqlx::query_as!( FlowConversation, "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by, title) VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (id) DO NOTHING RETURNING id, workspace_id, flow_path, title, created_at, updated_at, created_by", conversation_id, w_id, @@ -67,10 +57,41 @@ pub async fn get_or_create_conversation_with_id( username, title ) - .fetch_one(&mut **tx) + .fetch_optional(&mut **tx) .await?; + if let Some(conversation) = created { + return Ok(conversation); + } - Ok(conversation) + lock_conversation(tx, w_id, conversation_id) + .await? + .ok_or_else(|| { + crate::error::Error::BadRequest(format!( + "conversation {conversation_id} belongs to another workspace" + )) + }) +} + +/// Locked, so a turn orders against retention collecting the conversation +/// (windmill_common::jobs::delete_jobs): either the turn goes first and the collector then +/// sees its message, or it waits and finds the row gone and creates it again. Unlocked, the +/// message insert would wait on the parent row's lock instead and then fail its FK check. +async fn lock_conversation( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + w_id: &str, + conversation_id: Uuid, +) -> Result> { + Ok(sqlx::query_as!( + FlowConversation, + "SELECT id, workspace_id, flow_path, title, created_at, updated_at, created_by + FROM flow_conversation + WHERE id = $1 AND workspace_id = $2 + FOR UPDATE", + conversation_id, + w_id + ) + .fetch_optional(&mut **tx) + .await?) } /// Add a message to a conversation using an existing transaction diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 5541615838..6f3248f3fb 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -478,6 +478,12 @@ pub static WORKER_INTERNAL_SERVER_INLINE_UTILS: OnceCell error::Result<()> { sqlx::query!( "DELETE FROM dispatch_event WHERE producer_job_id = ANY($1)", @@ -485,12 +491,55 @@ pub async fn delete_jobs(conn: &mut sqlx::PgConnection, ids: &[uuid::Uuid]) -> e ) .execute(&mut *conn) .await?; - sqlx::query!( - "DELETE FROM flow_conversation_message WHERE job_id = ANY($1)", + let mut conversation_ids: Vec = sqlx::query_scalar!( + "DELETE FROM flow_conversation_message WHERE job_id = ANY($1) RETURNING conversation_id", ids ) - .execute(&mut *conn) + .fetch_all(&mut *conn) .await?; + conversation_ids.sort_unstable(); + conversation_ids.dedup(); + if !conversation_ids.is_empty() { + // A conversation is a view over its messages: once the last one goes with its job, + // the row and the agent's memory for it are all that is left, and nothing else + // collects them — `ai_agent_memory` carries no job id for retention to match on. + // Two statements rather than one CTE: a data-modifying CTE reads the snapshot from + // before the delete above, so every conversation would still look non-empty. + // Two calls each deleting one of a conversation's last messages would each still see + // the other's row — uncommitted deletes are invisible across transactions — so + // neither would collect it and nothing would try again. Taking the conversation row + // first serialises them: the second reads the first's delete and finds it empty. + sqlx::query_scalar!( + "SELECT id FROM flow_conversation WHERE id = ANY($1) ORDER BY id FOR UPDATE", + &conversation_ids + ) + .fetch_all(&mut *conn) + .await?; + // Memory first, since it reads the conversation row for its workspace. + sqlx::query!( + "DELETE FROM ai_agent_memory a + USING flow_conversation c + WHERE c.id = ANY($1) + AND a.conversation_id = c.id + AND a.workspace_id = c.workspace_id + AND NOT EXISTS ( + SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id + )", + &conversation_ids + ) + .execute(&mut *conn) + .await?; + sqlx::query!( + "DELETE FROM flow_conversation c + WHERE c.id = ANY($1) + AND NOT EXISTS ( + SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id + )", + &conversation_ids + ) + .execute(&mut *conn) + .await?; + } sqlx::query!("DELETE FROM zombie_job_counter WHERE job_id = ANY($1)", ids) .execute(&mut *conn) .await?; From 68f2248018fc218a090bf939e1eb22ff97d5bc22 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Thu, 17 Sep 2026 09:29:05 +0200 Subject: [PATCH 19/77] feat: keep flow inputs and seed the agent when chat mode is enabled (#11177) * feat: keep flow inputs and seed the agent when chat mode is enabled Co-Authored-By: Claude Fable 5.1 * fix: leave a linked agent's memory and streaming to the agent when enabling chat mode Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 --- CONTEXT.md | 19 + cli/src/guidance/skills.gen.ts | 36 ++ frontend/src/lib/common.ts | 2 + frontend/src/lib/components/ArgInput.svelte | 7 + .../lib/components/EditableSchemaForm.svelte | 9 +- .../lib/components/InputTransformForm.svelte | 5 + .../components/flows/content/FlowInput.svelte | 330 +++++++++++------- .../src/lib/components/flows/flowInfers.ts | 6 + system_prompts/auto-generated/flow.md | 36 ++ system_prompts/auto-generated/prompts.ts | 36 ++ .../auto-generated/skills/write-flow/SKILL.md | 36 ++ system_prompts/base/flow-base.md | 36 ++ 12 files changed, 433 insertions(+), 125 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index 64aa1b93d8..fbc2592cc9 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -37,6 +37,25 @@ _Avoid_: argument field, param Any other place a property can be picked into: the loop iterator, skip and early-stop predicates, the retry condition, a branch predicate, timeout. Its prop picker opens in a popover from the connect button rather than taking a pane. _Avoid_: JS field, code input +### Flow chat + +**Conversation**: +One thread of messages against one chat-enabled flow, with its own agent memory. A flow has +many; the chat shows one at a time. +_Avoid_: thread, session (that names an AI session, a different thing), chat (that names the surface) + +**Turn**: +One question and the answer to it: the run the question started, the handle that stops it, +and the rows it is writing. At most one per conversation, and the chat is held for its whole +length — from the moment the question takes the chat, before it has a job, until it is ended. +_Avoid_: request, exchange, message round + +**Transcript**: +The rows a conversation's chat holds. Not the conversation: it is the newest page plus +whatever older pages the reader has scrolled back through, so a question it cannot answer +from what it holds is one to ask the server rather than to guess at. +_Avoid_: history, messages (too easily read as "all of them") + ### Permissions **Member**: diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index f58092fdac..e4490276ce 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -5280,6 +5280,42 @@ tool, \`websearch\` for web search. "resource": "$res:", "model": }\`. Required unless the module links to a saved agent through \`value.agent\` +### Chat-Mode Flows + +A flow with \`value.chat_input_enabled: true\` is run from a chat instead of a form: the composer +sends one message per turn and renders the conversation. It needs a required \`user_message\` string +input, read by the agent. Any other flow input stays and is asked for under Configure inputs. + +\`\`\`json +{ + "id": "chat_agent", + "value": { + "type": "aiagent", + "input_transforms": { + "provider": { + "type": "static", + "value": { "kind": "anthropic", "resource": "$res:f/ai/claude", "model": "claude-sonnet-5" } + }, + "user_message": { "type": "javascript", "expr": "flow_input.user_message" }, + "user_attachments": { "type": "javascript", "expr": "flow_input.files" }, + "memory": { "type": "static", "value": { "kind": "auto", "context_length": 10 } }, + "streaming": { "type": "static", "value": true }, + "output_type": { "type": "static", "value": "text" } + }, + "tools": [] + } +} +\`\`\` + +- \`memory\` is what lets the agent see earlier turns; without it every message starts from nothing +- \`streaming\` on makes the answer and its thinking appear token by token instead of all at once +- \`user_attachments\` points at a flow input typed as an array of s3 objects + (\`{ "type": "array", "items": { "type": "object", "resourceType": "s3object" } }\`), so files + sent with a message reach the agent +- Running one needs a \`memory_id\` **query parameter** — not a flow argument — naming the + conversation the turn belongs to: a fresh UUID starts one, reusing a UUID continues it. The chat + supplies it itself; a run driven any other way has to pass it or the server refuses the job + ### Tool Naming Rules These rules cover \`flowmodule\` tools, the ones the agent calls by name. A \`websearch\` tool's diff --git a/frontend/src/lib/common.ts b/frontend/src/lib/common.ts index 0d76b1a0b8..4f82fd252e 100644 --- a/frontend/src/lib/common.ts +++ b/frontend/src/lib/common.ts @@ -45,6 +45,8 @@ export interface SchemaProperty { required?: string[] showExpr?: string hideWhenChatEnabled?: boolean + /** Why the oneOf variant is chat mode's to pick. Set = selector disabled, reason shown. */ + lockOneOfWhenChatEnabled?: string password?: boolean order?: string[] nullable?: boolean diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index b7001059c2..eaca31c761 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -123,6 +123,8 @@ workspace?: string | undefined s3StorageConfigured?: boolean chatInputEnabled?: boolean + /** Why the oneOf variant is fixed. Set = the selector is disabled and says so. */ + oneOfLockedReason?: string actions?: import('svelte').Snippet innerBottomSnippet?: import('svelte').Snippet fieldHeaderActions?: import('svelte').Snippet @@ -184,6 +186,7 @@ workspace = undefined, s3StorageConfigured = true, chatInputEnabled = false, + oneOfLockedReason = undefined, actions, innerBottomSnippet, fieldHeaderActions, @@ -1104,11 +1107,15 @@ {:else if inputCat == 'object' || inputCat == 'resource-object' || isListJson} {#if oneOf && oneOf.length >= 2}
+ {#if oneOfLockedReason !== undefined} +
{oneOfLockedReason}
+ {/if} {#if oneOf && oneOf.length >= 2} { oneOfSelected = detail const selectedObjProperties = diff --git a/frontend/src/lib/components/EditableSchemaForm.svelte b/frontend/src/lib/components/EditableSchemaForm.svelte index 67dfeed0b8..bd3399d9d2 100644 --- a/frontend/src/lib/components/EditableSchemaForm.svelte +++ b/frontend/src/lib/components/EditableSchemaForm.svelte @@ -43,6 +43,8 @@ interface Props { schema: Schema | any hiddenArgs?: string[] + /** Fields another part of the app owns: shown, but not renameable, deletable or retypeable. */ + lockedArgs?: string[] args?: Record shouldHideNoInputs?: boolean noVariablePicker?: boolean @@ -89,6 +91,7 @@ let { schema = $bindable(), hiddenArgs = [], + lockedArgs = [], args = $bindable(undefined), shouldHideNoInputs = false, noVariablePicker = false, @@ -587,6 +590,7 @@ > {#if keys.length > 0} {#each keys as argName, i (argName)} + {@const locked = lockedArgs.includes(argName)}
@@ -605,7 +609,7 @@ >
{argName} - {#if !uiOnly} + {#if !uiOnly && !locked}
{#snippet trigger()} @@ -654,7 +658,7 @@ Required {/if} - {#if !uiOnly} + {#if !uiOnly && !locked} + + {#snippet children({ item })} + + + {/snippet} + {/if}
{/if} @@ -696,11 +796,15 @@
{#if flowStore.val.value?.chat_input_enabled}
- {#if showAdditionalInputs} -
+ {#if chatPanelTab === 'inputs'} + +
{#snippet openEditTab()} - + {@render inputsAddTrigger()} {/snippet} {/snippet}
{/if} - + +
+ +
{:else}
@@ -815,22 +917,9 @@
{#snippet close_button()} -
@@ -884,12 +973,7 @@ }} > {#snippet trigger()} -
- -
+ {@render inputsAddTrigger()} {/snippet} {/if} diff --git a/frontend/src/lib/components/flows/flowInfers.ts b/frontend/src/lib/components/flows/flowInfers.ts index bce923c77a..3c38682350 100644 --- a/frontend/src/lib/components/flows/flowInfers.ts +++ b/frontend/src/lib/components/flows/flowInfers.ts @@ -39,6 +39,12 @@ export const AI_AGENT_SCHEMA: Schema = { }, memory: { type: 'object', + // Chat mode keys memory on the conversation, so a chat whose agent has memory off + // forgets every turn. Enabling chat mode sets `auto`; this keeps it there. A step + // sitting at `off` stays switchable, or a flow that reached that state before — + // an agent added to an already-chat-enabled flow — would have no way out of it. + lockOneOfWhenChatEnabled: + "Chat mode keys this agent's history on the conversation, so memory stays on while it is enabled.", description: 'History sent between the system message and the user message.', oneOf: [ { diff --git a/system_prompts/auto-generated/flow.md b/system_prompts/auto-generated/flow.md index dd0048ec9c..8518068992 100644 --- a/system_prompts/auto-generated/flow.md +++ b/system_prompts/auto-generated/flow.md @@ -106,6 +106,42 @@ tool, `websearch` for web search. "resource": "$res:", "model": }`. Required unless the module links to a saved agent through `value.agent` +### Chat-Mode Flows + +A flow with `value.chat_input_enabled: true` is run from a chat instead of a form: the composer +sends one message per turn and renders the conversation. It needs a required `user_message` string +input, read by the agent. Any other flow input stays and is asked for under Configure inputs. + +```json +{ + "id": "chat_agent", + "value": { + "type": "aiagent", + "input_transforms": { + "provider": { + "type": "static", + "value": { "kind": "anthropic", "resource": "$res:f/ai/claude", "model": "claude-sonnet-5" } + }, + "user_message": { "type": "javascript", "expr": "flow_input.user_message" }, + "user_attachments": { "type": "javascript", "expr": "flow_input.files" }, + "memory": { "type": "static", "value": { "kind": "auto", "context_length": 10 } }, + "streaming": { "type": "static", "value": true }, + "output_type": { "type": "static", "value": "text" } + }, + "tools": [] + } +} +``` + +- `memory` is what lets the agent see earlier turns; without it every message starts from nothing +- `streaming` on makes the answer and its thinking appear token by token instead of all at once +- `user_attachments` points at a flow input typed as an array of s3 objects + (`{ "type": "array", "items": { "type": "object", "resourceType": "s3object" } }`), so files + sent with a message reach the agent +- Running one needs a `memory_id` **query parameter** — not a flow argument — naming the + conversation the turn belongs to: a fresh UUID starts one, reusing a UUID continues it. The chat + supplies it itself; a run driven any other way has to pass it or the server refuses the job + ### Tool Naming Rules These rules cover `flowmodule` tools, the ones the agent calls by name. A `websearch` tool's diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 1e92c8e69b..9ae13dafbe 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -137,6 +137,42 @@ tool, \`websearch\` for web search. "resource": "$res:", "model": }\`. Required unless the module links to a saved agent through \`value.agent\` +### Chat-Mode Flows + +A flow with \`value.chat_input_enabled: true\` is run from a chat instead of a form: the composer +sends one message per turn and renders the conversation. It needs a required \`user_message\` string +input, read by the agent. Any other flow input stays and is asked for under Configure inputs. + +\`\`\`json +{ + "id": "chat_agent", + "value": { + "type": "aiagent", + "input_transforms": { + "provider": { + "type": "static", + "value": { "kind": "anthropic", "resource": "$res:f/ai/claude", "model": "claude-sonnet-5" } + }, + "user_message": { "type": "javascript", "expr": "flow_input.user_message" }, + "user_attachments": { "type": "javascript", "expr": "flow_input.files" }, + "memory": { "type": "static", "value": { "kind": "auto", "context_length": 10 } }, + "streaming": { "type": "static", "value": true }, + "output_type": { "type": "static", "value": "text" } + }, + "tools": [] + } +} +\`\`\` + +- \`memory\` is what lets the agent see earlier turns; without it every message starts from nothing +- \`streaming\` on makes the answer and its thinking appear token by token instead of all at once +- \`user_attachments\` points at a flow input typed as an array of s3 objects + (\`{ "type": "array", "items": { "type": "object", "resourceType": "s3object" } }\`), so files + sent with a message reach the agent +- Running one needs a \`memory_id\` **query parameter** — not a flow argument — naming the + conversation the turn belongs to: a fresh UUID starts one, reusing a UUID continues it. The chat + supplies it itself; a run driven any other way has to pass it or the server refuses the job + ### Tool Naming Rules These rules cover \`flowmodule\` tools, the ones the agent calls by name. A \`websearch\` tool's diff --git a/system_prompts/auto-generated/skills/write-flow/SKILL.md b/system_prompts/auto-generated/skills/write-flow/SKILL.md index 3556cbfff0..5988dd2f01 100644 --- a/system_prompts/auto-generated/skills/write-flow/SKILL.md +++ b/system_prompts/auto-generated/skills/write-flow/SKILL.md @@ -194,6 +194,42 @@ tool, `websearch` for web search. "resource": "$res:", "model": }`. Required unless the module links to a saved agent through `value.agent` +### Chat-Mode Flows + +A flow with `value.chat_input_enabled: true` is run from a chat instead of a form: the composer +sends one message per turn and renders the conversation. It needs a required `user_message` string +input, read by the agent. Any other flow input stays and is asked for under Configure inputs. + +```json +{ + "id": "chat_agent", + "value": { + "type": "aiagent", + "input_transforms": { + "provider": { + "type": "static", + "value": { "kind": "anthropic", "resource": "$res:f/ai/claude", "model": "claude-sonnet-5" } + }, + "user_message": { "type": "javascript", "expr": "flow_input.user_message" }, + "user_attachments": { "type": "javascript", "expr": "flow_input.files" }, + "memory": { "type": "static", "value": { "kind": "auto", "context_length": 10 } }, + "streaming": { "type": "static", "value": true }, + "output_type": { "type": "static", "value": "text" } + }, + "tools": [] + } +} +``` + +- `memory` is what lets the agent see earlier turns; without it every message starts from nothing +- `streaming` on makes the answer and its thinking appear token by token instead of all at once +- `user_attachments` points at a flow input typed as an array of s3 objects + (`{ "type": "array", "items": { "type": "object", "resourceType": "s3object" } }`), so files + sent with a message reach the agent +- Running one needs a `memory_id` **query parameter** — not a flow argument — naming the + conversation the turn belongs to: a fresh UUID starts one, reusing a UUID continues it. The chat + supplies it itself; a run driven any other way has to pass it or the server refuses the job + ### Tool Naming Rules These rules cover `flowmodule` tools, the ones the agent calls by name. A `websearch` tool's diff --git a/system_prompts/base/flow-base.md b/system_prompts/base/flow-base.md index e70ac3d42a..a102710911 100644 --- a/system_prompts/base/flow-base.md +++ b/system_prompts/base/flow-base.md @@ -106,6 +106,42 @@ tool, `websearch` for web search. "resource": "$res:", "model": }`. Required unless the module links to a saved agent through `value.agent` +### Chat-Mode Flows + +A flow with `value.chat_input_enabled: true` is run from a chat instead of a form: the composer +sends one message per turn and renders the conversation. It needs a required `user_message` string +input, read by the agent. Any other flow input stays and is asked for under Configure inputs. + +```json +{ + "id": "chat_agent", + "value": { + "type": "aiagent", + "input_transforms": { + "provider": { + "type": "static", + "value": { "kind": "anthropic", "resource": "$res:f/ai/claude", "model": "claude-sonnet-5" } + }, + "user_message": { "type": "javascript", "expr": "flow_input.user_message" }, + "user_attachments": { "type": "javascript", "expr": "flow_input.files" }, + "memory": { "type": "static", "value": { "kind": "auto", "context_length": 10 } }, + "streaming": { "type": "static", "value": true }, + "output_type": { "type": "static", "value": "text" } + }, + "tools": [] + } +} +``` + +- `memory` is what lets the agent see earlier turns; without it every message starts from nothing +- `streaming` on makes the answer and its thinking appear token by token instead of all at once +- `user_attachments` points at a flow input typed as an array of s3 objects + (`{ "type": "array", "items": { "type": "object", "resourceType": "s3object" } }`), so files + sent with a message reach the agent +- Running one needs a `memory_id` **query parameter** — not a flow argument — naming the + conversation the turn belongs to: a fresh UUID starts one, reusing a UUID continues it. The chat + supplies it itself; a run driven any other way has to pass it or the server refuses the job + ### Tool Naming Rules These rules cover `flowmodule` tools, the ones the agent calls by name. A `websearch` tool's From 189793c2e4db7f1c853695ebcc895c1ec82ed19f Mon Sep 17 00:00:00 2001 From: Guilhem Date: Thu, 17 Sep 2026 09:59:05 +0200 Subject: [PATCH 20/77] feat: flow chat model picker on a shared model-settings component (#11187) * refactor: render the session chat model menu from a shared ChatModelSettings config Co-Authored-By: Claude Fable 5.1 * feat: pick the flow chat's model and thinking from the provider fields the flow exposes Co-Authored-By: Claude Fable 5.1 * fix: name only the thinking level the flow run will send on the model button Co-Authored-By: Claude Opus 5 (1M context) * fix: let the flow chat take a typed model id and keep a shared thinking input editable Co-Authored-By: Claude Opus 5 (1M context) * fix: promote a flow input to the model button only where its control can edit it Co-Authored-By: Claude Opus 5 (1M context) * fix: drop any reasoning token the chosen model rejects before a flow chat run Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Fable 5.1 --- cli/src/guidance/skills.gen.ts | 25 +- .../components/AIReasoningEffortPicker.svelte | 2 +- .../lib/components/FlowPreviewContent.svelte | 1 + .../copilot/ChatModelSettings.svelte | 314 +++++++++++ .../copilot/ReasoningEffortSlider.svelte | 172 ++++++ .../copilot/chat/AIChatModelSettings.svelte | 316 +++-------- .../copilot/chatModelSettings.test.ts | 246 +++++++++ .../components/copilot/chatModelSettings.ts | 219 ++++++++ .../copilot/reasoningRegistry.test.ts | 26 +- .../components/copilot/reasoningRegistry.ts | 33 +- .../components/flows/content/FlowInput.svelte | 1 + .../flows/conversations/FlowChat.svelte | 5 + .../conversations/FlowChatInterface.svelte | 141 ++++- .../FlowChatModelSettings.svelte | 299 +++++++++++ .../conversations/agentChatInputs.test.ts | 492 ++++++++++++++++++ .../flows/conversations/agentChatInputs.ts | 441 ++++++++++++++++ .../(logged)/flows/get/[...path]/+page.svelte | 1 + system_prompts/auto-generated/flow.md | 25 +- system_prompts/auto-generated/prompts.ts | 25 +- .../auto-generated/skills/write-flow/SKILL.md | 25 +- system_prompts/base/flow-base.md | 25 +- 21 files changed, 2510 insertions(+), 324 deletions(-) create mode 100644 frontend/src/lib/components/copilot/ChatModelSettings.svelte create mode 100644 frontend/src/lib/components/copilot/ReasoningEffortSlider.svelte create mode 100644 frontend/src/lib/components/copilot/chatModelSettings.test.ts create mode 100644 frontend/src/lib/components/copilot/chatModelSettings.ts create mode 100644 frontend/src/lib/components/flows/conversations/FlowChatModelSettings.svelte create mode 100644 frontend/src/lib/components/flows/conversations/agentChatInputs.test.ts create mode 100644 frontend/src/lib/components/flows/conversations/agentChatInputs.ts diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index e4490276ce..993f546d3a 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -5276,15 +5276,26 @@ tool, \`websearch\` for web search. } \`\`\` -- \`provider\` is a static object, not a bare resource string: \`{ "kind": , +- \`provider\` is an object, not a bare resource string: \`{ "kind": , "resource": "$res:", "model": }\`. Required unless the module links to a saved - agent through \`value.agent\` + agent through \`value.agent\`. Static is right for a flow run from a form; a chat flow wires its + fields to flow inputs instead — see below ### Chat-Mode Flows A flow with \`value.chat_input_enabled: true\` is run from a chat instead of a form: the composer sends one message per turn and renders the conversation. It needs a required \`user_message\` string -input, read by the agent. Any other flow input stays and is asked for under Configure inputs. +input, read by the agent. Any other flow input the composer does not edit itself is asked for +under Configure inputs. + +**A static \`provider\` gives a chat that cannot change its model.** Feed it from flow inputs +instead, either way round: one input carrying the whole object (\`"expr": "flow_input.model_config"\`) +makes every field editable, or wire it field by field to fix some and expose others. A field the +chat can write becomes a control in the composer — a provider picker, a model list, a thinking +control — and a field left static is fixed, with no control drawn for it. \`kind\` is the one +exception: the composer writes it only together with \`resource\`, since a provider is picked as a +pair, so a \`kind\` input wired on its own stays askable under Configure inputs and nothing the run +needs becomes unreachable. \`\`\`json { @@ -5293,8 +5304,8 @@ input, read by the agent. Any other flow input stays and is asked for under Conf "type": "aiagent", "input_transforms": { "provider": { - "type": "static", - "value": { "kind": "anthropic", "resource": "$res:f/ai/claude", "model": "claude-sonnet-5" } + "type": "javascript", + "expr": "({ kind: 'anthropic', resource: '$res:f/ai/claude', model: flow_input.model, reasoning_effort: flow_input.thinking })" }, "user_message": { "type": "javascript", "expr": "flow_input.user_message" }, "user_attachments": { "type": "javascript", "expr": "flow_input.files" }, @@ -5307,6 +5318,10 @@ input, read by the agent. Any other flow input stays and is asked for under Conf } \`\`\` +- Wiring field by field means one object literal whose values are literals or bare \`flow_input.x\` + references. A spread, a call or a computed key leaves the composer unable to tell which input + feeds which field, so it offers no control at all — a bare \`flow_input.x\` for the whole object + is read instead as that one input carrying every field - \`memory\` is what lets the agent see earlier turns; without it every message starts from nothing - \`streaming\` on makes the answer and its thinking appear token by token instead of all at once - \`user_attachments\` points at a flow input typed as an array of s3 objects diff --git a/frontend/src/lib/components/AIReasoningEffortPicker.svelte b/frontend/src/lib/components/AIReasoningEffortPicker.svelte index 3ebf94c9bf..39b4e80011 100644 --- a/frontend/src/lib/components/AIReasoningEffortPicker.svelte +++ b/frontend/src/lib/components/AIReasoningEffortPicker.svelte @@ -27,7 +27,7 @@ let capability = $derived( provider && model ? getReasoningCapability(provider, model) - : { supported: false, levels: [], canDisable: false } + : { supported: false, levels: [], canDisable: false, known: false } ) // The token that turns reasoning off on a model that reasons by default diff --git a/frontend/src/lib/components/FlowPreviewContent.svelte b/frontend/src/lib/components/FlowPreviewContent.svelte index f6f6da0cc9..cd2460b1b0 100644 --- a/frontend/src/lib/components/FlowPreviewContent.svelte +++ b/frontend/src/lib/components/FlowPreviewContent.svelte @@ -473,6 +473,7 @@ hideSidebar={true} path={$pathStore} inputSchema={flowStore.val.schema} + flowModules={flowStore.val.value?.modules} />
{:else} diff --git a/frontend/src/lib/components/copilot/ChatModelSettings.svelte b/frontend/src/lib/components/copilot/ChatModelSettings.svelte new file mode 100644 index 0000000000..a1e677d46b --- /dev/null +++ b/frontend/src/lib/components/copilot/ChatModelSettings.svelte @@ -0,0 +1,314 @@ + + +{#snippet trigger()} +
+ +
+{/snippet} + +{#snippet typedField( + value: string, + placeholder: string, + onCommit: (value: string) => void, + close: () => void +)} + {#key value} + onCommit(e.currentTarget.value.trim()), + // Capture, not bubble: Svelte delegates `keydown` to the root, which sits above the + // menu — so a bubble handler here would run only after melt's own listener had read + // the key as typeahead and moved focus. A capture key is not delegatable, so this + // becomes a real listener on the input and sees the event first. + onkeydowncapture: (e) => { + // Escape cancels: let it reach the menu with the value untouched. + if (e.key === 'Escape') return + // Tab closes the menu, unmounting this field before focus moves, so no change + // event would ever fire. Commit on the way past. + if (e.key === 'Tab') { + onCommit(e.currentTarget.value.trim()) + return + } + // Enter means done: commit and close, rather than leaving the menu open around a + // field the commit is about to rebuild. + if (e.key === 'Enter') { + e.preventDefault() + onCommit(e.currentTarget.value.trim()) + close() + return + } + // Everything else is typing; the menu reads loose keys as typeahead. + e.stopPropagation() + } + }} + /> + {/key} +{/snippet} + +{#snippet section(sec: ChoiceSection, item: MeltItem, close: () => void)} +
{sec.label}
+ {#if sec.loading} +
+ Loading... +
+ {:else if sec.options.length === 0} +
{sec.emptyMessage ?? 'Nothing to choose from'}
+ {:else} +
+ {#each sec.options as option (option.key)} + option.onSelect()}> + {option.label} + {#if option.hint} + {option.hint} + {/if} + {#if option.selected} + + {/if} + + {/each} +
+ {/if} + {#if sec.custom && !sec.loading} + {@const custom = sec.custom} +
+ {@render typedField( + '', + custom.placeholder, + (value) => { + if (value) custom.onCommit(value) + }, + close + )} +
+ {/if} +{/snippet} + +{#snippet rows(items: Item[], item: MeltItem, builders: MeltBuilders)} + {#each items.filter((row) => !row.hide) as row (row.displayName)} + {#if row.separatorTop} +
+ {/if} + {#if row.submenuItems} + + + {:else} + row.action?.(e)}> + {#if row.icon} + + {/if} + {row.displayName} + {#if row.selected} + + {/if} + + {/if} + {/each} +{/snippet} + +{#if config.readOnly} + {@render trigger()} +{:else} + + {#snippet buttonReplacement()} + {@render trigger()} + {/snippet} + {#snippet menu({ item, builders, close })} +
+ {#if config.topItems} +
+ {@render rows(config.topItems(close), item, builders)} +
+ {/if} + {#each config.sections ?? [] as sec (sec.label)} +
+ {@render section(sec, item, close)} +
+ {/each} + {#if reasoning} +
+ {#if controlState === 'fixed'} + {}} + unsupportedReason={fixedReason} + /> + {:else if controlState === 'awaiting-model'} + {}} + unsupportedReason="Pick a model first" + /> + {:else if controlState === 'unknown'} + +
+
Thinking
+ {@render typedField(reasoning.value ?? '', 'none', reasoning.onSelect, close)} +
+ Windmill has no thinking levels for this provider — type what it accepts. +
+
+ {:else if controlState === 'ladder'} + + effortSlider?.adjust(e)} + class="block group" + > + (stop === reasoning?.offToken ? 'off' : stop)} + overrideLabel={stops.includes(currentStop) ? undefined : effortLabel} + /> + + {:else} + + {}} + unsupportedReason="Not supported by this model" + /> + {/if} +
+ {/if} + {#if config.bottomItems} +
+ {@render rows(config.bottomItems(close), item, builders)} +
+ {/if} +
+ {/snippet} + +{/if} diff --git a/frontend/src/lib/components/copilot/ReasoningEffortSlider.svelte b/frontend/src/lib/components/copilot/ReasoningEffortSlider.svelte new file mode 100644 index 0000000000..24ea4cf8aa --- /dev/null +++ b/frontend/src/lib/components/copilot/ReasoningEffortSlider.svelte @@ -0,0 +1,172 @@ + + +{#if unsupportedReason} + +
+
Thinking
+
{unsupportedReason}
+
+{:else} +
+ Thinking + {overrideLabel ?? format(current)} +
+ {#if stops.length > 1} + +
+ onSelect(stops[+e.currentTarget.value])} + onclick={(e) => { + // `click`, not `pointerup`: it is the event that means pressed and released on + // the track, so a press that began on the row above cannot commit an effort + // nobody chose. Only the click that moved nothing — any other stop has already + // committed through `oninput`, and doing it again would write it twice. + if (!hasPosition && +e.currentTarget.value === stopIndex) { + onSelect(stops[stopIndex]) + } + }} + use:isolatePointer + class="lean-range no-default-style w-full" + aria-label="Reasoning effort" + /> +
+ {/if} +{/if} + + diff --git a/frontend/src/lib/components/copilot/chat/AIChatModelSettings.svelte b/frontend/src/lib/components/copilot/chat/AIChatModelSettings.svelte index 9dd94fa4ea..914ad51bc9 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatModelSettings.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatModelSettings.svelte @@ -1,10 +1,12 @@ {#snippet externalLinkIcon()} {/snippet} - - {#snippet buttonReplacement()} -
- -
- {/snippet} - {#snippet menu({ item, builders, close })} -
- - {#if promptSettings} - - {/if} + -
-
Model
-
- {#each models as m (m.provider + m.model)} - selectModel(m)} - > - {m.model} - {#if m.model === providerModel.model && m.provider === providerModel.provider} - - {/if} - - {/each} -
- -
- {#if capability.supported} - - -
- Thinking - {currentStop} -
- {#if stops.length > 1} - -
- selectReasoning(stops[+e.currentTarget.value])} - use:isolatePointer - class="lean-range no-default-style w-full" - aria-label="Reasoning effort" - /> -
- {/if} - - {:else} - -
-
Thinking
-
Not supported by this model
-
- {/if} - - - (thinkingPreferences.expandByDefault = !thinkingPreferences.expandByDefault)} - > - Always expand thinking - {#if thinkingPreferences.expandByDefault} - - {/if} - -
- {/snippet} - - - {#if promptSettings} {/if} - - diff --git a/frontend/src/lib/components/copilot/chatModelSettings.test.ts b/frontend/src/lib/components/copilot/chatModelSettings.test.ts new file mode 100644 index 0000000000..04d1834556 --- /dev/null +++ b/frontend/src/lib/components/copilot/chatModelSettings.test.ts @@ -0,0 +1,246 @@ +import { describe, expect, it } from 'vitest' +import { + carriedReasoning, + fixedReasoningReason, + reasoningControlState, + reasoningDisplay, + REASONING_PROVIDER_DEFAULT, + type ChatModelSettingsReasoning +} from './chatModelSettings' +import { + getReasoningCapability, + REASONING_OFF, + resolveEffectiveReasoning +} from './reasoningRegistry' + +/** + * The trigger's suffix and the slider's stops are read side by side, so they have to agree + * about one value — the provider-native off token must not read as `none` on one and `off` + * on the other, and an effort the run does not send must not be named at all. + */ +function display( + reasoning: Partial & { provider: any; model: string } +) { + const full = { + value: undefined, + offToken: undefined, + sendsDefaultWhenUnset: false, + onSelect: () => {}, + ...reasoning + } as ChatModelSettingsReasoning + const capability = getReasoningCapability(full.provider, full.model) + // Composed exactly as the component composes it, so the test exercises the real pair. + const effective = resolveEffectiveReasoning({ + provider: full.provider, + model: full.model, + reasoning: full.value + }) + return reasoningDisplay(full, capability, effective) +} + +describe('reasoningDisplay', () => { + it('says nothing for a model that cannot reason', () => { + const shown = display({ provider: 'openai', model: 'gpt-4o' }) + expect(shown.label).toBeUndefined() + expect(shown.stops).toEqual([]) + }) + + // The session chat's own sentinel: what it stores is already the word the reader sees. + it('reads the session chat off sentinel as off', () => { + const shown = display({ + provider: 'openai', + model: 'gpt-5.1', + offToken: REASONING_OFF, + value: REASONING_OFF, + sendsDefaultWhenUnset: true + }) + expect(shown.label).toBe(REASONING_OFF) + expect(shown.currentStop).toBe(REASONING_OFF) + }) + + // An agent writes the provider's own token, which can read as anything. + it('reads a provider-native off token as off too', () => { + const shown = display({ + provider: 'openai', + model: 'gpt-5.1', + offToken: 'none', + value: 'none' + }) + expect(shown.label).toBe(REASONING_OFF) + expect(shown.currentStop).toBe('none') + expect(shown.stops[0]).toBe('none') + }) + + it('names the level a chat that fills one in will send', () => { + const shown = display({ + provider: 'openai', + model: 'gpt-5.1', + offToken: REASONING_OFF, + value: undefined, + sendsDefaultWhenUnset: true + }) + expect(shown.label).toBe('high') + }) + + // An agent step omits the field, so naming a level would claim something untrue. + it('names no level where an unset effort is simply not sent', () => { + const shown = display({ + provider: 'anthropic', + model: 'claude-sonnet-5', + offToken: 'none', + value: undefined + }) + expect(shown.label).toBe(REASONING_PROVIDER_DEFAULT) + expect(shown.currentStop).toBe('') + }) + + // Claude 4.x only thinks when asked, so an absent effort is already off — and the flow + // chat must be able to get back to it after a level has been picked. + it('offers omission as the off stop where that is how the model disables', () => { + const unset = display({ provider: 'anthropic', model: 'claude-opus-4-6', offToken: '' }) + expect(unset.label).toBe(REASONING_OFF) + expect(unset.stops[0]).toBe('') + const picked = display({ + provider: 'anthropic', + model: 'claude-opus-4-6', + offToken: '', + value: 'high' + }) + expect(picked.currentStop).toBe('high') + expect(picked.stops).toContain('') + }) + + // gpt-5 reasons at medium with no effort sent, so an empty off token buys no off stop. + it('offers no off where the model cannot stop thinking', () => { + const shown = display({ provider: 'openai', model: 'gpt-5', offToken: '' }) + expect(shown.stops).not.toContain('') + expect(shown.label).toBe(REASONING_PROVIDER_DEFAULT) + }) + + // The run sends an explicitly set effort whatever the model, so a token typed against a + // provider we have no rules for has to reach the trigger — silence would hide it. + it('names a set effort even where it can offer no ladder', () => { + const shown = display({ provider: 'customai', model: 'deepseek-r1', value: 'high' }) + expect(shown.label).toBe('high') + expect(shown.stops).toEqual([]) + }) +}) + +describe('carriedReasoning', () => { + const cap = (model: string) => getReasoningCapability('openai', model) + + // A level carried onto a model that cannot think stays in the flow input, and the run sends it. + it('drops a level the new model does not have', () => { + expect(carriedReasoning('high', '', cap('gpt-4o'))).toBeUndefined() + expect(carriedReasoning('xhigh', '', cap('gpt-5.1'))).toBeUndefined() + }) + + it('keeps a level the new model does have', () => { + expect(carriedReasoning('high', '', cap('gpt-5.1'))).toBe('high') + }) + + it('carries off only onto a model that can truly stop thinking', () => { + expect(carriedReasoning(REASONING_OFF, REASONING_OFF, cap('gpt-5.1'))).toBe(REASONING_OFF) + expect(carriedReasoning(REASONING_OFF, REASONING_OFF, cap('gpt-5'))).toBeUndefined() + }) + + // A provider the registry has no rules for draws no thinking control, so a carried level + // would be invisible and unclearable — and still sent, since an explicitly set effort + // goes out whatever the model. + it('drops the effort where it has no rules for the provider', () => { + expect( + carriedReasoning('high', REASONING_OFF, getReasoningCapability('customai', 'deepseek-r1')) + ).toBeUndefined() + }) + + it('has nothing to carry when no effort is set', () => { + expect(carriedReasoning(undefined, '', cap('gpt-5.1'))).toBeUndefined() + expect(carriedReasoning('', '', cap('gpt-5.1'))).toBeUndefined() + }) +}) + +const asReasoning = (over: Partial): ChatModelSettingsReasoning => + ({ + provider: 'openai', + model: 'gpt-5.1', + value: undefined, + offToken: REASONING_OFF, + sendsDefaultWhenUnset: false, + writable: true, + typedWhenUnknown: true, + onSelect: () => {}, + ...over + }) as ChatModelSettingsReasoning + +/** The control is always drawn; this is the only thing that decides what it draws. */ +describe('reasoningControlState', () => { + const cap = (model: string) => getReasoningCapability('openai', model) + + it('shows the ladder for a model with levels', () => { + expect(reasoningControlState(asReasoning({}), cap('gpt-5.1'))).toBe('ladder') + }) + + it('says a model cannot think when the registry knows it cannot', () => { + expect(reasoningControlState(asReasoning({ model: 'gpt-4o' }), cap('gpt-4o'))).toBe( + 'unsupported' + ) + }) + + // Not the same as "cannot think": we have no rules for the provider, so the flow's own + // token is typed rather than picked. + it('asks for a typed token where it has no rules for the provider', () => { + expect( + reasoningControlState( + asReasoning({ provider: 'customai', model: 'deepseek-r1' }), + getReasoningCapability('customai', 'deepseek-r1') + ) + ).toBe('unknown') + }) + + // The session chat has no typed effort: a provider with no rules reads as unable to think. + it('offers no typed token to a chat that does not take one', () => { + expect( + reasoningControlState( + asReasoning({ provider: 'customai', model: 'deepseek-r1', typedWhenUnknown: false }), + getReasoningCapability('customai', 'deepseek-r1') + ) + ).toBe('unsupported') + }) + + // A provider with a full ladder must not be described as unreadable just because no + // model has been picked yet — which is the state right after choosing a resource. + it('waits for a model rather than blaming the provider', () => { + expect( + reasoningControlState(asReasoning({ model: undefined }), { supported: false, known: false }) + ).toBe('awaiting-model') + }) + + it('shows what the flow fixed when this chat cannot write it', () => { + expect(reasoningControlState(asReasoning({ writable: false }), cap('gpt-5.1'))).toBe('fixed') + }) +}) + +describe('fixedReasoningReason', () => { + it('names the level the run will use', () => { + expect( + fixedReasoningReason(asReasoning({ value: 'high' }), { supported: true, known: true }) + ).toBe('high · set in the flow') + }) + + // The step naming no effort at all is the common shape; saying it was "set in the flow" + // would describe a line the flow does not contain. + it('does not claim a level the step never set', () => { + expect( + fixedReasoningReason(asReasoning({ value: undefined }), { supported: true, known: true }) + ).toBe('Not set in the flow, so the provider decides') + }) + + it('surfaces a level fixed on a model that cannot use it', () => { + expect( + fixedReasoningReason(asReasoning({ value: 'high', model: 'gpt-4o' }), { + supported: false, + known: true + }) + ).toContain('cannot think') + }) +}) diff --git a/frontend/src/lib/components/copilot/chatModelSettings.ts b/frontend/src/lib/components/copilot/chatModelSettings.ts new file mode 100644 index 0000000000..3fce0270d1 --- /dev/null +++ b/frontend/src/lib/components/copilot/chatModelSettings.ts @@ -0,0 +1,219 @@ +import type { AIProvider } from '$lib/gen' +import type { Item } from '$lib/utils' +import { REASONING_OFF } from './reasoningRegistry' + +/** + * The contract between a chat and its model button. + * + * One component renders this menu for every chat — the copilot's own session chat and + * the flow chat — so the component knows only about rows, choices and a reasoning + * ladder. What a row means (a workspace AI resource, a prompt to edit, a reading + * preference) is the caller's business, and each caller derives its own config: a fixed + * one for the session chat, one derived from the flow's exposed inputs for flow chat. + */ + +export type ModelChoice = { + /** Stable across rebuilds of the config; used as the `{#each}` key. */ + key: string + label: string + /** Muted trailing text, e.g. the provider a resource speaks. */ + hint?: string + selected: boolean + onSelect: () => void +} + +export type ChoiceSection = { + /** Section heading, e.g. 'Provider' or 'Model'. */ + label: string + options: ModelChoice[] + /** Fetched lists render one consistent loading line instead of the options. */ + loading?: boolean + /** Shown when the list is empty and settled. */ + emptyMessage?: string + maxHeight?: string + /** + * A typed entry under the options, for a value the list does not hold: an endpoint with no + * listing, or a model newer than the catalogue. An empty entry commits nothing. + */ + custom?: { placeholder: string; onCommit: (value: string) => void } +} + +export type ChatModelSettingsConfig = { + /** The trigger's main text: the chosen model, or an invitation to choose one. */ + label: string + title?: string + /** Trailing pill on the trigger, e.g. the free-tier grant this chat is spending. */ + badge?: { text: string; warn?: boolean } + /** Nothing here is editable — the trigger still names the model, but no menu opens. */ + readOnly?: boolean + readOnlyReason?: string + /** + * Rows above and below the choice sections. Given the menu's own `close` because a + * row that opens a modal must close the menu first, while a row that toggles a + * preference must not. + */ + topItems?: (close: () => void) => Item[] + sections?: ChoiceSection[] + bottomItems?: (close: () => void) => Item[] + /** + * The thinking slider. The ladder is derived from the provider and model here rather + * than by each caller, so a new provider's effort levels reach every chat at once. + * `value` is the raw stored effort (undefined meaning the model's default), and + * `offToken` the token this caller stores for "off" — the copilot keeps its own + * sentinel and translates when it calls the provider, an agent writes the + * provider-native token straight into its step. + */ + reasoning?: ChatModelSettingsReasoning +} + +export type ChatModelSettingsReasoning = { + /** Absent until the chat knows what it will run; the ladder then has nothing to stand on. */ + provider: AIProvider | undefined + model: string | undefined + value: string | undefined + offToken: string | undefined + /** + * What an unset value means on the wire. The copilot fills one in before calling the + * provider, so unset really runs at the default effort and the button says so. An agent + * step omits the field entirely, so unset means whatever the provider does by itself — + * naming a level there would state something the run does not do. + */ + sendsDefaultWhenUnset: boolean + /** + * Whether this chat can write the effort back. False where the flow fixes it in the step: + * the run still uses it, so the button shows it and refuses to pretend otherwise — only + * the flow editor can change it. + */ + writable: boolean + /** + * Whether a provider the registry has no rules for gets a typed effort field. True for an + * agent, which writes the token straight into its step. False for the copilot, which then + * shows the model as unable to think. + */ + typedWhenUnknown: boolean + onSelect: (token: string) => void +} + +/** What an unset agent effort reads as: the provider decides, and we do not know what. */ +export const REASONING_PROVIDER_DEFAULT = 'default' + +/** + * The effort to keep when the model changes, or nothing where the new model has no such + * level. Dropped rather than carried because a model that cannot think at that level either + * rejects the request or quietly runs at another one, and the button would name a level the + * run never used. Off survives only onto a model that can truly disable. + * + * A model the registry has no rules for drops it too, for the same reason: a chat without + * `typedWhenUnknown` draws no thinking control there, so a carried level would be invisible + * and unclearable while still going out on the wire — `resolveEffectiveReasoning` sends an explicitly set effort + * whatever the model, and a provider that rejects the field would then fail every turn with + * nothing on screen to explain it. + */ +export function carriedReasoning( + current: string | undefined, + offToken: string | undefined, + capability: { levels: string[]; canDisable: boolean } +): string | undefined { + if (current === undefined || current === '') return undefined + if (offToken !== undefined && current === offToken) { + return capability.canDisable ? current : undefined + } + return capability.levels.includes(current) ? current : undefined +} + +/** + * What the menu shows for the reasoning ladder: the stops the slider offers, the one it + * sits on, and the suffix on the trigger. + * + * Pure and here rather than in the component because these three have to agree — a stop + * the slider renders as `off` must not read as the provider's own `none` on the button — + * and because the rules are provider-shaped enough to be worth testing directly. + */ +export function reasoningDisplay( + reasoning: ChatModelSettingsReasoning | undefined, + capability: { supported: boolean; levels: string[]; canDisable: boolean }, + effective: string | undefined +): { + stops: string[] + currentStop: string + /** Trigger suffix, or undefined when there is nothing truthful to say. */ + label: string | undefined +} { + if (!reasoning) return { stops: [], currentStop: '', label: undefined } + if (!capability.supported) { + // No ladder to place it on, but an effort that is explicitly set still goes out — + // `resolveEffectiveReasoning` sends one whatever the model — so the button names it. + // Saying nothing would hide from the reader what the run is about to do. + const set = reasoning.value ? reasoning.value : undefined + return { stops: [], currentStop: '', label: set } + } + // An off position only where the model can truly disable, else the provider would + // coerce it to the lowest level; then the provider-native levels. + const offToken = capability.canDisable ? reasoning.offToken : undefined + const stops = [...(offToken !== undefined ? [offToken] : []), ...capability.levels] + // An agent whose model disables by omission stores the empty string, which the run + // treats as no effort at all — so an unset value already sits on that stop. + const isOff = offToken !== undefined && (reasoning.value ?? '') === offToken + if (isOff) { + // The off token is provider-native and can read as anything ('none', 'disabled'); + // on the button and on the slider it always reads as off. + return { stops, currentStop: offToken as string, label: REASONING_OFF } + } + if (reasoning.value === undefined || reasoning.value === '') { + // Where the provider takes an explicit disable, unset is a third state — the + // provider's own level, above off — that the ladder has no position for. The + // button still names it, and every stop the ladder does offer stays reachable. + return reasoning.sendsDefaultWhenUnset + ? { stops, currentStop: effective ?? '', label: effective ?? REASONING_OFF } + : { stops, currentStop: '', label: REASONING_PROVIDER_DEFAULT } + } + return { stops, currentStop: reasoning.value, label: reasoning.value } +} + +/** Which thinking control a chat should draw. */ +export type ReasoningControlState = + /** The flow sets the effort itself; show what the run will use. */ + | 'fixed' + /** No model chosen yet, so nothing can be said about its levels. */ + | 'awaiting-model' + /** No rules for this provider, and the chat takes a typed token. */ + | 'unknown' + /** Known levels — the ladder. */ + | 'ladder' + /** Known to have none. */ + | 'unsupported' + +/** + * The control is always drawn; only its state varies. Decided here rather than in the markup + * so the states sit in one readable, testable place. + */ +export function reasoningControlState( + reasoning: ChatModelSettingsReasoning | undefined, + capability: { supported: boolean; known: boolean } +): ReasoningControlState { + if (!reasoning || !reasoning.writable) return 'fixed' + if (!reasoning.model) return 'awaiting-model' + if (!capability.known && reasoning.typedWhenUnknown) return 'unknown' + return capability.supported ? 'ladder' : 'unsupported' +} + +/** + * What the row says when the chat cannot write the effort. Naming the level is the point: a + * button that names the model a run will use should name its thinking too, and a level fixed + * on a model that cannot use one is a broken flow worth seeing rather than a silent row. + */ +export function fixedReasoningReason( + reasoning: ChatModelSettingsReasoning | undefined, + capability: { supported: boolean; known: boolean } +): string { + const cannotThink = capability.known && !capability.supported + const model = reasoning?.model ?? 'this model' + if (!reasoning?.value) { + // The step names no effort, so the provider decides — saying it was "set in the flow" + // would describe a line the flow does not contain. + return cannotThink ? `${model} cannot think` : 'Not set in the flow, so the provider decides' + } + return cannotThink + ? `${reasoning.value} · set in the flow, but ${model} cannot think` + : `${reasoning.value} · set in the flow` +} diff --git a/frontend/src/lib/components/copilot/reasoningRegistry.test.ts b/frontend/src/lib/components/copilot/reasoningRegistry.test.ts index b4c935bcaa..faf6f60264 100644 --- a/frontend/src/lib/components/copilot/reasoningRegistry.test.ts +++ b/frontend/src/lib/components/copilot/reasoningRegistry.test.ts @@ -77,9 +77,9 @@ describe('supportsReasoning (static registry)', () => { } // Bedrock translates the same sentinel on its Converse path, but only for // Opus 5 — AWS documents Bedrock's Sonnet 5 as always thinking. - expect( - getReasoningCapability('aws_bedrock', 'global.anthropic.claude-opus-5').canDisable - ).toBe(true) + expect(getReasoningCapability('aws_bedrock', 'global.anthropic.claude-opus-5').canDisable).toBe( + true + ) expect( resolveRequestReasoning({ provider: 'aws_bedrock', @@ -189,13 +189,29 @@ describe('supportsReasoning (static registry)', () => { expect(supportsReasoning('mistral', 'mistral-medium-3.5')).toBe(true) expect(getReasoningCapability('mistral', 'mistral-medium-3-5').canDisable).toBe(true) }) - it('returns no levels for providers without a registry entry', () => { + it('returns no levels for a model its provider family has no entry for', () => { + // The family is known, so the `false` is an answer: codestral does not reason. expect(getReasoningCapability('mistral', 'codestral-latest')).toEqual({ supported: false, levels: [], - canDisable: false + canDisable: false, + known: true }) }) + + // `customai` fronts any OpenAI-compatible endpoint, so `supported: false` there is an + // absence of rules rather than a fact about the model. A caller that shows the reader + // "this model cannot think" has to tell the two apart. + it('admits when it has no rules for the provider at all', () => { + expect(getReasoningCapability('customai', 'deepseek-r1')).toEqual({ + supported: false, + levels: [], + canDisable: false, + known: false + }) + expect(getReasoningCapability('openai', 'gpt-4o').known).toBe(true) + expect(getReasoningCapability('anthropic', 'claude-sonnet-5').known).toBe(true) + }) it('only offers off where the model can truly disable thinking', () => { // Gemini Pro enforces a thinking floor — no off option. expect(getReasoningCapability('googleai', 'gemini-2.5-pro').canDisable).toBe(false) diff --git a/frontend/src/lib/components/copilot/reasoningRegistry.ts b/frontend/src/lib/components/copilot/reasoningRegistry.ts index 95a00e28c8..242f6cb329 100644 --- a/frontend/src/lib/components/copilot/reasoningRegistry.ts +++ b/frontend/src/lib/components/copilot/reasoningRegistry.ts @@ -231,14 +231,35 @@ export type ReasoningCapability = { * level, making the switch a lie. */ canDisable: boolean + /** + * Whether `supported` is an answer or an absence of one. The registry has rules per + * provider family and falls through to `false` for the rest — `customai` above all, + * which fronts any OpenAI-compatible endpoint and may well serve a thinking model. A + * caller that presents `supported: false` as a fact must check this first, or it tells + * the reader a model cannot think when all we know is that we have never heard of it. + */ + known: boolean } +/** Provider families the registry has real rules for; everything else is a shrug. */ +const KNOWN_REASONING_FAMILIES: ReadonlySet = new Set([ + 'anthropic', + 'aws_bedrock', + 'openai', + 'azure_openai', + 'openrouter', + 'googleai', + 'deepseek', + 'mistral' +]) + /** Resolve the reasoning capability of a model from the static registry. */ export function getReasoningCapability(provider: AIProvider, model: string): ReasoningCapability { const bareModel = stripLegacyThinkingSuffix(model) + const known = KNOWN_REASONING_FAMILIES.has(reasoningProviderFamily(provider, bareModel)) const supported = supportsReasoningStatic(provider, bareModel) if (!supported) { - return { supported: false, levels: [], canDisable: false } + return { supported: false, levels: [], canDisable: false, known } } const family = reasoningProviderFamily(provider, bareModel) const levels = @@ -251,7 +272,7 @@ export function getReasoningCapability(provider: AIProvider, model: string): Rea : family === 'openrouter' ? openrouterReasoningLevels(bareModel) : (PROVIDER_REASONING_LEVELS[family] ?? ['low', 'medium', 'high']) - return { supported, levels, canDisable: canDisableReasoning(provider, bareModel) } + return { supported, levels, canDisable: canDisableReasoning(provider, bareModel), known } } /** @@ -362,15 +383,11 @@ export function explicitOffToken(provider: AIProvider, model: string): Reasoning // real off there and stays the wire form. Only the 5 family, which // thinks when the field is absent, needs the explicit disable — // Fable and Mythos reject it outright and get no off token at all. - return /claude-(opus|sonnet)-5/.test(model.toLowerCase()) - ? ANTHROPIC_OFF_SENTINEL - : undefined + return /claude-(opus|sonnet)-5/.test(model.toLowerCase()) ? ANTHROPIC_OFF_SENTINEL : undefined case 'aws_bedrock': // Bedrock's Sonnet 5 cannot be disabled at all, so only Opus 5 gets // the sentinel; the rest keep omission. - return model.toLowerCase().includes('claude-opus-5') - ? ANTHROPIC_OFF_SENTINEL - : undefined + return model.toLowerCase().includes('claude-opus-5') ? ANTHROPIC_OFF_SENTINEL : undefined case 'googleai': // Gemini 2.5/3 think by default (dynamic budget / level). The backend // proxy maps 'none' to off on Flash, or the floor on Pro (only diff --git a/frontend/src/lib/components/flows/content/FlowInput.svelte b/frontend/src/lib/components/flows/content/FlowInput.svelte index 7d4483d577..eb78dbdc89 100644 --- a/frontend/src/lib/components/flows/content/FlowInput.svelte +++ b/frontend/src/lib/components/flows/content/FlowInput.svelte @@ -851,6 +851,7 @@ path={$pathStore} hideSidebar={true} inputSchema={flowStore.val.schema} + flowModules={flowStore.val.value?.modules} />
diff --git a/frontend/src/lib/components/flows/conversations/FlowChat.svelte b/frontend/src/lib/components/flows/conversations/FlowChat.svelte index dc4eb92b57..1025ddaa15 100644 --- a/frontend/src/lib/components/flows/conversations/FlowChat.svelte +++ b/frontend/src/lib/components/flows/conversations/FlowChat.svelte @@ -6,6 +6,7 @@ import FlowChatInterface from './FlowChatInterface.svelte' import { getContext } from 'svelte' import type { FlowEditorContext } from '../types' + import type { FlowModule } from '$lib/gen' interface Props { /** @@ -22,6 +23,8 @@ path: string hideSidebar?: boolean inputSchema?: Record + /** The flow's modules, read for the provider wiring of its AI agent steps. */ + flowModules?: FlowModule[] /** The flow's description, shown under the empty transcript's prompt. */ description?: string wideLayout?: boolean @@ -33,6 +36,7 @@ path, hideSidebar = false, inputSchema = undefined, + flowModules = undefined, description = undefined, wideLayout = false }: Props = $props() @@ -102,6 +106,7 @@ {chat} {deploymentInProgress} {additionalInputsSchema} + {flowModules} {path} {workspace} {description} diff --git a/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte b/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte index 535be3ca83..f6aa0670c4 100644 --- a/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte +++ b/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte @@ -10,11 +10,23 @@ import { emptyString, type DynamicInput } from '$lib/utils' import { onDestroy, tick, untrack } from 'svelte' import type { Chat } from 'windmill-chat' + import type { FlowModule } from '$lib/gen' + import { deepEqual } from 'fast-equals' + import FlowChatModelSettings from './FlowChatModelSettings.svelte' + import { + agentModelGap, + composerOwnedInputs, + resolveAgentModelWiring, + showsModelButton, + withoutRejectedEffort + } from './agentChatInputs' interface Props { chat: Chat deploymentInProgress?: boolean additionalInputsSchema?: Record + /** The flow's modules, read for the provider wiring of its AI agent steps. */ + flowModules?: FlowModule[] path: string workspace?: string /** The flow's description, shown under the empty transcript's prompt. */ @@ -26,6 +38,7 @@ chat, deploymentInProgress = false, additionalInputsSchema, + flowModules, path, workspace = undefined, description = undefined, @@ -42,14 +55,45 @@ return undefined }) + // The model gets its own button, shaped like the copilot's model settings, driven by + // whichever provider fields the flow exposes. Every other flow input is asked for in + // the Configure-inputs modal. + const modelWiring = $derived(resolveAgentModelWiring(flowModules)) + // An agent with nothing to call cannot answer, and the composer cannot fix it, so the + // chat says what to go and do instead of offering controls that write nowhere. + const modelGap = $derived(agentModelGap(modelWiring)) + const showModelButton = $derived(showsModelButton(modelWiring)) + // LocalStorage helpers const STORAGE_KEY_PREFIX = 'windmill_flow_chat_inputs_' - // State for additional inputs modal let showInputsModal = $state(false) - let additionalInputsValues = $state | undefined>( - loadInputsFromStorage() ?? undefined - ) + // Conversation settings, persisted per flow: what the reader chose, and nothing else. + let inputValues = $state>(loadInputsFromStorage() ?? {}) + let modalDraft = $state>({}) + + /** What the flow's own form would open on. */ + function schemaDefaults(schema: Record | undefined): Record { + const properties: Record = schema?.properties ?? {} + return Object.fromEntries( + Object.entries(properties) + .filter(([, property]) => property?.default !== undefined) + .map(([name, property]) => [name, property.default]) + ) + } + + // Derived rather than seeded into `inputValues`: the schema arrives with the flow, which + // on the deployed page is after this mounts, and only what the reader actually chose + // belongs in storage. A stored value wins over the default, including a deliberate empty. + const effectiveInputs = $derived({ + ...schemaDefaults(additionalInputsSchema), + ...inputValues + }) + + // What the run actually gets. The composer's own controls keep themselves consistent as + // they are used; this is where a pair that was never chosen through them — a stored + // value, an author's default — is made safe before it reaches the provider. + const runInputs = $derived(withoutRejectedEffort(modelWiring, effectiveInputs)) function getStorageKey(): string { return `${STORAGE_KEY_PREFIX}${path}` @@ -73,40 +117,72 @@ } } + function setInputValue(name: string, value: any) { + inputValues = { ...inputValues, [name]: value } + saveInputsToStorage(inputValues) + } + function handleModalConfirm() { - saveInputsToStorage(additionalInputsValues ?? {}) + // The modal opens on `effectiveInputs`, so its draft carries a value for every + // defaulted input whether or not the reader touched one. Storing those would pin + // today's defaults for good — `effectiveInputs` gives a stored value precedence, so + // a later change to the flow's schema would never reach this reader again. + const defaults = schemaDefaults(additionalInputsSchema) + const kept = Object.fromEntries( + Object.entries({ ...inputValues, ...modalDraft }).filter( + ([name, value]) => !deepEqual(value, defaults[name]) + ) + ) + inputValues = kept + saveInputsToStorage(inputValues) showInputsModal = false } function openInputsModal() { - const stored = loadInputsFromStorage() - if (stored) additionalInputsValues = stored + modalDraft = { ...effectiveInputs, ...(loadInputsFromStorage() ?? inputValues) } showInputsModal = true } - const hasMissingRequired = $derived.by(() => { - if (!additionalInputsSchema?.required?.length) return false - const values = additionalInputsValues ?? {} - return additionalInputsSchema.required.some( - (field: string) => - values[field] === undefined || values[field] === '' || values[field] === null - ) - }) - // The host follows the chat it was built on for the life of this component: FlowChat // remounts the interface under `{#key chat}`, so a later value of the prop never reaches it. const chatHost = new FlowChatViewHost( untrack(() => chat), { - additionalInputs: () => - additionalInputsSchema ? (loadInputsFromStorage() ?? additionalInputsValues) : undefined, + additionalInputs: () => (additionalInputsSchema ? { ...runInputs } : undefined), workspace: () => workspace, - sendDisabled: () => deploymentInProgress + sendDisabled: () => deploymentInProgress || !!modelGap } ) setChatViewHost(chatHost) onDestroy(() => chatHost.dispose()) + // What the Configure-inputs modal asks for: every flow input the composer does not + // edit itself. + const modalSchema = $derived.by(() => { + if (!additionalInputsSchema) return undefined + const promoted = new Set(composerOwnedInputs(modelWiring, undefined)) + const properties = Object.fromEntries( + Object.entries(additionalInputsSchema.properties ?? {}).filter(([key]) => !promoted.has(key)) + ) + if (Object.keys(properties).length === 0) return undefined + const required: string[] = Array.isArray(additionalInputsSchema.required) + ? additionalInputsSchema.required + : [] + return { + ...additionalInputsSchema, + properties, + required: required.filter((key) => !promoted.has(key)) + } + }) + + const modalMissingRequired = $derived.by(() => { + if (!modalSchema?.required?.length) return false + return modalSchema.required.some((field: string) => { + const value = effectiveInputs[field] + return value === undefined || value === '' || value === null + }) + }) + // Older pages load when the reader reaches the top; the viewport stays where it was. let scrollElement = $state(undefined) let loadingOlder = false @@ -126,12 +202,11 @@ } - -{#if additionalInputsSchema} +{#if modalSchema} @@ -159,7 +234,7 @@ {/snippet} {#snippet footerSettings()} - {#if additionalInputsSchema} + {#if modalSchema}
- {#if hasMissingRequired} + {#if modalMissingRequired} {/if}
{/if} + {#if modelWiring && showModelButton} + + + {/if} {/snippet}
{/if} +--> Date: Wed, 9 Sep 2026 18:41:35 +0200 Subject: [PATCH 42/77] fix(datatables): honour `-- role: x`, and fix the DuckDB attach test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review findings, both real. `attach_datatable_parses_name_and_role` never compiled: `parse_attach_datatable` returns `Result>` now and one call site kept a single `unwrap`. Its `?Role=analytics` case also asserted a refusal, contradicting the parser in the same commit, which matches the key case-insensitively. Replaced with the cases that are genuinely malformed, and a positive one for the cased key. `-- role: analytics` fell through to the default role — the silent fallback the strict parser exists to remove, for the spelling most likely to be typed. The keyword now accepts an optional colon, attached or spaced, while a word that merely starts with it (`rolebased`) is still not an attempt. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb --- backend/windmill-common/src/worker.rs | 37 +++++++++++++++---- .../windmill-worker/src/duckdb_executor.rs | 30 ++++++++++++--- 2 files changed, 54 insertions(+), 13 deletions(-) diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 59d76a9d99..6b3c63e3c0 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -1105,16 +1105,27 @@ impl SqlAnnotations { if !line.starts_with("--") { break; } - let mut tokens = line[2..].split_whitespace(); - if !tokens - .next() - .is_some_and(|t| t.eq_ignore_ascii_case("role")) - { + // `role`, `Role`, `role:` and `role:name` all open an attempt; `rolexyz` does not. + // The colon is worth accepting rather than skipping past: `-- role: x` is the likelier + // spelling, and skipping it is exactly the silent fallback this refuses. + let body = line[2..].trim_start(); + let Some(after) = body + .get(..4) + .filter(|kw| kw.eq_ignore_ascii_case("role")) + .map(|_| &body[4..]) + else { + continue; + }; + let colon = after.starts_with(':'); + let after = after.strip_prefix(':').unwrap_or(after); + if !after.is_empty() && !colon && !after.starts_with(char::is_whitespace) { continue; } + // Past this point the line is an attempt to name a role, so a malformed one is an // error rather than a miss. Falling through would run the query as the data table's // default role — quietly, and under a login the author did not choose. + let mut tokens = after.split_whitespace(); let role = tokens .next() .map(|role| role.strip_suffix(';').unwrap_or(role)); @@ -2731,9 +2742,15 @@ mod tests { assert_eq!(role("SELECT 1;\n-- role analytics").unwrap(), None); assert_eq!(role("SELECT 1").unwrap(), None); - // Unambiguous intent is honoured: the keyword matches case-insensitively, and a trailing - // semicolon is a habit carried over from SQL rather than a different role. - for accepted in ["-- Role operator\nSELECT 1", "-- role operator;\nSELECT 1"] { + // Unambiguous intent is honoured: the keyword matches case-insensitively, a trailing + // semicolon is a habit carried over from SQL rather than a different role, and the colon + // spelling is the one most likely to be typed. + for accepted in [ + "-- Role operator\nSELECT 1", + "-- role operator;\nSELECT 1", + "-- role: operator\nSELECT 1", + "-- role:operator\nSELECT 1", + ] { assert_eq!( role(accepted).unwrap(), Some("operator".to_string()), @@ -2747,10 +2764,14 @@ mod tests { "-- role operator -- why\nSELECT 1", "-- role an;alytics\nSELECT 1", "-- role\nSELECT 1", + "-- role:\nSELECT 1", "-- role based access is handled below\nSELECT 1", ] { assert!(role(near_miss).is_err(), "silently ignored: {near_miss}"); } + + // A word that merely starts with the keyword is not an attempt. + assert_eq!(role("-- rolebased notes\nSELECT 1").unwrap(), None); } fn matcher(id: &str) -> WorkspaceMatcher { diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index 7fb973d3ce..764859d29b 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -2792,8 +2792,9 @@ mod tests { #[test] fn attach_datatable_parses_name_and_role() { - let named = - parse_attach_datatable("ATTACH 'datatable://sales?role=analytics' AS dt").unwrap(); + let named = parse_attach_datatable("ATTACH 'datatable://sales?role=analytics' AS dt") + .unwrap() + .unwrap(); assert_eq!( (named.name, named.role, named.alias), ("sales", Some("analytics"), "dt") @@ -2807,11 +2808,30 @@ mod tests { .unwrap() .unwrap(); assert_eq!((no_role.name, no_role.role), ("sales", None)); - let bare = parse_attach_datatable("ATTACH 'datatable' AS dt").unwrap().unwrap(); + let bare = parse_attach_datatable("ATTACH 'datatable' AS dt") + .unwrap() + .unwrap(); assert_eq!((bare.name, bare.role), ("main", None)); assert!(parse_attach_datatable("SELECT 1").unwrap().is_none()); - // A malformed role is refused rather than attached under the default one. - assert!(parse_attach_datatable("ATTACH 'datatable://sales?Role=analytics' AS dt").is_err()); + + // The key matches case-insensitively, as the `-- role` annotation does. + let cased = parse_attach_datatable("ATTACH 'datatable://sales?Role=analytics' AS dt") + .unwrap() + .unwrap(); + assert_eq!(cased.role, Some("analytics")); + + // A query string that does not parse is refused rather than attached under the default + // role: the statement asked for a specific one. + for malformed in [ + "ATTACH 'datatable://sales?role=' AS dt", + "ATTACH 'datatable://sales?role=an;alytics' AS dt", + "ATTACH 'datatable://sales?x=1&role=analytics' AS dt", + ] { + assert!( + parse_attach_datatable(malformed).is_err(), + "silently ignored: {malformed}" + ); + } } #[test] From 2abcd122a83222de540e894f47eef520adca64fa Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Thu, 10 Sep 2026 08:01:25 +0200 Subject: [PATCH 43/77] fix(datatables): clone a fork's pointer instead of failing after the copy Forking a fork with cloning left an orphan database. The preflight resolves the pointer and sees the governing entry, so both endpoints ran and filled the new database; `apply_forked_datatable` then refused the inherited pointer and rolled the fork back, stranding a registered `wm_fork_*` that no entry names and whose name blocks the retry. Refusing earlier would have been the smaller change, but forking a fork and cloning worked before pointers existed, so it would trade an orphan for a regression. Resolve what the pointer names and write the terminal entry the clone needs: the whole `database` object rather than a patch of its `resource_path`, since a pointer has none, and `reference` removed with it. Also accepts `-- role=x` and `-- Role = x`, two more spellings that fell through to the default role. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb --- ...b9d1f440692f183c7c378f59e4b73f1c6e241.json | 17 ++++++ .../windmill-api-workspaces/src/workspaces.rs | 53 ++++++++++++------- backend/windmill-common/src/worker.rs | 18 ++++--- 3 files changed, 63 insertions(+), 25 deletions(-) create mode 100644 backend/.sqlx/query-ebaf3ed3097621da59dd201b5a4b9d1f440692f183c7c378f59e4b73f1c6e241.json diff --git a/backend/.sqlx/query-ebaf3ed3097621da59dd201b5a4b9d1f440692f183c7c378f59e4b73f1c6e241.json b/backend/.sqlx/query-ebaf3ed3097621da59dd201b5a4b9d1f440692f183c7c378f59e4b73f1c6e241.json new file mode 100644 index 0000000000..e5de67e73f --- /dev/null +++ b/backend/.sqlx/query-ebaf3ed3097621da59dd201b5a4b9d1f440692f183c7c378f59e4b73f1c6e241.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings\n SET datatable = jsonb_set(\n jsonb_set(\n datatable #- ARRAY['datatables', $2, 'reference'],\n ARRAY['datatables', $2, 'database'], $3::jsonb),\n ARRAY['datatables', $2, 'forked_from'], $4::jsonb\n )\n WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "ebaf3ed3097621da59dd201b5a4b9d1f440692f183c7c378f59e4b73f1c6e241" +} diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 9526c526fa..e6dc50b789 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -45,12 +45,12 @@ use windmill_common::workspaces::GitRepositorySettings; #[cfg(feature = "enterprise")] use windmill_common::workspaces::WorkspaceDeploymentUISettings; use windmill_common::workspaces::{ - check_deploy_rules, check_user_against_rule, get_datatable_resource_from_db, - datatable_ref_name, get_datatable_resource_from_db_unchecked, resolve_governing_datatable, - validate_dev_workspace_id, validate_fork_workspace_id, validate_workspace_name, DataTable, - DataTableCatalogResourceType, DataTableForkBehavior, DatatableAccess, ProtectionRuleKind, - ProtectionRules, ProtectionRuleset, RuleCheckResult, WorkspaceGitSyncSettings, - DEV_WORKSPACE_LOCK_RULE_NAME, + check_deploy_rules, check_user_against_rule, datatable_ref_name, + get_datatable_resource_from_db, get_datatable_resource_from_db_unchecked, + resolve_governing_datatable, validate_dev_workspace_id, validate_fork_workspace_id, + validate_workspace_name, DataTable, DataTableCatalogResourceType, DataTableForkBehavior, + DatatableAccess, ProtectionRuleKind, ProtectionRules, ProtectionRuleset, RuleCheckResult, + WorkspaceGitSyncSettings, DEV_WORKSPACE_LOCK_RULE_NAME, }; use windmill_common::workspaces::{Ducklake, DucklakeCatalogResourceType}; use windmill_common::PgDatabase; @@ -7962,29 +7962,44 @@ async fn apply_forked_datatable( let dt: DataTable = serde_json::from_value(config_val) .map_err(|e| Error::internal_err(format!("Failed to parse datatable config: {}", e)))?; - // A cloned data table owns its copy, so the fork's entry must be terminal. It arrived that way - // from the settings clone; a pointer here would mean the parent's own entry was one, and the - // clone has to name its new database rather than follow anything. - let database = dt.database.as_ref().ok_or_else(|| { - Error::BadRequest(format!( - "Data table '{}' points at another workspace's data table and cannot be cloned; \ - fork it from the workspace that owns it.", - fdt.name - )) - })?; + // A clone owns its copy, so the fork's entry has to be terminal. When the parent was itself a + // fork the settings clone hands down a pointer instead, and the database this clone just + // created is already filled — so resolve what it points at and name the copy, rather than + // refusing after the fact. Pointers are only ever written for instance databases + // (`point_kept_datatables_at_parent`), so the resolved entry is one. + let database = match dt.database.clone() { + Some(database) => database, + None => resolve_governing_datatable(db, parent_w_id, &fdt.name) + .await? + .datatable + .database + .ok_or_else(|| { + Error::internal_err(format!( + "Data table '{}' resolves to an entry that owns no database", + fdt.name + )) + })?, + }; if database.resource_type == DataTableCatalogResourceType::Instance { - // Instance: update resource_path to the new dbname + // The whole `database` object, not just its `resource_path`: a pointer entry has none to + // patch. `reference` goes with it — exactly one of the two may be set. + let new_database = serde_json::json!({ + "resource_type": "instance", + "resource_path": &fdt.new_dbname, + }); sqlx::query!( r#"UPDATE workspace_settings SET datatable = jsonb_set( - jsonb_set(datatable, ARRAY['datatables', $2, 'database', 'resource_path'], to_jsonb($3::text)), + jsonb_set( + datatable #- ARRAY['datatables', $2, 'reference'], + ARRAY['datatables', $2, 'database'], $3::jsonb), ARRAY['datatables', $2, 'forked_from'], $4::jsonb ) WHERE workspace_id = $1"#, forked_w_id, &fdt.name, - &fdt.new_dbname, + new_database, forked_from, ) .execute(&mut **tx) diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 6b3c63e3c0..c742a3efcb 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -1105,9 +1105,10 @@ impl SqlAnnotations { if !line.starts_with("--") { break; } - // `role`, `Role`, `role:` and `role:name` all open an attempt; `rolexyz` does not. - // The colon is worth accepting rather than skipping past: `-- role: x` is the likelier - // spelling, and skipping it is exactly the silent fallback this refuses. + // The keyword may be followed by whitespace, `:` or `=` — `role x`, `role: x`, + // `role=x`, `Role = x` all open an attempt, while `rolexyz` does not. Each accepted + // separator is one spelling that would otherwise take the `continue` below and run the + // query as the data table's default role, which is the silence this exists to remove. let body = line[2..].trim_start(); let Some(after) = body .get(..4) @@ -1116,15 +1117,18 @@ impl SqlAnnotations { else { continue; }; - let colon = after.starts_with(':'); - let after = after.strip_prefix(':').unwrap_or(after); - if !after.is_empty() && !colon && !after.starts_with(char::is_whitespace) { + if !after.is_empty() + && !after.starts_with(char::is_whitespace) + && !after.starts_with([':', '=']) + { continue; } // Past this point the line is an attempt to name a role, so a malformed one is an // error rather than a miss. Falling through would run the query as the data table's // default role — quietly, and under a login the author did not choose. + let after = after.trim_start(); + let after = after.strip_prefix([':', '=']).unwrap_or(after); let mut tokens = after.split_whitespace(); let role = tokens .next() @@ -2750,6 +2754,8 @@ mod tests { "-- role operator;\nSELECT 1", "-- role: operator\nSELECT 1", "-- role:operator\nSELECT 1", + "-- role=operator\nSELECT 1", + "-- Role = operator\nSELECT 1", ] { assert_eq!( role(accepted).unwrap(), From e6794984d2002a06bfddb3f3cf069ecb85f26484 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Thu, 10 Sep 2026 08:16:58 +0200 Subject: [PATCH 44/77] fix(datatables): refuse to roll back the catalog while roles exist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The down migration dropped the table and left every role behind: live Postgres logins whose passwords only that table carried, so after a revert Windmill could neither use, disable nor delete them, and re-applying could not recreate them because the names were taken. Cleaning up here is not possible either — dropping a role means reassigning what it owns in every instance database, and a migration runs in one — so it now refuses while the catalog is non-empty and says to delete the roles through instance settings, which does the cluster work. Also enforces the instance-only invariant the resolved-pointer clone relies on rather than only asserting it in a comment. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb --- ...0908142148_datatable_role_catalog.down.sql | 15 +++++++++ .../windmill-api-workspaces/src/workspaces.rs | 32 +++++++++++++------ 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/backend/migrations/20260908142148_datatable_role_catalog.down.sql b/backend/migrations/20260908142148_datatable_role_catalog.down.sql index 0bbf50bc89..26ad1c62d4 100644 --- a/backend/migrations/20260908142148_datatable_role_catalog.down.sql +++ b/backend/migrations/20260908142148_datatable_role_catalog.down.sql @@ -1 +1,16 @@ +-- Refuse while the catalog holds anything. Each row is a live Postgres login with a password +-- only this table carries, so dropping it would leave credentials on the cluster that Windmill can +-- no longer disable, delete or even name — and re-applying could not recreate them, because the +-- role names would already be taken. Cleaning them up here is not an option either: dropping a +-- role means reassigning what it owns in *every* instance database, and a migration runs in one. +-- +-- Delete the roles through instance settings first; that path does the cluster work. +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM datatable_role) THEN + RAISE EXCEPTION 'Cannot roll back: % data table role(s) still exist as Postgres logins. Delete them in instance settings first, which drops them from the cluster.', + (SELECT count(*) FROM datatable_role); + END IF; +END $$; + DROP TABLE IF EXISTS datatable_role; diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index e6dc50b789..74c6f833be 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -7969,16 +7969,30 @@ async fn apply_forked_datatable( // (`point_kept_datatables_at_parent`), so the resolved entry is one. let database = match dt.database.clone() { Some(database) => database, - None => resolve_governing_datatable(db, parent_w_id, &fdt.name) - .await? - .datatable - .database - .ok_or_else(|| { - Error::internal_err(format!( - "Data table '{}' resolves to an entry that owns no database", + None => { + let resolved = resolve_governing_datatable(db, parent_w_id, &fdt.name) + .await? + .datatable + .database + .ok_or_else(|| { + Error::internal_err(format!( + "Data table '{}' resolves to an entry that owns no database", + fdt.name + )) + })?; + // The resource branch below rewrites a resource this workspace owns; a pointer names + // one it does not, so following it there would move the fork onto someone else's + // database. Checked rather than assumed: only `point_kept_datatables_at_parent` + // writes pointers and only for instance entries, but nothing here enforces that. + if resolved.resource_type != DataTableCatalogResourceType::Instance { + return Err(Error::BadRequest(format!( + "Data table '{}' points at a resource-backed data table in another \ + workspace and cannot be cloned; fork it from the workspace that owns it.", fdt.name - )) - })?, + ))); + } + resolved + } }; if database.resource_type == DataTableCatalogResourceType::Instance { From 432ccc0fbd192a7f01be721bd4f07f3b1b5a1a90 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Thu, 10 Sep 2026 08:27:29 +0200 Subject: [PATCH 45/77] refactor(datatables): settle clonability in one place, before anything is created MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A clone is three stages a workspace apart — `create_pg_database`, then `import_pg_database`, then `apply_forked_datatable` inside the fork transaction. Only the third can roll back, and `CREATE DATABASE` is not transactional, so any refusal that lives there strands a registered `wm_fork_*` that no entry names and whose name blocks the retry. That orphan has now been fixed three times, most recently reintroduced by a guard added one commit ago. Patching each new refusal into the first endpoint is not the fix; having two places that can refuse is. `ensure_datatable_is_clonable` now answers every reason a copy can be refused and returns what it resolved, and the stage that writes the entry only does the work. Also takes an ACCESS EXCLUSIVE lock before the rollback guard counts, so a role created concurrently cannot slip between the check and the drop. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb --- ...0908142148_datatable_role_catalog.down.sql | 2 + .../windmill-api-workspaces/src/workspaces.rs | 71 ++++++++++--------- 2 files changed, 40 insertions(+), 33 deletions(-) diff --git a/backend/migrations/20260908142148_datatable_role_catalog.down.sql b/backend/migrations/20260908142148_datatable_role_catalog.down.sql index 26ad1c62d4..8d3f12bc4a 100644 --- a/backend/migrations/20260908142148_datatable_role_catalog.down.sql +++ b/backend/migrations/20260908142148_datatable_role_catalog.down.sql @@ -5,6 +5,8 @@ -- role means reassigning what it owns in *every* instance database, and a migration runs in one. -- -- Delete the roles through instance settings first; that path does the cluster work. +LOCK TABLE datatable_role IN ACCESS EXCLUSIVE MODE; + DO $$ BEGIN IF EXISTS (SELECT 1 FROM datatable_role) THEN diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 74c6f833be..4082a50900 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -49,8 +49,8 @@ use windmill_common::workspaces::{ get_datatable_resource_from_db, get_datatable_resource_from_db_unchecked, resolve_governing_datatable, validate_dev_workspace_id, validate_fork_workspace_id, validate_workspace_name, DataTable, DataTableCatalogResourceType, DataTableForkBehavior, - DatatableAccess, ProtectionRuleKind, ProtectionRules, ProtectionRuleset, RuleCheckResult, - WorkspaceGitSyncSettings, DEV_WORKSPACE_LOCK_RULE_NAME, + DatatableAccess, GoverningDatatable, ProtectionRuleKind, ProtectionRules, ProtectionRuleset, + RuleCheckResult, WorkspaceGitSyncSettings, DEV_WORKSPACE_LOCK_RULE_NAME, }; use windmill_common::workspaces::{Ducklake, DucklakeCatalogResourceType}; use windmill_common::PgDatabase; @@ -3399,7 +3399,18 @@ struct ImportPgDatabaseRequest { /// exists, refusing is the honest answer. Dropping `permissions` from the clone instead would be /// the unsafe half: the copy holds the parent's rows, so an unpermissioned clone hands all of them /// to everyone in the fork. -async fn ensure_datatable_is_clonable(db: &DB, w_id: &str, name: &str) -> Result<()> { +/// Every reason a copy can be refused, answered here and nowhere else. +/// +/// A clone is three stages a workspace apart: `create_pg_database`, `import_pg_database`, then +/// `apply_forked_datatable` inside the fork transaction. Only the third can roll back, and the +/// database the first created is not transactional — so a refusal that lives there strands a +/// registered `wm_fork_*` that no entry names and whose name blocks the retry. Both endpoints call +/// this before touching the cluster; the stage that writes the entry must only ever do the work. +async fn ensure_datatable_is_clonable( + db: &DB, + w_id: &str, + name: &str, +) -> Result { let governing = resolve_governing_datatable(db, w_id, name).await?; if governing.datatable.permissions.is_some() { return Err(Error::BadRequest(format!( @@ -3409,7 +3420,20 @@ async fn ensure_datatable_is_clonable(db: &DB, w_id: &str, name: &str) -> Result its roles off first." ))); } - Ok(()) + // The copy has to name a database of its own. A resource-backed entry reached through a + // pointer names one this workspace does not own, so there is nothing here to repoint. + let is_instance = governing + .datatable + .database + .as_ref() + .is_some_and(|d| d.resource_type == DataTableCatalogResourceType::Instance); + if governing.workspace_id != w_id && !is_instance { + return Err(Error::BadRequest(format!( + "Data table '{name}' points at a resource-backed data table in another workspace \ + and cannot be copied; fork it from the workspace that owns it." + ))); + } + Ok(governing) } /// Import (pg_dump/pg_import) from source to target @@ -7930,7 +7954,7 @@ async fn apply_forked_datatable( &DatatableAccess::Authed(authed.to_authed_ref()), ) .await?; - ensure_datatable_is_clonable(db, parent_w_id, &fdt.name).await?; + let governing = ensure_datatable_is_clonable(db, parent_w_id, &fdt.name).await?; windmill_common::validate_dbname(&fdt.new_dbname)?; if !fdt.new_dbname.starts_with("wm_fork_") { return Err(Error::BadRequest(format!( @@ -7963,36 +7987,17 @@ async fn apply_forked_datatable( .map_err(|e| Error::internal_err(format!("Failed to parse datatable config: {}", e)))?; // A clone owns its copy, so the fork's entry has to be terminal. When the parent was itself a - // fork the settings clone hands down a pointer instead, and the database this clone just - // created is already filled — so resolve what it points at and name the copy, rather than - // refusing after the fact. Pointers are only ever written for instance databases - // (`point_kept_datatables_at_parent`), so the resolved entry is one. + // fork the settings clone hands down a pointer instead, and what it points at is what the copy + // was taken from. `ensure_datatable_is_clonable` already settled that this shape can be + // cloned, so there is nothing left to refuse here — by now the database exists and is filled. let database = match dt.database.clone() { Some(database) => database, - None => { - let resolved = resolve_governing_datatable(db, parent_w_id, &fdt.name) - .await? - .datatable - .database - .ok_or_else(|| { - Error::internal_err(format!( - "Data table '{}' resolves to an entry that owns no database", - fdt.name - )) - })?; - // The resource branch below rewrites a resource this workspace owns; a pointer names - // one it does not, so following it there would move the fork onto someone else's - // database. Checked rather than assumed: only `point_kept_datatables_at_parent` - // writes pointers and only for instance entries, but nothing here enforces that. - if resolved.resource_type != DataTableCatalogResourceType::Instance { - return Err(Error::BadRequest(format!( - "Data table '{}' points at a resource-backed data table in another \ - workspace and cannot be cloned; fork it from the workspace that owns it.", - fdt.name - ))); - } - resolved - } + None => governing.datatable.database.clone().ok_or_else(|| { + Error::internal_err(format!( + "Data table '{}' resolves to an entry that owns no database", + fdt.name + )) + })?, }; if database.resource_type == DataTableCatalogResourceType::Instance { From 29fccb09781908a2f4f0caf885ff81e11c85682a Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Thu, 10 Sep 2026 18:06:28 +0200 Subject: [PATCH 46/77] fix(datatables): let a retried clone reclaim its own leftover database MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A clone creates its target database one request before it copies into it, and the fork that would name it is written a request after that. Any failure in between — a pg_dump error, a bad restore, a dropped connection, the source's roles changing mid-flow — left a registered `wm_fork_*` that no entry names, and every retry then failed on its name. This predates data table roles. `create_pg_database` now reclaims such a leftover before creating: only a `wm_fork_*` database Windmill registered as a data table database and that no data table or ducklake entry names, in any workspace, archived ones included. The drop never terminates connections, so a clone still copying into it makes the reclaim fail instead of being cut off. It is limited to callers who administer the source — reaching it is not enough, since on a data table without roles every member reaches it — and anyone else gets the refusal an existing database always got. Co-Authored-By: Claude Opus 5 (1M context) --- ...c5ab54e8b28037dcfc83d8b03a930c8ba6589.json | 22 ++++++ .../tests/datatable_roles.rs | 76 +++++++++++++++++++ .../windmill-api-workspaces/src/workspaces.rs | 25 +++++- backend/windmill-common/src/lib.rs | 71 +++++++++++++++++ 4 files changed, 190 insertions(+), 4 deletions(-) create mode 100644 backend/.sqlx/query-661462c1a96cab2367b2b2bf452c5ab54e8b28037dcfc83d8b03a930c8ba6589.json diff --git a/backend/.sqlx/query-661462c1a96cab2367b2b2bf452c5ab54e8b28037dcfc83d8b03a930c8ba6589.json b/backend/.sqlx/query-661462c1a96cab2367b2b2bf452c5ab54e8b28037dcfc83d8b03a930c8ba6589.json new file mode 100644 index 0000000000..fc4681c6ca --- /dev/null +++ b/backend/.sqlx/query-661462c1a96cab2367b2b2bf452c5ab54e8b28037dcfc83d8b03a930c8ba6589.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n COALESCE(value->'databases'->$1::text->>'tag' = 'datatable', false)\n AND NOT EXISTS (\n SELECT 1 FROM workspace_settings ws\n CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt\n WHERE dt.value->'database'->>'resource_type' = 'instance'\n AND dt.value->'database'->>'resource_path' = $1::text)\n AND NOT EXISTS (\n SELECT 1 FROM workspace_settings ws\n CROSS JOIN LATERAL jsonb_each(COALESCE(ws.ducklake->'ducklakes', '{}'::jsonb)) dl\n WHERE dl.value->'catalog'->>'resource_type' = 'instance'\n AND dl.value->'catalog'->>'resource_path' = $1::text)\n AS \"reclaimable!\"\n FROM global_settings WHERE name = 'custom_instance_pg_databases'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "reclaimable!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "661462c1a96cab2367b2b2bf452c5ab54e8b28037dcfc83d8b03a930c8ba6589" +} diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs index 787bd950fd..2a9245ce3e 100644 --- a/backend/windmill-api-integration-tests/tests/datatable_roles.rs +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -598,3 +598,79 @@ async fn a_data_table_under_roles_is_not_copied_into_a_fork( ); Ok(()) } + +async fn register_instance_database( + db: &Pool, + name: &str, + tag: &str, +) -> anyhow::Result<()> { + sqlx::query( + "UPDATE global_settings SET value = jsonb_set(value, ARRAY['databases', $1], \ + jsonb_build_object('tag', $2::text)) WHERE name = 'custom_instance_pg_databases'", + ) + .bind(name) + .bind(tag) + .execute(db) + .await?; + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn only_an_unnamed_fork_database_is_reclaimable(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + // Reclaiming a clone's leftover is a `DROP DATABASE`, so this guard is its whole safety story: + // only a database Windmill registered for a data table, and that no entry anywhere names. + for (name, tag) in [ + ("wm_fork_orphan", "datatable"), + ("wm_fork_named", "datatable"), + ("wm_fork_lake_catalog", "datatable"), + ("wm_fork_archived", "datatable"), + ("wm_fork_ducklake", "ducklake"), + ("dt_orphan", "datatable"), + ] { + register_instance_database(&db, name, tag).await?; + } + sqlx::query( + r#"UPDATE workspace_settings SET datatable = jsonb_set(datatable, '{datatables,clone}', + '{"database": {"resource_type": "instance", "resource_path": "wm_fork_named"}}') + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + // A ducklake catalog on an instance database names it through `resource_path` as well. + sqlx::query( + r#"UPDATE workspace_settings SET ducklake = '{"ducklakes": {"lake": {"catalog": + {"resource_type": "instance", "resource_path": "wm_fork_lake_catalog"}}}}' + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + // An archived workspace still owns what it names: unarchiving it must find its data intact. + sqlx::query( + r#"UPDATE workspace_settings SET datatable = jsonb_set(datatable, '{datatables,clone}', + '{"database": {"resource_type": "instance", "resource_path": "wm_fork_archived"}}') + WHERE workspace_id = 'wm-fork-dt'"#, + ) + .execute(&db) + .await?; + sqlx::query("UPDATE workspace SET deleted = true WHERE id = 'wm-fork-dt'") + .execute(&db) + .await?; + + for (name, reclaimable) in [ + ("wm_fork_orphan", true), + ("wm_fork_named", false), + ("wm_fork_lake_catalog", false), + ("wm_fork_archived", false), + ("wm_fork_ducklake", false), + ("wm_fork_unregistered", false), + ("dt_orphan", false), + ] { + assert_eq!( + windmill_common::is_reclaimable_fork_database(&db, name).await?, + reclaimable, + "{name}" + ); + } + Ok(()) +} diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 4082a50900..df12ca5e00 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -3312,10 +3312,12 @@ async fn create_pg_database( // The copy this database is for is refused a call later, and nothing collects an instance // database that no data table entry names. Refuse here too, so the clone stops before one // exists rather than leaving an empty registered `wm_fork_…` behind. - if let Some(reference) = req.source.strip_prefix("datatable://") { - let name = datatable_ref_name(reference); - ensure_datatable_is_clonable(&db, &w_id, name).await?; - } + let governing = match req.source.strip_prefix("datatable://") { + Some(reference) => { + Some(ensure_datatable_is_clonable(&db, &w_id, datatable_ref_name(reference)).await?) + } + None => None, + }; // Non-superadmin: restrict dbname to wm_fork_ prefix if !windmill_api_auth::is_super_admin_authed(&db, &authed).await? { @@ -3328,6 +3330,21 @@ async fn create_pg_database( } if is_instance_datatable_source(&db, &w_id, &req.source).await? { + // A retry after a clone that failed past this point finds its own leftover here. Reclaiming + // it is a `DROP DATABASE`, so it is for whoever administers the source — not merely whoever + // reaches it, which on a data table without roles is every member. Anyone else gets the + // refusal an existing database always got. + let may_reclaim = match &governing { + Some(governing) => crate::datatable_permissions::ensure_governs_datatable( + &db, &authed, &w_id, governing, + ) + .await + .is_ok(), + None => false, + }; + if may_reclaim { + windmill_common::reclaim_orphaned_fork_database(&db, &req.target_dbname).await?; + } windmill_common::create_custom_instance_database(&db, &req.target_dbname, "datatable") .await?; } else { diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index a7e84c55d7..99cf7de1bb 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -1457,6 +1457,77 @@ pub fn validate_dbname(dbname: &str) -> error::Result<()> { Ok(()) } +/// Whether `dbname` is a clone's leftover: a `wm_fork_*` database Windmill registered as a data +/// table database and that no data table or ducklake entry names, in any workspace, archived ones +/// included. A clone creates its database a request before any entry names it, so a failure in +/// between leaves exactly this, and its name then blocks every retry. +pub async fn is_reclaimable_fork_database(db: &DB, dbname: &str) -> error::Result { + if !dbname.starts_with("wm_fork_") { + return Ok(false); + } + Ok(sqlx::query_scalar!( + r#"SELECT + COALESCE(value->'databases'->$1::text->>'tag' = 'datatable', false) + AND NOT EXISTS ( + SELECT 1 FROM workspace_settings ws + CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt + WHERE dt.value->'database'->>'resource_type' = 'instance' + AND dt.value->'database'->>'resource_path' = $1::text) + AND NOT EXISTS ( + SELECT 1 FROM workspace_settings ws + CROSS JOIN LATERAL jsonb_each(COALESCE(ws.ducklake->'ducklakes', '{}'::jsonb)) dl + WHERE dl.value->'catalog'->>'resource_type' = 'instance' + AND dl.value->'catalog'->>'resource_path' = $1::text) + AS "reclaimable!" + FROM global_settings WHERE name = 'custom_instance_pg_databases'"#, + dbname + ) + .fetch_optional(db) + .await? + .unwrap_or(false)) +} + +/// Drop `dbname` if [`is_reclaimable_fork_database`], so a retried clone can recreate it, and +/// return whether it did. Unlike [`drop_custom_instance_database`] this never terminates +/// connections: a clone still copying into the database must make the drop fail, not be cut off +/// mid-copy. +pub async fn reclaim_orphaned_fork_database(db: &DB, dbname: &str) -> error::Result { + let dbname = dbname.trim(); + validate_dbname(dbname)?; + + let exists = sqlx::query_scalar!( + "SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_database WHERE datname = $1)", + dbname + ) + .fetch_one(db) + .await? + .unwrap_or(false); + if !exists || !is_reclaimable_fork_database(db, dbname).await? { + return Ok(false); + } + + // SAFETY: `dbname` has been validated via validate_dbname() above. + sqlx::query(&format!("DROP DATABASE IF EXISTS \"{}\"", dbname)) + .execute(db) + .await + .map_err(|e| { + error::Error::BadRequest(format!( + "Database '{dbname}' is left over from an earlier clone and could not be \ + reclaimed, most often because a clone is still copying into it: {e}" + )) + })?; + // A registered database that no longer exists makes every later per-database pass fail on it. + sqlx::query!( + r#"UPDATE global_settings SET value = value #- ARRAY['databases', $1] WHERE name = 'custom_instance_pg_databases'"#, + dbname + ) + .execute(db) + .await?; + + tracing::info!("Reclaimed orphaned fork database '{dbname}'"); + Ok(true) +} + /// Drop a custom instance database: validate, terminate connections, DROP DATABASE, remove from global_settings. pub async fn drop_custom_instance_database(db: &DB, dbname: &str) -> error::Result<()> { let dbname = dbname.trim(); From 643bc1b51a0389ff2aa7060dc8a8fb1055805f8b Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Thu, 10 Sep 2026 18:22:36 +0200 Subject: [PATCH 47/77] Revert "fix(datatables): let a retried clone reclaim its own leftover database" This reverts commit 7dd3275a10. The reclaim tied the caller to the source they administer, but not to the database it dropped. Between another workspace's import and its final fork request, that workspace's target is full, registered, unnamed and has no open connection, so an admin of any instance data table could name it and have it dropped and recreated empty. The victim's fork would then commit pointing at the empty copy. Safe reclaim needs durable clone ownership and serialization with the request that names the database; until then the leftover stays, as it did before this PR. Co-Authored-By: Claude Opus 5 (1M context) --- ...c5ab54e8b28037dcfc83d8b03a930c8ba6589.json | 22 ------ .../tests/datatable_roles.rs | 76 ------------------- .../windmill-api-workspaces/src/workspaces.rs | 25 +----- backend/windmill-common/src/lib.rs | 71 ----------------- 4 files changed, 4 insertions(+), 190 deletions(-) delete mode 100644 backend/.sqlx/query-661462c1a96cab2367b2b2bf452c5ab54e8b28037dcfc83d8b03a930c8ba6589.json diff --git a/backend/.sqlx/query-661462c1a96cab2367b2b2bf452c5ab54e8b28037dcfc83d8b03a930c8ba6589.json b/backend/.sqlx/query-661462c1a96cab2367b2b2bf452c5ab54e8b28037dcfc83d8b03a930c8ba6589.json deleted file mode 100644 index fc4681c6ca..0000000000 --- a/backend/.sqlx/query-661462c1a96cab2367b2b2bf452c5ab54e8b28037dcfc83d8b03a930c8ba6589.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n COALESCE(value->'databases'->$1::text->>'tag' = 'datatable', false)\n AND NOT EXISTS (\n SELECT 1 FROM workspace_settings ws\n CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt\n WHERE dt.value->'database'->>'resource_type' = 'instance'\n AND dt.value->'database'->>'resource_path' = $1::text)\n AND NOT EXISTS (\n SELECT 1 FROM workspace_settings ws\n CROSS JOIN LATERAL jsonb_each(COALESCE(ws.ducklake->'ducklakes', '{}'::jsonb)) dl\n WHERE dl.value->'catalog'->>'resource_type' = 'instance'\n AND dl.value->'catalog'->>'resource_path' = $1::text)\n AS \"reclaimable!\"\n FROM global_settings WHERE name = 'custom_instance_pg_databases'", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "reclaimable!", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "661462c1a96cab2367b2b2bf452c5ab54e8b28037dcfc83d8b03a930c8ba6589" -} diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs index 2a9245ce3e..787bd950fd 100644 --- a/backend/windmill-api-integration-tests/tests/datatable_roles.rs +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -598,79 +598,3 @@ async fn a_data_table_under_roles_is_not_copied_into_a_fork( ); Ok(()) } - -async fn register_instance_database( - db: &Pool, - name: &str, - tag: &str, -) -> anyhow::Result<()> { - sqlx::query( - "UPDATE global_settings SET value = jsonb_set(value, ARRAY['databases', $1], \ - jsonb_build_object('tag', $2::text)) WHERE name = 'custom_instance_pg_databases'", - ) - .bind(name) - .bind(tag) - .execute(db) - .await?; - Ok(()) -} - -#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] -async fn only_an_unnamed_fork_database_is_reclaimable(db: Pool) -> anyhow::Result<()> { - initialize_tracing().await; - // Reclaiming a clone's leftover is a `DROP DATABASE`, so this guard is its whole safety story: - // only a database Windmill registered for a data table, and that no entry anywhere names. - for (name, tag) in [ - ("wm_fork_orphan", "datatable"), - ("wm_fork_named", "datatable"), - ("wm_fork_lake_catalog", "datatable"), - ("wm_fork_archived", "datatable"), - ("wm_fork_ducklake", "ducklake"), - ("dt_orphan", "datatable"), - ] { - register_instance_database(&db, name, tag).await?; - } - sqlx::query( - r#"UPDATE workspace_settings SET datatable = jsonb_set(datatable, '{datatables,clone}', - '{"database": {"resource_type": "instance", "resource_path": "wm_fork_named"}}') - WHERE workspace_id = 'test-workspace'"#, - ) - .execute(&db) - .await?; - // A ducklake catalog on an instance database names it through `resource_path` as well. - sqlx::query( - r#"UPDATE workspace_settings SET ducklake = '{"ducklakes": {"lake": {"catalog": - {"resource_type": "instance", "resource_path": "wm_fork_lake_catalog"}}}}' - WHERE workspace_id = 'test-workspace'"#, - ) - .execute(&db) - .await?; - // An archived workspace still owns what it names: unarchiving it must find its data intact. - sqlx::query( - r#"UPDATE workspace_settings SET datatable = jsonb_set(datatable, '{datatables,clone}', - '{"database": {"resource_type": "instance", "resource_path": "wm_fork_archived"}}') - WHERE workspace_id = 'wm-fork-dt'"#, - ) - .execute(&db) - .await?; - sqlx::query("UPDATE workspace SET deleted = true WHERE id = 'wm-fork-dt'") - .execute(&db) - .await?; - - for (name, reclaimable) in [ - ("wm_fork_orphan", true), - ("wm_fork_named", false), - ("wm_fork_lake_catalog", false), - ("wm_fork_archived", false), - ("wm_fork_ducklake", false), - ("wm_fork_unregistered", false), - ("dt_orphan", false), - ] { - assert_eq!( - windmill_common::is_reclaimable_fork_database(&db, name).await?, - reclaimable, - "{name}" - ); - } - Ok(()) -} diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index df12ca5e00..4082a50900 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -3312,12 +3312,10 @@ async fn create_pg_database( // The copy this database is for is refused a call later, and nothing collects an instance // database that no data table entry names. Refuse here too, so the clone stops before one // exists rather than leaving an empty registered `wm_fork_…` behind. - let governing = match req.source.strip_prefix("datatable://") { - Some(reference) => { - Some(ensure_datatable_is_clonable(&db, &w_id, datatable_ref_name(reference)).await?) - } - None => None, - }; + if let Some(reference) = req.source.strip_prefix("datatable://") { + let name = datatable_ref_name(reference); + ensure_datatable_is_clonable(&db, &w_id, name).await?; + } // Non-superadmin: restrict dbname to wm_fork_ prefix if !windmill_api_auth::is_super_admin_authed(&db, &authed).await? { @@ -3330,21 +3328,6 @@ async fn create_pg_database( } if is_instance_datatable_source(&db, &w_id, &req.source).await? { - // A retry after a clone that failed past this point finds its own leftover here. Reclaiming - // it is a `DROP DATABASE`, so it is for whoever administers the source — not merely whoever - // reaches it, which on a data table without roles is every member. Anyone else gets the - // refusal an existing database always got. - let may_reclaim = match &governing { - Some(governing) => crate::datatable_permissions::ensure_governs_datatable( - &db, &authed, &w_id, governing, - ) - .await - .is_ok(), - None => false, - }; - if may_reclaim { - windmill_common::reclaim_orphaned_fork_database(&db, &req.target_dbname).await?; - } windmill_common::create_custom_instance_database(&db, &req.target_dbname, "datatable") .await?; } else { diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 99cf7de1bb..a7e84c55d7 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -1457,77 +1457,6 @@ pub fn validate_dbname(dbname: &str) -> error::Result<()> { Ok(()) } -/// Whether `dbname` is a clone's leftover: a `wm_fork_*` database Windmill registered as a data -/// table database and that no data table or ducklake entry names, in any workspace, archived ones -/// included. A clone creates its database a request before any entry names it, so a failure in -/// between leaves exactly this, and its name then blocks every retry. -pub async fn is_reclaimable_fork_database(db: &DB, dbname: &str) -> error::Result { - if !dbname.starts_with("wm_fork_") { - return Ok(false); - } - Ok(sqlx::query_scalar!( - r#"SELECT - COALESCE(value->'databases'->$1::text->>'tag' = 'datatable', false) - AND NOT EXISTS ( - SELECT 1 FROM workspace_settings ws - CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt - WHERE dt.value->'database'->>'resource_type' = 'instance' - AND dt.value->'database'->>'resource_path' = $1::text) - AND NOT EXISTS ( - SELECT 1 FROM workspace_settings ws - CROSS JOIN LATERAL jsonb_each(COALESCE(ws.ducklake->'ducklakes', '{}'::jsonb)) dl - WHERE dl.value->'catalog'->>'resource_type' = 'instance' - AND dl.value->'catalog'->>'resource_path' = $1::text) - AS "reclaimable!" - FROM global_settings WHERE name = 'custom_instance_pg_databases'"#, - dbname - ) - .fetch_optional(db) - .await? - .unwrap_or(false)) -} - -/// Drop `dbname` if [`is_reclaimable_fork_database`], so a retried clone can recreate it, and -/// return whether it did. Unlike [`drop_custom_instance_database`] this never terminates -/// connections: a clone still copying into the database must make the drop fail, not be cut off -/// mid-copy. -pub async fn reclaim_orphaned_fork_database(db: &DB, dbname: &str) -> error::Result { - let dbname = dbname.trim(); - validate_dbname(dbname)?; - - let exists = sqlx::query_scalar!( - "SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_database WHERE datname = $1)", - dbname - ) - .fetch_one(db) - .await? - .unwrap_or(false); - if !exists || !is_reclaimable_fork_database(db, dbname).await? { - return Ok(false); - } - - // SAFETY: `dbname` has been validated via validate_dbname() above. - sqlx::query(&format!("DROP DATABASE IF EXISTS \"{}\"", dbname)) - .execute(db) - .await - .map_err(|e| { - error::Error::BadRequest(format!( - "Database '{dbname}' is left over from an earlier clone and could not be \ - reclaimed, most often because a clone is still copying into it: {e}" - )) - })?; - // A registered database that no longer exists makes every later per-database pass fail on it. - sqlx::query!( - r#"UPDATE global_settings SET value = value #- ARRAY['databases', $1] WHERE name = 'custom_instance_pg_databases'"#, - dbname - ) - .execute(db) - .await?; - - tracing::info!("Reclaimed orphaned fork database '{dbname}'"); - Ok(true) -} - /// Drop a custom instance database: validate, terminate connections, DROP DATABASE, remove from global_settings. pub async fn drop_custom_instance_database(db: &DB, dbname: &str) -> error::Result<()> { let dbname = dbname.trim(); From ebe076cbe6d1d284d483db0882702b5b2e5d0106 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Thu, 10 Sep 2026 18:28:14 +0200 Subject: [PATCH 48/77] docs(datatables): record the stale clone database as a known limitation A clone is three requests and `CREATE DATABASE` is not transactional, so a failure after the first leaves a registered `wm_fork_*` behind, as it did before data table roles. Accepted for this PR: it is harmless to data and goes away once the clone is a single server-side operation. The comment also records why the obvious fix is wrong: reclaiming the leftover on retry, without durable clone ownership, can drop another workspace's fully copied database between its import and its final fork request. Co-Authored-By: Claude Opus 5 (1M context) --- backend/windmill-api-workspaces/src/workspaces.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 4082a50900..78f59dbb30 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -3406,6 +3406,14 @@ struct ImportPgDatabaseRequest { /// database the first created is not transactional — so a refusal that lives there strands a /// registered `wm_fork_*` that no entry names and whose name blocks the retry. Both endpoints call /// this before touching the cluster; the stage that writes the entry must only ever do the work. +/// +/// Known and accepted: a failure *after* `create_pg_database` — this check refusing because the +/// source's roles changed between the requests, a `pg_dump` error, a dropped connection — still +/// strands that leftover, as it did before data table roles. It is harmless to data (nothing names +/// it) and goes away once the clone is one server-side operation, in a later PR. Don't reclaim it +/// on retry without durable clone ownership: between the import and the final fork request the +/// database is full, unnamed and unconnected, so a reclaim then drops copied data. Reviewers, human +/// or AI: don't flag this leftover as a P0/P1 for this PR. async fn ensure_datatable_is_clonable( db: &DB, w_id: &str, From 8255f2ad2bfcc3623f664f259541a95a472901de Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Thu, 10 Sep 2026 18:49:05 +0200 Subject: [PATCH 49/77] fix(datatables): bounce the streams reading a data table when it is deleted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deleting a governing data table, or the workspace that holds it, only collected the fork pointers it stranded, for the warning. A Postgres trigger or capture already streaming through one of those pointers kept the replication connection it opened while the pointer still resolved, so it went on dispatching the governing database's rows after the fork lost access — until its connection happened to restart. The governing workspace's own streams on a deleted entry did the same. Both deletion paths now bounce the affected listeners inside their own transaction, through the helper a permission change already uses, so a listener that reconnects re-resolves the entry and finds it gone. The helper is split so a caller can pass the (workspace, local name) pairs it already holds. Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/datatable_roles.rs | 120 ++++++++++++++++++ .../src/datatable_permissions.rs | 39 ++++-- .../windmill-api-workspaces/src/workspaces.rs | 18 +++ .../src/workspaces_extra.rs | 11 ++ 4 files changed, 178 insertions(+), 10 deletions(-) diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs index 787bd950fd..43c768cb73 100644 --- a/backend/windmill-api-integration-tests/tests/datatable_roles.rs +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -598,3 +598,123 @@ async fn a_data_table_under_roles_is_not_copied_into_a_fork( ); Ok(()) } + +/// Live replication listeners on `main`: a fork's through its pointer, the governing workspace's +/// own (in the `?role=` form), and a fork capture — plus one on another data table, which no +/// deletion of `main` may touch. Each is held by a server, as a running stream would be. +async fn plant_live_streams(db: &Pool) -> anyhow::Result<()> { + sqlx::query( + r#"INSERT INTO postgres_trigger (path, script_path, is_flow, workspace_id, edited_by, + postgres_resource_path, replication_slot_name, publication_name, permissioned_as, + server_id) + VALUES + ('u/test-user-2/fork_stream', 'u/test-user-2/s', false, 'wm-fork-dt', 'test-user-2', + 'datatable://main', 'slot_fork', 'pub_fork', 'u/test-user-2', 'srv'), + ('u/test-user/own_stream', 'u/test-user/s', false, 'test-workspace', 'test-user', + 'datatable://main?role=admin', 'slot_own', 'pub_own', 'u/test-user', 'srv'), + ('u/test-user-2/unrelated', 'u/test-user-2/s', false, 'wm-fork-dt', 'test-user-2', + 'datatable://other', 'slot_other', 'pub_other', 'u/test-user-2', 'srv')"#, + ) + .execute(db) + .await?; + sqlx::query( + r#"INSERT INTO capture_config (workspace_id, path, is_flow, trigger_kind, owner, email, + trigger_config, server_id) + VALUES ('wm-fork-dt', 'u/test-user-2/s', false, 'postgres', 'u/test-user-2', + 'test2@windmill.dev', '{"postgres_resource_path": "datatable://main"}', 'srv')"#, + ) + .execute(db) + .await?; + Ok(()) +} + +/// Which server holds each planted listener; `None` is a bounced one, free for a reconnect. +async fn stream_servers( + db: &Pool, +) -> anyhow::Result>> { + let rows: Vec<(String, Option)> = sqlx::query_as( + "SELECT path, server_id FROM postgres_trigger + UNION ALL + SELECT 'capture:' || path, server_id FROM capture_config", + ) + .fetch_all(db) + .await?; + Ok(rows.into_iter().collect()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn deleting_a_governing_data_table_bounces_the_streams_reading_it( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + // A replication stream keeps the connection it opened while its entry resolved. Unbounced, a + // fork's stream reads on through a pointer that no longer resolves to anything. + plant_live_streams(&db).await?; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/w/test-workspace/workspaces/edit_datatable_config" + )), + "SECRET_TOKEN", + ) + .json(&json!({"settings": {"datatables": {}}, "renames": [], "deleted_datatables": ["main"]})) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + let servers = stream_servers(&db).await?; + for bounced in [ + "u/test-user-2/fork_stream", + "u/test-user/own_stream", + "capture:u/test-user-2/s", + ] { + assert_eq!( + servers[bounced], None, + "{bounced} kept streaming: {servers:?}" + ); + } + assert_eq!( + servers["u/test-user-2/unrelated"].as_deref(), + Some("srv"), + "a stream on another data table was bounced: {servers:?}" + ); + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn deleting_a_governing_workspace_bounces_its_forks_streams( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + // The governing workspace's own listeners go with it; each fork's stays behind on the + // connection it opened while this workspace still governed the pointer. + plant_live_streams(&db).await?; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let resp = authed( + client().delete(format!( + "http://localhost:{port}/api/workspaces/delete/test-workspace" + )), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + let servers = stream_servers(&db).await?; + for bounced in ["u/test-user-2/fork_stream", "capture:u/test-user-2/s"] { + assert_eq!( + servers[bounced], None, + "{bounced} kept streaming: {servers:?}" + ); + } + assert_eq!( + servers["u/test-user-2/unrelated"].as_deref(), + Some("srv"), + "a stream on another data table was bounced: {servers:?}" + ); + Ok(()) +} diff --git a/backend/windmill-api-workspaces/src/datatable_permissions.rs b/backend/windmill-api-workspaces/src/datatable_permissions.rs index a1493c4b98..1457be5322 100644 --- a/backend/windmill-api-workspaces/src/datatable_permissions.rs +++ b/backend/windmill-api-workspaces/src/datatable_permissions.rs @@ -438,7 +438,12 @@ async fn set_datatable_permissions( // A live replication stream holds a connection it opened under the old decision. Bouncing the // rows makes every listener reconnect and re-authorize. - restart_streams_reaching(&db, &governing).await?; + restart_streams_reaching( + &mut *db.acquire().await?, + &governing.workspace_id, + &governing.name, + ) + .await?; windmill_common::feature_usage::log_feature_usage( "datatable", @@ -456,29 +461,43 @@ async fn set_datatable_permissions( /// Make every Postgres trigger and capture reading this data table reconnect, so a revoked tenant /// stops streaming rather than living on inside an already-open replication connection. pub(crate) async fn restart_streams_reaching( - db: &DB, - governing: &GoverningDatatable, + conn: &mut sqlx::PgConnection, + governing_workspace_id: &str, + governing_name: &str, ) -> Result<()> { // Every workspace holding an entry that resolves here, under the name it calls it: the // governing one, plus each fork pointing at it. A fork's trigger names its own local entry, so // filtering on the governing workspace alone would leave its stream running on the connection // it already opened under the old decision — which is the one window this function exists to // close. - let mut reached = vec![(governing.workspace_id.clone(), governing.name.clone())]; + let mut reached = vec![( + governing_workspace_id.to_string(), + governing_name.to_string(), + )]; let pointers = sqlx::query!( r#"SELECT ws.workspace_id AS "workspace_id!", dt.key AS "datatable!" FROM workspace_settings ws CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt WHERE dt.value->'reference'->>'workspace_id' = $1 AND dt.value->'reference'->>'datatable' = $2"#, - &governing.workspace_id, - &governing.name, + governing_workspace_id, + governing_name, ) - .fetch_all(db) + .fetch_all(&mut *conn) .await?; reached.extend(pointers.into_iter().map(|r| (r.workspace_id, r.datatable))); + restart_streams_named(conn, reached).await +} - for (w_id, name) in reached { +/// Make every Postgres trigger and capture reading one of `entries` — each a workspace and the name +/// that workspace gives the data table — drop its connection and re-resolve the data table as it +/// now stands. A caller deciding inside a transaction passes it, so the bounce and the decision +/// become visible together and no listener reconnects in between. +pub(crate) async fn restart_streams_named( + conn: &mut sqlx::PgConnection, + entries: Vec<(String, String)>, +) -> Result<()> { + for (w_id, name) in entries { let reference = format!("datatable://{name}"); let prefix = format!("{reference}?%"); @@ -490,7 +509,7 @@ pub(crate) async fn restart_streams_reaching( &reference, &prefix, ) - .execute(db) + .execute(&mut *conn) .await?; // A capture keeps the reference inside its `trigger_config` blob rather than in a column @@ -504,7 +523,7 @@ pub(crate) async fn restart_streams_reaching( &reference, &prefix, ) - .execute(db) + .execute(&mut *conn) .await?; } diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 78f59dbb30..ca17629f58 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -3927,6 +3927,24 @@ async fn edit_datatable_config( ); } + // A stream reading a deleted entry keeps the connection it opened while the entry resolved — + // this workspace's own, and every fork's through its pointer. Bounced in this transaction, so + // a listener that reconnects finds the entry gone instead of streaming on. + crate::datatable_permissions::restart_streams_named( + &mut *tx, + new_config + .deleted_datatables + .iter() + .map(|name| (w_id.clone(), name.clone())) + .chain( + stranded + .iter() + .map(|s| (s.workspace_id.clone(), s.datatable.clone())), + ) + .collect(), + ) + .await?; + tx.commit().await?; for substrate in created_substrates { diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index 1cb5188e5b..d66f2a13b0 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -1265,6 +1265,17 @@ pub(crate) async fn delete_workspace( None, ) .await?; + // Each fork pointing here keeps the replication stream it opened while this workspace + // governed it. Bounced in this transaction, so a listener that reconnects finds its pointer + // dangling instead of streaming on. + crate::datatable_permissions::restart_streams_named( + &mut *tx, + stranded_pointers + .iter() + .map(|r| (r.workspace_id.clone(), r.datatable.clone())) + .collect(), + ) + .await?; tx.commit().await?; // Physical ducklake-namespace cleanup, post-commit, from the pre-read snapshot: fork From dc3ebfbe74c11a554e4518e91d304e052bc20612 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Thu, 10 Sep 2026 19:15:33 +0200 Subject: [PATCH 50/77] fix(datatables): keep the fork schema baseline, and bounce streams on every removal Three fixes from review. `edit_datatable_config` took `forked_from` wholesale from the stored entry, so the fork schema diff's save of an advanced baseline was silently discarded and an applied change was offered again. Whether an entry carries a clone stamp is still carried from the store, since that is what marks its database droppable, but the baseline inside it is now taken from the request. The stranded-pointer warning and the stream bounce ran over the optional `deleted_datatables` hint, which the settings-sync CLI never sends, so removing a governing data table through `wmill` bounced nothing. Removals are now derived from the stored configuration against the saved one. `delete_workspace` read the pointers to bounce before its transaction, so a fork committing a pointer during the deletion was missed. The read now happens inside the transaction, after the workspace row is deleted: a fork's insert key-share locks that row through its parent foreign key, so it is either seen or fails on the missing parent. Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/datatable_roles.rs | 76 ++++++++++++++++++- .../windmill-api-workspaces/src/workspaces.rs | 34 ++++++--- .../src/workspaces_extra.rs | 31 ++++---- 3 files changed, 115 insertions(+), 26 deletions(-) diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs index 43c768cb73..f91495ecef 100644 --- a/backend/windmill-api-integration-tests/tests/datatable_roles.rs +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -599,6 +599,79 @@ async fn a_data_table_under_roles_is_not_copied_into_a_fork( Ok(()) } +/// The fork's `forked_from` for one of its entries; `None` whether it is absent or `null`. +async fn forked_from_of(db: &Pool, name: &str) -> Option { + sqlx::query_scalar::<_, Option>( + "SELECT datatable->'datatables'->$1::text->'forked_from' + FROM workspace_settings WHERE workspace_id = 'wm-fork-dt'", + ) + .bind(name) + .fetch_one(db) + .await + .unwrap() + .filter(|v| !v.is_null()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_clone_stamp_is_carried_but_its_schema_baseline_advances( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + // Whether an entry is a clone is what marks its database droppable, so a save can neither + // stamp nor unstamp one. The schema baseline inside the stamp is what the fork's schema diff + // advances after applying a change; dropping it would offer that same change again. + sqlx::query( + r#"UPDATE workspace_settings SET datatable = '{"datatables": { + "clone": {"database": {"resource_type": "instance", "resource_path": "wm_fork_dt__clone"}, + "forked_from": {"schema": {}}}, + "plain": {"database": {"resource_type": "instance", "resource_path": "dt_plain"}}}}'::jsonb + WHERE workspace_id = 'wm-fork-dt'"#, + ) + .execute(&db) + .await?; + let server = ApiServer::start(db.clone()).await?; + let url = format!( + "http://localhost:{}/api/w/wm-fork-dt/workspaces/edit_datatable_config", + server.addr.port() + ); + let clone_db = json!({"resource_type": "instance", "resource_path": "wm_fork_dt__clone"}); + let plain_db = json!({"resource_type": "instance", "resource_path": "dt_plain"}); + let baseline = json!({"schema": {"public": {"orders": {"id": "int4"}}}}); + + let resp = authed(client().post(&url), "SECRET_TOKEN_2") + .json(&json!({"settings": {"datatables": { + "clone": {"database": clone_db, "forked_from": baseline}, + "plain": {"database": plain_db, "forked_from": {"schema": {}}} + }}})) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + assert_eq!( + forked_from_of(&db, "clone").await, + Some(baseline.clone()), + "the schema diff's baseline did not advance" + ); + assert_eq!( + forked_from_of(&db, "plain").await, + None, + "a save stamped a clone" + ); + + let resp = authed(client().post(&url), "SECRET_TOKEN_2") + .json(&json!({"settings": {"datatables": { + "clone": {"database": clone_db}, "plain": {"database": plain_db} + }}})) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + assert_eq!( + forked_from_of(&db, "clone").await, + Some(baseline), + "a save unstamped a clone" + ); + Ok(()) +} + /// Live replication listeners on `main`: a fork's through its pointer, the governing workspace's /// own (in the `?role=` form), and a fork capture — plus one on another data table, which no /// deletion of `main` may touch. Each is held by a server, as a running stream would be. @@ -659,7 +732,8 @@ async fn deleting_a_governing_data_table_bounces_the_streams_reading_it( )), "SECRET_TOKEN", ) - .json(&json!({"settings": {"datatables": {}}, "renames": [], "deleted_datatables": ["main"]})) + // The way the settings-sync CLI saves: the new document alone, with no deletion hint. + .json(&json!({"settings": {"datatables": {}}})) .send() .await?; assert_eq!(resp.status(), 200, "{}", resp.text().await?); diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index ca17629f58..6ac96e3f0b 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -3799,15 +3799,18 @@ async fn edit_datatable_config( Some(true) } }; - // Three fields this form does not own, carried across from the stored entry rather than - // taken from the request. `permissions` is an access decision, edited through its own - // endpoint; `reference` is what makes a fork answer to the workspace that governs its data - // table, and letting a save clear it would hand the fork the database outright; and - // `forked_from` is the clone stamp the fork flow writes. Only fork creation writes any of - // them, so a settings save can neither widen nor lose them. + // Carried across from the stored entry rather than taken from the request. `permissions` + // is an access decision, edited through its own endpoint; `reference` is what makes a fork + // answer to the workspace that governs its data table, and letting a save clear it would + // hand the fork the database outright. `forked_from` is the clone stamp the fork flow + // writes: whether an entry has one is carried the same way, since it is what marks the + // database droppable, but the schema baseline inside it is the diff view's to advance. dt.permissions = old.and_then(|old| old.permissions.clone()); dt.reference = old.and_then(|old| old.reference.clone()); - dt.forked_from = old.and_then(|old| old.forked_from.clone()); + dt.forked_from = match old.and_then(|old| old.forked_from.as_ref()) { + Some(stored) => Some(dt.forked_from.take().unwrap_or_else(|| stored.clone())), + None => None, + }; // Carrying the block onto a resource-backed entry would produce a data table the chokepoint // refuses on every job — a save that succeeds and breaks everything afterwards. Refuse it // instead: turning roles off first is one step, and it keeps discarding an access decision @@ -3868,6 +3871,18 @@ async fn edit_datatable_config( } } + // Every entry this save removes, derived rather than taken from `deleted_datatables`: that list + // is a hint the settings-sync CLI never sends, and the stranded-pointer warning and the stream + // bounce below must run for a removal whether or not the caller named it. + let removed: Vec = old_datatables + .keys() + .filter(|name| { + !new_config.settings.datatables.contains_key(*name) + && !new_config.renames.iter().any(|r| &r.from == *name) + }) + .cloned() + .collect(); + let config: serde_json::Value = serde_json::to_value(new_config.settings) .map_err(|err| Error::internal_err(err.to_string()))?; @@ -3907,7 +3922,7 @@ async fn edit_datatable_config( // A deletion cannot be followed the same way — there is nothing to point at any more. Read who // is left stranded so the caller is told, the way deleting a workspace does. let mut stranded: Vec = Vec::new(); - for name in &new_config.deleted_datatables { + for name in &removed { let rows = sqlx::query!( r#"SELECT ws.workspace_id AS "workspace_id!", dt.key AS "datatable!" FROM workspace_settings ws @@ -3932,8 +3947,7 @@ async fn edit_datatable_config( // a listener that reconnects finds the entry gone instead of streaming on. crate::datatable_permissions::restart_streams_named( &mut *tx, - new_config - .deleted_datatables + removed .iter() .map(|name| (w_id.clone(), name.clone())) .chain( diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index d66f2a13b0..371b80555c 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -995,21 +995,6 @@ pub(crate) async fn delete_workspace( // but the destructive cleanup itself runs only after the commit below: a delete that // fails mid-way must never leave a live workspace with its fork data destroyed and no // registry row to retry from. Read-only: nothing is dropped here. - // Read before the delete: another workspace's data table entry can point at one of this - // workspace's, and deleting the workspace it names leaves that pointer resolving to nothing. - // Nothing sweeps them — turning them back into copies would hand each fork the database - // outright — so the deleter is told which data tables they just stranded. - let stranded_pointers = sqlx::query!( - r#"SELECT ws.workspace_id AS "workspace_id!", dt.key AS "datatable!" - FROM workspace_settings ws - CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt - WHERE dt.value->'reference'->>'workspace_id' = $1 - ORDER BY ws.workspace_id, dt.key"#, - &w_id, - ) - .fetch_all(&db) - .await - .unwrap_or_default(); let fork_ducklake_cleanups = prepare_fork_ducklake_cleanups(&db, &w_id, None) .await @@ -1265,6 +1250,22 @@ pub(crate) async fn delete_workspace( None, ) .await?; + // Who points here, read after the delete above and inside this transaction. A fork writes its + // pointer in the transaction that inserts it, and that insert key-share locks this row through + // the parent foreign key: so the fork either committed before the delete and is seen here, or + // waits on it and then fails on the missing parent. No pointer can escape this list. Nothing + // sweeps them afterwards — turning them back into copies would hand each fork the database + // outright — so the deleter is told which data tables they stranded. + let stranded_pointers = sqlx::query!( + r#"SELECT ws.workspace_id AS "workspace_id!", dt.key AS "datatable!" + FROM workspace_settings ws + CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt + WHERE dt.value->'reference'->>'workspace_id' = $1 + ORDER BY ws.workspace_id, dt.key"#, + &w_id, + ) + .fetch_all(&mut *tx) + .await?; // Each fork pointing here keeps the replication stream it opened while this workspace // governed it. Bounced in this transaction, so a listener that reconnects finds its pointer // dangling instead of streaming on. From fa1e8bd3aa1ebfef324305989591e17741960b15 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Fri, 11 Sep 2026 10:35:36 +0200 Subject: [PATCH 51/77] refactor(datatables): keep Postgres triggers and data table roles apart A replication stream reads every row of every table whatever the data table's roles grant, and its listener checks access only when it connects. Rather than chase every way access can change and bounce the streams each one affects, a data table now carries one or the other: - a Postgres trigger or capture cannot be created on, or connect to, a data table under roles; - roles cannot be turned on while an enabled trigger or a live capture reads the data table, its own or a fork's through its pointer. The refusal names each one to disable. This removes the stream bounces on roles edits and on data table and workspace deletion, and the trigger gate that admitted admins. The fork schema baseline fix from the same review round is kept. Co-Authored-By: Claude Opus 5 (1M context) --- ...8961cfbd0171f835faa756762919c021bfd2b.json | 24 +++ .../tests/datatable_roles.rs | 156 ++++++------------ .../src/datatable_permissions.rs | 111 ++++++------- .../windmill-api-workspaces/src/workspaces.rs | 31 +--- .../src/workspaces_extra.rs | 42 ++--- backend/windmill-trigger-postgres/src/lib.rs | 28 ++-- 6 files changed, 148 insertions(+), 244 deletions(-) create mode 100644 backend/.sqlx/query-bc87664b87a84e1033589f729148961cfbd0171f835faa756762919c021bfd2b.json diff --git a/backend/.sqlx/query-bc87664b87a84e1033589f729148961cfbd0171f835faa756762919c021bfd2b.json b/backend/.sqlx/query-bc87664b87a84e1033589f729148961cfbd0171f835faa756762919c021bfd2b.json new file mode 100644 index 0000000000..b452b6fc16 --- /dev/null +++ b/backend/.sqlx/query-bc87664b87a84e1033589f729148961cfbd0171f835faa756762919c021bfd2b.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id || '/' || path AS \"stream!\" FROM postgres_trigger\n WHERE workspace_id = $1 AND mode <> 'disabled'::TRIGGER_MODE\n AND (postgres_resource_path = $2 OR starts_with(postgres_resource_path, $3))\n UNION ALL\n SELECT workspace_id || '/' || path || ' (capture)' FROM capture_config\n WHERE workspace_id = $1 AND trigger_kind = 'postgres'\n AND last_client_ping > now() - interval '10 seconds'\n AND (trigger_config->>'postgres_resource_path' = $2\n OR starts_with(trigger_config->>'postgres_resource_path', $3))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "stream!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "bc87664b87a84e1033589f729148961cfbd0171f835faa756762919c021bfd2b" +} diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs index f91495ecef..e619e73f23 100644 --- a/backend/windmill-api-integration-tests/tests/datatable_roles.rs +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -672,123 +672,59 @@ async fn a_clone_stamp_is_carried_but_its_schema_baseline_advances( Ok(()) } -/// Live replication listeners on `main`: a fork's through its pointer, the governing workspace's -/// own (in the `?role=` form), and a fork capture — plus one on another data table, which no -/// deletion of `main` may touch. Each is held by a server, as a running stream would be. -async fn plant_live_streams(db: &Pool) -> anyhow::Result<()> { +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn roles_cannot_be_turned_on_while_a_trigger_streams_the_data_table( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + // A replication stream reads every row whatever the roles grant, so a data table carries one + // or the other. An enabled trigger on it — here a fork's, through its pointer — keeps roles + // from being turned on, and disabling it is what lets them on. + sqlx::query( + "UPDATE workspace_settings SET datatable = datatable #- '{datatables,main,permissions}' + WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; sqlx::query( r#"INSERT INTO postgres_trigger (path, script_path, is_flow, workspace_id, edited_by, - postgres_resource_path, replication_slot_name, publication_name, permissioned_as, - server_id) - VALUES - ('u/test-user-2/fork_stream', 'u/test-user-2/s', false, 'wm-fork-dt', 'test-user-2', - 'datatable://main', 'slot_fork', 'pub_fork', 'u/test-user-2', 'srv'), - ('u/test-user/own_stream', 'u/test-user/s', false, 'test-workspace', 'test-user', - 'datatable://main?role=admin', 'slot_own', 'pub_own', 'u/test-user', 'srv'), - ('u/test-user-2/unrelated', 'u/test-user-2/s', false, 'wm-fork-dt', 'test-user-2', - 'datatable://other', 'slot_other', 'pub_other', 'u/test-user-2', 'srv')"#, + postgres_resource_path, replication_slot_name, publication_name, permissioned_as, mode) + VALUES ('u/test-user-2/fork_stream', 'u/test-user-2/s', false, 'wm-fork-dt', + 'test-user-2', 'datatable://main', 'slot_fork', 'pub_fork', 'u/test-user-2', + 'enabled')"#, ) - .execute(db) + .execute(&db) .await?; + + let server = ApiServer::start(db.clone()).await?; + let url = format!( + "http://localhost:{}/api/w/test-workspace/workspaces/datatable_permissions/main", + server.addr.port() + ); + let turn_on = json!({"permissioned": true, "default_role": "admin", + "roles": [{"id": "admin", "tenants": ["*"]}]}); + + let resp = authed(client().post(&url), "SECRET_TOKEN") + .json(&turn_on) + .send() + .await?; + assert_eq!(resp.status(), 400); + assert!( + resp.text() + .await? + .contains("wm-fork-dt/u/test-user-2/fork_stream"), + "the refusal does not name the trigger to disable" + ); + sqlx::query( - r#"INSERT INTO capture_config (workspace_id, path, is_flow, trigger_kind, owner, email, - trigger_config, server_id) - VALUES ('wm-fork-dt', 'u/test-user-2/s', false, 'postgres', 'u/test-user-2', - 'test2@windmill.dev', '{"postgres_resource_path": "datatable://main"}', 'srv')"#, + "UPDATE postgres_trigger SET mode = 'disabled' WHERE path = 'u/test-user-2/fork_stream'", ) - .execute(db) - .await?; - Ok(()) -} - -/// Which server holds each planted listener; `None` is a bounced one, free for a reconnect. -async fn stream_servers( - db: &Pool, -) -> anyhow::Result>> { - let rows: Vec<(String, Option)> = sqlx::query_as( - "SELECT path, server_id FROM postgres_trigger - UNION ALL - SELECT 'capture:' || path, server_id FROM capture_config", - ) - .fetch_all(db) - .await?; - Ok(rows.into_iter().collect()) -} - -#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] -async fn deleting_a_governing_data_table_bounces_the_streams_reading_it( - db: Pool, -) -> anyhow::Result<()> { - initialize_tracing().await; - // A replication stream keeps the connection it opened while its entry resolved. Unbounced, a - // fork's stream reads on through a pointer that no longer resolves to anything. - plant_live_streams(&db).await?; - let server = ApiServer::start(db.clone()).await?; - let port = server.addr.port(); - - let resp = authed( - client().post(format!( - "http://localhost:{port}/api/w/test-workspace/workspaces/edit_datatable_config" - )), - "SECRET_TOKEN", - ) - // The way the settings-sync CLI saves: the new document alone, with no deletion hint. - .json(&json!({"settings": {"datatables": {}}})) - .send() + .execute(&db) .await?; + let resp = authed(client().post(&url), "SECRET_TOKEN") + .json(&turn_on) + .send() + .await?; assert_eq!(resp.status(), 200, "{}", resp.text().await?); - - let servers = stream_servers(&db).await?; - for bounced in [ - "u/test-user-2/fork_stream", - "u/test-user/own_stream", - "capture:u/test-user-2/s", - ] { - assert_eq!( - servers[bounced], None, - "{bounced} kept streaming: {servers:?}" - ); - } - assert_eq!( - servers["u/test-user-2/unrelated"].as_deref(), - Some("srv"), - "a stream on another data table was bounced: {servers:?}" - ); - Ok(()) -} - -#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] -async fn deleting_a_governing_workspace_bounces_its_forks_streams( - db: Pool, -) -> anyhow::Result<()> { - initialize_tracing().await; - // The governing workspace's own listeners go with it; each fork's stays behind on the - // connection it opened while this workspace still governed the pointer. - plant_live_streams(&db).await?; - let server = ApiServer::start(db.clone()).await?; - let port = server.addr.port(); - - let resp = authed( - client().delete(format!( - "http://localhost:{port}/api/workspaces/delete/test-workspace" - )), - "SECRET_TOKEN", - ) - .send() - .await?; - assert_eq!(resp.status(), 200, "{}", resp.text().await?); - - let servers = stream_servers(&db).await?; - for bounced in ["u/test-user-2/fork_stream", "capture:u/test-user-2/s"] { - assert_eq!( - servers[bounced], None, - "{bounced} kept streaming: {servers:?}" - ); - } - assert_eq!( - servers["u/test-user-2/unrelated"].as_deref(), - Some("srv"), - "a stream on another data table was bounced: {servers:?}" - ); Ok(()) } diff --git a/backend/windmill-api-workspaces/src/datatable_permissions.rs b/backend/windmill-api-workspaces/src/datatable_permissions.rs index 1457be5322..b04fb209e7 100644 --- a/backend/windmill-api-workspaces/src/datatable_permissions.rs +++ b/backend/windmill-api-workspaces/src/datatable_permissions.rs @@ -346,6 +346,12 @@ async fn set_datatable_permissions( ))); } + // Turning roles on is refused while a replication stream reads this data table. One already + // under roles cannot have any: the listener refuses to open a stream on it. + if req.permissioned && governing.datatable.permissions.is_none() { + ensure_no_streams_reaching(&db, &governing).await?; + } + let permissions = if req.permissioned { let catalog = windmill_common::datatable_roles::read_role_catalog_tx(&mut tx).await?; let mut roles: BTreeMap = BTreeMap::new(); @@ -436,15 +442,6 @@ async fn set_datatable_permissions( } } - // A live replication stream holds a connection it opened under the old decision. Bouncing the - // rows makes every listener reconnect and re-authorize. - restart_streams_reaching( - &mut *db.acquire().await?, - &governing.workspace_id, - &governing.name, - ) - .await?; - windmill_common::feature_usage::log_feature_usage( "datatable", "roles_toggled", @@ -458,75 +455,61 @@ async fn set_datatable_permissions( }) } -/// Make every Postgres trigger and capture reading this data table reconnect, so a revoked tenant -/// stops streaming rather than living on inside an already-open replication connection. -pub(crate) async fn restart_streams_reaching( - conn: &mut sqlx::PgConnection, - governing_workspace_id: &str, - governing_name: &str, -) -> Result<()> { +/// Refuse to put a data table under roles while a Postgres trigger or capture streams it. A +/// replication stream reads every row whatever the roles grant, so a data table carries one or the +/// other; the listener side refuses a data table already under roles. +async fn ensure_no_streams_reaching(db: &DB, governing: &GoverningDatatable) -> Result<()> { // Every workspace holding an entry that resolves here, under the name it calls it: the // governing one, plus each fork pointing at it. A fork's trigger names its own local entry, so - // filtering on the governing workspace alone would leave its stream running on the connection - // it already opened under the old decision — which is the one window this function exists to - // close. - let mut reached = vec![( - governing_workspace_id.to_string(), - governing_name.to_string(), - )]; + // looking in the governing workspace alone would miss every stream a fork opened. + let mut reached = vec![(governing.workspace_id.clone(), governing.name.clone())]; let pointers = sqlx::query!( r#"SELECT ws.workspace_id AS "workspace_id!", dt.key AS "datatable!" FROM workspace_settings ws CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt WHERE dt.value->'reference'->>'workspace_id' = $1 AND dt.value->'reference'->>'datatable' = $2"#, - governing_workspace_id, - governing_name, + &governing.workspace_id, + &governing.name, ) - .fetch_all(&mut *conn) + .fetch_all(db) .await?; reached.extend(pointers.into_iter().map(|r| (r.workspace_id, r.datatable))); - restart_streams_named(conn, reached).await -} -/// Make every Postgres trigger and capture reading one of `entries` — each a workspace and the name -/// that workspace gives the data table — drop its connection and re-resolve the data table as it -/// now stands. A caller deciding inside a transaction passes it, so the bounce and the decision -/// become visible together and no listener reconnects in between. -pub(crate) async fn restart_streams_named( - conn: &mut sqlx::PgConnection, - entries: Vec<(String, String)>, -) -> Result<()> { - for (w_id, name) in entries { + let mut streams = Vec::new(); + for (w_id, name) in reached { let reference = format!("datatable://{name}"); - let prefix = format!("{reference}?%"); - - sqlx::query!( - "UPDATE postgres_trigger SET server_id = NULL, last_server_ping = NULL - WHERE workspace_id = $1 - AND (postgres_resource_path = $2 OR postgres_resource_path LIKE $3)", - &w_id, - &reference, - &prefix, - ) - .execute(&mut *conn) - .await?; - - // A capture keeps the reference inside its `trigger_config` blob rather than in a column - // of its own, and only a postgres capture has one there at all. - sqlx::query!( - "UPDATE capture_config SET server_id = NULL, last_server_ping = NULL - WHERE workspace_id = $1 AND trigger_kind = 'postgres' - AND (trigger_config->>'postgres_resource_path' = $2 - OR trigger_config->>'postgres_resource_path' LIKE $3)", - &w_id, - &reference, - &prefix, - ) - .execute(&mut *conn) - .await?; + let with_query = format!("{reference}?"); + // A suspended trigger keeps its listener, so only a disabled one is not streaming; a + // capture streams for as long as its client keeps pinging. + streams.extend( + sqlx::query_scalar!( + r#"SELECT workspace_id || '/' || path AS "stream!" FROM postgres_trigger + WHERE workspace_id = $1 AND mode <> 'disabled'::TRIGGER_MODE + AND (postgres_resource_path = $2 OR starts_with(postgres_resource_path, $3)) + UNION ALL + SELECT workspace_id || '/' || path || ' (capture)' FROM capture_config + WHERE workspace_id = $1 AND trigger_kind = 'postgres' + AND last_client_ping > now() - interval '10 seconds' + AND (trigger_config->>'postgres_resource_path' = $2 + OR starts_with(trigger_config->>'postgres_resource_path', $3))"#, + &w_id, + &reference, + &with_query, + ) + .fetch_all(db) + .await?, + ); + } + if !streams.is_empty() { + return Err(Error::BadRequest(format!( + "Data table '{}' cannot be put under roles while a Postgres trigger or capture streams \ + it: a replication stream reads every row whatever the roles grant. Disable them \ + first: {}", + governing.name, + streams.join(", ") + ))); } - Ok(()) } diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 6ac96e3f0b..ab81e50b9d 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -3871,18 +3871,6 @@ async fn edit_datatable_config( } } - // Every entry this save removes, derived rather than taken from `deleted_datatables`: that list - // is a hint the settings-sync CLI never sends, and the stranded-pointer warning and the stream - // bounce below must run for a removal whether or not the caller named it. - let removed: Vec = old_datatables - .keys() - .filter(|name| { - !new_config.settings.datatables.contains_key(*name) - && !new_config.renames.iter().any(|r| &r.from == *name) - }) - .cloned() - .collect(); - let config: serde_json::Value = serde_json::to_value(new_config.settings) .map_err(|err| Error::internal_err(err.to_string()))?; @@ -3922,7 +3910,7 @@ async fn edit_datatable_config( // A deletion cannot be followed the same way — there is nothing to point at any more. Read who // is left stranded so the caller is told, the way deleting a workspace does. let mut stranded: Vec = Vec::new(); - for name in &removed { + for name in &new_config.deleted_datatables { let rows = sqlx::query!( r#"SELECT ws.workspace_id AS "workspace_id!", dt.key AS "datatable!" FROM workspace_settings ws @@ -3942,23 +3930,6 @@ async fn edit_datatable_config( ); } - // A stream reading a deleted entry keeps the connection it opened while the entry resolved — - // this workspace's own, and every fork's through its pointer. Bounced in this transaction, so - // a listener that reconnects finds the entry gone instead of streaming on. - crate::datatable_permissions::restart_streams_named( - &mut *tx, - removed - .iter() - .map(|name| (w_id.clone(), name.clone())) - .chain( - stranded - .iter() - .map(|s| (s.workspace_id.clone(), s.datatable.clone())), - ) - .collect(), - ) - .await?; - tx.commit().await?; for substrate in created_substrates { diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index 371b80555c..1cb5188e5b 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -995,6 +995,21 @@ pub(crate) async fn delete_workspace( // but the destructive cleanup itself runs only after the commit below: a delete that // fails mid-way must never leave a live workspace with its fork data destroyed and no // registry row to retry from. Read-only: nothing is dropped here. + // Read before the delete: another workspace's data table entry can point at one of this + // workspace's, and deleting the workspace it names leaves that pointer resolving to nothing. + // Nothing sweeps them — turning them back into copies would hand each fork the database + // outright — so the deleter is told which data tables they just stranded. + let stranded_pointers = sqlx::query!( + r#"SELECT ws.workspace_id AS "workspace_id!", dt.key AS "datatable!" + FROM workspace_settings ws + CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt + WHERE dt.value->'reference'->>'workspace_id' = $1 + ORDER BY ws.workspace_id, dt.key"#, + &w_id, + ) + .fetch_all(&db) + .await + .unwrap_or_default(); let fork_ducklake_cleanups = prepare_fork_ducklake_cleanups(&db, &w_id, None) .await @@ -1250,33 +1265,6 @@ pub(crate) async fn delete_workspace( None, ) .await?; - // Who points here, read after the delete above and inside this transaction. A fork writes its - // pointer in the transaction that inserts it, and that insert key-share locks this row through - // the parent foreign key: so the fork either committed before the delete and is seen here, or - // waits on it and then fails on the missing parent. No pointer can escape this list. Nothing - // sweeps them afterwards — turning them back into copies would hand each fork the database - // outright — so the deleter is told which data tables they stranded. - let stranded_pointers = sqlx::query!( - r#"SELECT ws.workspace_id AS "workspace_id!", dt.key AS "datatable!" - FROM workspace_settings ws - CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt - WHERE dt.value->'reference'->>'workspace_id' = $1 - ORDER BY ws.workspace_id, dt.key"#, - &w_id, - ) - .fetch_all(&mut *tx) - .await?; - // Each fork pointing here keeps the replication stream it opened while this workspace - // governed it. Bounced in this transaction, so a listener that reconnects finds its pointer - // dangling instead of streaming on. - crate::datatable_permissions::restart_streams_named( - &mut *tx, - stranded_pointers - .iter() - .map(|r| (r.workspace_id.clone(), r.datatable.clone())) - .collect(), - ) - .await?; tx.commit().await?; // Physical ducklake-namespace cleanup, post-commit, from the pre-read snapshot: fork diff --git a/backend/windmill-trigger-postgres/src/lib.rs b/backend/windmill-trigger-postgres/src/lib.rs index c77f7ce0d3..41e82614a8 100644 --- a/backend/windmill-trigger-postgres/src/lib.rs +++ b/backend/windmill-trigger-postgres/src/lib.rs @@ -11,10 +11,7 @@ use serde::{Deserialize, Deserializer, Serialize}; use serde_json::value::RawValue; use sqlx::FromRow; use windmill_api_auth::ApiAuthed; -use windmill_common::workspaces::{ - ensure_datatable_admin_access, get_datatable_replication_resource_from_db_unchecked, - DatatableAccess, -}; +use windmill_common::workspaces::get_datatable_replication_resource_from_db_unchecked; use windmill_common::{ db::UserDB, error::{to_anyhow, Error, Result}, @@ -386,15 +383,20 @@ pub async fn resolve_postgres_resource( ) -> Result { if let Some(datatable_name) = postgres_resource_path.strip_prefix("datatable://") { // A replication stream reads every row of every table whatever the data table's roles - // grant, so it is not something a role can be tenanted into: only someone who could have - // connected as `admin` may open one. - ensure_datatable_admin_access( - db, - w_id, - datatable_name, - &DatatableAccess::Authed(authed.to_authed_ref()), - ) - .await?; + // grant, so the two don't mix: a data table under roles takes no triggers or captures, and + // roles cannot be turned on while one is enabled on it. + if windmill_common::workspaces::resolve_governing_datatable(db, w_id, datatable_name) + .await? + .datatable + .permissions + .is_some() + { + return Err(Error::BadRequest(format!( + "Data table '{datatable_name}' is under roles, and a Postgres trigger or capture \ + cannot read one: a replication stream sees every row whatever the roles grant. \ + Turn its roles off to stream it." + ))); + } // Trigger connections (publication/slot management + logical replication) run // as the dedicated replication user on custom-instance databases. let resource_value = From 7279ee15e2dd19c15ffd3a169d880b961fa74107 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Fri, 11 Sep 2026 10:45:55 +0200 Subject: [PATCH 52/77] fix(datatables): refuse a Postgres trigger on a data table under roles when it is saved Creating or editing a trigger that points at a data table under roles was accepted, and its listener then retried the refused connection every 30 seconds forever. The save is now refused, and a trigger that reaches such a data table anyway (re-enabled, or cloned into a fork) is disabled by its listener with the reason, as a missing replication slot is. Co-Authored-By: Claude Opus 5 (1M context) --- .../windmill-trigger-postgres/src/handler.rs | 12 +++++- backend/windmill-trigger-postgres/src/lib.rs | 42 ++++++++++++------- .../windmill-trigger-postgres/src/listener.rs | 15 +++++-- 3 files changed, 50 insertions(+), 19 deletions(-) diff --git a/backend/windmill-trigger-postgres/src/handler.rs b/backend/windmill-trigger-postgres/src/handler.rs index 92dd2e1e19..6bc23514dd 100644 --- a/backend/windmill-trigger-postgres/src/handler.rs +++ b/backend/windmill-trigger-postgres/src/handler.rs @@ -25,7 +25,8 @@ use windmill_trigger::{Trigger, TriggerCrud, TriggerData}; use super::{ check_if_valid_publication_for_postgres_version, create_logical_replication_slot, - create_pg_publication, drop_publication, generate_random_string, get_default_pg_connection, + create_pg_publication, drop_publication, ensure_not_under_roles, generate_random_string, + get_default_pg_connection, mapper::{Mapper, MappingInfo}, PostgresConfig, PostgresConfigRequest, PostgresPublicationReplication, PostgresTrigger, PublicationData, Relations, Slot, SlotList, TableToTrack, TemplateScript, TestPostgresConfig, @@ -64,6 +65,15 @@ impl TriggerCrud for PostgresTrigger { DeployedObject::PostgresTrigger { path, parent_path } } + async fn validate_config( + &self, + db: &DB, + config: &Self::TriggerConfigRequest, + workspace_id: &str, + ) -> Result<()> { + ensure_not_under_roles(db, workspace_id, &config.postgres_resource_path).await + } + async fn create_trigger( &self, db: &DB, diff --git a/backend/windmill-trigger-postgres/src/lib.rs b/backend/windmill-trigger-postgres/src/lib.rs index 41e82614a8..25b19e6c1b 100644 --- a/backend/windmill-trigger-postgres/src/lib.rs +++ b/backend/windmill-trigger-postgres/src/lib.rs @@ -374,6 +374,32 @@ pub async fn get_raw_postgres_connection( Ok(client) } +/// A replication stream reads every row of every table whatever the data table's roles grant, so +/// the two don't mix: a data table under roles takes no triggers or captures, and roles cannot be +/// turned on while one is enabled on it. +pub async fn ensure_not_under_roles( + db: &DB, + w_id: &str, + postgres_resource_path: &str, +) -> Result<()> { + let Some(datatable_name) = postgres_resource_path.strip_prefix("datatable://") else { + return Ok(()); + }; + if windmill_common::workspaces::resolve_governing_datatable(db, w_id, datatable_name) + .await? + .datatable + .permissions + .is_some() + { + return Err(Error::BadRequest(format!( + "Data table '{datatable_name}' is under roles, and a Postgres trigger or capture \ + cannot read one: a replication stream sees every row whatever the roles grant. \ + Turn its roles off to stream it." + ))); + } + Ok(()) +} + pub async fn resolve_postgres_resource( authed: &ApiAuthed, user_db: Option, @@ -382,21 +408,7 @@ pub async fn resolve_postgres_resource( w_id: &str, ) -> Result { if let Some(datatable_name) = postgres_resource_path.strip_prefix("datatable://") { - // A replication stream reads every row of every table whatever the data table's roles - // grant, so the two don't mix: a data table under roles takes no triggers or captures, and - // roles cannot be turned on while one is enabled on it. - if windmill_common::workspaces::resolve_governing_datatable(db, w_id, datatable_name) - .await? - .datatable - .permissions - .is_some() - { - return Err(Error::BadRequest(format!( - "Data table '{datatable_name}' is under roles, and a Postgres trigger or capture \ - cannot read one: a replication stream sees every row whatever the roles grant. \ - Turn its roles off to stream it." - ))); - } + ensure_not_under_roles(db, w_id, postgres_resource_path).await?; // Trigger connections (publication/slot management + logical replication) run // as the dedicated replication user on custom-instance databases. let resource_value = diff --git a/backend/windmill-trigger-postgres/src/listener.rs b/backend/windmill-trigger-postgres/src/listener.rs index 14d348b5f2..876b75661e 100644 --- a/backend/windmill-trigger-postgres/src/listener.rs +++ b/backend/windmill-trigger-postgres/src/listener.rs @@ -20,7 +20,8 @@ use windmill_common::{ use windmill_trigger::{listener::ListeningTrigger, trigger_helpers::TriggerJobArgs, Listener}; use super::{ - drop_publication, get_default_pg_connection, get_raw_postgres_connection, + drop_publication, ensure_not_under_roles, get_default_pg_connection, + get_raw_postgres_connection, handler::drop_logical_replication_slot, relation::RelationConverter, replication_message::{ @@ -135,8 +136,8 @@ impl PostgresSimpleClient { /// Resolves the Postgres resource, validates that the configured publication and /// replication slot still exist, and opens a fresh logical replication stream. /// -/// Returns `Error::BadConfig` when the publication or slot is missing (an -/// unrecoverable misconfiguration). Any other error is treated as transient +/// Returns `Error::BadConfig` when the publication or slot is missing, or the +/// data table is under roles (unrecoverable misconfigurations). Any other error is treated as transient /// (connection refused, network interruption, ...) and is retried by the caller. /// The resource is re-resolved on every call so credential rotations are picked /// up across reconnections. @@ -149,6 +150,14 @@ async fn connect_logical_replication_stream( let PostgresConfig { postgres_resource_path, publication_name, replication_slot_name, .. } = trigger_config; + // Retrying cannot lift roles, so this disables the trigger like a missing slot does. + ensure_not_under_roles(db, workspace_id, postgres_resource_path) + .await + .map_err(|e| match e { + Error::BadRequest(msg) => Error::BadConfig(msg), + e => e, + })?; + let database = resolve_postgres_resource( authed, Some(UserDB::new(db.clone())), From 2f95c22242e2a6e5e4e9e0777db3aea736a4ac22 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Fri, 11 Sep 2026 11:34:51 +0200 Subject: [PATCH 53/77] fix(datatables): disable a data table role before deleting it Deleting a role reassigns and drops what it owns in each registered database on its own connection, and each of those passes commits as it goes. A database failing part-way left the role enabled in the catalog and able to log in, but already stripped in the databases reached before it. The role is now disabled in its own commit first, so a failed delete leaves a disabled role to retry. Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/datatable_roles.rs | 70 +++++++++++++++++++ backend/windmill-api-settings/src/lib.rs | 54 ++++++++++---- .../windmill-common/src/datatable_roles.rs | 4 +- 3 files changed, 115 insertions(+), 13 deletions(-) diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs index e619e73f23..f709637750 100644 --- a/backend/windmill-api-integration-tests/tests/datatable_roles.rs +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -407,6 +407,76 @@ async fn concurrent_role_creations_both_survive(db: Pool) -> anyhow::R outcome } +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_role_delete_that_fails_part_way_leaves_the_role_disabled( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let suffix: String = uuid::Uuid::new_v4().simple().to_string()[..8].to_string(); + let name = format!("wmtest_del_{suffix}"); + + let outcome = async { + let created: Value = authed( + client().post(format!( + "http://localhost:{port}/api/settings/datatable_roles" + )), + "SECRET_TOKEN", + ) + .json(&json!({ "name": name })) + .send() + .await? + .error_for_status()? + .json() + .await?; + let id = created["id"].as_str().unwrap().to_string(); + + // Each database's pass commits on its own, so one that cannot be reached fails the delete + // after the others may already have stripped the role. + sqlx::query( + "UPDATE global_settings SET value = jsonb_set(value, '{databases,wm_unreachable}', '{}') + WHERE name = 'custom_instance_pg_databases'", + ) + .execute(&db) + .await?; + + let resp = authed( + client().delete(format!( + "http://localhost:{port}/api/settings/datatable_roles/{id}" + )), + "SECRET_TOKEN", + ) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, 400, "{body}"); + + let catalog = windmill_common::datatable_roles::read_role_catalog(&db).await?; + let role = catalog + .get(&id) + .expect("a failed delete keeps the entry to retry"); + assert!( + !role.enabled, + "a half-deleted role is still enabled in the catalog" + ); + let can_login: bool = + sqlx::query_scalar("SELECT rolcanlogin FROM pg_roles WHERE rolname = $1") + .bind(&name) + .fetch_one(&db) + .await?; + assert!(!can_login, "a half-deleted role can still log in"); + Ok::<_, anyhow::Error>(()) + } + .await; + + let _ = sqlx::query(&format!("DROP ROLE IF EXISTS \"{name}\"")) + .execute(&db) + .await; + outcome +} + #[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] async fn renaming_a_governing_data_table_carries_its_forks( db: Pool, diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index fc3dddde2a..d8bcf0174f 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -151,7 +151,10 @@ pub fn global_service() -> Router { "/list_custom_instance_pg_databases", post(list_custom_instance_pg_databases), ) - .route("/datatable_roles", get(list_datatable_roles).post(create_datatable_role)) + .route( + "/datatable_roles", + get(list_datatable_roles).post(create_datatable_role), + ) .route( "/datatable_roles/{id}", post(update_datatable_role).delete(delete_datatable_role), @@ -2690,7 +2693,11 @@ async fn create_datatable_role( ) .await?; - Ok(Json(DatatableRoleInfo { id, name: req.name, enabled: true })) + Ok(Json(DatatableRoleInfo { + id, + name: req.name, + enabled: true, + })) } async fn update_datatable_role( @@ -2753,24 +2760,45 @@ async fn update_datatable_role( })) } -/// Drop the Postgres role, then forget it, then strip it from every workspace that tenanted it. +/// Disable the role in its own commit, then drop the Postgres role, then forget it, then strip it +/// from every workspace that tenanted it. /// -/// Dropping first is what makes the catalog trustworthy: the drop refuses while any instance -/// database is unreachable, so a failure leaves the entry in place to retry rather than a live -/// Postgres login nothing names. +/// Dropping before forgetting is what makes the catalog trustworthy: the drop refuses while any +/// instance database is unreachable, so a failure leaves the entry in place to retry rather than a +/// live Postgres login nothing names. async fn delete_datatable_role( authed: ApiAuthed, Extension(db): Extension, Path(id): Path, ) -> JsonResult<()> { require_super_admin(&db, &authed).await?; + let find = |catalog: &windmill_common::datatable_roles::DatatableRoleCatalog| { + catalog + .get(&id) + .cloned() + .ok_or_else(|| error::Error::NotFound(format!("No data table role with id '{id}'"))) + }; + let mut tx = db.begin().await?; windmill_common::datatable_roles::lock_role_catalog(&mut tx).await?; - let catalog = windmill_common::datatable_roles::read_role_catalog_tx(&mut tx).await?; - let role = catalog - .get(&id) - .cloned() - .ok_or_else(|| error::Error::NotFound(format!("No data table role with id '{id}'")))?; + let mut role = find(&windmill_common::datatable_roles::read_role_catalog_tx(&mut tx).await?)?; + if role.enabled { + windmill_common::datatable_roles::set_instance_role_login(&mut tx, &role.name, false) + .await?; + role.enabled = false; + windmill_common::datatable_roles::update_role_catalog_entry(&mut tx, &id, &role).await?; + } + tx.commit().await?; + + let mut tx = db.begin().await?; + windmill_common::datatable_roles::lock_role_catalog(&mut tx).await?; + let role = find(&windmill_common::datatable_roles::read_role_catalog_tx(&mut tx).await?)?; + if role.enabled { + return Err(error::Error::BadRequest(format!( + "Data table role '{}' was re-enabled while being deleted", + role.name + ))); + } windmill_common::datatable_roles::drop_instance_role(&db, &mut tx, &role.name).await?; windmill_common::datatable_roles::delete_role_catalog_entry(&mut tx, &id).await?; @@ -2810,7 +2838,9 @@ async fn converge_connect_grants_everywhere( windmill_common::datatable_roles::converge_connect_grants_with(db, &dbname, catalog) .await { - tracing::warn!("Could not converge CONNECT grants on instance database '{dbname}': {e}"); + tracing::warn!( + "Could not converge CONNECT grants on instance database '{dbname}': {e}" + ); } } } diff --git a/backend/windmill-common/src/datatable_roles.rs b/backend/windmill-common/src/datatable_roles.rs index ab5e519c3a..07e3e16114 100644 --- a/backend/windmill-common/src/datatable_roles.rs +++ b/backend/windmill-common/src/datatable_roles.rs @@ -403,7 +403,9 @@ pub async fn rename_instance_role( /// /// The per-database passes open their own connections and cannot join `tx`; the lock is what keeps /// a concurrent mutation out while they run. Only the final `DROP ROLE` is on `tx`, so it commits -/// or rolls back with the catalog write that forgets the role. +/// or rolls back with the catalog write that forgets the role. Those passes commit as they go, so +/// callers MUST have disabled the role in an earlier committed transaction: a failure part-way +/// then leaves a disabled role to retry, not an enabled one already stripped in some databases. pub async fn drop_instance_role( db: &DB, tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, From 122a3bd16422bf317161cb0e76f171eb777dc7d1 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Fri, 11 Sep 2026 11:59:27 +0200 Subject: [PATCH 54/77] fix(datatables): serialize roles going on with a stream starting Turning roles on looked for enabled triggers and live captures once, without a lock anything starting a stream also took. A trigger enabled in that window could have its listener connect before roles committed, and a healthy listener never checks again. Both transitions now serialize on one advisory lock: roles going on hold it exclusive while they look, and trigger create, edit and enable, and capture setup and ping hold it shared while they commit. Either the look sees the stream, or the listener connects after roles are committed and refuses. Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/datatable_roles.rs | 54 +++++++++++++++++++ .../src/datatable_permissions.rs | 1 + backend/windmill-api/src/capture.rs | 6 +++ .../windmill-common/src/datatable_roles.rs | 16 ++++++ .../windmill-trigger-postgres/src/handler.rs | 18 ++++++- 5 files changed, 94 insertions(+), 1 deletion(-) diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs index f709637750..9179095c6a 100644 --- a/backend/windmill-api-integration-tests/tests/datatable_roles.rs +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -798,3 +798,57 @@ async fn roles_cannot_be_turned_on_while_a_trigger_streams_the_data_table( assert_eq!(resp.status(), 200, "{}", resp.text().await?); Ok(()) } + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn roles_going_on_wait_for_a_trigger_being_enabled(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + sqlx::query( + "UPDATE workspace_settings SET datatable = datatable #- '{datatables,main,permissions}' + WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + + // A trigger enable in flight: it holds the stream lock and its row is not committed yet, so a + // roles save that looked for streams now would miss it and its listener would connect to a + // data table it is about to be refused. + let mut enabling = db.begin().await?; + windmill_common::datatable_roles::lock_datatable_streams(&mut *enabling, false).await?; + sqlx::query( + r#"INSERT INTO postgres_trigger (path, script_path, is_flow, workspace_id, edited_by, + postgres_resource_path, replication_slot_name, publication_name, permissioned_as, mode) + VALUES ('u/test-user-2/racing_stream', 'u/test-user-2/s', false, 'wm-fork-dt', + 'test-user-2', 'datatable://main', 'slot_race', 'pub_race', 'u/test-user-2', + 'enabled')"#, + ) + .execute(&mut *enabling) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let url = format!( + "http://localhost:{}/api/w/test-workspace/workspaces/datatable_permissions/main", + server.addr.port() + ); + let save = tokio::spawn( + authed(client().post(&url), "SECRET_TOKEN") + .json(&json!({"permissioned": true, "default_role": "admin", + "roles": [{"id": "admin", "tenants": ["*"]}]})) + .send(), + ); + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + assert!( + !save.is_finished(), + "roles went on while a trigger was being enabled" + ); + enabling.commit().await?; + + let resp = save.await??; + assert_eq!(resp.status(), 400); + assert!( + resp.text() + .await? + .contains("wm-fork-dt/u/test-user-2/racing_stream"), + "the roles save missed the trigger enabled while it waited" + ); + Ok(()) +} diff --git a/backend/windmill-api-workspaces/src/datatable_permissions.rs b/backend/windmill-api-workspaces/src/datatable_permissions.rs index b04fb209e7..4014e81f1e 100644 --- a/backend/windmill-api-workspaces/src/datatable_permissions.rs +++ b/backend/windmill-api-workspaces/src/datatable_permissions.rs @@ -349,6 +349,7 @@ async fn set_datatable_permissions( // Turning roles on is refused while a replication stream reads this data table. One already // under roles cannot have any: the listener refuses to open a stream on it. if req.permissioned && governing.datatable.permissions.is_none() { + windmill_common::datatable_roles::lock_datatable_streams(&mut *tx, true).await?; ensure_no_streams_reaching(&db, &governing).await?; } diff --git a/backend/windmill-api/src/capture.rs b/backend/windmill-api/src/capture.rs index 2e2058ea80..6255b5eb13 100644 --- a/backend/windmill-api/src/capture.rs +++ b/backend/windmill-api/src/capture.rs @@ -567,6 +567,9 @@ async fn set_config( }; let mut tx = user_db.begin(&authed).await?; + if matches!(nc.trigger_kind, TriggerKind::Postgres) { + windmill_common::datatable_roles::lock_datatable_streams(&mut *tx, false).await?; + } sqlx::query!( r#" @@ -614,6 +617,9 @@ async fn ping_config( )>, ) -> Result<()> { let mut tx = user_db.begin(&authed).await?; + if matches!(trigger_kind, TriggerKind::Postgres) { + windmill_common::datatable_roles::lock_datatable_streams(&mut *tx, false).await?; + } sqlx::query!( r#" diff --git a/backend/windmill-common/src/datatable_roles.rs b/backend/windmill-common/src/datatable_roles.rs index 07e3e16114..3a7709ccd1 100644 --- a/backend/windmill-common/src/datatable_roles.rs +++ b/backend/windmill-common/src/datatable_roles.rs @@ -127,6 +127,22 @@ pub async fn lock_role_catalog(tx: &mut sqlx::Transaction<'_, sqlx::Postgres>) - Ok(()) } +/// A replication stream reads every row whatever a data table's roles grant. Turning roles on looks +/// for streams holding this exclusive; whatever can start a Postgres trigger or capture streaming +/// holds it shared on the transaction that commits it. So either the look sees the stream, or the +/// stream's listener connects after roles are committed and refuses. Held for the transaction. +pub async fn lock_datatable_streams(conn: &mut sqlx::PgConnection, exclusive: bool) -> Result<()> { + let lock = if exclusive { + "pg_advisory_xact_lock" + } else { + "pg_advisory_xact_lock_shared" + }; + sqlx::query(&format!("SELECT {lock}(hashtext('datatable_streams'))")) + .execute(conn) + .await?; + Ok(()) +} + /// Disclosure: returns every role's stored Postgres password in plaintext. Any server path that /// has to resolve or name a role may call it — including handlers open to a workspace member, who /// need the names — but callers MUST NOT let `pwd` reach a response, a log line, an audit record diff --git a/backend/windmill-trigger-postgres/src/handler.rs b/backend/windmill-trigger-postgres/src/handler.rs index 6bc23514dd..4294645b89 100644 --- a/backend/windmill-trigger-postgres/src/handler.rs +++ b/backend/windmill-trigger-postgres/src/handler.rs @@ -21,7 +21,7 @@ use windmill_common::{ use windmill_git_sync::DeployedObject; use windmill_api_auth::{check_scopes, ApiAuthed}; -use windmill_trigger::{Trigger, TriggerCrud, TriggerData}; +use windmill_trigger::{Trigger, TriggerCrud, TriggerData, TriggerMode}; use super::{ check_if_valid_publication_for_postgres_version, create_logical_replication_slot, @@ -74,6 +74,20 @@ impl TriggerCrud for PostgresTrigger { ensure_not_under_roles(db, workspace_id, &config.postgres_resource_path).await } + async fn authorize_set_trigger_mode( + &self, + _authed: &ApiAuthed, + tx: &mut PgConnection, + _workspace_id: &str, + _path: &str, + mode: &TriggerMode, + ) -> Result<()> { + if *mode != TriggerMode::Disabled { + windmill_common::datatable_roles::lock_datatable_streams(tx, false).await?; + } + Ok(()) + } + async fn create_trigger( &self, db: &DB, @@ -82,6 +96,7 @@ impl TriggerCrud for PostgresTrigger { w_id: &str, trigger: TriggerData, ) -> Result<()> { + windmill_common::datatable_roles::lock_datatable_streams(&mut *tx, false).await?; let resolved_edited_by = trigger.base.resolve_edited_by(authed); let resolved_permissioned_as = trigger.base.resolve_permissioned_as(authed); let Self::TriggerConfigRequest { @@ -171,6 +186,7 @@ impl TriggerCrud for PostgresTrigger { path: &str, trigger: TriggerData, ) -> Result<()> { + windmill_common::datatable_roles::lock_datatable_streams(&mut *tx, false).await?; let resolved_edited_by = trigger.base.resolve_edited_by(authed); let resolved_permissioned_as = trigger.base.resolve_permissioned_as(authed); let Self::TriggerConfigRequest { From df471e37d8db477c0832c94c0e9359b95b1298e8 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Fri, 11 Sep 2026 12:28:48 +0200 Subject: [PATCH 55/77] fix(datatables): wait out live listeners, and resolve stored names containing `?` Turning roles on counted a trigger as gone once disabled, and a capture once its client stopped pinging, but the listener keeps its replication connection until its next heartbeat notices. A trigger or capture whose listener pinged in the last 15 seconds, the window a server holds a listener for, now still counts as streaming. Data table names could contain `?` before they were restricted, and such entries are still stored. Splitting `?role=` off a reference misread them: `a?b` became `a` with an unknown parameter, and the clone checks looked at a different entry than the one copied. An entry stored under the whole reference is now looked up first, in the Postgres executor, DuckDB ATTACH and the clone checks. Agent workers cannot read the workspace and keep the strict parse, which refuses such a name rather than misreading it. Co-Authored-By: Claude Opus 5 (1M context) --- ...8961cfbd0171f835faa756762919c021bfd2b.json | 24 ---- .../tests/datatable_roles.rs | 54 ++++++++- .../src/datatable_permissions.rs | 27 +++-- .../windmill-api-workspaces/src/workspaces.rs | 20 ++-- backend/windmill-common/src/workspaces.rs | 34 ++++-- .../windmill-worker/src/duckdb_executor.rs | 110 +++++++++--------- backend/windmill-worker/src/pg_executor.rs | 15 +-- 7 files changed, 164 insertions(+), 120 deletions(-) delete mode 100644 backend/.sqlx/query-bc87664b87a84e1033589f729148961cfbd0171f835faa756762919c021bfd2b.json diff --git a/backend/.sqlx/query-bc87664b87a84e1033589f729148961cfbd0171f835faa756762919c021bfd2b.json b/backend/.sqlx/query-bc87664b87a84e1033589f729148961cfbd0171f835faa756762919c021bfd2b.json deleted file mode 100644 index b452b6fc16..0000000000 --- a/backend/.sqlx/query-bc87664b87a84e1033589f729148961cfbd0171f835faa756762919c021bfd2b.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT workspace_id || '/' || path AS \"stream!\" FROM postgres_trigger\n WHERE workspace_id = $1 AND mode <> 'disabled'::TRIGGER_MODE\n AND (postgres_resource_path = $2 OR starts_with(postgres_resource_path, $3))\n UNION ALL\n SELECT workspace_id || '/' || path || ' (capture)' FROM capture_config\n WHERE workspace_id = $1 AND trigger_kind = 'postgres'\n AND last_client_ping > now() - interval '10 seconds'\n AND (trigger_config->>'postgres_resource_path' = $2\n OR starts_with(trigger_config->>'postgres_resource_path', $3))", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "stream!", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "bc87664b87a84e1033589f729148961cfbd0171f835faa756762919c021bfd2b" -} diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs index 9179095c6a..719b8e42db 100644 --- a/backend/windmill-api-integration-tests/tests/datatable_roles.rs +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -786,8 +786,27 @@ async fn roles_cannot_be_turned_on_while_a_trigger_streams_the_data_table( "the refusal does not name the trigger to disable" ); + // Disabled, but its listener pinged just now and stops only at its next heartbeat. sqlx::query( - "UPDATE postgres_trigger SET mode = 'disabled' WHERE path = 'u/test-user-2/fork_stream'", + "UPDATE postgres_trigger SET mode = 'disabled', server_id = NULL, last_server_ping = now() + WHERE path = 'u/test-user-2/fork_stream'", + ) + .execute(&db) + .await?; + let resp = authed(client().post(&url), "SECRET_TOKEN") + .json(&turn_on) + .send() + .await?; + assert_eq!( + resp.status(), + 400, + "roles went on while a disabled trigger's listener was still attached: {}", + resp.text().await? + ); + + sqlx::query( + "UPDATE postgres_trigger SET last_server_ping = now() - interval '20 seconds' + WHERE path = 'u/test-user-2/fork_stream'", ) .execute(&db) .await?; @@ -852,3 +871,36 @@ async fn roles_going_on_wait_for_a_trigger_being_enabled(db: Pool) -> ); Ok(()) } + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_stored_name_containing_a_question_mark_resolves_as_itself( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + // Names could contain `?` before they were restricted, and such an entry is still stored. + sqlx::query( + "UPDATE workspace_settings + SET datatable = jsonb_set(datatable, '{datatables,legacy?dt}', datatable->'datatables'->'main') + WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + + let resolve = |reference: &'static str| { + let db = db.clone(); + async move { + windmill_common::workspaces::parse_datatable_ref_for(&db, "test-workspace", reference) + .await + } + }; + assert_eq!(resolve("legacy?dt").await?, ("legacy?dt".to_string(), None)); + assert_eq!( + resolve("main?role=analytics").await?, + ("main".to_string(), Some("analytics".to_string())) + ); + assert!( + resolve("main?dt").await.is_err(), + "an unknown parameter was ignored" + ); + Ok(()) +} diff --git a/backend/windmill-api-workspaces/src/datatable_permissions.rs b/backend/windmill-api-workspaces/src/datatable_permissions.rs index 4014e81f1e..f774448d64 100644 --- a/backend/windmill-api-workspaces/src/datatable_permissions.rs +++ b/backend/windmill-api-workspaces/src/datatable_permissions.rs @@ -481,23 +481,28 @@ async fn ensure_no_streams_reaching(db: &DB, governing: &GoverningDatatable) -> for (w_id, name) in reached { let reference = format!("datatable://{name}"); let with_query = format!("{reference}?"); - // A suspended trigger keeps its listener, so only a disabled one is not streaming; a - // capture streams for as long as its client keeps pinging. + // A suspended trigger keeps its listener, and a capture streams while its client pings. A + // listener also outlives its trigger being disabled, or its capture's client going quiet, + // until its next heartbeat notices; one that pinged within the 15 seconds a server holds a + // listener for may still be dispatching. streams.extend( - sqlx::query_scalar!( - r#"SELECT workspace_id || '/' || path AS "stream!" FROM postgres_trigger - WHERE workspace_id = $1 AND mode <> 'disabled'::TRIGGER_MODE + sqlx::query_scalar::<_, String>( + r#"SELECT workspace_id || '/' || path FROM postgres_trigger + WHERE workspace_id = $1 + AND (mode <> 'disabled'::TRIGGER_MODE + OR last_server_ping > now() - interval '15 seconds') AND (postgres_resource_path = $2 OR starts_with(postgres_resource_path, $3)) UNION ALL SELECT workspace_id || '/' || path || ' (capture)' FROM capture_config WHERE workspace_id = $1 AND trigger_kind = 'postgres' - AND last_client_ping > now() - interval '10 seconds' + AND (last_client_ping > now() - interval '10 seconds' + OR last_server_ping > now() - interval '15 seconds') AND (trigger_config->>'postgres_resource_path' = $2 OR starts_with(trigger_config->>'postgres_resource_path', $3))"#, - &w_id, - &reference, - &with_query, ) + .bind(&w_id) + .bind(&reference) + .bind(&with_query) .fetch_all(db) .await?, ); @@ -505,8 +510,8 @@ async fn ensure_no_streams_reaching(db: &DB, governing: &GoverningDatatable) -> if !streams.is_empty() { return Err(Error::BadRequest(format!( "Data table '{}' cannot be put under roles while a Postgres trigger or capture streams \ - it: a replication stream reads every row whatever the roles grant. Disable them \ - first: {}", + it: a replication stream reads every row whatever the roles grant. Disable them, then \ + allow their listeners up to 15 seconds to stop: {}", governing.name, streams.join(", ") ))); diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index ab81e50b9d..e5a1690c7b 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -45,12 +45,12 @@ use windmill_common::workspaces::GitRepositorySettings; #[cfg(feature = "enterprise")] use windmill_common::workspaces::WorkspaceDeploymentUISettings; use windmill_common::workspaces::{ - check_deploy_rules, check_user_against_rule, datatable_ref_name, - get_datatable_resource_from_db, get_datatable_resource_from_db_unchecked, - resolve_governing_datatable, validate_dev_workspace_id, validate_fork_workspace_id, - validate_workspace_name, DataTable, DataTableCatalogResourceType, DataTableForkBehavior, - DatatableAccess, GoverningDatatable, ProtectionRuleKind, ProtectionRules, ProtectionRuleset, - RuleCheckResult, WorkspaceGitSyncSettings, DEV_WORKSPACE_LOCK_RULE_NAME, + check_deploy_rules, check_user_against_rule, get_datatable_resource_from_db, + get_datatable_resource_from_db_unchecked, parse_datatable_ref_for, resolve_governing_datatable, + validate_dev_workspace_id, validate_fork_workspace_id, validate_workspace_name, DataTable, + DataTableCatalogResourceType, DataTableForkBehavior, DatatableAccess, GoverningDatatable, + ProtectionRuleKind, ProtectionRules, ProtectionRuleset, RuleCheckResult, + WorkspaceGitSyncSettings, DEV_WORKSPACE_LOCK_RULE_NAME, }; use windmill_common::workspaces::{Ducklake, DucklakeCatalogResourceType}; use windmill_common::PgDatabase; @@ -3313,8 +3313,8 @@ async fn create_pg_database( // database that no data table entry names. Refuse here too, so the clone stops before one // exists rather than leaving an empty registered `wm_fork_…` behind. if let Some(reference) = req.source.strip_prefix("datatable://") { - let name = datatable_ref_name(reference); - ensure_datatable_is_clonable(&db, &w_id, name).await?; + let (name, _) = parse_datatable_ref_for(&db, &w_id, reference).await?; + ensure_datatable_is_clonable(&db, &w_id, &name).await?; } // Non-superadmin: restrict dbname to wm_fork_ prefix @@ -3457,8 +3457,8 @@ async fn import_pg_database( } if let Some(reference) = req.source.strip_prefix("datatable://") { - let name = datatable_ref_name(reference); - ensure_datatable_is_clonable(&db, &w_id, name).await?; + let (name, _) = parse_datatable_ref_for(&db, &w_id, reference).await?; + ensure_datatable_is_clonable(&db, &w_id, &name).await?; } if req.fork_behavior == DataTableForkBehavior::SchemaAndData { diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index bddf85f0c1..0e9a7ed3c9 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -2106,13 +2106,30 @@ pub fn strip_datatable_permissions( Some(datatable) } -/// The data table a `datatable://` reference names, ignoring its query string. For callers that -/// only need to find the entry; use [`parse_datatable_ref`] wherever the role is acted on. -pub fn datatable_ref_name(reference: &str) -> &str { - reference - .split_once('?') - .map(|(name, _)| name) - .unwrap_or(reference) +/// As [`parse_datatable_ref`], except that an entry whose stored name itself contains `?` — which +/// names could before they were restricted — resolves by that exact name, without a role. It is +/// looked up first, so `sales?role=x` never reaches a different entry than the one stored so. +pub async fn parse_datatable_ref_for( + db: &DB, + w_id: &str, + reference: &str, +) -> Result<(String, Option)> { + if reference.contains('?') { + let exists = sqlx::query_scalar::<_, Option>( + "SELECT (datatable->'datatables') ? $2 FROM workspace_settings WHERE workspace_id = $1", + ) + .bind(w_id) + .bind(reference) + .fetch_optional(db) + .await? + .flatten() + .unwrap_or(false); + if exists { + return Ok((reference.to_string(), None)); + } + } + let (name, role) = parse_datatable_ref(reference)?; + Ok((name.to_string(), role.map(str::to_string))) } /// Split a `datatable://` reference into its name and the role its query string names. @@ -3431,9 +3448,6 @@ mod tests { "silently ignored: {malformed}" ); } - - // The name-only helper stays lenient — it is used where the role is never acted on. - assert_eq!(datatable_ref_name("sales?role="), "sales"); } #[test] diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index 764859d29b..bc5660b5fb 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -2602,31 +2602,28 @@ fn fork_defer_statements( } struct AttachedDatatable<'a> { - name: &'a str, - role: Option<&'a str>, + /// The data table reference, query string included; a bare `datatable` is `main`. + reference: String, alias: &'a str, } /// `ATTACH 'datatable[://][?role=]' AS `. A bare `datatable` names the default -/// data table, so the role query string has to be accepted with and without an explicit name. -fn parse_attach_datatable(query: &str) -> Result>> { +/// data table, so the role query string has to be accepted with and without an explicit name. The +/// reference is split only once the workspace can be read, because a stored name may contain `?`. +fn parse_attach_datatable(query: &str) -> Option> { lazy_static::lazy_static! { static ref RE: regex::Regex = regex::Regex::new( - r"(?i)ATTACH\s*'datatable(://[^'?:]+)?(\?[^':]*)?'\s*AS\s+([^ ;]+)" + r"(?i)ATTACH\s*'datatable(://[^':]+|\?[^':]*)?'\s*AS\s+([^ ;]+)" ).unwrap(); } - let Some(cap) = RE.captures(query) else { - return Ok(None); + let cap = RE.captures(query)?; + let reference = match cap.get(1).map(|m| m.as_str()) { + Some(named) if named.starts_with("://") => named[3..].to_string(), + Some(query) => format!("main{query}"), + None => "main".to_string(), }; - let name = cap.get(1).map(|m| &m.as_str()[3..]).unwrap_or("main"); - // A query string that does not parse is refused rather than dropped: attaching under the - // default role when the statement asked for another one is the failure this guards. - let role = match cap.get(2) { - Some(m) => windmill_common::workspaces::parse_datatable_ref(m.as_str())?.1, - None => None, - }; - let alias = cap.get(3).map(|m| m.as_str()).unwrap_or(""); - Ok(Some(AttachedDatatable { name, role, alias })) + let alias = cap.get(2).map(|m| m.as_str()).unwrap_or(""); + Some(AttachedDatatable { reference, alias }) } async fn transform_attach_datatable( @@ -2635,27 +2632,31 @@ async fn transform_attach_datatable( hidden_passwords: &mut Arc>>, job: &MiniPulledJob, ) -> Result>> { - let Some(attached) = parse_attach_datatable(query)? else { + let Some(attached) = parse_attach_datatable(query) else { return Ok(None); }; + // A query string that does not parse is refused rather than dropped: attaching under the + // default role when the statement asked for another one is the failure this guards. let db_resource = match conn { Connection::Http(client) => { - get_datatable_resource_from_agent_http( - client, - attached.name, - &job.workspace_id, - attached.role, - &job.id, - ) - .await? + let (name, role) = + windmill_common::workspaces::parse_datatable_ref(&attached.reference)?; + get_datatable_resource_from_agent_http(client, name, &job.workspace_id, role, &job.id) + .await? } Connection::Sql(db) => { + let (name, role) = windmill_common::workspaces::parse_datatable_ref_for( + db, + &job.workspace_id, + &attached.reference, + ) + .await?; get_datatable_resource_from_db( db, &job.workspace_id, - attached.name, - attached.role, + &name, + role.as_deref(), DatatableAccess::PermissionedAs { permissioned_as: &job.permissioned_as, email: &job.permissioned_as_email, @@ -2792,45 +2793,40 @@ mod tests { #[test] fn attach_datatable_parses_name_and_role() { - let named = parse_attach_datatable("ATTACH 'datatable://sales?role=analytics' AS dt") - .unwrap() - .unwrap(); + let reference_of = |q: &str| parse_attach_datatable(q).unwrap().reference; + let named = + parse_attach_datatable("ATTACH 'datatable://sales?role=analytics' AS dt").unwrap(); assert_eq!( - (named.name, named.role, named.alias), - ("sales", Some("analytics"), "dt") + (named.reference.as_str(), named.alias), + ("sales?role=analytics", "dt") ); // A bare `datatable` is the default one, and still takes a role. - let default = parse_attach_datatable("ATTACH 'datatable?role=analytics' AS dt") - .unwrap() - .unwrap(); - assert_eq!((default.name, default.role), ("main", Some("analytics"))); - let no_role = parse_attach_datatable("ATTACH 'datatable://sales' AS dt") - .unwrap() - .unwrap(); - assert_eq!((no_role.name, no_role.role), ("sales", None)); - let bare = parse_attach_datatable("ATTACH 'datatable' AS dt") - .unwrap() - .unwrap(); - assert_eq!((bare.name, bare.role), ("main", None)); - assert!(parse_attach_datatable("SELECT 1").unwrap().is_none()); + assert_eq!( + reference_of("ATTACH 'datatable?role=analytics' AS dt"), + "main?role=analytics" + ); + assert_eq!(reference_of("ATTACH 'datatable://sales' AS dt"), "sales"); + assert_eq!(reference_of("ATTACH 'datatable' AS dt"), "main"); + assert!(parse_attach_datatable("SELECT 1").is_none()); + // A stored name can contain `?`, so that is left to the workspace lookup to split. + assert_eq!(reference_of("ATTACH 'datatable://a?b' AS dt"), "a?b"); - // The key matches case-insensitively, as the `-- role` annotation does. - let cased = parse_attach_datatable("ATTACH 'datatable://sales?Role=analytics' AS dt") - .unwrap() - .unwrap(); - assert_eq!(cased.role, Some("analytics")); - - // A query string that does not parse is refused rather than attached under the default - // role: the statement asked for a specific one. + // The key matches case-insensitively, as the `-- role` annotation does, and a query string + // that does not parse is refused rather than attached under the default role. + let parse = |q: &str| { + windmill_common::workspaces::parse_datatable_ref(&reference_of(q)) + .map(|(name, role)| (name.to_string(), role.map(str::to_string))) + }; + assert_eq!( + parse("ATTACH 'datatable://sales?Role=analytics' AS dt").unwrap(), + ("sales".to_string(), Some("analytics".to_string())) + ); for malformed in [ "ATTACH 'datatable://sales?role=' AS dt", "ATTACH 'datatable://sales?role=an;alytics' AS dt", "ATTACH 'datatable://sales?x=1&role=analytics' AS dt", ] { - assert!( - parse_attach_datatable(malformed).is_err(), - "silently ignored: {malformed}" - ); + assert!(parse(malformed).is_err(), "silently ignored: {malformed}"); } } diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index 03560690f3..69aa3ee0fb 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -29,7 +29,7 @@ use windmill_common::worker::{ to_raw_value, Connection, SqlAnnotations, SqlResultCollectionStrategy, CLOUD_HOSTED, }; use windmill_common::workspaces::{ - get_datatable_resource_from_db, parse_datatable_ref, DatatableAccess, + get_datatable_resource_from_db, parse_datatable_ref, parse_datatable_ref_for, DatatableAccess, }; use windmill_common::{PgDatabase, PrepareQueryColumnInfo, PrepareQueryResult, DB}; use windmill_parser::{Arg, Typ}; @@ -683,28 +683,29 @@ pub async fn do_postgresql( match pg_args.get("database").cloned() { Some(Value::String(db_str)) if db_str.starts_with("datatable://") => { let reference = db_str.trim_start_matches("datatable://"); - let (db_str, uri_role) = parse_datatable_ref(reference)?; // The annotation wins: a generated query can carry a `?role=` in the reference it // was handed, but only the script's author writes the leading comment block. let annotated = SqlAnnotations::datatable_role(&query)?; - let role = annotated.as_deref().or(uri_role); Some(match conn { Connection::Http(client) => { + let (name, uri_role) = parse_datatable_ref(reference)?; get_datatable_resource_from_agent_http( client, - db_str, + name, &job.workspace_id, - role, + annotated.as_deref().or(uri_role), &job.id, ) .await? } Connection::Sql(db) => { + let (name, uri_role) = + parse_datatable_ref_for(db, &job.workspace_id, reference).await?; get_datatable_resource_from_db( db, &job.workspace_id, - db_str, - role, + &name, + annotated.as_deref().or(uri_role.as_deref()), DatatableAccess::PermissionedAs { permissioned_as: &job.permissioned_as, email: &job.permissioned_as_email, From 1dbc1e775955f188b2311a26071628e328ed54a9 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Fri, 11 Sep 2026 14:36:37 +0200 Subject: [PATCH 56/77] fix(datatables): warn when a settings sync strands fork pointers A settings save reported the fork pointers left resolving to nothing only for the names in `deleted_datatables`, which `wmill sync push` never sends. The save now works out what it removed from the locked entries, and the CLI prints the stranded pointers it returns. Also correct the replication helper's contract: no role or admin check makes a replication connection safe, so a data table under roles is refused outright rather than gated as an admin operation. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb --- .../tests/datatable_roles.rs | 30 +++++++++++++++++++ .../windmill-api-workspaces/src/workspaces.rs | 14 ++++++++- backend/windmill-common/src/workspaces.rs | 10 ++++--- cli/src/core/settings.ts | 9 +++++- 4 files changed, 57 insertions(+), 6 deletions(-) diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs index 719b8e42db..d8a44d7a55 100644 --- a/backend/windmill-api-integration-tests/tests/datatable_roles.rs +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -904,3 +904,33 @@ async fn a_stored_name_containing_a_question_mark_resolves_as_itself( ); Ok(()) } + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_settings_save_dropping_a_governing_entry_names_the_forks_it_strands( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + // The whole map and no `deleted_datatables`, as a settings sync sends it. + let resp = authed( + client().post(format!( + "http://localhost:{}/api/w/test-workspace/workspaces/edit_datatable_config", + server.addr.port() + )), + "SECRET_TOKEN", + ) + .json(&json!({ "settings": { "datatables": {} } })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, 200, "{body}"); + let result: Value = serde_json::from_str(&body)?; + assert!( + result["stranded_references"] + .as_array() + .is_some_and(|refs| refs.iter().any(|r| r["workspace_id"] == "wm-fork-dt")), + "the fork left pointing at nothing was not named: {body}" + ); + Ok(()) +} diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index e5a1690c7b..cfd3c47593 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -3871,6 +3871,18 @@ async fn edit_datatable_config( } } + // Worked out from the locked entries rather than taken from `deleted_datatables`: a settings + // sync sends the whole map without that list, and dropping a governing entry strands every + // fork pointing at it all the same. + let removed: Vec = old_datatables + .keys() + .filter(|name| { + !new_config.settings.datatables.contains_key(*name) + && !new_config.renames.iter().any(|r| &r.from == *name) + }) + .cloned() + .collect(); + let config: serde_json::Value = serde_json::to_value(new_config.settings) .map_err(|err| Error::internal_err(err.to_string()))?; @@ -3910,7 +3922,7 @@ async fn edit_datatable_config( // A deletion cannot be followed the same way — there is nothing to point at any more. Read who // is left stranded so the caller is told, the way deleting a workspace does. let mut stranded: Vec = Vec::new(); - for name in &new_config.deleted_datatables { + for name in &removed { let rows = sqlx::query!( r#"SELECT ws.workspace_id AS "workspace_id!", dt.key AS "datatable!" FROM workspace_settings ws diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 0e9a7ed3c9..837bff6c6d 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -1582,8 +1582,9 @@ pub async fn get_datatable_resource_from_db_unchecked( /// datatables resolve to the user's own resource unchanged; configuring it for /// replication there is the user's responsibility. /// -/// Authorization: a replication connection reads every row whatever the roles grant, so callers -/// must gate it with [`ensure_datatable_admin_access`] rather than a role check. +/// Authorization: a replication connection reads every row whatever the roles grant, so no role or +/// admin check makes it safe. Callers MUST refuse a data table under roles outright — the Postgres +/// trigger crate's `ensure_not_under_roles` — and turning roles on is refused while one streams. pub async fn get_datatable_replication_resource_from_db_unchecked( db: &DB, w_id: &str, @@ -1887,8 +1888,9 @@ pub async fn ensure_can_use_datatable_role( ))) } -/// Gate the operations that see the whole database whatever the roles grant: replication streams, -/// a migration that declares no role, exports, and editing the permissions themselves. Passing +/// Gate the operations that see the whole database whatever the roles grant: a migration that +/// declares no role, exports, and editing the permissions themselves. Not replication, which a +/// data table under roles refuses whoever asks (see `ensure_not_under_roles`). Passing /// means the caller could have connected as `admin` anyway. pub async fn ensure_datatable_admin_access( db: &DB, diff --git a/cli/src/core/settings.ts b/cli/src/core/settings.ts index 4c95090bcf..4174ec0358 100644 --- a/cli/src/core/settings.ts +++ b/cli/src/core/settings.ts @@ -393,10 +393,17 @@ export async function pushWorkspaceSettings( if (!deepEqual(localSettings.datatable, settings.datatable)) { log.debug(`Updating datatable config...`); - await wmill.editDataTableConfig({ + const { stranded_references } = await wmill.editDataTableConfig({ workspace, requestBody: { settings: localSettings.datatable ?? { datatables: {} } }, }); + if (stranded_references?.length) { + log.warn( + `Removed data tables governed data tables in other workspaces, which no longer resolve: ${stranded_references + .map((r) => `${r.workspace_id}/${r.datatable}`) + .join(", ")}. A superadmin can point them somewhere else.`, + ); + } } if (localSettings.slack_command_script != settings.slack_command_script) { From 6d9ef5f6b7e79a337558fcf5b366c6f72b633c4f Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Mon, 14 Sep 2026 21:21:41 +0200 Subject: [PATCH 57/77] fix(datatables): refuse a save that drops a data table's roles through an undeclared rename A data table's roles follow its entry only through a declared rename. A settings sync sends the whole map and never declares one, so renaming a data table under roles there read as a delete and a new entry on the same database: the new entry carried no roles, and every caller connected as admin. Such a save is now refused, naming both entries. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb --- .../tests/datatable_roles.rs | 36 +++++++++++++++++++ .../windmill-api-workspaces/src/workspaces.rs | 34 ++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs index d8a44d7a55..e3269c8138 100644 --- a/backend/windmill-api-integration-tests/tests/datatable_roles.rs +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -934,3 +934,39 @@ async fn a_settings_save_dropping_a_governing_entry_names_the_forks_it_strands( ); Ok(()) } + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_save_replacing_an_entry_under_roles_on_its_database_is_refused( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + // A rename as a settings sync sends it: the whole map, no `renames`. + let resp = authed( + client().post(format!( + "http://localhost:{}/api/w/test-workspace/workspaces/edit_datatable_config", + server.addr.port() + )), + "SECRET_TOKEN", + ) + .json(&json!({ "settings": { "datatables": { + "main_renamed": { "database": { "resource_type": "instance", "resource_path": "dt_main" } } + } } })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!( + status, 400, + "a save dropped the roles of the database it kept: {body}" + ); + + let still_governed: bool = sqlx::query_scalar( + "SELECT (datatable->'datatables'->'main') ? 'permissions' FROM workspace_settings + WHERE workspace_id = 'test-workspace'", + ) + .fetch_one(&db) + .await?; + assert!(still_governed, "the refused save still took effect"); + Ok(()) +} diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index cfd3c47593..1e591e742e 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -3883,6 +3883,40 @@ async fn edit_datatable_config( .cloned() .collect(); + // Roles follow an entry only through a declared rename. A save that drops an entry under roles + // and adds another on the same database without one — which is how a settings sync sends a + // rename — would leave that database answering everyone as `admin`. + for name in &removed { + let Some(old_db) = old_datatables + .get(name) + .filter(|old| old.permissions.is_some()) + .and_then(|old| old.database.as_ref()) + else { + continue; + }; + let added_on_same_database = + new_config + .settings + .datatables + .iter() + .find_map(|(added, dt)| { + let db = dt.database.as_ref()?; + (!old_datatables.contains_key(added) + && !new_config.renames.iter().any(|r| &r.to == added) + && db.resource_type == old_db.resource_type + && db.resource_path == old_db.resource_path) + .then_some(added) + }); + if let Some(added) = added_on_same_database { + return Err(Error::BadRequest(format!( + "Data table '{name}' is under roles, and this save removes it while adding '{added}' \ + on the same database. Its roles would not carry over, leaving that database open to \ + everyone as `admin`. Rename it from the data table settings, which carries its \ + roles, or turn its roles off first." + ))); + } + } + let config: serde_json::Value = serde_json::to_value(new_config.settings) .map_err(|err| Error::internal_err(err.to_string()))?; From 8a0babd7a607a36c78164756825ce43aaebd86a9 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Mon, 14 Sep 2026 21:36:50 +0200 Subject: [PATCH 58/77] fix(datatables): no entry without roles may newly reach a database under roles The previous guard only caught a new name replacing an entry under roles. A whole-map save could also repoint an existing entry without roles at that database, or another workspace could point one there, and every caller of that entry would connect as admin. The rule is now stated on the saved entries: one that carries no roles and newly points at an instance database any entry under roles uses, in this workspace or another, is refused. A declared rename carries its roles and passes. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb --- .../tests/datatable_roles.rs | 66 ++++++++++++----- .../windmill-api-workspaces/src/workspaces.rs | 70 ++++++++++++------- 2 files changed, 91 insertions(+), 45 deletions(-) diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs index e3269c8138..0a00ed8bb6 100644 --- a/backend/windmill-api-integration-tests/tests/datatable_roles.rs +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -936,30 +936,58 @@ async fn a_settings_save_dropping_a_governing_entry_names_the_forks_it_strands( } #[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] -async fn a_save_replacing_an_entry_under_roles_on_its_database_is_refused( +async fn an_entry_without_roles_cannot_newly_reach_a_database_under_roles( db: Pool, ) -> anyhow::Result<()> { initialize_tracing().await; - let server = ApiServer::start(db.clone()).await?; - // A rename as a settings sync sends it: the whole map, no `renames`. - let resp = authed( - client().post(format!( - "http://localhost:{}/api/w/test-workspace/workspaces/edit_datatable_config", - server.addr.port() - )), - "SECRET_TOKEN", + sqlx::query( + r#"UPDATE workspace_settings SET datatable = jsonb_set(datatable, '{datatables,other}', + '{"database": {"resource_type": "instance", "resource_path": "dt_other"}}') + WHERE workspace_id = 'test-workspace'"#, ) - .json(&json!({ "settings": { "datatables": { - "main_renamed": { "database": { "resource_type": "instance", "resource_path": "dt_main" } } - } } })) - .send() + .execute(&db) .await?; - let status = resp.status(); - let body = resp.text().await?; - assert_eq!( - status, 400, - "a save dropped the roles of the database it kept: {body}" - ); + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Whole-map saves with no `renames`, as a settings sync sends them. + let dt_main = + json!({ "database": { "resource_type": "instance", "resource_path": "dt_main" } }); + let dt_other = + json!({ "database": { "resource_type": "instance", "resource_path": "dt_other" } }); + for (case, w_id, datatables) in [ + ( + "a rename to a new name", + "test-workspace", + json!({ "main_renamed": dt_main, "other": dt_other }), + ), + ( + "an existing name repointed", + "test-workspace", + json!({ "other": dt_main }), + ), + ( + "another workspace's entry", + "wm-fork-dt", + json!({ "direct": dt_main }), + ), + ] { + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/w/{w_id}/workspaces/edit_datatable_config" + )), + "SECRET_TOKEN", + ) + .json(&json!({ "settings": { "datatables": datatables } })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert!( + status == 400 && body.contains("which a data table under roles uses"), + "{case} reached the database under roles without them ({status}): {body}" + ); + } let still_governed: bool = sqlx::query_scalar( "SELECT (datatable->'datatables'->'main') ? 'permissions' FROM workspace_settings diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 1e591e742e..11f6958322 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -3883,36 +3883,54 @@ async fn edit_datatable_config( .cloned() .collect(); - // Roles follow an entry only through a declared rename. A save that drops an entry under roles - // and adds another on the same database without one — which is how a settings sync sends a - // rename — would leave that database answering everyone as `admin`. - for name in &removed { - let Some(old_db) = old_datatables - .get(name) - .filter(|old| old.permissions.is_some()) - .and_then(|old| old.database.as_ref()) + // A database under roles is reached only through an entry that carries them. Roles follow an + // entry through a declared rename alone, and a settings sync never declares one, so an entry + // without roles that newly points at such a database — a name added, or an existing one + // repointed — would answer everyone there as `admin`. That holds whichever workspace governs it. + let governed_elsewhere: Vec = sqlx::query_scalar( + "SELECT DISTINCT dt.value->'database'->>'resource_path' FROM workspace_settings ws + CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt + WHERE ws.workspace_id <> $1 AND dt.value ? 'permissions' + AND dt.value->'database'->>'resource_type' = 'instance'", + ) + .bind(&w_id) + .fetch_all(&mut *tx) + .await?; + for (name, dt) in new_config.settings.datatables.iter() { + if dt.permissions.is_some() { + continue; + } + let Some(db) = dt + .database + .as_ref() + .filter(|d| d.resource_type == DataTableCatalogResourceType::Instance) else { continue; }; - let added_on_same_database = - new_config - .settings - .datatables - .iter() - .find_map(|(added, dt)| { - let db = dt.database.as_ref()?; - (!old_datatables.contains_key(added) - && !new_config.renames.iter().any(|r| &r.to == added) - && db.resource_type == old_db.resource_type - && db.resource_path == old_db.resource_path) - .then_some(added) - }); - if let Some(added) = added_on_same_database { + let lookup = rename_src + .get(name.as_str()) + .copied() + .unwrap_or(name.as_str()); + let repointed = old_datatables + .get(lookup) + .and_then(|old| old.database.as_ref()) + .is_none_or(|old_db| { + old_db.resource_type != db.resource_type || old_db.resource_path != db.resource_path + }); + let governed_here = old_datatables.values().any(|old| { + old.permissions.is_some() + && old.database.as_ref().is_some_and(|d| { + d.resource_type == DataTableCatalogResourceType::Instance + && d.resource_path == db.resource_path + }) + }); + if repointed && (governed_here || governed_elsewhere.contains(&db.resource_path)) { return Err(Error::BadRequest(format!( - "Data table '{name}' is under roles, and this save removes it while adding '{added}' \ - on the same database. Its roles would not carry over, leaving that database open to \ - everyone as `admin`. Rename it from the data table settings, which carries its \ - roles, or turn its roles off first." + "Data table '{name}' would point at database '{}', which a data table under roles \ + uses, without carrying those roles: everyone reaching '{name}' would connect there \ + as `admin`. Rename the data table under roles from the data table settings, which \ + carries its roles, or turn its roles off first.", + db.resource_path ))); } } From c7269e975f6cb019411e81356653c4c30a720aa3 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Tue, 15 Sep 2026 12:47:46 +0200 Subject: [PATCH 59/77] feat(datatables): move data table role catalog and resolution to the enterprise edition Roles are an Enterprise Edition feature. The catalog, the Postgres logins, CONNECT convergence, tenant evaluation and the role half of connection resolution move to windmill-ee-private. Every public function keeps its path and signature and forwards through datatable_roles_oss, which re-exports the enterprise implementation or, without it, refuses. Without the enterprise edition a data table under roles, or a caller naming a role, is refused a connection rather than resolved as admin, and the reach and admin-access checks refuse one under roles. A data table not under roles resolves as before in every edition, and an instance database keeps the CONNECT grants it was created with. The catalog lock, the stream lock, the tenant cascades and the permissions stripping stay in OSS: they only restrict. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb --- backend/ee-repo-ref.txt | 2 +- .../windmill-common/src/datatable_roles.rs | 201 +------------ .../src/datatable_roles_oss.rs | 206 +++++++++++++ backend/windmill-common/src/lib.rs | 27 +- backend/windmill-common/src/workspaces.rs | 281 ++---------------- 5 files changed, 255 insertions(+), 462 deletions(-) create mode 100644 backend/windmill-common/src/datatable_roles_oss.rs diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 8a75d8815e..e4502b5bd8 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -475e60445f44506c754db3f17e307875d9075ce3 +e6e5dc7d20bb7143231a2bede7f53c430aeaa2c1 diff --git a/backend/windmill-common/src/datatable_roles.rs b/backend/windmill-common/src/datatable_roles.rs index 3a7709ccd1..3c269d1ada 100644 --- a/backend/windmill-common/src/datatable_roles.rs +++ b/backend/windmill-common/src/datatable_roles.rs @@ -102,16 +102,13 @@ pub fn validate_role_name(name: &str) -> Result<()> { Ok(()) } -/// SAFETY: every caller must have run [`validate_role_name`] first — the charset it enforces is -/// what makes this quoting sufficient. -fn quote_ident(name: &str) -> String { +/// A double-quoted Postgres identifier. Doubling `"` is Postgres's own escaping inside one, so this +/// quotes any name — schema, table or role. Role names are validated as well +/// ([`validate_role_name`]) because they also travel unquoted, in `-- role ` and `?role=`. +pub fn quote_ident(name: &str) -> String { format!("\"{}\"", name.replace('"', "\"\"")) } -fn quote_literal(value: &str) -> String { - format!("'{}'", value.replace('\'', "''")) -} - /// Serialize the mutations that are not already serialized by the row itself. /// /// A create is an insert and a delete is a delete, which Postgres orders for us — the unique index @@ -149,18 +146,7 @@ pub async fn lock_datatable_streams(conn: &mut sqlx::PgConnection, exclusive: bo /// or an export. Nothing about who may call it: the credential is the whole risk, and `Debug` is /// hand-written to redact it for the same reason. pub async fn read_role_catalog(db: &DB) -> Result { - let rows = sqlx::query!("SELECT id, name, enabled, pwd FROM datatable_role") - .fetch_all(db) - .await?; - Ok(rows - .into_iter() - .map(|r| { - ( - r.id, - InstanceDatatableRole { name: r.name, enabled: r.enabled, pwd: r.pwd }, - ) - }) - .collect()) + crate::datatable_roles_oss::read_role_catalog(db).await } /// As [`read_role_catalog`], reading inside the caller's transaction so the value is the one @@ -168,18 +154,7 @@ pub async fn read_role_catalog(db: &DB) -> Result { pub async fn read_role_catalog_tx( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, ) -> Result { - let rows = sqlx::query!("SELECT id, name, enabled, pwd FROM datatable_role") - .fetch_all(&mut **tx) - .await?; - Ok(rows - .into_iter() - .map(|r| { - ( - r.id, - InstanceDatatableRole { name: r.name, enabled: r.enabled, pwd: r.pwd }, - ) - }) - .collect()) + crate::datatable_roles_oss::read_role_catalog_tx(tx).await } /// Record a role, in the caller's transaction so it commits with the `CREATE ROLE` it describes. @@ -191,16 +166,7 @@ pub async fn insert_role_catalog_entry( id: &str, role: &InstanceDatatableRole, ) -> Result<()> { - sqlx::query!( - "INSERT INTO datatable_role (id, name, enabled, pwd) VALUES ($1, $2, $3, $4)", - id, - role.name, - role.enabled, - role.pwd, - ) - .execute(&mut **tx) - .await?; - Ok(()) + crate::datatable_roles_oss::insert_role_catalog_entry(tx, id, role).await } /// Update a role's recorded name, login flag and password. Same contract as @@ -210,16 +176,7 @@ pub async fn update_role_catalog_entry( id: &str, role: &InstanceDatatableRole, ) -> Result<()> { - sqlx::query!( - "UPDATE datatable_role SET name = $2, enabled = $3, pwd = $4 WHERE id = $1", - id, - role.name, - role.enabled, - role.pwd, - ) - .execute(&mut **tx) - .await?; - Ok(()) + crate::datatable_roles_oss::update_role_catalog_entry(tx, id, role).await } /// Forget a role. Same contract as [`insert_role_catalog_entry`]; run it in the transaction that @@ -228,10 +185,7 @@ pub async fn delete_role_catalog_entry( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, id: &str, ) -> Result<()> { - sqlx::query!("DELETE FROM datatable_role WHERE id = $1", id) - .execute(&mut **tx) - .await?; - Ok(()) + crate::datatable_roles_oss::delete_role_catalog_entry(tx, id).await } /// Resolve the role a caller named to its catalog id. A disabled role is an error rather than a @@ -261,13 +215,7 @@ pub fn role_id_by_name<'a>(catalog: &'a DatatableRoleCatalog, name: &str) -> Res /// Every instance database the registry knows about. Role provisioning has to reach all of them: /// a role that cannot `CONNECT` to a database is refused by Postgres before any grant matters. pub async fn registered_instance_databases(db: &DB) -> Result> { - let names = sqlx::query_scalar!( - "SELECT jsonb_object_keys(value->'databases') FROM global_settings - WHERE name = 'custom_instance_pg_databases'" - ) - .fetch_all(db) - .await?; - Ok(names.into_iter().flatten().collect()) + crate::datatable_roles_oss::registered_instance_databases(db).await } /// `CONNECT` on `dbname` for every enabled role, and none for `PUBLIC`. Run at role creation, at @@ -278,8 +226,7 @@ pub async fn registered_instance_databases(db: &DB) -> Result> { /// Callers MUST have authorized administration of `dbname` — superadmin, or an admin of the /// workspace governing a data table on it. pub async fn converge_connect_grants(db: &DB, dbname: &str) -> Result<()> { - let catalog = read_role_catalog(db).await?; - converge_connect_grants_with(db, dbname, &catalog).await + crate::datatable_roles_oss::converge_connect_grants(db, dbname).await } /// As [`converge_connect_grants`], with a catalog the caller already read. Same contract. @@ -288,18 +235,7 @@ pub async fn converge_connect_grants_with( dbname: &str, catalog: &DatatableRoleCatalog, ) -> Result<()> { - crate::validate_dbname(dbname)?; - let quoted_db = quote_ident(dbname); - let mut sql = format!("REVOKE CONNECT ON DATABASE {quoted_db} FROM PUBLIC;\n"); - for role in catalog.values().filter(|r| r.enabled) { - validate_role_name(&role.name)?; - sql.push_str(&format!( - "GRANT CONNECT ON DATABASE {quoted_db} TO {};\n", - quote_ident(&role.name) - )); - } - sqlx::raw_sql(&sql).execute(db).await?; - Ok(()) + crate::datatable_roles_oss::converge_connect_grants_with(db, dbname, catalog).await } /// `CREATE ROLE LOGIN PASSWORD ...; GRANT TO custom_instance_user`, and `CONNECT` on @@ -313,36 +249,7 @@ pub async fn create_instance_role( name: &str, password: &str, ) -> Result<()> { - validate_role_name(name)?; - let exists = sqlx::query_scalar!( - "SELECT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = $1)", - name - ) - .fetch_one(&mut **tx) - .await? - .unwrap_or(false); - if exists { - return Err(Error::BadRequest(format!( - "A Postgres role named '{name}' already exists on this cluster" - ))); - } - let quoted = quote_ident(name); - // One statement per call rather than a batch: `raw_sql` takes the simple protocol, which is - // only needed for genuinely multi-statement SQL, and its future is not `Send` — which an axum - // handler holding this transaction requires. - sqlx::query(&format!( - "CREATE ROLE {quoted} LOGIN PASSWORD {}", - quote_literal(password) - )) - .execute(&mut **tx) - .await?; - sqlx::query(&format!( - "GRANT {quoted} TO {}", - quote_ident(CUSTOM_INSTANCE_USER) - )) - .execute(&mut **tx) - .await?; - Ok(()) + crate::datatable_roles_oss::create_instance_role(tx, name, password).await } /// Authorization: alters a cluster-wide Postgres login. Callers MUST restrict this to superadmin @@ -352,15 +259,7 @@ pub async fn set_instance_role_login( name: &str, enabled: bool, ) -> Result<()> { - validate_role_name(name)?; - sqlx::query(&format!( - "ALTER ROLE {} {}", - quote_ident(name), - if enabled { "LOGIN" } else { "NOLOGIN" } - )) - .execute(&mut **tx) - .await?; - Ok(()) + crate::datatable_roles_oss::set_instance_role_login(tx, name, enabled).await } /// A rename discards an md5-hashed password, so the caller has to hand over a fresh one. @@ -373,35 +272,7 @@ pub async fn rename_instance_role( to: &str, password: &str, ) -> Result<()> { - validate_role_name(from)?; - validate_role_name(to)?; - let taken = sqlx::query_scalar!( - "SELECT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = $1)", - to - ) - .fetch_one(&mut **tx) - .await? - .unwrap_or(false); - if taken { - return Err(Error::BadRequest(format!( - "A Postgres role named '{to}' already exists on this cluster" - ))); - } - sqlx::query(&format!( - "ALTER ROLE {} RENAME TO {}", - quote_ident(from), - quote_ident(to) - )) - .execute(&mut **tx) - .await?; - sqlx::query(&format!( - "ALTER ROLE {} PASSWORD {}", - quote_ident(to), - quote_literal(password) - )) - .execute(&mut **tx) - .await?; - Ok(()) + crate::datatable_roles_oss::rename_instance_role(tx, from, to, password).await } /// A role owning anything in any database blocks its own `DROP ROLE`, and both its objects and the @@ -427,47 +298,7 @@ pub async fn drop_instance_role( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, name: &str, ) -> Result<()> { - validate_role_name(name)?; - let quoted = quote_ident(name); - let reassign = format!( - "REASSIGN OWNED BY {quoted} TO {};\nDROP OWNED BY {quoted};", - quote_ident(CUSTOM_INSTANCE_USER) - ); - - let base = crate::PgDatabase::parse_uri(&crate::get_database_url().await?.as_str().await)?; - for dbname in registered_instance_databases(db).await? { - let creds = crate::PgDatabase { dbname: dbname.clone(), ..base.clone() }; - let (client, connection) = creds.connect(Some(db)).await.map_err(|e| { - Error::BadRequest(format!( - "Cannot delete role '{name}': instance database '{dbname}' is unreachable ({e}). \ - Objects it owns there would be orphaned." - )) - })?; - let join_handle = tokio::spawn(async move { connection.await }); - let result = client.batch_execute(&reassign).await; - drop(client); - crate::shutdown_pg_connection(join_handle).await?; - result.map_err(|e| { - Error::internal_err(format!( - "Reassigning what role '{name}' owns in '{dbname}': {}", - crate::error::pg_error_message(&e) - )) - })?; - } - - sqlx::query(&format!( - "REASSIGN OWNED BY {quoted} TO {}", - quote_ident(CUSTOM_INSTANCE_USER) - )) - .execute(&mut **tx) - .await?; - sqlx::query(&format!("DROP OWNED BY {quoted}")) - .execute(&mut **tx) - .await?; - sqlx::query(&format!("DROP ROLE {quoted}")) - .execute(&mut **tx) - .await?; - Ok(()) + crate::datatable_roles_oss::drop_instance_role(db, tx, name).await } #[cfg(test)] diff --git a/backend/windmill-common/src/datatable_roles_oss.rs b/backend/windmill-common/src/datatable_roles_oss.rs new file mode 100644 index 0000000000..98fef4260f --- /dev/null +++ b/backend/windmill-common/src/datatable_roles_oss.rs @@ -0,0 +1,206 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Where data table roles come from: the enterprise implementation, or a refusal. +//! +//! Roles are an Enterprise Edition feature. An edition without them creates, grants and connects +//! as none, and a data table saved under roles — by an enterprise build, before a downgrade — is +//! refused rather than resolved as `admin`. A data table not under roles, asked for no role, +//! resolves as it always has. `private` alone is not that edition: community builds carry it. + +use crate::error::Error; + +/// What every roles path answers without the Enterprise Edition. +pub fn datatable_roles_unavailable() -> Error { + Error::BadRequest("Data table roles are a Windmill Enterprise Edition feature".to_string()) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub(crate) use crate::datatable_roles_ee::{ + can_use_datatable_role, can_use_datatable_role_in_governing_workspace, converge_connect_grants, + converge_connect_grants_with, create_instance_role, delete_role_catalog_entry, + drop_instance_role, ensure_can_use_datatable_role, ensure_datatable_admin_access, + ensure_instance_db_grant_options_unchecked, forget_datatable_role_everywhere, + insert_role_catalog_entry, read_role_catalog, read_role_catalog_tx, + registered_instance_databases, rename_instance_role, resolve_datatable_role_connection, + set_instance_role_login, update_role_catalog_entry, +}; + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub(crate) use ce::*; + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +mod ce { + use super::datatable_roles_unavailable as unavailable; + use crate::{ + datatable_roles::{DatatableRoleCatalog, InstanceDatatableRole}, + db::AuthedRef, + error::Result, + workspaces::{ + resolve_governing_datatable, DataTableRoleTenants, DatatableAccess, GoverningDatatable, + }, + DB, + }; + + type Tx<'a> = sqlx::Transaction<'a, sqlx::Postgres>; + + pub(crate) async fn read_role_catalog(_db: &DB) -> Result { + Err(unavailable()) + } + + pub(crate) async fn read_role_catalog_tx(_tx: &mut Tx<'_>) -> Result { + Err(unavailable()) + } + + pub(crate) async fn insert_role_catalog_entry( + _tx: &mut Tx<'_>, + _id: &str, + _role: &InstanceDatatableRole, + ) -> Result<()> { + Err(unavailable()) + } + + pub(crate) async fn update_role_catalog_entry( + _tx: &mut Tx<'_>, + _id: &str, + _role: &InstanceDatatableRole, + ) -> Result<()> { + Err(unavailable()) + } + + pub(crate) async fn delete_role_catalog_entry(_tx: &mut Tx<'_>, _id: &str) -> Result<()> { + Err(unavailable()) + } + + pub(crate) async fn registered_instance_databases(_db: &DB) -> Result> { + Err(unavailable()) + } + + /// Nothing to converge: with no roles to admit, an instance database keeps the `CONNECT` + /// grants it was created with, `PUBLIC`'s included, as it did before roles existed. + pub(crate) async fn converge_connect_grants(_db: &DB, _dbname: &str) -> Result<()> { + Ok(()) + } + + /// As [`converge_connect_grants`]. + pub(crate) async fn converge_connect_grants_with( + _db: &DB, + _dbname: &str, + _catalog: &DatatableRoleCatalog, + ) -> Result<()> { + Ok(()) + } + + pub(crate) async fn create_instance_role( + _tx: &mut Tx<'_>, + _name: &str, + _password: &str, + ) -> Result<()> { + Err(unavailable()) + } + + pub(crate) async fn set_instance_role_login( + _tx: &mut Tx<'_>, + _name: &str, + _enabled: bool, + ) -> Result<()> { + Err(unavailable()) + } + + pub(crate) async fn rename_instance_role( + _tx: &mut Tx<'_>, + _from: &str, + _to: &str, + _password: &str, + ) -> Result<()> { + Err(unavailable()) + } + + pub(crate) async fn drop_instance_role(_db: &DB, _tx: &mut Tx<'_>, _name: &str) -> Result<()> { + Err(unavailable()) + } + + pub(crate) async fn ensure_instance_db_grant_options_unchecked( + _db: &DB, + _dbname: &str, + ) -> Result<()> { + Err(unavailable()) + } + + /// No tenant list covers anyone: there is no role to connect as. + pub(crate) fn can_use_datatable_role( + _tenants: &DataTableRoleTenants, + _authed: &AuthedRef<'_>, + ) -> bool { + false + } + + pub(crate) async fn can_use_datatable_role_in_governing_workspace( + _db: &DB, + _governing_w_id: &str, + _w_id: &str, + _tenants: &DataTableRoleTenants, + _access: &DatatableAccess<'_>, + ) -> Result { + Err(unavailable()) + } + + /// Reached only for a data table under roles or a caller naming a role: both are refused. + pub(crate) async fn resolve_datatable_role_connection( + _db: &DB, + _w_id: &str, + _name: &str, + _governing: &GoverningDatatable, + _db_resource: serde_json::Value, + _role: Option<&str>, + _access: DatatableAccess<'_>, + ) -> Result { + Err(unavailable()) + } + + /// A data table not under roles, asked for no role, is not a role decision and passes, as it + /// did before roles existed. Anything else is refused. + pub(crate) async fn ensure_can_use_datatable_role( + db: &DB, + w_id: &str, + name: &str, + role: Option<&str>, + _access: &DatatableAccess<'_>, + _context: &str, + ) -> Result<()> { + let governing = resolve_governing_datatable(db, w_id, name).await?; + if governing.datatable.permissions.is_none() && role.is_none() { + Ok(()) + } else { + Err(unavailable()) + } + } + + /// A data table not under roles is the `admin` connection for anyone who reaches it, as before + /// roles existed. One under roles is refused. + pub(crate) async fn ensure_datatable_admin_access( + db: &DB, + w_id: &str, + name: &str, + _access: &DatatableAccess<'_>, + ) -> Result<()> { + let governing = resolve_governing_datatable(db, w_id, name).await?; + if governing.datatable.permissions.is_none() { + Ok(()) + } else { + Err(unavailable()) + } + } + + pub(crate) async fn forget_datatable_role_everywhere( + _tx: &mut Tx<'_>, + _role_id: &str, + ) -> Result<()> { + Err(unavailable()) + } +} diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index a7e84c55d7..52d1a85aec 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -38,6 +38,9 @@ pub mod cache; pub mod client; pub mod data_metrics; pub mod datatable_roles; +#[cfg(all(feature = "private", feature = "enterprise"))] +mod datatable_roles_ee; +pub mod datatable_roles_oss; pub mod db; #[cfg(all(feature = "enterprise", feature = "private"))] mod db_entra_ee; @@ -1521,7 +1524,7 @@ pub async fn drop_custom_instance_database(db: &DB, dbname: &str) -> error::Resu /// and it is the one that hands privileges to data table roles. Postgres refuses to let a role pass /// on a privilege it does not itself hold with grant option, so without these an admin could own /// the database and still be unable to grant `SELECT` on it to `analytics`. -fn instance_db_grants(dbname: &str) -> String { +pub(crate) fn instance_db_grants(dbname: &str) -> String { format!( "GRANT CONNECT ON DATABASE \"{dbname}\" TO custom_instance_user WITH GRANT OPTION; GRANT CREATE ON DATABASE \"{dbname}\" TO custom_instance_user WITH GRANT OPTION; @@ -1543,23 +1546,11 @@ fn instance_db_grants(dbname: &str) -> String { /// Authorization: reaches an instance database with the server's own credentials and checks /// nothing. Callers MUST have authorized administration of `dbname` — superadmin, or an admin of /// the workspace governing a data table on it. -pub async fn ensure_instance_db_grant_options_unchecked(db: &DB, dbname: &str) -> error::Result<()> { - let dbname = dbname.trim(); - validate_dbname(dbname)?; - let wmill_pg_creds = PgDatabase::parse_uri(&get_database_url().await?.as_str().await)?; - let creds = PgDatabase { dbname: dbname.to_string(), ..wmill_pg_creds }; - let (client, connection) = creds.connect(Some(db)).await?; - let join_handle = tokio::spawn(async move { connection.await }); - let result = client.batch_execute(&instance_db_grants(dbname)).await; - drop(client); - shutdown_pg_connection(join_handle).await?; - result.map_err(|e| { - error::Error::internal_err(format!( - "Failed to grant permissions on '{}': {}", - dbname, - crate::error::pg_error_message(&e) - )) - }) +pub async fn ensure_instance_db_grant_options_unchecked( + db: &DB, + dbname: &str, +) -> error::Result<()> { + crate::datatable_roles_oss::ensure_instance_db_grant_options_unchecked(db, dbname).await } /// Create a custom instance database: CREATE DATABASE, grant permissions, register in global_settings. diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 837bff6c6d..41cf2b728f 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -1620,18 +1620,7 @@ pub fn can_use_datatable_role( tenants: &DataTableRoleTenants, authed: &crate::db::AuthedRef<'_>, ) -> bool { - *authed.is_admin - || tenants.tenants.iter().any(|tenant| { - if tenant == DATATABLE_TENANT_WILDCARD { - return true; - } - match tenant.split_once('/') { - Some(("u", user)) => authed.username == user, - Some(("g", group)) => authed.groups.iter().any(|g| g == group), - Some(("f", folder)) => authed.folders.iter().any(|(f, _, _)| f == folder), - _ => false, - } - }) + crate::datatable_roles_oss::can_use_datatable_role(tenants, authed) } /// Evaluate a tenant list **as a member of the governing workspace**, whoever is calling. @@ -1648,101 +1637,14 @@ pub async fn can_use_datatable_role_in_governing_workspace( tenants: &DataTableRoleTenants, access: &DatatableAccess<'_>, ) -> Result { - let (permissioned_as, email): (String, String) = match access { - DatatableAccess::Unchecked => return Ok(true), - DatatableAccess::NoIdentity => return Ok(false), - DatatableAccess::Authed(authed) => { - if w_id == governing_w_id { - return Ok(can_use_datatable_role(tenants, authed)); - } - (format!("u/{}", authed.username), authed.email.to_string()) - } - DatatableAccess::PermissionedAs { permissioned_as, email } => { - (permissioned_as.to_string(), email.to_string()) - } - DatatableAccess::Job(job_id) => { - let job = sqlx::query!( - "SELECT permissioned_as, permissioned_as_email FROM v2_job - WHERE id = $1 AND workspace_id = $2", - job_id, - w_id, - ) - .fetch_optional(db) - .await? - .ok_or_else(|| Error::NotFound(format!("job {job_id} not found in {w_id}")))?; - (job.permissioned_as, job.permissioned_as_email) - } - }; - - if w_id == governing_w_id { - let authed = - crate::auth::fetch_authed_from_permissioned_as(&permissioned_as, &email, w_id, db) - .await?; - return Ok(can_use_datatable_role(tenants, &authed.to_authed_ref())); - } - if crate::auth::is_super_admin_email(db, &email).await? { - return Ok(true); - } - if !permissioned_as.starts_with("u/") { - return Ok(false); - } - let Some(username) = sqlx::query_scalar!( - "SELECT username FROM usr WHERE workspace_id = $1 AND email = $2 AND disabled = false", - governing_w_id, - &email - ) - .fetch_optional(db) - .await? - else { - return Ok(false); - }; - let authed = crate::auth::fetch_authed_from_permissioned_as( - &format!("u/{username}"), - &email, - governing_w_id, + crate::datatable_roles_oss::can_use_datatable_role_in_governing_workspace( db, + governing_w_id, + w_id, + tenants, + access, ) - .await?; - Ok(can_use_datatable_role(tenants, &authed.to_authed_ref())) -} - -/// Which tenant list a caller's role selection lands on. `Ok(None)` means the data table is -/// unpermissioned and resolves through its own `admin` connection, as it did before roles existed. -/// -/// `role` is the **name** a caller wrote (`-- role analytics`); it is mapped to the catalog id the -/// tenant lists are keyed by here, so a rename moves nothing. -fn datatable_role_entry<'a>( - permissions: Option<&'a DataTablePermissions>, - catalog: &crate::datatable_roles::DatatableRoleCatalog, - name: &str, - role: Option<&str>, -) -> Result> { - let Some(permissions) = permissions else { - return match role { - Some(role) if role != ADMIN_DATATABLE_ROLE => Err(Error::BadRequest(format!( - "Cannot use role '{role}': data table '{name}' is not under roles. \ - Put it under roles in its permissions drawer first." - ))), - _ => Ok(None), - }; - }; - let role_id = match role { - None => permissions.default_role().to_string(), - Some(ADMIN_DATATABLE_ROLE) => ADMIN_DATATABLE_ROLE.to_string(), - Some(role) => crate::datatable_roles::role_id_by_name(catalog, role)?.to_string(), - }; - let tenants = permissions.roles.get(&role_id).ok_or_else(|| { - let display = role.map(str::to_string).unwrap_or_else(|| { - catalog - .get(&role_id) - .map(|r| r.name.clone()) - .unwrap_or_else(|| role_id.clone()) - }); - Error::NotFound(format!( - "Role '{display}' is not among the roles of data table '{name}'" - )) - })?; - Ok(Some((role_id, tenants))) + .await } /// Resolve a data table to connection credentials for one identity. @@ -1761,83 +1663,22 @@ pub async fn get_datatable_resource_from_db( access: DatatableAccess<'_>, ) -> Result { let governing = resolve_governing_datatable(db, w_id, name).await?; - let mut db_resource = resolve_datatable_connection_unchecked(db, &governing, false).await?; - - // A data table role is a login on Windmill's own cluster, so it is only meaningful against an - // instance database. Substituting its password into a resource-backed connection would hand a - // real cluster credential to whatever host that resource names — which a workspace admin - // chooses. Refused rather than ignored: an entry that reached this state was never a shape the - // permissions endpoint accepts, so silently resolving it as admin would hide a broken record. - if !governing.is_instance() { - return match (governing.datatable.permissions.as_ref(), role) { - (None, None | Some(ADMIN_DATATABLE_ROLE)) => Ok(db_resource), - _ => Err(Error::BadRequest(format!( - "Data table '{name}' is backed by a Postgres resource, which cannot be put under \ - data table roles" - ))), - }; - } - - let catalog = crate::datatable_roles::read_role_catalog(db).await?; - let Some((role_id, tenants)) = datatable_role_entry( - governing.datatable.permissions.as_ref(), - &catalog, - name, - role, - )? - else { + let db_resource = resolve_datatable_connection_unchecked(db, &governing, false).await?; + // Not under roles and asked for none: the `admin` connection, as before roles existed, in + // every edition. Anything else is a role decision. + if governing.datatable.permissions.is_none() && role.is_none() { return Ok(db_resource); - }; - - // Only for a data table actually under roles: whether people name a role or ride the default - // is what says if the `-- role` annotation is carrying its weight. - crate::feature_usage::log_feature_usage( - "datatable", - "role_connection", - if role.is_some() { "named" } else { "default" }, - ); - - if !can_use_datatable_role_in_governing_workspace( + } + crate::datatable_roles_oss::resolve_datatable_role_connection( db, - &governing.workspace_id, w_id, - tenants, - &access, + name, + &governing, + db_resource, + role, + access, ) - .await? - { - let display = catalog - .get(&role_id) - .map(|r| r.name.as_str()) - .unwrap_or(role_id.as_str()); - return Err(Error::NotAuthorized(format!( - "Not allowed to use role '{display}' of data table '{name}'" - ))); - } - - if role_id == ADMIN_DATATABLE_ROLE { - return Ok(db_resource); - } - let entry = catalog.get(&role_id).ok_or_else(|| { - Error::NotFound(format!( - "Data table '{name}' names a role that no longer exists on this instance" - )) - })?; - if !entry.enabled { - return Err(Error::BadRequest(format!( - "Data table role '{}' is disabled on this instance", - entry.name - ))); - } - let pwd = entry.pwd.as_ref().ok_or_else(|| { - Error::internal_err(format!( - "Data table role '{}' has no stored credential; recreate it in instance settings", - entry.name - )) - })?; - db_resource["user"] = serde_json::Value::String(entry.name.clone()); - db_resource["password"] = serde_json::Value::String(pwd.clone()); - Ok(db_resource) + .await } /// Would the chokepoint accept this identity connecting as this role? Answers without resolving @@ -1854,38 +1695,8 @@ pub async fn ensure_can_use_datatable_role( access: &DatatableAccess<'_>, context: &str, ) -> Result<()> { - let governing = resolve_governing_datatable(db, w_id, name).await?; - if !governing.is_instance() { - return Ok(()); - } - let catalog = crate::datatable_roles::read_role_catalog(db).await?; - let Some((role_id, tenants)) = datatable_role_entry( - governing.datatable.permissions.as_ref(), - &catalog, - name, - role, - )? - else { - return Ok(()); - }; - if can_use_datatable_role_in_governing_workspace( - db, - &governing.workspace_id, - w_id, - tenants, - access, - ) - .await? - { - return Ok(()); - } - let display = catalog - .get(&role_id) - .map(|r| r.name.as_str()) - .unwrap_or(role_id.as_str()); - Err(Error::NotAuthorized(format!( - "{context} runs as role '{display}' of data table '{name}', which you are not allowed to use" - ))) + crate::datatable_roles_oss::ensure_can_use_datatable_role(db, w_id, name, role, access, context) + .await } /// Gate the operations that see the whole database whatever the roles grant: a migration that @@ -1898,35 +1709,7 @@ pub async fn ensure_datatable_admin_access( name: &str, access: &DatatableAccess<'_>, ) -> Result<()> { - let governing = resolve_governing_datatable(db, w_id, name).await?; - if !governing.is_instance() { - return Ok(()); - } - let Some(permissions) = governing.datatable.permissions.as_ref() else { - return Ok(()); - }; - let admin = permissions - .roles - .get(ADMIN_DATATABLE_ROLE) - .cloned() - .unwrap_or_default(); - if can_use_datatable_role_in_governing_workspace( - db, - &governing.workspace_id, - w_id, - &admin, - access, - ) - .await? - { - Ok(()) - } else { - Err(Error::NotAuthorized(format!( - "Data table '{name}' is under roles; this reaches the whole database, so it is for \ - the admins of workspace '{}', which governs it.", - governing.workspace_id - ))) - } + crate::datatable_roles_oss::ensure_datatable_admin_access(db, w_id, name, access).await } /// Rewrite the `permissions` of every data table entry of one workspace, in the caller's @@ -2062,25 +1845,7 @@ pub async fn forget_datatable_role_everywhere( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, role_id: &str, ) -> Result<()> { - let workspaces = sqlx::query_scalar!( - "SELECT workspace_id FROM workspace_settings WHERE datatable::text LIKE $1", - format!("%{}%", role_id) - ) - .fetch_all(&mut **tx) - .await?; - - for w_id in workspaces { - update_datatable_permissions_in_workspace(tx, &w_id, |permissions| { - let mut touched = permissions.roles.remove(role_id).is_some(); - if permissions.default_role.as_deref() == Some(role_id) { - permissions.default_role = Some(ADMIN_DATATABLE_ROLE.to_string()); - touched = true; - } - touched - }) - .await?; - } - Ok(()) + crate::datatable_roles_oss::forget_datatable_role_everywhere(tx, role_id).await } /// Drop the `permissions` block from a `workspace_settings.datatable` value before it leaves the From 7ae3335f035d48ccf1c5bd940baf7a85f8905adc Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Tue, 15 Sep 2026 12:47:46 +0200 Subject: [PATCH 60/77] feat(datatables): move the data table permissions endpoints to the enterprise edition The permissions read, save and usable-roles handlers move to windmill-ee-private; the routes stay registered and, without the enterprise edition, answer that data table roles are an Enterprise Edition feature. ensure_governs_datatable and ensure_reaches_datatable keep their paths: the first refuses, the second passes a data table not under roles. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb --- .../src/datatable_permissions.rs | 537 +----------------- .../src/datatable_permissions_oss.rs | 73 +++ backend/windmill-api-workspaces/src/lib.rs | 4 + 3 files changed, 90 insertions(+), 524 deletions(-) create mode 100644 backend/windmill-api-workspaces/src/datatable_permissions_oss.rs diff --git a/backend/windmill-api-workspaces/src/datatable_permissions.rs b/backend/windmill-api-workspaces/src/datatable_permissions.rs index f774448d64..5cb1f3c1c3 100644 --- a/backend/windmill-api-workspaces/src/datatable_permissions.rs +++ b/backend/windmill-api-workspaces/src/datatable_permissions.rs @@ -14,125 +14,40 @@ //! a save is tenant lists and a default, and the Postgres roles themselves are the instance //! catalog's business. -use std::collections::BTreeMap; +use axum::{routing::get, Router}; -use axum::{ - extract::{Extension, Path}, - routing::get, - Json, Router, -}; -use serde::{Deserialize, Serialize}; - -use windmill_api_auth::{require_super_admin, ApiAuthed}; -use windmill_audit::audit_oss::audit_log; -use windmill_audit::ActionKind; -use windmill_common::datatable_roles::{read_role_catalog, ADMIN_DATATABLE_ROLE}; -use windmill_common::error::{Error, JsonResult, Result}; -use windmill_common::workspaces::{ - can_use_datatable_role_in_governing_workspace, resolve_governing_datatable, - DataTableCatalogResourceType, DataTablePermissions, DataTableRoleTenants, DatatableAccess, - GoverningDatatable, DATATABLE_TENANT_WILDCARD, -}; +use windmill_api_auth::ApiAuthed; +use windmill_common::error::Result; +use windmill_common::workspaces::GoverningDatatable; use windmill_common::DB; +use crate::datatable_permissions_oss as roles; + pub(crate) fn routes() -> Router { Router::new() .route( "/datatable_permissions/{datatable_name}", - get(get_datatable_permissions).post(set_datatable_permissions), + get(roles::get_datatable_permissions).post(roles::set_datatable_permissions), ) .route( "/datatable_usable_roles/{datatable_name}", - get(list_usable_datatable_roles), + get(roles::list_usable_datatable_roles), ) } -/// One row of the permissions drawer: an instance role (or the reserved `admin`) and who may -/// connect as it here. -#[derive(Serialize, Deserialize, Debug)] -pub struct DatatableRoleTenantsInfo { - /// The instance catalog id, or `admin`. - pub id: String, - /// The role's current name, for display. Absent when the catalog no longer has the id — a - /// role deleted out from under this data table, which the drawer shows so it can be removed. - #[serde(skip_serializing_if = "Option::is_none")] - pub name: Option, - pub tenants: Vec, -} - -#[derive(Serialize)] -struct DatatablePermissionsInfo { - /// Whether this data table can be put under roles at all — only one on the instance database - /// can, since a role is a login on that cluster. - supported: bool, - /// Whether the data table is under roles at all. - permissioned: bool, - default_role: String, - roles: Vec, - /// The workspace whose entry this is, when it is not the one asking. - #[serde(skip_serializing_if = "Option::is_none")] - governing_workspace_id: Option, - /// Whether this caller may save. False from a fork, and for a non-admin. - editable: bool, - /// Every instance role the instance defines, to pick from. - available_roles: Vec, - /// Other workspaces whose own entry reaches the same database without being governed by this - /// one — a legacy fork's copy, or a second entry a superadmin pointed here. They keep their own - /// access, so a save here does not reach them. - #[serde(skip_serializing_if = "Vec::is_empty")] - ungoverned_reachers: Vec, -} - -#[derive(Serialize)] -struct AvailableRole { - id: String, - name: String, - enabled: bool, -} - -#[derive(Serialize)] -struct UngovernedReacher { - workspace_id: String, - datatable: String, -} - -#[derive(Deserialize)] -pub struct SetDatatablePermissions { - /// False clears the block: the data table goes back to everyone connecting as `admin`. - pub permissioned: bool, - #[serde(default)] - pub default_role: Option, - #[serde(default)] - pub roles: Vec, -} - -#[derive(Serialize)] -struct UsableDatatableRoles { - permissioned: bool, - /// Names, not ids: this is what a caller writes in `-- role `. - roles: Vec, - default_role: String, -} - /// Administering a data table — its permissions, its migrations that declare no role, its exports /// — is for the admins of the workspace that governs it. A fork can use the data table; it never /// administers it. +// The gate for whatever administers a data table under roles, which the routes of this module alone +// do not always reach. +#[allow(dead_code)] pub(crate) async fn ensure_governs_datatable( db: &DB, authed: &ApiAuthed, w_id: &str, governing: &GoverningDatatable, ) -> Result<()> { - if governing.workspace_id == w_id && authed.is_admin { - return Ok(()); - } - if require_super_admin(db, authed).await.is_ok() { - return Ok(()); - } - Err(Error::NotAuthorized(format!( - "Data table '{}' is governed by workspace '{}'; this is for its admins.", - governing.name, governing.workspace_id - ))) + roles::ensure_governs_datatable(db, authed, w_id, governing).await } /// Refuse a caller that no tenant of this data table covers. @@ -148,431 +63,5 @@ pub(crate) async fn ensure_reaches_datatable( datatable_name: &str, authed: &ApiAuthed, ) -> Result<()> { - let governing = resolve_governing_datatable(db, w_id, datatable_name).await?; - let Some(permissions) = governing.datatable.permissions.as_ref() else { - return Ok(()); - }; - let catalog = read_role_catalog(db).await?; - let access = DatatableAccess::Authed(authed.to_authed_ref()); - for (id, tenants) in &permissions.roles { - // A role the instance no longer defines, or has disabled, cannot be connected as, so being - // tenanted into it is not reach. - if id != ADMIN_DATATABLE_ROLE && !catalog.get(id).is_some_and(|r| r.enabled) { - continue; - } - if can_use_datatable_role_in_governing_workspace( - db, - &governing.workspace_id, - w_id, - tenants, - &access, - ) - .await? - { - return Ok(()); - } - } - Err(Error::NotAuthorized(format!( - "Not allowed to use data table '{datatable_name}': no role of it covers you" - ))) -} - -fn validate_tenant(tenant: &str) -> Result<()> { - if tenant == DATATABLE_TENANT_WILDCARD { - return Ok(()); - } - match tenant.split_once('/') { - Some(("u" | "g" | "f", rest)) if !rest.is_empty() => Ok(()), - _ => Err(Error::BadRequest(format!( - "Invalid tenant '{tenant}': expected 'u/', 'g/', 'f/' or '*'" - ))), - } -} - -/// Entries in other workspaces that reach the same instance database without pointing at this one. -/// -/// They exist by construction and are allowed: a fork created before data table roles holds a -/// literal copy of its parent's entry, and a superadmin can point a second workspace at any -/// instance database. Each governs its own access, so a save here leaves them untouched — which is -/// exactly why they are worth naming at the moment someone turns roles on. -async fn ungoverned_reachers( - db: &DB, - governing: &GoverningDatatable, -) -> Result> { - let Some(database) = governing.datatable.database.as_ref() else { - return Ok(vec![]); - }; - if database.resource_type != DataTableCatalogResourceType::Instance { - return Ok(vec![]); - } - let rows = sqlx::query!( - r#" - SELECT ws.workspace_id AS "workspace_id!", dt.key AS "datatable!" - FROM workspace_settings ws - JOIN workspace w ON w.id = ws.workspace_id AND w.deleted = false - CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt - WHERE ws.workspace_id <> $1 - AND dt.value->'database'->>'resource_type' = 'instance' - AND dt.value->'database'->>'resource_path' = $2 - ORDER BY ws.workspace_id, dt.key - "#, - &governing.workspace_id, - &database.resource_path, - ) - .fetch_all(db) - .await?; - Ok(rows - .into_iter() - .map(|r| UngovernedReacher { workspace_id: r.workspace_id, datatable: r.datatable }) - .collect()) -} - -async fn get_datatable_permissions( - authed: ApiAuthed, - Extension(db): Extension, - Path((w_id, datatable_name)): Path<(String, String)>, -) -> JsonResult { - let governing = resolve_governing_datatable(&db, &w_id, &datatable_name).await?; - let catalog = read_role_catalog(&db).await?; - let editable = ensure_governs_datatable(&db, &authed, &w_id, &governing) - .await - .is_ok(); - - let permissions = governing.datatable.permissions.as_ref(); - let roles = permissions - .map(|p| { - p.roles - .iter() - .map(|(id, tenants)| DatatableRoleTenantsInfo { - id: id.clone(), - name: if id == ADMIN_DATATABLE_ROLE { - Some(ADMIN_DATATABLE_ROLE.to_string()) - } else { - catalog.get(id).map(|r| r.name.clone()) - }, - // Tenants name users, groups and folders of the governing workspace, so they - // are for the people who set them. Someone reading from a fork gets the shape - // of the decision, not the parent's membership; what they may use themselves - // is what `datatable_usable_roles` answers. - tenants: if editable { - tenants.tenants.clone() - } else { - vec![] - }, - }) - .collect() - }) - .unwrap_or_default(); - - Ok(Json(DatatablePermissionsInfo { - supported: governing.is_instance(), - permissioned: permissions.is_some(), - default_role: permissions - .map(|p| p.default_role().to_string()) - .unwrap_or_else(|| ADMIN_DATATABLE_ROLE.to_string()), - roles, - governing_workspace_id: (governing.workspace_id != w_id) - .then(|| governing.workspace_id.clone()), - editable, - // The instance's role names are only of use to someone who can pick from them, and - // enumerating them is the first step of anything that wants to name one it shouldn't. - available_roles: if editable { - catalog - .iter() - .map(|(id, role)| AvailableRole { - id: id.clone(), - name: role.name.clone(), - enabled: role.enabled, - }) - .collect() - } else { - vec![] - }, - ungoverned_reachers: if editable { - ungoverned_reachers(&db, &governing).await? - } else { - vec![] - }, - })) -} - -async fn set_datatable_permissions( - authed: ApiAuthed, - Extension(db): Extension, - Path((w_id, datatable_name)): Path<(String, String)>, - Json(req): Json, -) -> Result { - let governing = resolve_governing_datatable(&db, &w_id, &datatable_name).await?; - ensure_governs_datatable(&db, &authed, &w_id, &governing).await?; - - // A data table role is a login on Windmill's own cluster; a resource-backed data table dials a - // host the workspace admin chose, so it has no business naming one. - if req.permissioned && !governing.is_instance() { - return Err(Error::BadRequest(format!( - "Data table '{}' is backed by a Postgres resource. Data table roles are logins on the \ - Windmill instance's own Postgres, so only a data table on the instance database can \ - use them.", - governing.name - ))); - } - - // One transaction for the whole save, holding both locks the decision depends on: the role - // catalog, so a role cannot be deleted between validating an id and writing it back, and the - // workspace settings row, so a concurrent settings save cannot carry a stale copy of this - // block forward over what is written here. - let mut tx = db.begin().await?; - windmill_common::datatable_roles::lock_role_catalog(&mut tx).await?; - sqlx::query!( - "SELECT 1 AS one FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE", - &governing.workspace_id - ) - .fetch_optional(&mut *tx) - .await?; - - // Everything above was decided on a read taken before the locks. A settings save committing in - // between could have moved this data table onto a PostgreSQL resource — recreating the exact - // state the transition guard refuses — or renamed it, in which case the write below would - // target a key that no longer exists and report success having changed nothing. Re-resolve and - // re-check on the locked state; the earlier pass stays because it is what refuses without - // taking locks at all. - let governing = resolve_governing_datatable(&db, &w_id, &datatable_name).await?; - ensure_governs_datatable(&db, &authed, &w_id, &governing).await?; - if req.permissioned && !governing.is_instance() { - return Err(Error::BadRequest(format!( - "Data table '{}' is backed by a Postgres resource. Data table roles are logins on the \ - Windmill instance's own Postgres, so only a data table on the instance database can \ - use them.", - governing.name - ))); - } - - // Turning roles on is refused while a replication stream reads this data table. One already - // under roles cannot have any: the listener refuses to open a stream on it. - if req.permissioned && governing.datatable.permissions.is_none() { - windmill_common::datatable_roles::lock_datatable_streams(&mut *tx, true).await?; - ensure_no_streams_reaching(&db, &governing).await?; - } - - let permissions = if req.permissioned { - let catalog = windmill_common::datatable_roles::read_role_catalog_tx(&mut tx).await?; - let mut roles: BTreeMap = BTreeMap::new(); - for role in req.roles { - if role.id != ADMIN_DATATABLE_ROLE && !catalog.contains_key(&role.id) { - return Err(Error::BadRequest(format!( - "'{}' is not a data table role of this instance", - role.name.unwrap_or(role.id) - ))); - } - for tenant in &role.tenants { - validate_tenant(tenant)?; - } - roles.insert(role.id, DataTableRoleTenants { tenants: role.tenants }); - } - // `admin` is always a row: it is the connection every object in the database is owned by, - // and a save that dropped it would leave the data table with no way back in. - roles.entry(ADMIN_DATATABLE_ROLE.to_string()).or_default(); - - let default_role = req - .default_role - .unwrap_or_else(|| ADMIN_DATATABLE_ROLE.to_string()); - if !roles.contains_key(&default_role) { - return Err(Error::BadRequest(format!( - "The default role '{default_role}' is not among the data table's roles" - ))); - } - Some(DataTablePermissions { default_role: Some(default_role), roles }) - } else { - None - }; - - let value = match &permissions { - Some(p) => serde_json::to_value(p).map_err(|e| Error::internal_err(e.to_string()))?, - None => serde_json::Value::Null, - }; - // Written straight onto the governing workspace's entry rather than through the settings form, - // which deliberately carries this block across untouched. - sqlx::query!( - r#"UPDATE workspace_settings - SET datatable = CASE WHEN $3::jsonb = 'null'::jsonb - THEN datatable #- ARRAY['datatables', $2, 'permissions'] - ELSE jsonb_set(datatable, ARRAY['datatables', $2, 'permissions'], $3::jsonb) - END - WHERE workspace_id = $1"#, - &governing.workspace_id, - &governing.name, - value, - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "workspaces.set_datatable_permissions", - ActionKind::Update, - &governing.workspace_id, - Some(&authed.email), - Some([("datatable", governing.name.as_str())].into()), - ) - .await?; - tx.commit().await?; - - // An instance database provisioned before data table roles existed has neither the grant - // options the admin connection needs to delegate privileges, nor a CONNECT grant for any role - // — so a role would be refused at login however its tenants read. Repair it here, at the one - // moment someone is deciding this data table's roles. Best-effort: neither is worth failing a - // tenant edit over, and both converge again on the next save. - // - // Runs after the commit: it opens its own connections to other databases, which has no place - // inside a transaction holding two locks. - if permissions.is_some() { - if let Some(database) = governing.datatable.database.as_ref() { - if database.resource_type == DataTableCatalogResourceType::Instance { - let dbname = &database.resource_path; - if let Err(e) = - windmill_common::ensure_instance_db_grant_options_unchecked(&db, dbname).await - { - tracing::warn!("Could not refresh grant options on '{dbname}': {e}"); - } - if let Err(e) = - windmill_common::datatable_roles::converge_connect_grants(&db, dbname).await - { - tracing::warn!("Could not refresh CONNECT grants on '{dbname}': {e}"); - } - } - } - } - - windmill_common::feature_usage::log_feature_usage( - "datatable", - "roles_toggled", - if permissions.is_some() { "on" } else { "off" }, - ); - - Ok(if permissions.is_some() { - format!("Updated the roles of data table '{}'", governing.name) - } else { - format!("Data table '{}' is no longer under roles", governing.name) - }) -} - -/// Refuse to put a data table under roles while a Postgres trigger or capture streams it. A -/// replication stream reads every row whatever the roles grant, so a data table carries one or the -/// other; the listener side refuses a data table already under roles. -async fn ensure_no_streams_reaching(db: &DB, governing: &GoverningDatatable) -> Result<()> { - // Every workspace holding an entry that resolves here, under the name it calls it: the - // governing one, plus each fork pointing at it. A fork's trigger names its own local entry, so - // looking in the governing workspace alone would miss every stream a fork opened. - let mut reached = vec![(governing.workspace_id.clone(), governing.name.clone())]; - let pointers = sqlx::query!( - r#"SELECT ws.workspace_id AS "workspace_id!", dt.key AS "datatable!" - FROM workspace_settings ws - CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt - WHERE dt.value->'reference'->>'workspace_id' = $1 - AND dt.value->'reference'->>'datatable' = $2"#, - &governing.workspace_id, - &governing.name, - ) - .fetch_all(db) - .await?; - reached.extend(pointers.into_iter().map(|r| (r.workspace_id, r.datatable))); - - let mut streams = Vec::new(); - for (w_id, name) in reached { - let reference = format!("datatable://{name}"); - let with_query = format!("{reference}?"); - // A suspended trigger keeps its listener, and a capture streams while its client pings. A - // listener also outlives its trigger being disabled, or its capture's client going quiet, - // until its next heartbeat notices; one that pinged within the 15 seconds a server holds a - // listener for may still be dispatching. - streams.extend( - sqlx::query_scalar::<_, String>( - r#"SELECT workspace_id || '/' || path FROM postgres_trigger - WHERE workspace_id = $1 - AND (mode <> 'disabled'::TRIGGER_MODE - OR last_server_ping > now() - interval '15 seconds') - AND (postgres_resource_path = $2 OR starts_with(postgres_resource_path, $3)) - UNION ALL - SELECT workspace_id || '/' || path || ' (capture)' FROM capture_config - WHERE workspace_id = $1 AND trigger_kind = 'postgres' - AND (last_client_ping > now() - interval '10 seconds' - OR last_server_ping > now() - interval '15 seconds') - AND (trigger_config->>'postgres_resource_path' = $2 - OR starts_with(trigger_config->>'postgres_resource_path', $3))"#, - ) - .bind(&w_id) - .bind(&reference) - .bind(&with_query) - .fetch_all(db) - .await?, - ); - } - if !streams.is_empty() { - return Err(Error::BadRequest(format!( - "Data table '{}' cannot be put under roles while a Postgres trigger or capture streams \ - it: a replication stream reads every row whatever the roles grant. Disable them, then \ - allow their listeners up to 15 seconds to stop: {}", - governing.name, - streams.join(", ") - ))); - } - Ok(()) -} - -/// The roles this caller may connect as, by name, plus the one they get without asking. Drives the -/// role pickers; an empty list means the data table is not under roles. -async fn list_usable_datatable_roles( - authed: ApiAuthed, - Extension(db): Extension, - Path((w_id, datatable_name)): Path<(String, String)>, -) -> JsonResult { - let governing = resolve_governing_datatable(&db, &w_id, &datatable_name).await?; - let Some(permissions) = governing.datatable.permissions.as_ref() else { - return Ok(Json(UsableDatatableRoles { - permissioned: false, - roles: vec![], - default_role: ADMIN_DATATABLE_ROLE.to_string(), - })); - }; - let catalog = read_role_catalog(&db).await?; - let access = DatatableAccess::Authed(authed.to_authed_ref()); - - let mut roles = Vec::new(); - for (id, tenants) in &permissions.roles { - let name = if id == ADMIN_DATATABLE_ROLE { - ADMIN_DATATABLE_ROLE.to_string() - } else { - match catalog.get(id).filter(|r| r.enabled) { - Some(role) => role.name.clone(), - // Deleted or disabled instance-side: it cannot be connected as, so it is not - // offered, even to someone the tenants cover. - None => continue, - } - }; - if can_use_datatable_role_in_governing_workspace( - &db, - &governing.workspace_id, - &w_id, - tenants, - &access, - ) - .await? - { - roles.push(name); - } - } - - let default_role = permissions.default_role(); - Ok(Json(UsableDatatableRoles { - permissioned: true, - roles, - default_role: if default_role == ADMIN_DATATABLE_ROLE { - ADMIN_DATATABLE_ROLE.to_string() - } else { - catalog - .get(default_role) - .map(|r| r.name.clone()) - .unwrap_or_else(|| default_role.to_string()) - }, - })) + roles::ensure_reaches_datatable(db, w_id, datatable_name, authed).await } diff --git a/backend/windmill-api-workspaces/src/datatable_permissions_oss.rs b/backend/windmill-api-workspaces/src/datatable_permissions_oss.rs new file mode 100644 index 0000000000..f4d8c6a7ad --- /dev/null +++ b/backend/windmill-api-workspaces/src/datatable_permissions_oss.rs @@ -0,0 +1,73 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Where the data table permissions endpoints and their gates come from: the enterprise +//! implementation, or a refusal. Roles are an Enterprise Edition feature; see +//! `windmill_common::datatable_roles_oss`. + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub(crate) use crate::datatable_permissions_ee::{ + ensure_governs_datatable, ensure_reaches_datatable, get_datatable_permissions, + list_usable_datatable_roles, set_datatable_permissions, +}; + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub(crate) use ce::*; + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +mod ce { + use windmill_api_auth::ApiAuthed; + use windmill_common::{ + datatable_roles_oss::datatable_roles_unavailable as unavailable, + error::Result, + workspaces::{resolve_governing_datatable, GoverningDatatable}, + DB, + }; + + /// Nobody administers a data table's roles without them. + #[allow(dead_code)] + pub(crate) async fn ensure_governs_datatable( + _db: &DB, + _authed: &ApiAuthed, + _w_id: &str, + _governing: &GoverningDatatable, + ) -> Result<()> { + Err(unavailable()) + } + + /// A data table not under roles is reached as it was before roles existed. One under roles is + /// refused: no role of it can be connected as. + pub(crate) async fn ensure_reaches_datatable( + db: &DB, + w_id: &str, + datatable_name: &str, + _authed: &ApiAuthed, + ) -> Result<()> { + let governing = resolve_governing_datatable(db, w_id, datatable_name).await?; + if governing.datatable.permissions.is_none() { + Ok(()) + } else { + Err(unavailable()) + } + } + + // The routes stay registered so the API has one shape; each answers after authentication, + // before anything is read. + + pub(crate) async fn get_datatable_permissions(_authed: ApiAuthed) -> Result { + Err(unavailable()) + } + + pub(crate) async fn set_datatable_permissions(_authed: ApiAuthed) -> Result { + Err(unavailable()) + } + + pub(crate) async fn list_usable_datatable_roles(_authed: ApiAuthed) -> Result { + Err(unavailable()) + } +} diff --git a/backend/windmill-api-workspaces/src/lib.rs b/backend/windmill-api-workspaces/src/lib.rs index ac3f492ff0..c1eb6c1ffc 100644 --- a/backend/windmill-api-workspaces/src/lib.rs +++ b/backend/windmill-api-workspaces/src/lib.rs @@ -3,6 +3,7 @@ pub mod ai_session_backups; pub mod data_metrics; pub mod datatable_migrations; pub mod datatable_permissions; +pub mod datatable_permissions_oss; pub mod deployment_requests; pub mod workspaces; pub mod workspaces_extra; @@ -10,3 +11,6 @@ pub mod workspaces_oss; #[cfg(feature = "private")] pub mod workspaces_ee; + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub mod datatable_permissions_ee; From 11ba3cd47796f7401aca77f41f18f38beb4fc578 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Tue, 15 Sep 2026 12:47:46 +0200 Subject: [PATCH 61/77] feat(datatables): move the data table role catalog endpoints to the enterprise edition The superadmin list, create, update and delete handlers move to windmill-ee-private. The routes stay registered and, without the enterprise edition, refuse after authentication. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb --- .../src/datatable_roles_oss.rs | 44 +++ backend/windmill-api-settings/src/lib.rs | 262 +----------------- 2 files changed, 51 insertions(+), 255 deletions(-) create mode 100644 backend/windmill-api-settings/src/datatable_roles_oss.rs diff --git a/backend/windmill-api-settings/src/datatable_roles_oss.rs b/backend/windmill-api-settings/src/datatable_roles_oss.rs new file mode 100644 index 0000000000..885d13ee9d --- /dev/null +++ b/backend/windmill-api-settings/src/datatable_roles_oss.rs @@ -0,0 +1,44 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Where the data table role catalog endpoints come from: the enterprise implementation, or a +//! refusal. Roles are an Enterprise Edition feature; see `windmill_common::datatable_roles_oss`. + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub(crate) use crate::datatable_roles_ee::{ + create_datatable_role, delete_datatable_role, list_datatable_roles, update_datatable_role, +}; + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub(crate) use ce::*; + +// The routes stay registered so the API has one shape; each answers after authentication, before +// anything is read. +#[cfg(not(all(feature = "private", feature = "enterprise")))] +mod ce { + use windmill_api_auth::ApiAuthed; + use windmill_common::{ + datatable_roles_oss::datatable_roles_unavailable as unavailable, error::Result, + }; + + pub(crate) async fn list_datatable_roles(_authed: ApiAuthed) -> Result { + Err(unavailable()) + } + + pub(crate) async fn create_datatable_role(_authed: ApiAuthed) -> Result { + Err(unavailable()) + } + + pub(crate) async fn update_datatable_role(_authed: ApiAuthed) -> Result { + Err(unavailable()) + } + + pub(crate) async fn delete_datatable_role(_authed: ApiAuthed) -> Result { + Err(unavailable()) + } +} diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index d8bcf0174f..5d01baafbd 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -17,6 +17,9 @@ mod audit_logs_s3; mod audit_logs_s3_backfill; #[cfg(feature = "parquet")] mod background_task; +#[cfg(all(feature = "private", feature = "enterprise"))] +mod datatable_roles_ee; +mod datatable_roles_oss; #[cfg(feature = "private")] mod ee; pub mod ee_oss; @@ -153,11 +156,13 @@ pub fn global_service() -> Router { ) .route( "/datatable_roles", - get(list_datatable_roles).post(create_datatable_role), + get(datatable_roles_oss::list_datatable_roles) + .post(datatable_roles_oss::create_datatable_role), ) .route( "/datatable_roles/{id}", - post(update_datatable_role).delete(delete_datatable_role), + post(datatable_roles_oss::update_datatable_role) + .delete(datatable_roles_oss::delete_datatable_role), ) .route( "/refresh_custom_instance_user_pwd", @@ -2591,256 +2596,3 @@ mod object_storage_test_hardening { } } } - -// --------------------------------------------------------------------------- -// Data table roles -// --------------------------------------------------------------------------- - -use windmill_audit::audit_oss::audit_log; -use windmill_audit::ActionKind; - -/// One catalog entry as the settings UI sees it. The password never leaves the instance: it is a -/// Postgres credential Windmill mints and hands only to a resolved connection. -#[derive(Serialize)] -struct DatatableRoleInfo { - id: String, - name: String, - enabled: bool, -} - -#[derive(Deserialize)] -struct CreateDatatableRole { - name: String, -} - -#[derive(Deserialize)] -struct UpdateDatatableRole { - /// A rename. Absent leaves the name alone. - #[serde(default)] - name: Option, - /// `LOGIN` / `NOLOGIN`. Grants and ownership survive either way. - #[serde(default)] - enabled: Option, -} - -fn datatable_role_infos( - catalog: &windmill_common::datatable_roles::DatatableRoleCatalog, -) -> Vec { - catalog - .iter() - .map(|(id, role)| DatatableRoleInfo { - id: id.clone(), - name: role.name.clone(), - enabled: role.enabled, - }) - .collect() -} - -async fn list_datatable_roles( - authed: ApiAuthed, - Extension(db): Extension, -) -> JsonResult> { - require_super_admin(&db, &authed).await?; - let catalog = windmill_common::datatable_roles::read_role_catalog(&db).await?; - Ok(Json(datatable_role_infos(&catalog))) -} - -/// Create the Postgres role first, then record it. The cluster is the source of truth: a catalog -/// entry naming a role that does not exist would resolve to a login nothing can authenticate as. -async fn create_datatable_role( - authed: ApiAuthed, - Extension(db): Extension, - Json(req): Json, -) -> JsonResult { - require_super_admin(&db, &authed).await?; - windmill_common::datatable_roles::validate_role_name(&req.name)?; - - // The cluster DDL and the row that records it are one transaction under one lock, so a - // half-done create cannot leave a live login the catalog does not know about. - let mut tx = db.begin().await?; - windmill_common::datatable_roles::lock_role_catalog(&mut tx).await?; - let mut catalog = windmill_common::datatable_roles::read_role_catalog_tx(&mut tx).await?; - if catalog.values().any(|r| r.name == req.name) { - return Err(error::Error::BadRequest(format!( - "A data table role named '{}' already exists", - req.name - ))); - } - - let id = windmill_common::utils::rd_string(12); - let pwd = uuid::Uuid::new_v4().to_string(); - windmill_common::datatable_roles::create_instance_role(&mut tx, &req.name, &pwd).await?; - - let entry = windmill_common::datatable_roles::InstanceDatatableRole { - name: req.name.clone(), - enabled: true, - pwd: Some(pwd), - }; - windmill_common::datatable_roles::insert_role_catalog_entry(&mut tx, &id, &entry).await?; - catalog.insert(id.clone(), entry); - tx.commit().await?; - converge_connect_grants_everywhere(&db, &catalog).await; - windmill_common::feature_usage::log_feature_usage("datatable", "role_created", ""); - - audit_log( - &db, - &authed, - "settings.create_datatable_role", - ActionKind::Create, - "global", - Some(&authed.email), - Some([("name", req.name.as_str())].into()), - ) - .await?; - - Ok(Json(DatatableRoleInfo { - id, - name: req.name, - enabled: true, - })) -} - -async fn update_datatable_role( - authed: ApiAuthed, - Extension(db): Extension, - Path(id): Path, - Json(req): Json, -) -> JsonResult { - require_super_admin(&db, &authed).await?; - let mut tx = db.begin().await?; - windmill_common::datatable_roles::lock_role_catalog(&mut tx).await?; - let mut catalog = windmill_common::datatable_roles::read_role_catalog_tx(&mut tx).await?; - let role = catalog - .get(&id) - .cloned() - .ok_or_else(|| error::Error::NotFound(format!("No data table role with id '{id}'")))?; - - let mut updated = role.clone(); - if let Some(name) = req.name.filter(|n| n != &role.name) { - windmill_common::datatable_roles::validate_role_name(&name)?; - if catalog.values().any(|r| r.name == name) { - return Err(error::Error::BadRequest(format!( - "A data table role named '{name}' already exists" - ))); - } - // RENAME discards an md5-hashed password, so the role gets a fresh one in the same - // statement and the catalog records it. Tenants name the id, so nothing else moves. - let pwd = uuid::Uuid::new_v4().to_string(); - windmill_common::datatable_roles::rename_instance_role(&mut tx, &role.name, &name, &pwd) - .await?; - updated.name = name; - updated.pwd = Some(pwd); - } - if let Some(enabled) = req.enabled.filter(|e| *e != role.enabled) { - windmill_common::datatable_roles::set_instance_role_login(&mut tx, &updated.name, enabled) - .await?; - updated.enabled = enabled; - } - - windmill_common::datatable_roles::update_role_catalog_entry(&mut tx, &id, &updated).await?; - catalog.insert(id.clone(), updated.clone()); - tx.commit().await?; - converge_connect_grants_everywhere(&db, &catalog).await; - - audit_log( - &db, - &authed, - "settings.update_datatable_role", - ActionKind::Update, - "global", - Some(&authed.email), - Some([("name", updated.name.as_str())].into()), - ) - .await?; - - Ok(Json(DatatableRoleInfo { - id, - name: updated.name, - enabled: updated.enabled, - })) -} - -/// Disable the role in its own commit, then drop the Postgres role, then forget it, then strip it -/// from every workspace that tenanted it. -/// -/// Dropping before forgetting is what makes the catalog trustworthy: the drop refuses while any -/// instance database is unreachable, so a failure leaves the entry in place to retry rather than a -/// live Postgres login nothing names. -async fn delete_datatable_role( - authed: ApiAuthed, - Extension(db): Extension, - Path(id): Path, -) -> JsonResult<()> { - require_super_admin(&db, &authed).await?; - let find = |catalog: &windmill_common::datatable_roles::DatatableRoleCatalog| { - catalog - .get(&id) - .cloned() - .ok_or_else(|| error::Error::NotFound(format!("No data table role with id '{id}'"))) - }; - - let mut tx = db.begin().await?; - windmill_common::datatable_roles::lock_role_catalog(&mut tx).await?; - let mut role = find(&windmill_common::datatable_roles::read_role_catalog_tx(&mut tx).await?)?; - if role.enabled { - windmill_common::datatable_roles::set_instance_role_login(&mut tx, &role.name, false) - .await?; - role.enabled = false; - windmill_common::datatable_roles::update_role_catalog_entry(&mut tx, &id, &role).await?; - } - tx.commit().await?; - - let mut tx = db.begin().await?; - windmill_common::datatable_roles::lock_role_catalog(&mut tx).await?; - let role = find(&windmill_common::datatable_roles::read_role_catalog_tx(&mut tx).await?)?; - if role.enabled { - return Err(error::Error::BadRequest(format!( - "Data table role '{}' was re-enabled while being deleted", - role.name - ))); - } - - windmill_common::datatable_roles::drop_instance_role(&db, &mut tx, &role.name).await?; - windmill_common::datatable_roles::delete_role_catalog_entry(&mut tx, &id).await?; - windmill_common::workspaces::forget_datatable_role_everywhere(&mut tx, &id).await?; - tx.commit().await?; - - audit_log( - &db, - &authed, - "settings.delete_datatable_role", - ActionKind::Delete, - "global", - Some(&authed.email), - Some([("name", role.name.as_str())].into()), - ) - .await?; - - Ok(Json(())) -} - -/// Best-effort `CONNECT` convergence over the instance database registry. A database that is -/// unreachable right now is repaired the next time one of its data tables is administered, so a -/// role creation is not held hostage by an unrelated database being down. -async fn converge_connect_grants_everywhere( - db: &DB, - catalog: &windmill_common::datatable_roles::DatatableRoleCatalog, -) { - let dbnames = match windmill_common::datatable_roles::registered_instance_databases(db).await { - Ok(dbnames) => dbnames, - Err(e) => { - tracing::warn!("Could not list instance databases to grant CONNECT: {e}"); - return; - } - }; - for dbname in dbnames { - if let Err(e) = - windmill_common::datatable_roles::converge_connect_grants_with(db, &dbname, catalog) - .await - { - tracing::warn!( - "Could not converge CONNECT grants on instance database '{dbname}': {e}" - ); - } - } -} From db05c8b7c96696de8eceb3ccfe714e179379d715 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Tue, 15 Sep 2026 12:47:46 +0200 Subject: [PATCH 62/77] test(datatables): run the roles tests on the enterprise edition, refusals without it Each test that exercises roles runs with private and enterprise. Two tests run without them: every roles route answers the Enterprise refusal, and a data table saved under roles, or a named role, is refused a connection while one not under roles resolves as before. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb --- .../tests/datatable_roles.rs | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs index 0a00ed8bb6..a15439455f 100644 --- a/backend/windmill-api-integration-tests/tests/datatable_roles.rs +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -90,6 +90,7 @@ async fn freeing_a_principal_takes_its_datatable_tenant(db: Pool) -> a Ok(()) } +#[cfg(all(feature = "private", feature = "enterprise"))] #[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] async fn a_fork_uses_the_data_table_it_points_at_but_never_administers_it( db: Pool, @@ -168,6 +169,7 @@ async fn a_fork_uses_the_data_table_it_points_at_but_never_administers_it( Ok(()) } +#[cfg(all(feature = "private", feature = "enterprise"))] #[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] async fn a_second_entry_on_the_same_database_is_reported_rather_than_governed( db: Pool, @@ -202,6 +204,7 @@ async fn a_second_entry_on_the_same_database_is_reported_rather_than_governed( Ok(()) } +#[cfg(all(feature = "private", feature = "enterprise"))] #[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] async fn a_resource_backed_data_table_cannot_be_put_under_roles( db: Pool, @@ -282,6 +285,7 @@ async fn a_fork_renaming_its_own_entry_leaves_the_governing_bookkeeping_alone( Ok(()) } +#[cfg(all(feature = "private", feature = "enterprise"))] #[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] async fn a_caller_who_is_not_a_member_of_the_governing_workspace_reaches_nothing( db: Pool, @@ -312,6 +316,7 @@ async fn a_caller_who_is_not_a_member_of_the_governing_workspace_reaches_nothing Ok(()) } +#[cfg(all(feature = "private", feature = "enterprise"))] #[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] async fn a_caller_with_no_identity_reaches_a_permissioned_data_table_not_at_all( db: Pool, @@ -351,6 +356,7 @@ async fn a_caller_with_no_identity_reaches_a_permissioned_data_table_not_at_all( Ok(()) } +#[cfg(all(feature = "private", feature = "enterprise"))] #[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] async fn concurrent_role_creations_both_survive(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; @@ -407,6 +413,7 @@ async fn concurrent_role_creations_both_survive(db: Pool) -> anyhow::R outcome } +#[cfg(all(feature = "private", feature = "enterprise"))] #[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] async fn a_role_delete_that_fails_part_way_leaves_the_role_disabled( db: Pool, @@ -477,6 +484,7 @@ async fn a_role_delete_that_fails_part_way_leaves_the_role_disabled( outcome } +#[cfg(all(feature = "private", feature = "enterprise"))] #[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] async fn renaming_a_governing_data_table_carries_its_forks( db: Pool, @@ -742,6 +750,7 @@ async fn a_clone_stamp_is_carried_but_its_schema_baseline_advances( Ok(()) } +#[cfg(all(feature = "private", feature = "enterprise"))] #[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] async fn roles_cannot_be_turned_on_while_a_trigger_streams_the_data_table( db: Pool, @@ -818,6 +827,7 @@ async fn roles_cannot_be_turned_on_while_a_trigger_streams_the_data_table( Ok(()) } +#[cfg(all(feature = "private", feature = "enterprise"))] #[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] async fn roles_going_on_wait_for_a_trigger_being_enabled(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; @@ -998,3 +1008,121 @@ async fn an_entry_without_roles_cannot_newly_reach_a_database_under_roles( assert!(still_governed, "the refused save still took effect"); Ok(()) } + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +const ENTERPRISE_REFUSAL: &str = "Data table roles are a Windmill Enterprise Edition feature"; + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn every_roles_route_is_an_enterprise_feature(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let api = format!("http://localhost:{}/api", server.addr.port()); + let dt = format!("{api}/w/test-workspace/workspaces"); + + for (method, url, body) in [ + ( + reqwest::Method::GET, + format!("{api}/settings/datatable_roles"), + None, + ), + ( + reqwest::Method::POST, + format!("{api}/settings/datatable_roles"), + Some(json!({ "name": "wmtest_ce" })), + ), + ( + reqwest::Method::POST, + format!("{api}/settings/datatable_roles/role1"), + Some(json!({ "enabled": false })), + ), + ( + reqwest::Method::DELETE, + format!("{api}/settings/datatable_roles/role1"), + None, + ), + ( + reqwest::Method::GET, + format!("{dt}/datatable_permissions/main"), + None, + ), + ( + reqwest::Method::POST, + format!("{dt}/datatable_permissions/main"), + Some(json!({ "permissioned": false })), + ), + ( + reqwest::Method::GET, + format!("{dt}/datatable_usable_roles/main"), + None, + ), + ] { + let mut request = authed(client().request(method.clone(), &url), "SECRET_TOKEN"); + if let Some(body) = body { + request = request.json(&body); + } + let resp = request.send().await?; + let status = resp.status(); + let text = resp.text().await?; + assert!( + status == 400 && text.contains(ENTERPRISE_REFUSAL), + "{method} {url} answered {status}: {text}" + ); + } + + // Refused, not acted on: the catalog row and the data table's roles are where they were. + let untouched: (i64, bool) = sqlx::query_as( + "SELECT (SELECT count(*) FROM datatable_role), + (datatable->'datatables'->'main') ? 'permissions' + FROM workspace_settings WHERE workspace_id = 'test-workspace'", + ) + .fetch_one(&db) + .await?; + assert_eq!(untouched, (1, true)); + Ok(()) +} + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn without_the_enterprise_edition_a_data_table_under_roles_is_refused_a_connection( + db: Pool, +) -> anyhow::Result<()> { + use windmill_common::workspaces::{get_datatable_resource_from_db, DatatableAccess}; + initialize_tracing().await; + + // Saved under roles, as an enterprise build left it: refused whoever asks, never `admin`. + for access in [DatatableAccess::Unchecked, DatatableAccess::NoIdentity] { + let err = get_datatable_resource_from_db(&db, "test-workspace", "main", None, access) + .await + .expect_err("a data table under roles resolved"); + assert!(err.to_string().contains(ENTERPRISE_REFUSAL), "{err}"); + } + + // Not under roles, it resolves as it always has; naming a role on it is refused. + sqlx::query( + "UPDATE workspace_settings SET datatable = datatable #- '{datatables,main,permissions}' + WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + let resolved = get_datatable_resource_from_db( + &db, + "test-workspace", + "main", + None, + DatatableAccess::NoIdentity, + ) + .await?; + assert_eq!(resolved["dbname"], "dt_main", "{resolved}"); + let err = get_datatable_resource_from_db( + &db, + "test-workspace", + "main", + Some("analytics"), + DatatableAccess::Unchecked, + ) + .await + .expect_err("a named role resolved"); + assert!(err.to_string().contains(ENTERPRISE_REFUSAL), "{err}"); + Ok(()) +} From eafc551651c284f5a9fb5fb7f794de61575db651 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Tue, 15 Sep 2026 12:47:46 +0200 Subject: [PATCH 63/77] feat(datatables): gate the roles UI mount sites on an enterprise license Both mount sites are still commented out; the gate travels with them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb --- .../workspaceSettings/DataTableSettings.svelte | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte index 4cda99cb4a..867cc58d87 100644 --- a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte @@ -481,11 +481,13 @@ than rewriting it, and uncomment this together with the roles section at the bottom of this file and the two imports at the top. - + {#if $enterpriseLicense} + + {/if} -->
From d3ee67c9bc17e6d32c24fa19d2d6c8a0e85db3e5 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Tue, 15 Sep 2026 13:13:07 +0200 Subject: [PATCH 64/77] test(datatables): run the tenant matcher test on the enterprise edition The matcher it covers is enterprise code now, so without the enterprise edition the test hit the stub and failed the default windmill-common run. It runs with private and enterprise, and a counterpart without them asserts that no tenant list covers anyone, the wildcard and a workspace admin included. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb --- backend/windmill-common/src/workspaces.rs | 24 +++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 41cf2b728f..e8974da0e8 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -3152,6 +3152,7 @@ mod tests { DataTableRoleTenants { tenants: list.iter().map(|t| t.to_string()).collect() } } + #[cfg(all(feature = "private", feature = "enterprise"))] #[test] fn a_tenant_list_covers_users_groups_folders_and_the_wildcard() { let groups = vec!["analysts".to_string()]; @@ -3189,6 +3190,29 @@ mod tests { assert!(can_use_datatable_role(&tenants(&[]), &admin)); } + #[cfg(not(all(feature = "private", feature = "enterprise")))] + #[test] + fn without_the_enterprise_edition_no_tenant_list_covers_anyone() { + let groups = vec![]; + let folders = vec![]; + let scopes = None; + let token_prefix = None; + let is_admin = true; + let is_operator = false; + let admin = crate::db::AuthedRef { + email: "alice@windmill.dev", + username: "alice", + is_admin: &is_admin, + is_operator: &is_operator, + groups: &groups, + folders: &folders, + scopes: &scopes, + token_prefix: &token_prefix, + }; + assert!(!can_use_datatable_role(&tenants(&["*"]), &admin)); + assert!(!can_use_datatable_role(&tenants(&["u/alice"]), &admin)); + } + #[test] fn a_datatable_ref_splits_off_its_role() { assert_eq!(parse_datatable_ref("sales").unwrap(), ("sales", None)); From d1c2f801f41d7437d88b686e46733f7e385b66ee Mon Sep 17 00:00:00 2001 From: "windmill-internal-app[bot]" Date: Wed, 16 Sep 2026 10:07:44 +0000 Subject: [PATCH 65/77] chore: update ee-repo-ref to a1873dbb67f2302b85ff5362f8387b48eccdb607 This commit updates the EE repository reference after PR #783 was merged in windmill-ee-private. Previous ee-repo-ref: 5c853e2c20eca6b748415fc0d6862a6ebfb5fec4 New ee-repo-ref: a1873dbb67f2302b85ff5362f8387b48eccdb607 Automated by sync-ee-ref workflow. --- backend/ee-repo-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index e4502b5bd8..ed798f6d32 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -e6e5dc7d20bb7143231a2bede7f53c430aeaa2c1 +a1873dbb67f2302b85ff5362f8387b48eccdb607 From 31b5b0a062f4cdd00549944dac5e73992544fef6 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Wed, 16 Sep 2026 14:06:44 +0200 Subject: [PATCH 66/77] fix(datatables): refuse roles while a same-workspace alias reaches the database Co-Authored-By: Claude Opus 5 (1M context) --- ...8dc45020a2a553d8874c49f9eafedea5a9d40.json | 24 +++++++++++++++++++ backend/ee-repo-ref.txt | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 backend/.sqlx/query-d48ca62c86b1af7a9dd2450c1c28dc45020a2a553d8874c49f9eafedea5a9d40.json diff --git a/backend/.sqlx/query-d48ca62c86b1af7a9dd2450c1c28dc45020a2a553d8874c49f9eafedea5a9d40.json b/backend/.sqlx/query-d48ca62c86b1af7a9dd2450c1c28dc45020a2a553d8874c49f9eafedea5a9d40.json new file mode 100644 index 0000000000..49875acd78 --- /dev/null +++ b/backend/.sqlx/query-d48ca62c86b1af7a9dd2450c1c28dc45020a2a553d8874c49f9eafedea5a9d40.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT dt.key AS \"datatable!\"\n FROM workspace_settings ws\n CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt\n WHERE ws.workspace_id = $1\n AND dt.key <> $2\n AND NOT dt.value ? 'permissions'\n AND dt.value->'database'->>'resource_type' = 'instance'\n AND dt.value->'database'->>'resource_path' = $3\n ORDER BY dt.key", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "datatable!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "d48ca62c86b1af7a9dd2450c1c28dc45020a2a553d8874c49f9eafedea5a9d40" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index ed798f6d32..1d1f24af71 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -a1873dbb67f2302b85ff5362f8387b48eccdb607 +53614778ef0eec12ef22c6001c8905cd46f5ee17 From d191cfe4eb01ce99c2796d48748843cdd187bd9a Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Wed, 16 Sep 2026 17:25:42 +0200 Subject: [PATCH 67/77] fix(datatables): let CE migrations connect as an explicitly named admin Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/datatable_roles.rs | 23 +++++++++++-------- .../src/datatable_roles_oss.rs | 8 ++++--- backend/windmill-common/src/workspaces.rs | 7 +++--- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs index a15439455f..08790386a4 100644 --- a/backend/windmill-api-integration-tests/tests/datatable_roles.rs +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -1098,22 +1098,25 @@ async fn without_the_enterprise_edition_a_data_table_under_roles_is_refused_a_co assert!(err.to_string().contains(ENTERPRISE_REFUSAL), "{err}"); } - // Not under roles, it resolves as it always has; naming a role on it is refused. + // Not under roles, it resolves as it always has, including when `admin` is named — which every + // migration does; naming any other role on it is refused. sqlx::query( "UPDATE workspace_settings SET datatable = datatable #- '{datatables,main,permissions}' WHERE workspace_id = 'test-workspace'", ) .execute(&db) .await?; - let resolved = get_datatable_resource_from_db( - &db, - "test-workspace", - "main", - None, - DatatableAccess::NoIdentity, - ) - .await?; - assert_eq!(resolved["dbname"], "dt_main", "{resolved}"); + for role in [None, Some("admin")] { + let resolved = get_datatable_resource_from_db( + &db, + "test-workspace", + "main", + role, + DatatableAccess::NoIdentity, + ) + .await?; + assert_eq!(resolved["dbname"], "dt_main", "{resolved}"); + } let err = get_datatable_resource_from_db( &db, "test-workspace", diff --git a/backend/windmill-common/src/datatable_roles_oss.rs b/backend/windmill-common/src/datatable_roles_oss.rs index 98fef4260f..057a739bd4 100644 --- a/backend/windmill-common/src/datatable_roles_oss.rs +++ b/backend/windmill-common/src/datatable_roles_oss.rs @@ -163,8 +163,8 @@ mod ce { Err(unavailable()) } - /// A data table not under roles, asked for no role, is not a role decision and passes, as it - /// did before roles existed. Anything else is refused. + /// A data table not under roles, asked for no role or for `admin`, is not a role decision and + /// passes, as it did before roles existed. Anything else is refused. pub(crate) async fn ensure_can_use_datatable_role( db: &DB, w_id: &str, @@ -174,7 +174,9 @@ mod ce { _context: &str, ) -> Result<()> { let governing = resolve_governing_datatable(db, w_id, name).await?; - if governing.datatable.permissions.is_none() && role.is_none() { + if governing.datatable.permissions.is_none() + && role.is_none_or(|r| r == crate::datatable_roles::ADMIN_DATATABLE_ROLE) + { Ok(()) } else { Err(unavailable()) diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index e8974da0e8..3e8dd0aae5 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -1664,9 +1664,10 @@ pub async fn get_datatable_resource_from_db( ) -> Result { let governing = resolve_governing_datatable(db, w_id, name).await?; let db_resource = resolve_datatable_connection_unchecked(db, &governing, false).await?; - // Not under roles and asked for none: the `admin` connection, as before roles existed, in - // every edition. Anything else is a role decision. - if governing.datatable.permissions.is_none() && role.is_none() { + // Not under roles and asked for none, or for `admin` by name: the `admin` connection, as before + // roles existed, in every edition. Anything else is a role decision. Every migration names + // `admin` explicitly, so an edition without roles must not treat that as one. + if governing.datatable.permissions.is_none() && role.is_none_or(|r| r == ADMIN_DATATABLE_ROLE) { return Ok(db_resource); } crate::datatable_roles_oss::resolve_datatable_role_connection( From 79bb761c5ddf5f1eb0da176f4a52f46bc3d22bfd Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Wed, 16 Sep 2026 18:09:07 +0200 Subject: [PATCH 68/77] fix(datatables): serialize roles going on with aliases saved from other workspaces Co-Authored-By: Claude Opus 5 (1M context) --- backend/ee-repo-ref.txt | 2 +- .../tests/datatable_roles.rs | 62 ++++++++++++++ .../windmill-api-workspaces/src/workspaces.rs | 84 +++++++++++-------- .../windmill-common/src/datatable_roles.rs | 19 +++++ 4 files changed, 131 insertions(+), 36 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 1d1f24af71..9717f3f96f 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -53614778ef0eec12ef22c6001c8905cd46f5ee17 +38d6fcf2aeb39cfdac21814bbdbbcc02911e566a diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs index 08790386a4..f12daf1953 100644 --- a/backend/windmill-api-integration-tests/tests/datatable_roles.rs +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -1009,6 +1009,68 @@ async fn an_entry_without_roles_cannot_newly_reach_a_database_under_roles( Ok(()) } +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn an_alias_saved_elsewhere_waits_for_roles_going_on_for_its_database( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + sqlx::query( + r#"UPDATE workspace_settings SET datatable = jsonb_set(datatable, '{datatables,other}', + '{"database": {"resource_type": "instance", "resource_path": "dt_other"}}') + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + + // Roles going on for `dt_other`, not committed yet: it holds only its own workspace's settings + // row, so an alias saved from another workspace that looked for roles now would miss them. + let enabling = { + let mut tx = db.begin().await?; + windmill_common::datatable_roles::lock_instance_databases_governance( + &mut *tx, + ["dt_other"], + ) + .await?; + sqlx::query( + r#"UPDATE workspace_settings SET datatable = jsonb_set(datatable, + '{datatables,other,permissions}', + '{"default_role": "admin", "roles": {"admin": {"tenants": ["*"]}}}') + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&mut *tx) + .await?; + tx + }; + + let server = ApiServer::start(db.clone()).await?; + let url = format!( + "http://localhost:{}/api/w/wm-fork-dt/workspaces/edit_datatable_config", + server.addr.port() + ); + let save = tokio::spawn( + authed(client().post(&url), "SECRET_TOKEN") + .json(&json!({ "settings": { "datatables": { + "direct": { "database": { "resource_type": "instance", "resource_path": "dt_other" } } + } } })) + .send(), + ); + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + assert!( + !save.is_finished(), + "an alias was saved while roles were going on for its database" + ); + enabling.commit().await?; + + let resp = save.await??; + let status = resp.status(); + let body = resp.text().await?; + assert!( + status == 400 && body.contains("which a data table under roles uses"), + "the alias reached the database whose roles went on while it waited ({status}): {body}" + ); + Ok(()) +} + #[cfg(not(all(feature = "private", feature = "enterprise")))] const ENTERPRISE_REFUSAL: &str = "Data table roles are a Windmill Enterprise Edition feature"; diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 11f6958322..65fdc88212 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -3887,50 +3887,64 @@ async fn edit_datatable_config( // entry through a declared rename alone, and a settings sync never declares one, so an entry // without roles that newly points at such a database — a name added, or an existing one // repointed — would answer everyone there as `admin`. That holds whichever workspace governs it. - let governed_elsewhere: Vec = sqlx::query_scalar( - "SELECT DISTINCT dt.value->'database'->>'resource_path' FROM workspace_settings ws - CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt - WHERE ws.workspace_id <> $1 AND dt.value ? 'permissions' - AND dt.value->'database'->>'resource_type' = 'instance'", + let newly_pointed: Vec<(&String, &str)> = new_config + .settings + .datatables + .iter() + .filter(|(_, dt)| dt.permissions.is_none()) + .filter_map(|(name, dt)| { + let db = dt + .database + .as_ref() + .filter(|d| d.resource_type == DataTableCatalogResourceType::Instance)?; + let lookup = rename_src + .get(name.as_str()) + .copied() + .unwrap_or(name.as_str()); + let repointed = old_datatables + .get(lookup) + .and_then(|old| old.database.as_ref()) + .is_none_or(|old_db| { + old_db.resource_type != db.resource_type + || old_db.resource_path != db.resource_path + }); + repointed.then_some((name, db.resource_path.as_str())) + }) + .collect(); + // Another workspace turning roles on for the same database holds only its own settings row, so + // without this the scan below could read past its uncommitted write. + windmill_common::datatable_roles::lock_instance_databases_governance( + &mut *tx, + newly_pointed.iter().map(|(_, dbname)| *dbname), ) - .bind(&w_id) - .fetch_all(&mut *tx) .await?; - for (name, dt) in new_config.settings.datatables.iter() { - if dt.permissions.is_some() { - continue; - } - let Some(db) = dt - .database - .as_ref() - .filter(|d| d.resource_type == DataTableCatalogResourceType::Instance) - else { - continue; - }; - let lookup = rename_src - .get(name.as_str()) - .copied() - .unwrap_or(name.as_str()); - let repointed = old_datatables - .get(lookup) - .and_then(|old| old.database.as_ref()) - .is_none_or(|old_db| { - old_db.resource_type != db.resource_type || old_db.resource_path != db.resource_path - }); + let governed_elsewhere: Vec = if newly_pointed.is_empty() { + vec![] + } else { + sqlx::query_scalar( + "SELECT DISTINCT dt.value->'database'->>'resource_path' FROM workspace_settings ws + CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt + WHERE ws.workspace_id <> $1 AND dt.value ? 'permissions' + AND dt.value->'database'->>'resource_type' = 'instance'", + ) + .bind(&w_id) + .fetch_all(&mut *tx) + .await? + }; + for (name, dbname) in newly_pointed { let governed_here = old_datatables.values().any(|old| { old.permissions.is_some() && old.database.as_ref().is_some_and(|d| { d.resource_type == DataTableCatalogResourceType::Instance - && d.resource_path == db.resource_path + && d.resource_path == dbname }) }); - if repointed && (governed_here || governed_elsewhere.contains(&db.resource_path)) { + if governed_here || governed_elsewhere.iter().any(|g| g == dbname) { return Err(Error::BadRequest(format!( - "Data table '{name}' would point at database '{}', which a data table under roles \ - uses, without carrying those roles: everyone reaching '{name}' would connect there \ - as `admin`. Rename the data table under roles from the data table settings, which \ - carries its roles, or turn its roles off first.", - db.resource_path + "Data table '{name}' would point at database '{dbname}', which a data table under \ + roles uses, without carrying those roles: everyone reaching '{name}' would connect \ + there as `admin`. Rename the data table under roles from the data table settings, \ + which carries its roles, or turn its roles off first." ))); } } diff --git a/backend/windmill-common/src/datatable_roles.rs b/backend/windmill-common/src/datatable_roles.rs index 3c269d1ada..4dd9b06fde 100644 --- a/backend/windmill-common/src/datatable_roles.rs +++ b/backend/windmill-common/src/datatable_roles.rs @@ -140,6 +140,25 @@ pub async fn lock_datatable_streams(conn: &mut sqlx::PgConnection, exclusive: bo Ok(()) } +/// Whether an instance database is reached only through entries under roles is decided by two +/// writes that lock different workspaces' settings rows: turning roles on for one entry, and a +/// settings save pointing an entry without roles at the database. Each holds this for every +/// database it decides on, so neither reads past the other's uncommitted write. Held for the +/// transaction; the names are locked in sorted order so two holders cannot deadlock. +pub async fn lock_instance_databases_governance<'a>( + conn: &mut sqlx::PgConnection, + dbnames: impl IntoIterator, +) -> Result<()> { + let dbnames: std::collections::BTreeSet<&str> = dbnames.into_iter().collect(); + for dbname in dbnames { + sqlx::query("SELECT pg_advisory_xact_lock(hashtext('datatable_instance_database:' || $1))") + .bind(dbname) + .execute(&mut *conn) + .await?; + } + Ok(()) +} + /// Disclosure: returns every role's stored Postgres password in plaintext. Any server path that /// has to resolve or name a role may call it — including handlers open to a workspace member, who /// need the names — but callers MUST NOT let `pwd` reach a response, a log line, an audit record From 937882847281713b67fab96b12cf50fd1d47efef Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Wed, 16 Sep 2026 22:25:27 +0200 Subject: [PATCH 69/77] docs(datatables): note that legacy names with ? cannot be migrated Co-Authored-By: Claude Opus 5 (1M context) --- backend/windmill-api-workspaces/src/datatable_migrations.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/windmill-api-workspaces/src/datatable_migrations.rs b/backend/windmill-api-workspaces/src/datatable_migrations.rs index 8900e708e4..fa433b2cbc 100644 --- a/backend/windmill-api-workspaces/src/datatable_migrations.rs +++ b/backend/windmill-api-workspaces/src/datatable_migrations.rs @@ -174,6 +174,10 @@ async fn datatable_database_arg( // default role — which is what `ensure_migration_role_allowed` gated it as, and which is the // only role a DDL statement can be expected to succeed under. A migration that does declare a // role overrides this: the annotation wins over the reference. + // + // A legacy name containing `?` cannot be migrated through this reference: the appended query + // makes it neither an exact name nor a parseable one. Accepted on purpose, since such names can + // no longer be created and none are expected to carry migrations. Ok(to_raw_value(&format!( "datatable://{datatable_name}?role={ADMIN_DATATABLE_ROLE}" ))) From 93d346e54be61ff09ce8affdbf40ab7d6b577f51 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Wed, 16 Sep 2026 15:22:05 +0200 Subject: [PATCH 70/77] feat(datatables): add an ACL editor for data table roles Co-Authored-By: Claude Opus 5 (1M context) --- backend/ee-repo-ref.txt | 2 +- .../tests/datatable_acl.rs | 108 + .../src/datatable_acl.rs | 1908 +++++++++++++++++ .../src/datatable_acl_oss.rs | 44 + backend/windmill-api-workspaces/src/lib.rs | 5 + .../windmill-api-workspaces/src/workspaces.rs | 1 + backend/windmill-api/openapi.yaml | 340 +++ backend/windmill-common/src/lib.rs | 14 + .../lib/components/InstanceSettings.svelte | 12 +- .../datatableAcl/AclTargetPicker.svelte | 50 + .../datatableAcl/PgAclEditor.svelte | 289 +++ .../datatableAcl/PgGrantBuilder.svelte | 95 + .../components/datatableAcl/aclScopes.test.ts | 178 ++ .../lib/components/datatableAcl/aclScopes.ts | 211 ++ .../DataTablePermissionsButton.svelte | 75 +- .../DataTableRolesSection.svelte | 11 - .../DataTableSettings.svelte | 41 +- .../InstanceRolesButton.svelte | 29 + 18 files changed, 3362 insertions(+), 51 deletions(-) create mode 100644 backend/windmill-api-integration-tests/tests/datatable_acl.rs create mode 100644 backend/windmill-api-workspaces/src/datatable_acl.rs create mode 100644 backend/windmill-api-workspaces/src/datatable_acl_oss.rs create mode 100644 frontend/src/lib/components/datatableAcl/AclTargetPicker.svelte create mode 100644 frontend/src/lib/components/datatableAcl/PgAclEditor.svelte create mode 100644 frontend/src/lib/components/datatableAcl/PgGrantBuilder.svelte create mode 100644 frontend/src/lib/components/datatableAcl/aclScopes.test.ts create mode 100644 frontend/src/lib/components/datatableAcl/aclScopes.ts create mode 100644 frontend/src/lib/components/workspaceSettings/InstanceRolesButton.svelte diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 9717f3f96f..b878c8de1f 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -38d6fcf2aeb39cfdac21814bbdbbcc02911e566a +6f26308c67acf9fcc45773b373aa30a2593b665c diff --git a/backend/windmill-api-integration-tests/tests/datatable_acl.rs b/backend/windmill-api-integration-tests/tests/datatable_acl.rs new file mode 100644 index 0000000000..4310830ae2 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/datatable_acl.rs @@ -0,0 +1,108 @@ +//! Who may read and change a data table's grants and owners. On the Enterprise Edition: its +//! administrators, from the workspace that governs it. Without it: nobody. Each refusal is decided +//! before anything connects to the data table, so the fixture's database never has to exist. + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +fn grant_select_on_public() -> Value { + json!({ + "target": {"kind": "schema", "schema": "public"}, + "change": {"type": "grant", "role": "analytics", "privileges": ["SELECT"], + "scope": "all_tables"}, + "statements": [r#"GRANT SELECT ON ALL TABLES IN SCHEMA "public" TO "analytics""#] + }) +} + +async fn post_acl( + port: u16, + w_id: &str, + action: &str, + token: &str, +) -> anyhow::Result { + Ok(reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/{w_id}/workspaces/datatable_acl/main/{action}" + )) + .header("Authorization", format!("Bearer {token}")) + .json(&grant_select_on_public()) + .send() + .await?) +} + +/// A fork reaches the data table through a pointer: it may use it, never change what each role may +/// touch on it — not even as an admin of the fork. +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_fork_cannot_change_access_on_the_data_table_it_points_at( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + for action in ["plan", "apply"] { + let resp = post_acl(port, "wm-fork-dt", action, "SECRET_TOKEN_2").await?; + assert_eq!(resp.status(), 401, "{action}: {}", resp.text().await?); + } + Ok(()) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_member_who_is_not_an_admin_cannot_change_access( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + for action in ["plan", "apply"] { + let resp = post_acl(port, "test-workspace", action, "SECRET_TOKEN_2").await?; + assert_eq!(resp.status(), 401, "{action}: {}", resp.text().await?); + } + Ok(()) +} + +/// Not even reading, and not even on a data table that is not under roles — which any member +/// reaches, so only the edition stands between them and the instance's credentials. +#[cfg(not(all(feature = "private", feature = "enterprise")))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn only_the_enterprise_edition_has_the_access_editor( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + sqlx::query( + "UPDATE workspace_settings + SET datatable = datatable #- '{datatables,main,permissions}' + WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let read = reqwest::Client::new() + .get(format!( + "http://localhost:{port}/api/w/test-workspace/workspaces/datatable_acl/main?kind=database" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await?; + let mut responses = vec![("read", read)]; + for action in ["plan", "apply"] { + responses.push(( + action, + post_acl(port, "test-workspace", action, "SECRET_TOKEN").await?, + )); + } + for (action, resp) in responses { + assert_eq!(resp.status(), 400, "{action}"); + let body = resp.text().await?; + assert!( + body.contains("Data table roles are a Windmill Enterprise Edition feature"), + "{action}: {body}" + ); + } + Ok(()) +} diff --git a/backend/windmill-api-workspaces/src/datatable_acl.rs b/backend/windmill-api-workspaces/src/datatable_acl.rs new file mode 100644 index 0000000000..39fa8fac5d --- /dev/null +++ b/backend/windmill-api-workspaces/src/datatable_acl.rs @@ -0,0 +1,1908 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Ownership and grants on the objects of an instance data table. +//! +//! [`datatable_permissions`](crate::datatable_permissions) decides who may connect as which role; +//! this decides what each role may then touch. Every change is a real `GRANT`, `REVOKE`, +//! `ALTER ... OWNER TO` or `ALTER DEFAULT PRIVILEGES`, so Postgres is what enforces it. +//! +//! Reading is open to anyone who reaches the data table. Planning and applying are for those who +//! administer it — admins of the workspace that governs it, and superadmins. All of it is +//! Enterprise Edition ([`crate::datatable_acl_oss`]). + +use std::collections::BTreeMap; + +use axum::{ + extract::{Extension, Path, Query}, + routing::{get, post}, + Json, Router, +}; +use serde::{Deserialize, Serialize}; +use tokio::sync::mpsc; +use tokio_postgres::error::{DbError, SqlState}; +use tokio_postgres::AsyncMessage; + +use windmill_api_auth::ApiAuthed; +use windmill_audit::audit_oss::audit_log; +use windmill_audit::ActionKind; +use windmill_common::datatable_roles::{ + lock_role_catalog, quote_ident, read_role_catalog, read_role_catalog_tx, DatatableRoleCatalog, + ADMIN_DATATABLE_ROLE, CUSTOM_INSTANCE_USER, +}; +use windmill_common::error::{pg_error_message, Error, JsonResult, Result}; +use windmill_common::workspaces::{ + get_datatable_resource_from_db_unchecked, resolve_governing_datatable, GoverningDatatable, +}; +use windmill_common::{PgDatabase, DB}; + +use crate::datatable_permissions::{ensure_governs_datatable, ensure_reaches_datatable}; + +pub(crate) fn routes() -> Router { + Router::new() + .route("/datatable_acl/{datatable_name}", get(get_datatable_acl)) + .route( + "/datatable_acl/{datatable_name}/plan", + post(plan_datatable_acl), + ) + .route( + "/datatable_acl/{datatable_name}/apply", + post(apply_datatable_acl), + ) +} + +/// What a read or a change is about. +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum AclTarget { + /// The data table's own database — where the privilege to create schemas lives. + Database, + Schema { + schema: String, + }, + Table { + schema: String, + table: String, + }, +} + +impl AclTarget { + /// The schema the target is in, absent for the database itself. + pub(crate) fn schema(&self) -> Option<&str> { + match self { + AclTarget::Database => None, + AclTarget::Schema { schema } => Some(schema), + AclTarget::Table { schema, .. } => Some(schema), + } + } + + /// What it is called in a message. + pub(crate) fn label(&self, dbname: &str) -> String { + match self { + AclTarget::Database => dbname.to_string(), + AclTarget::Schema { schema } => schema.clone(), + AclTarget::Table { schema, table } => format!("{schema}.{table}"), + } + } +} + +#[derive(Deserialize, Debug)] +pub struct AclTargetQuery { + kind: String, + schema: Option, + table: Option, +} + +impl TryFrom for AclTarget { + type Error = Error; + fn try_from(q: AclTargetQuery) -> Result { + match (q.kind.as_str(), q.schema, q.table) { + ("database", _, _) => Ok(AclTarget::Database), + ("schema", Some(schema), _) => Ok(AclTarget::Schema { schema }), + ("table", Some(schema), Some(table)) => Ok(AclTarget::Table { schema, table }), + ("schema" | "table", None, _) => { + Err(Error::BadRequest("This target needs a schema".to_string())) + } + ("table", _, None) => Err(Error::BadRequest( + "A table target needs a table".to_string(), + )), + (kind, _, _) => Err(Error::BadRequest(format!("Unknown ACL target '{kind}'"))), + } + } +} + +/// Where a set of privileges applies, relative to the target. +/// +/// `Future*` covers what does not exist yet: those become `ALTER DEFAULT PRIVILEGES`, which only +/// binds objects created by the roles it names. +#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum GrantScope { + /// The target itself — the database, the schema, or the table. + Target, + AllTables, + AllSequences, + AllFunctions, + FutureTables, + FutureSequences, + FutureFunctions, +} + +impl GrantScope { + pub(crate) fn is_future(&self) -> bool { + matches!( + self, + GrantScope::FutureTables | GrantScope::FutureSequences | GrantScope::FutureFunctions + ) + } +} + +/// A change to plan. One at a time: each is confirmed against its own SQL. +#[derive(Deserialize, Serialize, Debug, Clone)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum AclChange { + /// Hand the target — and, for a schema, everything already in it but an extension's members, + /// which stay with the extension — to another role. + SetOwner { + role: String, + }, + Grant { + role: String, + privileges: Vec, + scope: GrantScope, + }, + Revoke { + role: String, + privileges: Vec, + scope: GrantScope, + /// Objects inside the target, empty for the target itself. `ON ALL TABLES` grants read + /// back per object, so they are revoked per object — and the same privileges on several + /// of them are revoked together. + #[serde(default)] + objects: Vec, + }, +} + +impl AclChange { + /// The role the change is about, as the editor names it. + fn role(&self) -> &str { + match self { + AclChange::SetOwner { role } + | AclChange::Grant { role, .. } + | AclChange::Revoke { role, .. } => role, + } + } +} + +#[derive(Deserialize, Debug)] +pub struct AclChangeRequest { + pub target: AclTarget, + pub change: AclChange, + /// The statements the plan showed. An apply runs only those: it plans again and refuses if the + /// result differs. + #[serde(default)] + pub statements: Option>, +} + +/// An object inside a schema, named the way `REVOKE ... ON ` needs it. +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] +pub struct AclObject { + pub name: String, + /// `TABLE`, `SEQUENCE`, `FUNCTION`, `PROCEDURE` or `TYPE`: what the object is. + /// [`object_keyword`] turns it into the keyword a revoke takes; a type has none. + pub kind: String, + /// A routine is identified by its argument types, not by its name: two `f` in one schema are + /// two objects. Absent for everything else. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub args: Option, +} + +/// A grant as the database has it, under the role names the editor uses. +#[derive(Serialize, Debug, PartialEq)] +pub struct AclGrant { + /// A data table role's name, `admin` for `custom_instance_user`, else the raw Postgres role + /// (`PUBLIC` included). + pub grantee: String, + pub privileges: Vec, + /// `None` for the target itself, else the object inside it. + #[serde(skip_serializing_if = "Option::is_none")] + pub object: Option, + /// `TABLES` / `SEQUENCES` / `FUNCTIONS` / `TYPES` (or, on the database, `SCHEMAS`) when this + /// is a default privilege, which applies to objects that do not exist yet. + #[serde(skip_serializing_if = "Option::is_none")] + pub future: Option, + /// Where the grant comes from, each role once: who granted it, or for a default privilege the + /// role whose future objects it covers. A revoke of some of the privileges takes them back from + /// every source that gave them. + pub sources: Vec, +} + +/// One role a grant comes from. +#[derive(Serialize, Debug, PartialEq)] +pub struct AclSource { + /// Under the same names as [`AclGrant::grantee`]. + pub role: String, + /// What `role` gave of the grant's privileges. A revoke is held back only by a source out of + /// reach that gave some of what it takes back. + pub privileges: Vec, + /// Whether this data table's connection can take back what `role` gave: on an object only the + /// owner's grants when it acts for the owner, or else its own (`grant_source!`); for a default + /// privilege, a creating role it acts for. What a source out of reach gave is not revocable + /// from here; privileges only other sources gave still are. + pub reachable: bool, +} + +#[derive(Serialize, Debug)] +pub struct DatatableAclInfo { + /// Under the same names as [`AclGrant::grantee`]. + pub owner: String, + /// The roles a change may name: `admin`, then every role of the instance catalog. Only for a + /// caller who may change anything, as the catalog is in the permissions drawer. + pub roles: Vec, + /// Whether this caller may plan and apply changes: they administer the data table, on an + /// edition that has the planner. + pub editable: bool, + /// Whether the server is Postgres 17 or later, which added the `MAINTAIN` table privilege. + pub supports_maintain: bool, + /// The database the target lives in, which no target carries itself. + pub dbname: String, + pub grants: Vec, + /// What the target holds that is a target of its own: a database's schemas, a schema's tables. + pub children: Vec, +} + +#[derive(Serialize, Debug)] +pub struct AclPlan { + pub statements: Vec, + pub warnings: Vec, +} + +/// The Postgres role a role name stands for. A data table role is a login named exactly like the +/// role, so this is the identity — except `admin`, which is `custom_instance_user`. +/// +/// Anything else is refused, never resolved to some default: every statement a plan writes names +/// the role it is about. +pub(crate) fn pg_role_of(name: &str, catalog: &DatatableRoleCatalog) -> Result { + if name == ADMIN_DATATABLE_ROLE { + return Ok(CUSTOM_INSTANCE_USER.to_string()); + } + if catalog.values().any(|r| r.name == name) { + return Ok(name.to_string()); + } + Err(Error::BadRequest(format!( + "'{name}' is not a data table role of this instance" + ))) +} + +/// The reverse of [`pg_role_of`], for display. A role that is not a data table role reads back as +/// itself. +pub(crate) fn role_name_of(pg_role: &str) -> String { + if pg_role == CUSTOM_INSTANCE_USER { + ADMIN_DATATABLE_ROLE.to_string() + } else { + pg_role.to_string() + } +} + +/// Every role a change may name: `admin` first, then the catalog. +fn role_names(catalog: &DatatableRoleCatalog) -> Vec { + let mut names: Vec = catalog.values().map(|r| r.name.clone()).collect(); + names.sort(); + names.insert(0, ADMIN_DATATABLE_ROLE.to_string()); + names +} + +fn ensure_instance(governing: &GoverningDatatable) -> Result<()> { + if governing.is_instance() { + return Ok(()); + } + Err(Error::BadRequest(format!( + "Data table '{}' is backed by a Postgres resource, so its access is managed on that \ + server directly. Only a data table on the Windmill instance's own database has data \ + table roles to grant to.", + governing.name + ))) +} + +/// The data table's `admin` connection, and the notices Postgres sends on it. +/// +/// Authorization: connects as `custom_instance_user` with the instance's own credentials and checks +/// nothing. Callers MUST have authorized the request first — a request about to be refused must +/// not get as far as this connection. +async fn connect_as_admin_unchecked( + db: &DB, + governing: &GoverningDatatable, +) -> Result<( + tokio_postgres::Client, + mpsc::UnboundedReceiver, + String, +)> { + let resource = + get_datatable_resource_from_db_unchecked(db, &governing.workspace_id, &governing.name) + .await?; + let pg: PgDatabase = serde_json::from_value(resource) + .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {e}")))?; + let dbname = pg.dbname.clone(); + let (client, mut connection) = pg.connect(Some(db)).await?; + // Unbounded: the driver must never wait on the receiver, which only drains once the statement + // the driver is carrying has completed. + let (notices_tx, notices) = mpsc::unbounded_channel(); + tokio::spawn(async move { + loop { + match std::future::poll_fn(|cx| connection.poll_message(cx)).await { + Some(Ok(AsyncMessage::Notice(notice))) => { + let _ = notices_tx.send(notice); + } + Some(Ok(_)) => {} + Some(Err(e)) => { + tracing::error!("Datatable ACL connection error: {e}"); + break; + } + None => break, + } + } + }); + Ok((client, notices, dbname)) +} + +/// An object whose ownership follows the schema's. +#[derive(Debug, PartialEq)] +pub(crate) struct OwnedObject { + /// The keyword `ALTER ... OWNER TO` takes for this kind of object. + pub(crate) keyword: &'static str, + /// How Postgres names the object (`pg_identify_object`): schema-qualified, quoted where + /// needed, with a routine's arguments or an operator class's access method. It goes into the + /// statement as it is. + pub(crate) identity: String, +} + +/// Everything a schema's change of owner takes along, in schema `$1`, as (kind, identity, owner +/// oid), the first two as `pg_identify_object` gives them. The plan, the check of what this +/// connection may move and the check after the move all read this one list, so what is moved and +/// what is checked cannot differ. +/// +/// Read from what depends on the schema rather than catalog by catalog, so no kind of object is +/// left out by omission; array types, row types and indexes depend on another object instead. +/// Left out: an object's internal parts (a range type's constructors and multirange, an identity +/// column's sequence), a sequence tied to a column (it follows its table, and refuses an owner of +/// its own), an extension's members, and what has no `ALTER ... OWNER` at all (an extension, a text +/// search parser or template). Postgres records no owner for the bootstrap superuser, oid 10. +macro_rules! schema_owned_objects { + () => { + "SELECT o.type AS kind, o.identity, COALESCE(s.refobjid, 10::oid) AS owner + FROM pg_depend d + CROSS JOIN LATERAL pg_identify_object(d.classid, d.objid, 0) o + LEFT JOIN pg_shdepend s + ON s.dbid = (SELECT oid FROM pg_database WHERE datname = current_database()) + AND s.classid = d.classid AND s.objid = d.objid AND s.deptype = 'o' + WHERE d.refclassid = 'pg_namespace'::regclass AND d.deptype = 'n' + AND d.refobjid = (SELECT oid FROM pg_namespace WHERE nspname = $1) + AND d.classid <> ALL(ARRAY['pg_extension'::regclass, 'pg_ts_parser'::regclass, + 'pg_ts_template'::regclass]::oid[]) + AND NOT EXISTS ( + SELECT 1 FROM pg_depend x + WHERE x.classid = d.classid AND x.objid = d.objid AND x.objsubid = 0 + AND (x.deptype IN ('i', 'e') + OR (x.deptype = 'a' AND d.classid = 'pg_class'::regclass + AND x.refobjsubid <> 0)))" + }; +} + +/// The keyword `ALTER ... OWNER TO` takes for a kind of object, as `pg_identify_object` names the +/// kind. A kind missing here is refused rather than skipped, which would leave it behind. +fn owned_keyword(kind: &str) -> Option<&'static str> { + Some(match kind { + "table" => "TABLE", + "view" => "VIEW", + "materialized view" => "MATERIALIZED VIEW", + "sequence" => "SEQUENCE", + "foreign table" => "FOREIGN TABLE", + "type" => "TYPE", + "function" | "procedure" | "aggregate" => "ROUTINE", + "collation" => "COLLATION", + "conversion" => "CONVERSION", + "operator" => "OPERATOR", + "operator class" => "OPERATOR CLASS", + "operator family" => "OPERATOR FAMILY", + "statistics object" => "STATISTICS", + "text search dictionary" => "TEXT SEARCH DICTIONARY", + "text search configuration" => "TEXT SEARCH CONFIGURATION", + _ => return None, + }) +} + +async fn read_owned_objects( + client: &tokio_postgres::Client, + schema: &str, +) -> Result> { + let rows = client + .query( + concat!( + "SELECT kind, identity FROM (", + schema_owned_objects!(), + ") o ORDER BY kind, identity" + ), + &[&schema], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to list the objects of schema '{schema}': {}", + pg_error_message(&e) + )) + })?; + rows.into_iter() + .map(|row| { + let kind: &str = row.get(0); + let identity: String = row.get(1); + match owned_keyword(kind) { + Some(keyword) => Ok(OwnedObject { keyword, identity }), + None => Err(Error::BadRequest(format!( + "{identity} is a {kind}, whose owner cannot be changed from here, and schema \ + {schema} would change hands without it. Move it to another schema first." + ))), + } + }) + .collect() +} + +/// Default privileges set database-wide (no `IN SCHEMA`), as `(defaclrole, defaclobjtype, grantee, +/// privilege_type)`. Postgres stores such an entry as a whole acl, the creator's own privileges +/// included, so only what goes beyond the built-in default is a grant. It applies in every schema +/// on top of the schema's own defaults, which cannot take it back. +macro_rules! database_wide_defaults { + () => { + "SELECT d.defaclrole, d.defaclobjtype, a.grantee, a.privilege_type + FROM pg_default_acl d, aclexplode(d.defaclacl) a + WHERE d.defaclnamespace = 0 + AND NOT EXISTS ( + SELECT 1 FROM aclexplode(acldefault( + CASE d.defaclobjtype WHEN 'S' THEN 's' ELSE d.defaclobjtype END, d.defaclrole)) x + WHERE x.grantee = a.grantee AND x.privilege_type = a.privilege_type)" + }; +} + +/// `TABLES`, `SEQUENCES`, `FUNCTIONS` or `TYPES`, as a `defaclobjtype` names them. +fn default_objects_keyword(objtype: &str) -> &'static str { + match objtype { + "r" => "TABLES", + "S" => "SEQUENCES", + "T" => "TYPES", + _ => "FUNCTIONS", + } +} + +/// What a schema's owner holds on what gets created there later. A change of owner hands the new +/// owner the same and takes these back: otherwise every former owner keeps reaching whatever the +/// other roles create there. +#[derive(Debug, PartialEq)] +pub(crate) struct FormerOwnerDefaults { + pub(crate) pg_role: String, + /// (creating role, `TABLES` / `SEQUENCES` / `FUNCTIONS` / `TYPES`), one per default privilege + /// it holds. + pub(crate) defaults: Vec<(String, &'static str)>, +} + +/// The default privileges schema `schema`'s owner holds there — `None` when it holds none, or is +/// `new_owner` already. Refused when one was set by a role this connection cannot act for: only +/// a member of the creating role may change its defaults, so the revoke would fail at apply. Also +/// refused when the owner holds defaults set database-wide, which no change to this schema takes +/// back. +async fn read_former_owner_defaults( + client: &tokio_postgres::Client, + schema: &str, + new_owner: &str, +) -> Result> { + let database_wide = client + .query_opt( + concat!( + "SELECT pg_get_userbyid(n.nspowner), pg_get_userbyid(g.defaclrole), + g.defaclobjtype::text + FROM pg_namespace n, (", + database_wide_defaults!(), + ") g + WHERE n.nspname = $1 AND g.grantee = n.nspowner + AND g.defaclobjtype IN ('r', 'S', 'f', 'T') + AND n.nspowner <> (SELECT oid FROM pg_roles WHERE rolname = $2) + ORDER BY 2, 3 + LIMIT 1" + ), + &[&schema, &new_owner], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to read the database-wide default privileges: {}", + pg_error_message(&e) + )) + })?; + if let Some(row) = database_wide { + let pg_role: String = row.get(0); + let creator: String = row.get(1); + return Err(Error::BadRequest(format!( + "{} holds default privileges set database-wide by {}, which no default of schema \ + {schema} takes back, so it would keep reaching what is created here after the change \ + of owner. Revoke them database-wide first: ALTER DEFAULT PRIVILEGES FOR ROLE {} \ + REVOKE ALL PRIVILEGES ON {} FROM {}", + role_name_of(&pg_role), + role_name_of(&creator), + quote_ident(&creator), + default_objects_keyword(row.get(2)), + quote_ident(&pg_role) + ))); + } + + let rows = client + .query( + "SELECT DISTINCT pg_get_userbyid(n.nspowner), pg_get_userbyid(d.defaclrole), + d.defaclobjtype::text, pg_has_role(d.defaclrole, 'USAGE') + FROM pg_namespace n + JOIN pg_default_acl d ON d.defaclnamespace = n.oid + CROSS JOIN LATERAL aclexplode(d.defaclacl) a + WHERE n.nspname = $1 AND a.grantee = n.nspowner + -- What the owner gives itself is about the objects it creates, not about owning + -- the schema, so a move leaves it alone. + AND a.grantee <> d.defaclrole + AND d.defaclobjtype IN ('r', 'S', 'f', 'T') + AND n.nspowner <> (SELECT oid FROM pg_roles WHERE rolname = $2) + ORDER BY 2, 3", + &[&schema, &new_owner], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to read the default privileges of schema '{schema}': {}", + pg_error_message(&e) + )) + })?; + let Some(first) = rows.first() else { + return Ok(None); + }; + let pg_role: String = first.get(0); + let plural = |row: &tokio_postgres::Row| default_objects_keyword(row.get(2)); + if let Some(row) = rows.iter().find(|row| !row.get::<_, bool>(3)) { + let creator: String = row.get(1); + return Err(Error::BadRequest(format!( + "{} holds default privileges in schema {schema} from {creator}, which this data \ + table's connection cannot act for, so they would outlive the change of owner. \ + Revoke them as {creator} first: ALTER DEFAULT PRIVILEGES FOR ROLE {} IN SCHEMA {} \ + REVOKE ALL PRIVILEGES ON {} FROM {}", + role_name_of(&pg_role), + quote_ident(&creator), + quote_ident(schema), + plural(row), + quote_ident(&pg_role) + ))); + } + Ok(Some(FormerOwnerDefaults { + defaults: rows + .iter() + .map(|row| (row.get::<_, String>(1), plural(row))) + .collect(), + pg_role, + })) +} + +/// What the catalog holds that a plan depends on, read before planning so the planner stays pure. +#[derive(Debug, Default, PartialEq)] +pub(crate) struct CatalogFacts { + /// Every role that may create objects here, but the one the change is about. + pub(crate) other_pg_roles: Vec, + /// For a schema's change of owner: what moves along with it. + pub(crate) existing_objects: Vec, + /// For a schema's change of owner: the defaults it takes back from the owner it replaces. + pub(crate) former_owner: Option, + /// For a revoke: each grant it takes back. + pub(crate) revoked_grants: Vec, +} + +/// A grant a revoke takes back, as the catalog records it: what `source` gave on `object` (the +/// target itself when `None`), or for a default privilege on what `source` creates later. +#[derive(Debug, PartialEq)] +pub(crate) struct RevokedGrant { + pub(crate) object: Option, + /// The role that made the grant: its grantor, or the creating role of a default privilege. + pub(crate) source: String, + pub(crate) privileges: Vec, +} + +/// An `aclexplode` row's source and whether this connection can take back what it gave, on an +/// object owned by `$owner`. A REVOKE speaks for the grantor Postgres picks itself — the owner, +/// when the connection acts for the owner, and otherwise the connection — and `GRANTED BY` names +/// nobody else, so a grant any other role made stays whatever the connection runs. +macro_rules! grant_source { + ($owner:literal) => { + concat!( + "pg_get_userbyid(a.grantor), a.grantor = CASE WHEN pg_has_role(", + $owner, + ", 'USAGE') THEN ", + $owner, + " ELSE (SELECT oid FROM pg_roles WHERE rolname = current_user) END" + ) + }; +} + +/// What relation `$2` of schema `$1` grants role `$3`, as (source, whether this connection can +/// take it back, privilege). +const RELATION_GRANTS: &str = concat!( + "SELECT ", + grant_source!("c.relowner"), + ", a.privilege_type + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace, + aclexplode(COALESCE(c.relacl, acldefault( + CASE c.relkind WHEN 'S' THEN 's' ELSE 'r' END::\"char\", c.relowner))) a + WHERE n.nspname = $1 AND c.relname = $2 + AND c.relkind = ANY(ARRAY['r','p','v','m','S','f']::\"char\"[]) + AND a.grantee = (SELECT oid FROM pg_roles WHERE rolname = $3)" +); + +/// The grants a revoke takes back, read from the catalog rather than from the request: one per +/// object and source that gave `pg_role` any of `privileges`. Refused when a source's grant is out +/// of this connection's reach (`grant_source!`, or for a default privilege a creating role it does +/// not act for): the revoke would leave that grant in place. +async fn read_revoked_grants( + client: &tokio_postgres::Client, + dbname: &str, + target: &AclTarget, + scope: GrantScope, + objects: &[AclObject], + privileges: &[String], + pg_role: &str, +) -> Result> { + let read_error = |e: tokio_postgres::Error| { + Error::internal_err(format!( + "Failed to read what the revoke takes back: {}", + pg_error_message(&e) + )) + }; + let wanted: Vec = privileges.iter().map(|p| p.to_uppercase()).collect(); + // (object, how it reads in a refusal, one row per source and privilege) + let mut read = Vec::new(); + match (scope, target) { + (scope, AclTarget::Schema { schema }) if scope.is_future() => { + let (objtype, plural) = match scope { + GrantScope::FutureTables => ("r", "tables"), + GrantScope::FutureSequences => ("S", "sequences"), + _ => ("f", "functions"), + }; + let database_wide = client + .query( + concat!( + "SELECT pg_get_userbyid(g.defaclrole), g.privilege_type FROM (", + database_wide_defaults!(), + ") g + WHERE g.defaclobjtype::text = $1 + AND g.grantee = (SELECT oid FROM pg_roles WHERE rolname = $2) + ORDER BY 1, 2" + ), + &[&objtype, &pg_role], + ) + .await + .map_err(read_error)?; + let mut still_granted: BTreeMap> = BTreeMap::new(); + for row in database_wide { + let privilege: String = row.get(1); + if wanted.contains(&privilege) { + still_granted.entry(row.get(0)).or_default().push(privilege); + } + } + if let Some((creator, taken)) = still_granted.into_iter().next() { + return Err(Error::BadRequest(format!( + "{} also receives {} on {plural} {} creates through a default privilege set \ + database-wide, which no default of schema {schema} takes back. Revoke it \ + database-wide first: ALTER DEFAULT PRIVILEGES FOR ROLE {} REVOKE {} ON {} \ + FROM {}", + role_name_of(pg_role), + taken.join(", "), + role_name_of(&creator), + quote_ident(&creator), + taken.join(", "), + default_objects_keyword(objtype), + quote_ident(pg_role) + ))); + } + let rows = client + .query( + "SELECT pg_get_userbyid(d.defaclrole), pg_has_role(d.defaclrole, 'USAGE'), + a.privilege_type + FROM pg_default_acl d + JOIN pg_namespace n ON n.oid = d.defaclnamespace, + aclexplode(d.defaclacl) a + WHERE n.nspname = $1 AND d.defaclobjtype::text = $2 + AND a.grantee = (SELECT oid FROM pg_roles WHERE rolname = $3)", + &[schema, &objtype, &pg_role], + ) + .await + .map_err(read_error)?; + read.push(( + None, + format!("{plural} created later in schema {schema}"), + rows, + )); + } + (GrantScope::Target, _) if objects.is_empty() => { + let rows = match target { + AclTarget::Database => { + client + .query( + concat!( + "SELECT ", + grant_source!("d.datdba"), + ", a.privilege_type + FROM pg_database d, + aclexplode(COALESCE(d.datacl, acldefault('d', d.datdba))) a + WHERE d.datname = current_database() + AND a.grantee = (SELECT oid FROM pg_roles WHERE rolname = $1)" + ), + &[&pg_role], + ) + .await + } + AclTarget::Schema { schema } => { + client + .query( + concat!( + "SELECT ", + grant_source!("n.nspowner"), + ", a.privilege_type + FROM pg_namespace n, + aclexplode(COALESCE(n.nspacl, acldefault('n', n.nspowner))) a + WHERE n.nspname = $1 + AND a.grantee = (SELECT oid FROM pg_roles WHERE rolname = $2)" + ), + &[schema, &pg_role], + ) + .await + } + AclTarget::Table { schema, table } => { + client + .query(RELATION_GRANTS, &[schema, table, &pg_role]) + .await + } + } + .map_err(read_error)?; + read.push((None, target.label(dbname), rows)); + } + (GrantScope::Target, AclTarget::Schema { schema }) => { + for object in objects { + let rows = match object_keyword(&object.kind)? { + "ROUTINE" => { + client + .query( + concat!( + "SELECT ", + grant_source!("p.proowner"), + ", a.privilege_type + FROM pg_proc p + JOIN pg_namespace n ON n.oid = p.pronamespace, + aclexplode(COALESCE(p.proacl, acldefault('f', p.proowner))) a + WHERE n.nspname = $1 AND p.proname = $2 + AND pg_get_function_identity_arguments(p.oid) = $3 + AND a.grantee = (SELECT oid FROM pg_roles WHERE rolname = $4)" + ), + &[ + schema, + &object.name, + &object.args.as_deref().unwrap_or(""), + &pg_role, + ], + ) + .await + } + _ => { + client + .query(RELATION_GRANTS, &[schema, &object.name, &pg_role]) + .await + } + } + .map_err(read_error)?; + read.push(( + Some(object.clone()), + format!("{} {schema}.{}", object.kind.to_lowercase(), object.name), + rows, + )); + } + } + // Every other scope and target is the planner's to refuse. + _ => {} + } + + let mut revoked = Vec::new(); + for (object, label, rows) in read { + let mut by_source: BTreeMap)> = BTreeMap::new(); + for row in rows { + let privilege: String = row.get(2); + if wanted.contains(&privilege) { + by_source + .entry(row.get(0)) + .or_insert_with(|| (row.get(1), vec![])) + .1 + .push(privilege); + } + } + for (source, (reachable, mut privileges)) in by_source { + if !reachable { + return Err(Error::BadRequest(format!( + "{} on {label} was granted to {} by {source}, and Postgres takes a grant \ + back only through the role that made it, which this data table's \ + connection cannot speak for here. Revoke it as {source}.", + privileges.join(", "), + role_name_of(pg_role), + ))); + } + privileges.sort(); + privileges.dedup(); + revoked.push(RevokedGrant { object: object.clone(), source, privileges }); + } + } + Ok(revoked) +} + +/// The keyword a `REVOKE ... ON` takes for one object, checked rather than interpolated: it lands +/// in SQL unquoted. +pub(crate) fn object_keyword(kind: &str) -> Result<&'static str> { + match kind.to_uppercase().as_str() { + "TABLE" | "VIEW" | "MATERIALIZED VIEW" | "FOREIGN TABLE" => Ok("TABLE"), + "SEQUENCE" => Ok("SEQUENCE"), + // `FUNCTION` names no procedure; `ROUTINE` names either. + "FUNCTION" | "PROCEDURE" | "ROUTINE" => Ok("ROUTINE"), + other => Err(Error::BadRequest(format!("Unknown object kind '{other}'"))), + } +} + +/// Every object of a schema, named the way the catalog names it. +async fn read_schema_objects( + client: &tokio_postgres::Client, + schema: &str, +) -> Result> { + let rows = client + .query( + "SELECT CASE c.relkind WHEN 'S' THEN 'SEQUENCE' ELSE 'TABLE' END, c.relname, NULL::text + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relkind = ANY(ARRAY['r','p','v','m','S','f']::\"char\"[]) + UNION ALL + SELECT CASE p.prokind WHEN 'p' THEN 'PROCEDURE' ELSE 'FUNCTION' END, p.proname, + pg_get_function_identity_arguments(p.oid) + FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = $1", + &[&schema], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to list the objects of schema '{schema}': {}", + pg_error_message(&e) + )) + })?; + Ok(rows + .into_iter() + .map(|row| AclObject { kind: row.get(0), name: row.get(1), args: row.get(2) }) + .collect()) +} + +/// Replace the objects a revoke names with the catalog's own entry for each. +/// +/// A routine is identified by its argument types, and those go into the statement as written — +/// there is no quoting for them — so the request may name an object but never spell one: what +/// reaches the SQL is read back from Postgres. An object that resolves to nothing is refused rather +/// than dropped, since a revoke that silently covers less than it says is worse than an error. +async fn resolve_acl_objects( + client: &tokio_postgres::Client, + target: &AclTarget, + objects: &[AclObject], +) -> Result> { + if objects.is_empty() { + return Ok(vec![]); + } + let Some(schema) = target.schema() else { + return Err(Error::BadRequest( + "A database has no objects of its own to revoke on".to_string(), + )); + }; + let known = read_schema_objects(client, schema).await?; + objects + .iter() + .map(|requested| { + let keyword = object_keyword(&requested.kind)?; + known + .iter() + .find(|k| { + k.name == requested.name + && k.args == requested.args + && object_keyword(&k.kind).is_ok_and(|k| k == keyword) + }) + .cloned() + .ok_or_else(|| { + Error::NotFound(format!( + "'{}' is not an object of schema '{schema}'", + requested.name + )) + }) + }) + .collect() +} + +async fn read_owner(client: &tokio_postgres::Client, target: &AclTarget) -> Result> { + let row = match target { + AclTarget::Database => { + client + .query_opt( + "SELECT pg_get_userbyid(datdba) FROM pg_database WHERE datname = current_database()", + &[], + ) + .await + } + AclTarget::Schema { schema } => { + client + .query_opt( + // `public` is owned by `pg_database_owner`, a placeholder role whose membership + // is whoever owns the database — naming it back would say nothing, so resolve + // it to that owner. + "SELECT pg_get_userbyid(owner) FROM ( + SELECT CASE WHEN n.nspowner = (SELECT oid FROM pg_roles WHERE rolname = 'pg_database_owner') + THEN (SELECT d.datdba FROM pg_database d WHERE d.datname = current_database()) + ELSE n.nspowner END AS owner + FROM pg_namespace n WHERE n.nspname = $1 + ) o", + &[schema], + ) + .await + } + AclTarget::Table { schema, table } => { + client + .query_opt( + "SELECT pg_get_userbyid(c.relowner) + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relname = $2", + &[schema, table], + ) + .await + } + } + .map_err(|e| Error::internal_err(format!("Failed to read the owner: {}", pg_error_message(&e))))?; + Ok(row.map(|row| row.get(0))) +} + +async fn read_children(client: &tokio_postgres::Client, target: &AclTarget) -> Result> { + let rows = match target { + AclTarget::Database => { + client + .query( + "SELECT nspname::text FROM pg_namespace + WHERE nspname <> 'information_schema' AND nspname NOT LIKE 'pg\\_%' + ORDER BY nspname", + &[], + ) + .await + } + AclTarget::Schema { schema } => { + client + .query( + "SELECT c.relname::text + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relkind = ANY(ARRAY['r','p']::\"char\"[]) + ORDER BY c.relname", + &[schema], + ) + .await + } + AclTarget::Table { .. } => return Ok(vec![]), + } + .map_err(|e| { + Error::internal_err(format!( + "Failed to list what the target holds: {}", + pg_error_message(&e) + )) + })?; + Ok(rows.into_iter().map(|row| row.get(0)).collect()) +} + +async fn get_datatable_acl( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, + Query(query): Query, +) -> JsonResult { + crate::datatable_acl_oss::ensure_datatable_acl_available()?; + let target: AclTarget = query.try_into()?; + ensure_reaches_datatable(&db, &w_id, &datatable_name, &authed).await?; + let governing = resolve_governing_datatable(&db, &w_id, &datatable_name).await?; + ensure_instance(&governing)?; + let editable = ensure_governs_datatable(&db, &authed, &w_id, &governing) + .await + .is_ok(); + let roles = if editable { + role_names(&read_role_catalog(&db).await?) + } else { + vec![] + }; + + let (client, _notices, dbname) = connect_as_admin_unchecked(&db, &governing).await?; + let owner = read_owner(&client, &target) + .await? + .ok_or_else(|| Error::NotFound(format!("{} not found", target.label(&dbname))))?; + let grants = read_grants(&client, &target).await?; + let supports_maintain: bool = client + .query_one( + "SELECT current_setting('server_version_num')::int >= 170000", + &[], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to read the server version: {}", + pg_error_message(&e) + )) + })? + .get(0); + let children = read_children(&client, &target).await?; + + Ok(Json(DatatableAclInfo { + owner: role_name_of(&owner), + roles, + editable, + supports_maintain, + dbname, + grants, + children, + })) +} + +async fn read_grants(client: &tokio_postgres::Client, target: &AclTarget) -> Result> { + // `aclexplode` turns an acl array into one row per (grantee, privilege); grantee 0 is PUBLIC, + // which has no name to resolve. A NULL acl is not "no access" but Postgres's built-in default — + // the owner holds everything and, on a routine, PUBLIC may EXECUTE — hence `acldefault`. The + // owner's own entries are left out: what it holds comes with ownership, which the owner shows, + // not with a grant a revoke here could take back. Each row ends with its source — the grantor, + // or a default privilege's creating role — and whether this connection can take back what it + // gave. + // Column-level grants (`pg_attribute.attacl`) are not supported yet: they are neither read here + // nor revocable from the editor. + let mut rows = match target { + AclTarget::Database => { + let mut out = client + .query( + concat!( + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, NULL::text, NULL::text, NULL::text, NULL::text, ", + grant_source!("d.datdba"), + " FROM pg_database d, aclexplode(COALESCE(d.datacl, acldefault('d', d.datdba))) a + WHERE d.datname = current_database() AND a.grantee <> d.datdba" + ), + &[], + ) + .await + .map_err(grant_read_error)?; + out.extend( + client + .query( + // Default privileges set database-wide reach what is created in every + // schema, so they are the database's to show rather than any schema's. + concat!( + "SELECT CASE WHEN g.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(g.grantee) END, + g.privilege_type, NULL::text, + CASE g.defaclobjtype + WHEN 'r' THEN 'TABLES' WHEN 'S' THEN 'SEQUENCES' + WHEN 'f' THEN 'FUNCTIONS' WHEN 'n' THEN 'SCHEMAS' + ELSE 'TYPES' END, NULL::text, NULL::text, + pg_get_userbyid(g.defaclrole), pg_has_role(g.defaclrole, 'USAGE') + FROM (", + database_wide_defaults!(), + ") g" + ), + &[], + ) + .await + .map_err(grant_read_error)?, + ); + out + } + AclTarget::Schema { schema } => { + let mut out = client + .query( + concat!( + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, NULL::text, NULL::text, NULL::text, NULL::text, ", + grant_source!("n.nspowner"), + " FROM pg_namespace n, aclexplode(COALESCE(n.nspacl, acldefault('n', n.nspowner))) a + WHERE n.nspname = $1 AND a.grantee <> n.nspowner" + ), + &[schema], + ) + .await + .map_err(grant_read_error)?; + out.extend( + client + .query( + concat!( + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, c.relname, NULL::text, + CASE c.relkind WHEN 'S' THEN 'SEQUENCE' ELSE 'TABLE' END, + NULL::text, ", + grant_source!("c.relowner"), + " FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace, + aclexplode(COALESCE(c.relacl, acldefault( + CASE c.relkind WHEN 'S' THEN 's' ELSE 'r' END::\"char\", c.relowner))) a + WHERE n.nspname = $1 + AND c.relkind = ANY(ARRAY['r','p','v','m','S','f']::\"char\"[]) + AND a.grantee <> c.relowner" + ), + &[schema], + ) + .await + .map_err(grant_read_error)?, + ); + out.extend( + client + .query( + // Routines carry their own acl in `pg_proc`; without this a grant made here + // would vanish on the next read and could never be revoked back. + concat!( + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, p.proname, NULL::text, + CASE p.prokind WHEN 'p' THEN 'PROCEDURE' ELSE 'FUNCTION' END, + pg_get_function_identity_arguments(p.oid), ", + grant_source!("p.proowner"), + " FROM pg_proc p + JOIN pg_namespace n ON n.oid = p.pronamespace, + aclexplode(COALESCE(p.proacl, acldefault('f', p.proowner))) a + WHERE n.nspname = $1 AND a.grantee <> p.proowner" + ), + &[schema], + ) + .await + .map_err(grant_read_error)?, + ); + out.extend( + client + .query( + // `USAGE` on a type is what lets a role use it in a column. Only a type the + // schema holds in its own right has an acl: an array, a row type and a + // multirange answer to their element, table or range. + concat!( + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, t.typname, NULL::text, 'TYPE', NULL::text, ", + grant_source!("t.typowner"), + " FROM pg_type t + JOIN pg_depend d ON d.classid = 'pg_type'::regclass AND d.objid = t.oid + AND d.refclassid = 'pg_namespace'::regclass AND d.deptype = 'n', + aclexplode(COALESCE(t.typacl, acldefault('T', t.typowner))) a + WHERE d.refobjid = (SELECT oid FROM pg_namespace WHERE nspname = $1) + AND a.grantee <> t.typowner + AND NOT EXISTS ( + SELECT 1 FROM pg_depend x + WHERE x.classid = 'pg_type'::regclass AND x.objid = t.oid + AND x.objsubid = 0 AND x.deptype = 'i')" + ), + &[schema], + ) + .await + .map_err(grant_read_error)?, + ); + out.extend( + client + .query( + // What a creating role set is taken back `FOR ROLE` that role, which only + // a role acting for it may do. + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, NULL::text, + CASE d.defaclobjtype + WHEN 'r' THEN 'TABLES' WHEN 'S' THEN 'SEQUENCES' + WHEN 'f' THEN 'FUNCTIONS' ELSE 'TYPES' END, NULL::text, NULL::text, + pg_get_userbyid(d.defaclrole), pg_has_role(d.defaclrole, 'USAGE') + FROM pg_default_acl d + JOIN pg_namespace n ON n.oid = d.defaclnamespace, + aclexplode(d.defaclacl) a + WHERE n.nspname = $1", + &[schema], + ) + .await + .map_err(grant_read_error)?, + ); + out + } + AclTarget::Table { schema, table } => client + .query( + concat!( + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, NULL::text, NULL::text, NULL::text, NULL::text, ", + grant_source!("c.relowner"), + " FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace, + aclexplode(COALESCE(c.relacl, acldefault( + CASE c.relkind WHEN 'S' THEN 's' ELSE 'r' END::\"char\", c.relowner))) a + WHERE n.nspname = $1 AND c.relname = $2 AND a.grantee <> c.relowner" + ), + &[schema, table], + ) + .await + .map_err(grant_read_error)?, + }; + + // One row per privilege and source — and, for default privileges, per creating role. Fold them + // back into one entry per grantee and object that keeps every source and what each gave: a + // revoke takes a privilege back from every source that gave it. + let mut folded: BTreeMap< + ( + String, + Option<(String, String, Option)>, + Option, + ), + (Vec, BTreeMap)>), + > = BTreeMap::new(); + for row in rows.drain(..) { + let grantee: String = row.get(0); + let privilege: String = row.get(1); + let object: Option = row.get(2); + let future: Option = row.get(3); + let object_kind: Option = row.get(4); + let object_args: Option = row.get(5); + let source: String = row.get(6); + let reachable: bool = row.get(7); + let (privileges, sources) = folded + .entry(( + role_name_of(&grantee), + object.map(|name| { + ( + name, + object_kind.unwrap_or_else(|| "TABLE".to_string()), + object_args, + ) + }), + future, + )) + .or_default(); + sources + .entry(role_name_of(&source)) + .or_insert_with(|| (reachable, vec![])) + .1 + .push(privilege.clone()); + privileges.push(privilege); + } + Ok(folded + .into_iter() + .map(|((grantee, object, future), (mut privileges, sources))| { + privileges.sort(); + privileges.dedup(); + AclGrant { + grantee, + privileges, + object: object.map(|(name, kind, args)| AclObject { name, kind, args }), + future, + sources: sources + .into_iter() + .map(|(role, (reachable, mut privileges))| { + privileges.sort(); + privileges.dedup(); + AclSource { role, privileges, reachable } + }) + .collect(), + } + }) + .collect()) +} + +fn grant_read_error(e: tokio_postgres::Error) -> Error { + Error::internal_err(format!("Failed to read grants: {}", pg_error_message(&e))) +} + +/// Changing a data table's access is administering it. Checked in full before anything connects +/// with the instance's credentials. +async fn authorize_acl_change( + db: &DB, + authed: &ApiAuthed, + w_id: &str, + datatable_name: &str, +) -> Result { + let governing = resolve_governing_datatable(db, w_id, datatable_name).await?; + ensure_governs_datatable(db, authed, w_id, &governing).await?; + ensure_instance(&governing)?; + Ok(governing) +} + +/// Plan one change against the catalog and the database as they are now. +async fn build_plan( + client: &tokio_postgres::Client, + dbname: &str, + catalog: &DatatableRoleCatalog, + target: &AclTarget, + change: &AclChange, +) -> Result { + let role = change.role(); + // `admin` is the login the data table itself reaches Postgres through, and the one every + // change here runs as: a revoke that lands leaves nothing able to grant it back. + if matches!(change, AclChange::Revoke { .. }) && role == ADMIN_DATATABLE_ROLE { + return Err(Error::BadRequest(format!( + "'{ADMIN_DATATABLE_ROLE}' is how this data table reaches its database; \ + its own access is not revocable from here" + ))); + } + let pg_role = pg_role_of(role, catalog)?; + let change = match change { + AclChange::Revoke { role, privileges, scope, objects } => AclChange::Revoke { + role: role.clone(), + privileges: privileges.clone(), + scope: *scope, + objects: resolve_acl_objects(client, target, objects).await?, + }, + change => change.clone(), + }; + // Default privileges are recorded per creating role, and a schema's new owner is kept in reach + // of what the others create there, so both are written for every role there is. + let other_pg_roles = role_names(catalog) + .iter() + .map(|name| pg_role_of(name, catalog)) + .filter(|r| r.as_ref().map_or(true, |r| *r != pg_role)) + .collect::>>()?; + let (existing_objects, former_owner) = match (&change, target) { + (AclChange::SetOwner { .. }, AclTarget::Schema { schema }) => ( + read_owned_objects(client, schema).await?, + read_former_owner_defaults(client, schema, &pg_role).await?, + ), + _ => (vec![], None), + }; + let revoked_grants = match &change { + AclChange::Revoke { privileges, scope, objects, .. } => { + read_revoked_grants( + client, dbname, target, *scope, objects, privileges, &pg_role, + ) + .await? + } + _ => vec![], + }; + if matches!(change, AclChange::SetOwner { .. }) { + if let Some((object, owner)) = unmanaged_owner(client, target).await? { + return Err(Error::BadRequest(format!( + "{object} is owned by {owner}, which this data table's connection cannot act \ + for, so its owner cannot be changed from here" + ))); + } + } + let facts = CatalogFacts { other_pg_roles, existing_objects, former_owner, revoked_grants }; + let mut plan = + crate::datatable_acl_oss::plan_statements(target, &change, dbname, &pg_role, &facts)?; + if matches!(change, AclChange::SetOwner { .. }) { + if let Some(missing) = missing_owner_privilege(client, target, &pg_role).await? { + plan.warnings.push(format!( + "{role} does not have {missing}, which Postgres requires of a new owner, so this \ + will be refused. Grant it first." + )); + } + } + Ok(plan) +} + +/// Postgres only hands an object to a role that could have created it: a table to one with +/// `CREATE` on its schema, a schema to one with `CREATE` on the database. +async fn missing_owner_privilege( + client: &tokio_postgres::Client, + target: &AclTarget, + pg_role: &str, +) -> Result> { + let (row, missing) = match target { + AclTarget::Table { schema, .. } => ( + client + .query_one( + "SELECT has_schema_privilege($1::name, $2::text, 'CREATE')", + &[&pg_role, schema], + ) + .await, + format!("CREATE on schema {schema}"), + ), + AclTarget::Schema { .. } => ( + client + .query_one( + "SELECT has_database_privilege($1::name, current_database(), 'CREATE')", + &[&pg_role], + ) + .await, + "CREATE on the database".to_string(), + ), + AclTarget::Database => return Ok(None), + }; + let has: bool = row + .map_err(|e| { + Error::internal_err(format!( + "Failed to read the new owner's privileges: {}", + pg_error_message(&e) + )) + })? + .get(0); + Ok((!has).then_some(missing)) +} + +/// The first thing a change of owner would move that this connection cannot act for, with its +/// owner. Postgres lets only a member of the current owner move an object, and every change runs as +/// `custom_instance_user`, so an object it does not hold the owner of — `public`, owned by the +/// database's owner, above all — is refused here rather than at apply. Running as the instance's +/// own user instead would reach objects Windmill never created. +async fn unmanaged_owner( + client: &tokio_postgres::Client, + target: &AclTarget, +) -> Result> { + let read_error = |e: tokio_postgres::Error| { + Error::internal_err(format!( + "Failed to read who owns what the change moves: {}", + pg_error_message(&e) + )) + }; + match target { + AclTarget::Database => Ok(None), + AclTarget::Table { schema, table } => { + let row = client + .query_opt( + "SELECT pg_get_userbyid(c.relowner) + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relname = $2 AND NOT pg_has_role(c.relowner, 'USAGE')", + &[schema, table], + ) + .await + .map_err(read_error)?; + Ok(row.map(|row| (format!("{schema}.{table}"), row.get(0)))) + } + AclTarget::Schema { schema } => { + let row = client + .query_opt( + concat!( + "SELECT ord, identity, pg_get_userbyid(owner) FROM ( + SELECT 0 AS ord, NULL::text AS identity, n.nspowner AS owner + FROM pg_namespace n WHERE n.nspname = $1 + UNION ALL + SELECT 1, identity, owner FROM (", + schema_owned_objects!(), + ") m + ) o WHERE NOT pg_has_role(owner, 'USAGE') ORDER BY ord, identity LIMIT 1" + ), + &[schema], + ) + .await + .map_err(read_error)?; + Ok(row.map(|row| { + let label = row + .get::<_, Option>(1) + .unwrap_or_else(|| format!("schema {schema}")); + (label, row.get(2)) + })) + } + } +} + +async fn plan_datatable_acl( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, + Json(req): Json, +) -> JsonResult { + crate::datatable_acl_oss::ensure_datatable_acl_available()?; + let governing = authorize_acl_change(&db, &authed, &w_id, &datatable_name).await?; + let catalog = read_role_catalog(&db).await?; + let (client, _notices, dbname) = connect_as_admin_unchecked(&db, &governing).await?; + Ok(Json( + build_plan(&client, &dbname, &catalog, &req.target, &req.change).await?, + )) +} + +async fn apply_datatable_acl( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, + Json(req): Json, +) -> Result { + crate::datatable_acl_oss::ensure_datatable_acl_available()?; + let confirmed = req.statements.as_ref().ok_or_else(|| { + Error::BadRequest( + "An apply runs exactly the statements its plan showed; plan the change first" + .to_string(), + ) + })?; + // Refuses without taking a lock; everything is checked again once they are held. + let governing = authorize_acl_change(&db, &authed, &w_id, &datatable_name).await?; + + // Held until the change is committed: a role renamed or dropped meanwhile would change what + // the plan names, and a settings save could move the entry onto another database. Taken in the + // same order as the permissions save, so the two cannot deadlock. + let mut tx = db.begin().await?; + lock_role_catalog(&mut tx).await?; + sqlx::query!( + "SELECT 1 AS one FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE", + &governing.workspace_id + ) + .fetch_optional(&mut *tx) + .await?; + let governing = authorize_acl_change(&db, &authed, &w_id, &datatable_name).await?; + let catalog = read_role_catalog_tx(&mut tx).await?; + + let (mut client, mut notices, dbname) = connect_as_admin_unchecked(&db, &governing).await?; + let plan = build_plan(&client, &dbname, &catalog, &req.target, &req.change).await?; + if &plan.statements != confirmed { + return Err(Error::BadRequest( + "The data table or its roles changed since this was planned, so it would no longer \ + run what was confirmed. Plan it again." + .to_string(), + )); + } + + // Postgres only lets a role pass on a privilege it holds with grant option, and an instance + // database provisioned before data table roles holds none. Best-effort: a grant this fails to + // enable is refused below rather than skipped. + if let Err(e) = windmill_common::ensure_instance_db_grant_options_unchecked(&db, &dbname).await + { + tracing::warn!("Could not refresh grant options on '{dbname}': {e}"); + } + + // One transaction: a half-applied ownership transfer leaves one schema's objects owned by two + // different roles. + let pg_tx = client.transaction().await.map_err(|e| { + Error::internal_err(format!( + "Failed to open a transaction on the data table: {}", + pg_error_message(&e) + )) + })?; + for statement in &plan.statements { + pg_tx.batch_execute(statement).await.map_err(|e| { + Error::ExecutionErr(format!( + "Failed to run `{statement}`: {}", + pg_error_message(&e) + )) + })?; + // A privilege the connection cannot pass on is only a warning to Postgres, which then + // carries on having changed nothing. Returning drops the transaction, rolling back + // everything before it. + while let Ok(notice) = notices.try_recv() { + if *notice.code() == SqlState::WARNING_PRIVILEGE_NOT_GRANTED + || *notice.code() == SqlState::WARNING_PRIVILEGE_NOT_REVOKED + { + return Err(Error::ExecutionErr(format!( + "`{statement}` did not take effect ({}), so nothing was applied", + notice.message() + ))); + } + } + } + + // A schema's objects were listed before the transaction opened; one committed since would stay + // with its old owner. One created while this transaction is still open can still slip past, as + // Postgres has no lock that holds creation in a schema back. That is benign: it stays with its + // creator, like anything created there later, and moving that table fixes it. + if let (AclChange::SetOwner { role }, AclTarget::Schema { schema }) = (&req.change, &req.target) + { + let new_owner = pg_role_of(role, &catalog)?; + let straggler = pg_tx + .query_opt( + concat!( + "SELECT identity FROM (", + schema_owned_objects!(), + ") m WHERE owner <> (SELECT oid FROM pg_roles WHERE rolname = $2) + ORDER BY 1 LIMIT 1" + ), + &[schema, &new_owner], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to check what the schema holds: {}", + pg_error_message(&e) + )) + })?; + if let Some(row) = straggler { + return Err(Error::BadRequest(format!( + "{} appeared while this ran and would keep its old owner, so nothing was \ + applied. Plan it again.", + row.get::<_, String>(0) + ))); + } + } + + let target_label = req.target.label(&dbname); + audit_log( + &mut *tx, + &authed, + "workspaces.datatable_acl", + ActionKind::Update, + &governing.workspace_id, + Some(&governing.name), + Some( + [ + ("target", target_label.as_str()), + ("change", change_kind(&req.change)), + ("role", req.change.role()), + ] + .into(), + ), + ) + .await?; + pg_tx.commit().await.map_err(|e| { + Error::internal_err(format!( + "Failed to commit the changes: {}", + pg_error_message(&e) + )) + })?; + tx.commit().await?; + + windmill_common::feature_usage::log_feature_usage( + "datatable", + "acl_applied", + change_kind(&req.change), + ); + + Ok(format!("Updated access on {target_label}")) +} + +/// What kind of change, never what it named: the telemetry key and the audit's summary. +fn change_kind(change: &AclChange) -> &'static str { + match change { + AclChange::SetOwner { .. } => "owner", + AclChange::Grant { scope, .. } | AclChange::Revoke { scope, .. } if scope.is_future() => { + "default_privileges" + } + AclChange::Grant { .. } => "grant", + AclChange::Revoke { .. } => "revoke", + } +} + +#[cfg(test)] +mod tests { + use super::*; + use windmill_common::datatable_roles::InstanceDatatableRole; + + #[test] + fn a_role_is_its_own_postgres_role_except_admin() { + let catalog: DatatableRoleCatalog = BTreeMap::from([( + "role1".to_string(), + InstanceDatatableRole { name: "analytics".to_string(), enabled: true, pwd: None }, + )]); + assert_eq!( + pg_role_of("admin", &catalog).unwrap(), + "custom_instance_user" + ); + assert_eq!(pg_role_of("analytics", &catalog).unwrap(), "analytics"); + assert_eq!(role_name_of("custom_instance_user"), "admin"); + assert_eq!(role_name_of("analytics"), "analytics"); + // A catalog id, a name the catalog lacks, or the admin login spelled out never stands for + // some other role. + for unknown in ["role1", "operator", "custom_instance_user", "PUBLIC", ""] { + assert!( + matches!(pg_role_of(unknown, &catalog), Err(Error::BadRequest(_))), + "{unknown}" + ); + } + } + + /// A connection to the test's own database, the way the handlers reach a data table's. + async fn catalog_client(pool: &sqlx::PgPool) -> tokio_postgres::Client { + let mut config: tokio_postgres::Config = + std::env::var("DATABASE_URL").unwrap().parse().unwrap(); + config.dbname(pool.connect_options().get_database().unwrap()); + let (client, connection) = config.connect(tokio_postgres::NoTls).await.unwrap(); + tokio::spawn(connection); + client + } + + /// What a revoke takes back is read from the catalog, per object and source, and only the + /// privileges it asks for: the planner renders exactly this, and refuses an empty read. + #[sqlx::test(migrations = false)] + async fn a_revoke_reads_back_only_what_it_asks_for(pool: sqlx::PgPool) { + let client = catalog_client(&pool).await; + // A predefined role, so that nothing is granted outside the test's own database. + client + .batch_execute( + "CREATE SCHEMA granted; + CREATE TABLE granted.g (id int); + GRANT SELECT, INSERT ON granted.g TO pg_read_all_data;", + ) + .await + .unwrap(); + let target = AclTarget::Table { schema: "granted".to_string(), table: "g".to_string() }; + let revoked = read_revoked_grants( + &client, + "db", + &target, + GrantScope::Target, + &[], + &["select".to_string()], + "pg_read_all_data", + ) + .await + .unwrap(); + assert_eq!(revoked.len(), 1, "{revoked:?}"); + assert_eq!(revoked[0].object, None); + assert_eq!(revoked[0].privileges, ["SELECT"]); + let held_none = read_revoked_grants( + &client, + "db", + &target, + GrantScope::Target, + &[], + &["update".to_string()], + "pg_read_all_data", + ) + .await + .unwrap(); + assert!(held_none.is_empty(), "{held_none:?}"); + // Each source says what it gave: that, and not the whole row, is what a revoke of some of + // its privileges is held back by. + let grants = read_grants( + &client, + &AclTarget::Schema { schema: "granted".to_string() }, + ) + .await + .unwrap(); + let on_g = grants + .iter() + .find(|g| { + g.grantee == "pg_read_all_data" && g.object.as_ref().is_some_and(|o| o.name == "g") + }) + .unwrap(); + assert_eq!(on_g.sources.len(), 1, "{:?}", on_g.sources); + assert_eq!(on_g.sources[0].privileges, ["INSERT", "SELECT"]); + assert!(on_g.sources[0].reachable); + } + + /// A default privilege set database-wide applies in every schema on top of the schema's own, + /// which cannot take it back: the database shows it, and neither a schema's revoke of it nor a + /// change of owner away from its grantee may go ahead as if it were gone. + #[sqlx::test(migrations = false)] + async fn a_schema_cannot_take_back_a_database_wide_default(pool: sqlx::PgPool) { + let client = catalog_client(&pool).await; + client + .batch_execute( + "CREATE SCHEMA owned AUTHORIZATION pg_read_all_data; + ALTER DEFAULT PRIVILEGES GRANT SELECT ON TABLES TO pg_read_all_data; + ALTER DEFAULT PRIVILEGES IN SCHEMA owned + GRANT SELECT, INSERT ON TABLES TO pg_read_all_data; + CREATE SCHEMA creators AUTHORIZATION pg_monitor; + ALTER DEFAULT PRIVILEGES FOR ROLE pg_monitor + GRANT SELECT ON TABLES TO pg_read_all_stats;", + ) + .await + .unwrap(); + let creator: String = client + .query_one("SELECT current_user::text", &[]) + .await + .unwrap() + .get(0); + + let grants = read_grants(&client, &AclTarget::Database).await.unwrap(); + let database_wide = grants + .iter() + .find(|g| g.grantee == "pg_read_all_data" && g.future.as_deref() == Some("TABLES")) + .unwrap_or_else(|| panic!("{grants:?}")); + assert_eq!(database_wide.privileges, ["SELECT"]); + // A database-wide entry also holds its creator's own privileges, which come with creating + // and are no grant: neither a row nor a refusal may stem from them. + assert!( + !grants + .iter() + .any(|g| g.future.is_some() && (g.grantee == creator || g.grantee == "pg_monitor")), + "{grants:?}" + ); + assert_eq!( + database_wide.sources.len(), + 1, + "{:?}", + database_wide.sources + ); + assert_eq!(database_wide.sources[0].role, creator); + + let schema = AclTarget::Schema { schema: "owned".to_string() }; + let still_granted = read_revoked_grants( + &client, + "db", + &schema, + GrantScope::FutureTables, + &[], + &["select".to_string()], + "pg_read_all_data", + ) + .await; + assert!( + matches!(&still_granted, Err(Error::BadRequest(m)) if m.contains("database-wide")), + "{still_granted:?}" + ); + let schema_only = read_revoked_grants( + &client, + "db", + &schema, + GrantScope::FutureTables, + &[], + &["insert".to_string()], + "pg_read_all_data", + ) + .await + .unwrap(); + assert_eq!(schema_only.len(), 1, "{schema_only:?}"); + assert_eq!(schema_only[0].privileges, ["INSERT"]); + + let moved = read_former_owner_defaults(&client, "owned", "pg_write_all_data").await; + assert!( + matches!(&moved, Err(Error::BadRequest(m)) if m.contains("database-wide")), + "{moved:?}" + ); + let moved_from_creator = + read_former_owner_defaults(&client, "creators", "pg_write_all_data").await; + assert!( + matches!(moved_from_creator, Ok(None)), + "{moved_from_creator:?}" + ); + } + + /// Defaults on types go with a schema's owner like the other kinds: once a database-wide + /// default takes PUBLIC's USAGE on types away, they are all that reaches a new type. + #[sqlx::test(migrations = false)] + async fn a_schemas_former_owner_defaults_include_types(pool: sqlx::PgPool) { + let client = catalog_client(&pool).await; + client + .batch_execute( + "CREATE SCHEMA typed AUTHORIZATION pg_read_all_data; + ALTER DEFAULT PRIVILEGES IN SCHEMA typed GRANT USAGE ON TYPES TO pg_read_all_data; + ALTER DEFAULT PRIVILEGES FOR ROLE pg_read_all_data IN SCHEMA typed + GRANT SELECT ON TABLES TO pg_read_all_data;", + ) + .await + .unwrap(); + let creator: String = client + .query_one("SELECT current_user::text", &[]) + .await + .unwrap() + .get(0); + let former = read_former_owner_defaults(&client, "typed", "pg_write_all_data") + .await + .unwrap(); + assert_eq!( + former, + Some(FormerOwnerDefaults { + pg_role: "pg_read_all_data".to_string(), + defaults: vec![(creator, "TYPES")], + }) + ); + } + + /// A kind of object the list misses stays with its old owner while the schema changes hands, + /// which only a real catalog shows. + #[sqlx::test(migrations = false)] + async fn a_schemas_owner_change_takes_every_object_in_it(pool: sqlx::PgPool) { + let client = catalog_client(&pool).await; + client + .batch_execute( + "CREATE SCHEMA moved; + CREATE TABLE moved.t (id serial PRIMARY KEY, a int, b int); + CREATE STATISTICS moved.st ON a, b FROM moved.t; + CREATE TABLE moved.pt (id int) PARTITION BY RANGE (id); + CREATE TABLE moved.pt1 PARTITION OF moved.pt FOR VALUES FROM (0) TO (10); + CREATE TYPE moved.r AS RANGE (subtype = float8); + CREATE TYPE moved.pair AS (a int, b int); + CREATE TYPE moved.mood AS ENUM ('ok'); + CREATE DOMAIN moved.tags AS text[]; + CREATE COLLATION moved.coll (provider = libc, locale = 'C');", + ) + .await + .unwrap(); + let owned = read_owned_objects(&client, "moved").await.unwrap(); + // Not the serial's sequence, the range's constructors and multirange, or any array or row + // type: each follows the object it belongs to. + assert_eq!( + owned + .iter() + .map(|o| (o.keyword, o.identity.as_str())) + .collect::>(), + [ + ("COLLATION", "moved.coll"), + ("STATISTICS", "moved.st"), + ("TABLE", "moved.pt"), + ("TABLE", "moved.pt1"), + ("TABLE", "moved.t"), + ("TYPE", "moved.mood"), + ("TYPE", "moved.pair"), + ("TYPE", "moved.r"), + ("TYPE", "moved.tags"), + ] + ); + } +} diff --git a/backend/windmill-api-workspaces/src/datatable_acl_oss.rs b/backend/windmill-api-workspaces/src/datatable_acl_oss.rs new file mode 100644 index 0000000000..59eb8cdbe0 --- /dev/null +++ b/backend/windmill-api-workspaces/src/datatable_acl_oss.rs @@ -0,0 +1,44 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Where the ACL planner comes from: the enterprise one, or a refusal. +//! +//! Data table roles are an Enterprise Edition feature, and so is everything here — reading who +//! owns what included. `private` alone is not that edition — community builds carry it — so the +//! planner is behind `enterprise` as well. + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub(crate) use crate::datatable_acl_ee::plan_statements; + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub(crate) fn ensure_datatable_acl_available() -> windmill_common::error::Result<()> { + Ok(()) +} + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +use { + crate::datatable_acl::{AclChange, AclPlan, AclTarget, CatalogFacts}, + windmill_common::{datatable_roles_oss::datatable_roles_unavailable, error::Result}, +}; + +/// Checked first by every ACL route, before anything is read or connected to. +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub(crate) fn ensure_datatable_acl_available() -> Result<()> { + Err(datatable_roles_unavailable()) +} + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub(crate) fn plan_statements( + _target: &AclTarget, + _change: &AclChange, + _dbname: &str, + _pg_role: &str, + _facts: &CatalogFacts, +) -> Result { + Err(datatable_roles_unavailable()) +} diff --git a/backend/windmill-api-workspaces/src/lib.rs b/backend/windmill-api-workspaces/src/lib.rs index c1eb6c1ffc..5dfdece644 100644 --- a/backend/windmill-api-workspaces/src/lib.rs +++ b/backend/windmill-api-workspaces/src/lib.rs @@ -1,6 +1,8 @@ #[cfg(feature = "parquet")] pub mod ai_session_backups; pub mod data_metrics; +pub mod datatable_acl; +pub mod datatable_acl_oss; pub mod datatable_migrations; pub mod datatable_permissions; pub mod datatable_permissions_oss; @@ -12,5 +14,8 @@ pub mod workspaces_oss; #[cfg(feature = "private")] pub mod workspaces_ee; +#[cfg(all(feature = "private", feature = "enterprise"))] +pub mod datatable_acl_ee; + #[cfg(all(feature = "private", feature = "enterprise"))] pub mod datatable_permissions_ee; diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 65fdc88212..e22e44ed84 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -144,6 +144,7 @@ pub fn workspaced_service() -> Router { ) .merge(crate::datatable_migrations::routes()) .merge(crate::datatable_permissions::routes()) + .merge(crate::datatable_acl::routes()) .route("/git_sync_enabled", get(get_git_sync_enabled)) .route("/git_sync_deploy_mode", get(get_git_sync_deploy_mode)) .route("/edit_git_sync_config", post(edit_git_sync_config)) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index c36236294c..fe067a536b 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -5294,6 +5294,97 @@ paths: schema: type: string + /w/{workspace}/workspaces/datatable_acl/{datatable_name}: + get: + summary: read the owner and grants of an instance data table's database, schema or table + operationId: getDatatableAcl + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + - name: kind + in: query + required: true + schema: + type: string + enum: [database, schema, table] + - name: schema + in: query + required: false + schema: + type: string + - name: table + in: query + required: false + schema: + type: string + responses: + "200": + description: owner and grants + content: + application/json: + schema: + $ref: "#/components/schemas/DatatableAclInfo" + + /w/{workspace}/workspaces/datatable_acl/{datatable_name}/plan: + post: + summary: preview the SQL an ownership or grant change would run (data table administrators only) + operationId: planDatatableAcl + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/AclChangeRequest" + responses: + "200": + description: statements that would run, in a single transaction + content: + application/json: + schema: + $ref: "#/components/schemas/AclPlan" + + /w/{workspace}/workspaces/datatable_acl/{datatable_name}/apply: + post: + summary: run an ownership or grant change exactly as planned (data table administrators only) + operationId: applyDatatableAcl + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/AclChangeRequest" + responses: + "200": + description: change applied + content: + text/plain: + schema: + type: string + /w/{workspace}/workspaces/datatable_usable_roles/{datatable_name}: get: summary: list the data table roles the caller may connect as @@ -33570,6 +33661,255 @@ components: datatable: type: string + AclTarget: + description: what access is read or changed on + oneOf: + - $ref: "#/components/schemas/AclTargetDatabase" + - $ref: "#/components/schemas/AclTargetSchema" + - $ref: "#/components/schemas/AclTargetTable" + discriminator: + propertyName: kind + mapping: + database: "#/components/schemas/AclTargetDatabase" + schema: "#/components/schemas/AclTargetSchema" + table: "#/components/schemas/AclTargetTable" + + AclTargetDatabase: + type: object + required: [kind] + properties: + kind: + type: string + enum: [database] + + AclTargetSchema: + type: object + required: [kind, schema] + properties: + kind: + type: string + enum: [schema] + schema: + type: string + + AclTargetTable: + type: object + required: [kind, schema, table] + properties: + kind: + type: string + enum: [table] + schema: + type: string + table: + type: string + + AclChange: + description: one change to plan or apply + oneOf: + - $ref: "#/components/schemas/AclChangeSetOwner" + - $ref: "#/components/schemas/AclChangeGrant" + - $ref: "#/components/schemas/AclChangeRevoke" + discriminator: + propertyName: type + mapping: + set_owner: "#/components/schemas/AclChangeSetOwner" + grant: "#/components/schemas/AclChangeGrant" + revoke: "#/components/schemas/AclChangeRevoke" + + AclChangeSetOwner: + type: object + description: >- + hands the target to role — for a schema, with everything already in it but an extension's + members, which stay with the extension + required: [type, role] + properties: + type: + type: string + enum: [set_owner] + role: + type: string + description: a data table role of the instance, or admin + + AclChangeGrant: + type: object + required: [type, role, privileges, scope] + properties: + type: + type: string + enum: [grant] + role: + type: string + description: a data table role of the instance, or admin + privileges: + type: array + items: + type: string + scope: + $ref: "#/components/schemas/AclGrantScope" + + AclChangeRevoke: + type: object + required: [type, role, privileges, scope] + properties: + type: + type: string + enum: [revoke] + role: + type: string + description: a data table role of the instance, other than admin + privileges: + type: array + items: + type: string + scope: + $ref: "#/components/schemas/AclGrantScope" + objects: + type: array + description: >- + objects inside the target the revoke covers, empty for the target itself. Only with the + target scope; a revoke on all objects of a kind is refused, since it cannot say which + grants it takes back. + items: + $ref: "#/components/schemas/AclObject" + + AclGrantScope: + type: string + enum: + [ + target, + all_tables, + all_sequences, + all_functions, + future_tables, + future_sequences, + future_functions, + ] + + AclChangeRequest: + type: object + required: [target, change] + properties: + target: + $ref: "#/components/schemas/AclTarget" + change: + $ref: "#/components/schemas/AclChange" + statements: + type: array + description: >- + The statements the plan showed. Required to apply, which plans again and refuses if + the result differs. + items: + type: string + + AclPlan: + type: object + required: [statements, warnings] + properties: + statements: + type: array + items: + type: string + warnings: + type: array + items: + type: string + + AclObject: + type: object + required: [name, kind] + properties: + name: + type: string + kind: + type: string + description: >- + TABLE, SEQUENCE, FUNCTION, PROCEDURE or TYPE — what the object is. A revoke turns it + into the keyword it takes, ROUTINE for both routine kinds; a type's grants are read + only. + args: + type: string + description: identity arguments of a routine, which is what tells two of the same name apart + + AclGrant: + type: object + required: [grantee, privileges, sources] + properties: + grantee: + type: string + privileges: + type: array + items: + type: string + object: + $ref: "#/components/schemas/AclObject" + future: + type: string + description: >- + set for a default privilege, naming the kind of object it covers (TABLES, SEQUENCES, + FUNCTIONS, TYPES, or SCHEMAS). On a schema, the defaults set in that schema; on the + database, the ones set database-wide, which apply in every schema and which no schema's + own defaults take back. + sources: + type: array + description: >- + the roles the grant comes from, each once — who granted it, or for a default privilege + the role whose future objects it covers. A revoke of some of the grant's privileges + takes them back from every source that gave them. + items: + $ref: "#/components/schemas/AclSource" + + AclSource: + type: object + required: [role, privileges, reachable] + properties: + role: + type: string + privileges: + type: array + description: >- + what role gave of the grant's privileges. A revoke is held back only by a source out of + reach that gave some of what it takes back. + items: + type: string + reachable: + type: boolean + description: >- + whether the data table's connection can take back what role gave. On an object that is + the owner, when the connection acts for the owner, and otherwise the connection itself; + for a default privilege, a creating role the connection acts for. What a source out of + reach gave is not revocable from here; privileges only other sources gave still are. + + DatatableAclInfo: + type: object + required: [owner, roles, editable, supports_maintain, dbname, grants, children] + properties: + owner: + type: string + roles: + type: array + description: the roles a change may name; empty unless the caller may change anything + items: + type: string + editable: + type: boolean + description: whether the caller may plan and apply changes + supports_maintain: + type: boolean + description: whether the server is Postgres 17+, which added the MAINTAIN table privilege + dbname: + type: string + description: the database the target lives in + grants: + type: array + items: + $ref: "#/components/schemas/AclGrant" + children: + type: array + description: a database's schemas, or a schema's tables + items: + type: string + CustomInstanceDb: type: object required: diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 52d1a85aec..a67bb6bfa1 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -1024,6 +1024,20 @@ impl Future for TokioPgConnection { } } +impl TokioPgConnection { + /// Drive the connection and hand back what the server sends outside of a query's response — + /// notices above all, which driving it as a future silently discards. + pub fn poll_message( + &mut self, + cx: &mut core::task::Context<'_>, + ) -> core::task::Poll>> { + match self { + TokioPgConnection::Tls(conn) => conn.poll_message(cx), + TokioPgConnection::NoTls(conn) => conn.poll_message(cx), + } + } +} + impl PgDatabase { /// The role the connection logs in as, whichever way it authenticates. pub fn login_name(&self) -> &str { diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index 6ceab37872..abe37c3537 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -1093,8 +1093,10 @@ migrations are set up and used, how often an empty workspace home is seen, how often the home page’s create menu and hub-project picker are opened and from which entry point, the name of any public hub project imported from the home page and how far that - import got, whether a pre-approved trial offer was opened, and whether data tables are - put under roles and whether callers name a role or take the default, last 30 days)
  • feature adoption (counts of which flow, script, trigger, worker and data table @@ -1160,8 +1162,10 @@ migrations are set up and used, how often an empty workspace home is seen, how often the home page’s create menu and hub-project picker are opened and from which entry point, the name of any public hub project imported from the home page and how far that - import got, whether a pre-approved trial offer was opened, and whether data tables are - put under roles and whether callers name a role or take the default, last 30 days)
  • feature adoption (counts of which flow, script, trigger, worker and data table diff --git a/frontend/src/lib/components/datatableAcl/AclTargetPicker.svelte b/frontend/src/lib/components/datatableAcl/AclTargetPicker.svelte new file mode 100644 index 0000000000..cbfb43ad39 --- /dev/null +++ b/frontend/src/lib/components/datatableAcl/AclTargetPicker.svelte @@ -0,0 +1,50 @@ + + +
    + ({ value: t, label: t }))} + bind:value={table} + placeholder="The whole schema" + clearable + size="sm" + class="w-56" + /> + {/if} +
    diff --git a/frontend/src/lib/components/datatableAcl/PgAclEditor.svelte b/frontend/src/lib/components/datatableAcl/PgAclEditor.svelte new file mode 100644 index 0000000000..e0ae596a3d --- /dev/null +++ b/frontend/src/lib/components/datatableAcl/PgAclEditor.svelte @@ -0,0 +1,289 @@ + + +{#if acl.error} + {errorText(acl.error)} +{:else if !info} + Loading… +{:else} +
    + {#if !info.editable} + + Read only: access is changed by the admins of the workspace that governs this data table, on + Windmill Enterprise Edition. + + {/if} + + {#if target.kind !== 'database'} +
    +
    + Owner + + {target.kind === 'schema' + ? 'The role that owns the schema and everything already in it, except what belongs to an extension, which stays with the extension. Changing it also keeps the new owner in reach of what the current roles create here from then on; a role added afterwards is not covered.' + : 'The role that owns the table. Its owner may always read and write it, and is who ALTER and DROP answer to.'} + +
    + {#if info.editable} + scope, + (s) => { + if (!s) return + scope = s + // A privilege only exists for some objects — SELECT means nothing on a function — + // so drop what the new scope cannot carry rather than send it. + const allowed = privilegesOf(s, target.kind, supportsMaintain) + privileges = privileges.filter((p) => allowed.includes(p)) + } + } + items={scopesOf(target.kind)} + {disabled} + size="sm" + class="w-52" + /> + TO +