Compare commits

...
Author SHA1 Message Date
Diego Imbert 7577c55b13 Merge remote-tracking branch 'origin/datatable-external-instance-4' into datatable-external-instance-ui 2026-09-18 01:03:57 +02:00
Diego Imbert 51224582c1 Merge commit '2661dadba63efcc0c5db88a7942d66caf34f20b9' into HEAD
# Conflicts:
#	backend/ee-repo-ref.txt
2026-09-18 01:00:23 +02:00
Diego Imbert 2661dadba6 Merge commit '8c554ef1337aa6cd99ab869443dfd30becc65a36' into HEAD 2026-09-18 01:00:09 +02:00
Diego ImbertandClaude Opus 5 8c554ef133 fix(datatables): take the fork data table lock once, before the external cluster's
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-18 01:00:06 +02:00
Diego Imbert cdd681c029 Merge commit '154bade187169ec5c403d300a160556b768a0ead' into HEAD 2026-09-18 00:58:35 +02:00
Diego Imbert 154bade187 Merge remote-tracking branch 'origin/fork-database-authorization' into HEAD 2026-09-18 00:58:33 +02:00
Diego ImbertandClaude Opus 5 e550374f06 fix(datatables): migrate fork reservations on workspace rename, and lock the parent's data tables for the whole fork
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-18 00:58:19 +02:00
Diego ImbertandClaude Opus 5 57211d6c95 fix(datatables): keep only the most recently used DuckDB root certificate files
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-18 00:55:35 +02:00
Diego Imbert 0dbf5eef74 Merge commit 'addd77eaf4' into HEAD
# Conflicts:
#	backend/ee-repo-ref.txt
2026-09-18 00:55:35 +02:00
Diego ImbertandClaude Opus 5 addd77eaf4 fix(datatables): create external databases only on a cluster setup succeeded on, and document the registry reader
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 18:43:49 +02:00
Diego Imbert 7f5628e775 Merge commit 'eab084b36fe14ecee3ecc9a78fb079760de133b9' into HEAD
# Conflicts:
#	backend/windmill-common/src/lib.rs
2026-09-17 18:43:49 +02:00
Diego Imbert eab084b36f Merge remote-tracking branch 'origin/fork-database-authorization' into HEAD 2026-09-17 18:41:27 +02:00
Diego ImbertandClaude Opus 5 4cba79b7f0 fix(datatables): authenticate instance database setup before writing its status, and keep a fork reservation across it
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 18:41:11 +02:00
Diego Imbert 8e780a7923 Merge remote-tracking branch 'origin/datatable-external-instance-4' into datatable-external-instance-ui 2026-09-17 18:25:57 +02:00
Diego Imbert 3fdb3047a7 Merge commit '5dfb2f30f74cedaae9f113383e42d3a35cde5189' into HEAD
# Conflicts:
#	backend/ee-repo-ref.txt
#	backend/windmill-common/src/datatable_roles.rs
2026-09-17 18:23:38 +02:00
Diego Imbert 5dfb2f30f7 Merge commit '76a10f8ee8' into HEAD
# Conflicts:
#	backend/ee-repo-ref.txt
#	backend/windmill-common/src/lib.rs
2026-09-17 18:22:16 +02:00
Diego Imbert 76a10f8ee8 Merge commit 'c9e2246e1c2480135884c9b3dcace11da0907dba' into HEAD
# Conflicts:
#	backend/windmill-api-workspaces/src/workspaces.rs
#	backend/windmill-api-workspaces/src/workspaces_extra.rs
2026-09-17 18:21:18 +02:00
Diego Imbert c9e2246e1c Merge remote-tracking branch 'origin/fork-database-authorization' into HEAD 2026-09-17 18:17:06 +02:00
Diego ImbertandClaude Opus 5 b68768084e fix(datatables): bind fork database copies to their workspace, and count every use before dropping one
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 18:16:48 +02:00
Diego Imbert 4a0d7c0c29 Merge commit 'd62807e9edc553b1eb08c10ce28521a71007a263' into HEAD 2026-09-17 18:04:22 +02:00
Diego ImbertandClaude Opus 5 d62807e9ed fix(datatables): stop counting storage-only fork cleanup rows as uses of an external database
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 18:04:19 +02:00
Diego Imbert 696b4ab5f8 Merge remote-tracking branch 'origin/datatable-external-instance-4' into datatable-external-instance-ui 2026-09-17 16:37:27 +02:00
Diego ImbertandClaude Opus 5 48f026eb0f fix(datatables): refuse rolling back while external data tables are under roles, and type external_instance in the CLI
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 16:35:45 +02:00
Diego Imbert 0e659898f3 Merge commit 'a296ca016a' into HEAD
# Conflicts:
#	backend/ee-repo-ref.txt
2026-09-17 16:35:45 +02:00
Diego Imbert a296ca016a Merge commit '7c16d482fd' into HEAD
# Conflicts:
#	backend/ee-repo-ref.txt
2026-09-17 16:34:50 +02:00
Diego ImbertandClaude Opus 5 7c16d482fd fix(datatables): refuse fork cleanup of an external database another workspace uses
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 16:34:14 +02:00
Diego ImbertandClaude Opus 5 947e44fe49 fix(settings): show the server's reason when saving instance settings fails
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 16:26:29 +02:00
Diego Imbert 489493dba2 Merge remote-tracking branch 'origin/datatable-external-instance-4' into datatable-external-instance-ui 2026-09-17 16:18:12 +02:00
Diego Imbert c55ff3141c Merge commit '5eec203099ba1034039f8b52e81400078cac919d' into HEAD 2026-09-17 16:17:18 +02:00
Diego ImbertandClaude Opus 5 5eec203099 fix(datatables): keep certificate verification when DuckDB attaches an external data table
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 16:17:15 +02:00
Diego ImbertandClaude Opus 5 0b3882f2b2 fix(datatables): keep certificate verification when DuckDB attaches an external data table
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 16:17:02 +02:00
Diego Imbert 58731b1828 Merge commit '53d62dd399' into HEAD
# Conflicts:
#	backend/ee-repo-ref.txt
2026-09-17 16:17:02 +02:00
Diego ImbertandClaude Opus 5 5624d350d5 feat(datatables): manage the external instance cluster's data table roles from the UI
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 16:09:25 +02:00
Diego ImbertandClaude Opus 5 53d62dd399 fix(datatables): register external fork catalogs under the lifecycle lock, and keep certificate verification in DuckDB attaches
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 16:07:40 +02:00
Diego Imbert e17b76c255 Merge commit '5ddaee5b78' into HEAD
# Conflicts:
#	backend/ee-repo-ref.txt
2026-09-17 16:07:40 +02:00
Diego ImbertandClaude Opus 5 5ddaee5b78 fix(datatables): write the external cluster setting under the lifecycle lock, and check fork targets are registered
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 16:04:18 +02:00
Diego Imbert cfada61f41 Merge remote-tracking branch 'origin/datatable-external-instance-4' into datatable-external-instance-ui 2026-09-17 16:00:06 +02:00
Diego ImbertandClaude Opus 5 0406f133c5 feat(datatables): per-cluster data table role catalogs, with roles on the external instance cluster
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 15:59:22 +02:00
Diego ImbertandClaude Opus 5 aebae79397 feat(datatables): configure the external instance cluster and pick its databases from the UI
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 15:45:22 +02:00
Diego ImbertandClaude Opus 5 b62d8b246a fix(datatables): protect external databases pending fork cleanup, and describe Ducklake usage in the API
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 15:30:25 +02:00
Diego ImbertandClaude Opus 5 3981b2eb1b fix(datatables): refuse repointing the external cluster while it is in use, and keep verify-ca working for pg_dump
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 15:28:31 +02:00
Diego Imbert 5bc2f13565 Merge commit '3981b2eb1b3bf77cdc1a5cbf43090e6b9706aa1f' into datatable-external-instance-3 2026-09-17 15:28:31 +02:00
Diego Imbert a928253af0 Merge remote-tracking branch 'origin/datatable-roles-redesign-part-2' into datatable-roles-redesign-part-4
# Conflicts:
#	backend/windmill-common/src/workspaces.rs
2026-09-17 15:22:45 +02:00
Diego Imbert 5d8f031cf6 Merge branch 'datatable-roles-redesign' into datatable-roles-redesign-part-2 2026-09-17 15:22:26 +02:00
Diego ImbertandClaude Opus 5 8a7f364cfe fix(datatables): hold the parent's settings while a fork points at its data tables
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 13:29:35 +02:00
Diego ImbertandClaude Opus 5 ee176e24d5 perf(datatables): resolve a workspace's data tables per pointer hop, not per entry
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 13:04:26 +02:00
Diego ImbertandClaude Opus 5 4bdc5a9e20 fix(datatables): drop a DuckDB data table secret once its ATTACH has used it
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 12:17:30 +02:00
Diego Imbert 0c53990d18 Merge commit '9b0046d8671697e6f6568c89170052367139d157' into datatable-external-instance-3
# Conflicts:
#	backend/ee-repo-ref.txt
#	backend/windmill-common/src/external_instance_pg.rs
2026-09-17 11:50:21 +02:00
Diego ImbertandClaude Opus 5 9b0046d867 docs(datatables): state the authorization contract of external database usage lookups
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 11:50:10 +02:00
Diego Imbert 1f78ce77f5 Merge commit 'dbe0230f483bbe10aeca055c4d39506fea41459d' into HEAD
# Conflicts:
#	backend/ee-repo-ref.txt
2026-09-17 11:50:00 +02:00
Diego ImbertandClaude Opus 5 dbe0230f48 fix(datatables): never grant CREATEROLE to custom_instance_user on the external cluster
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 11:49:20 +02:00
Diego ImbertandClaude Opus 5 67f488c676 feat(datatables): Ducklake catalogs on the external instance cluster
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 11:47:25 +02:00
Diego Imbert ced4913b33 Merge commit 'b3a3f1cf5a0e6f97af2679c8abf640c6b832d7db' into datatable-external-instance-2
# Conflicts:
#	backend/ee-repo-ref.txt
2026-09-17 11:34:32 +02:00
Diego Imbert b3a3f1cf5a Merge remote-tracking branch 'origin/datatable-roles-redesign-part-4' into HEAD
# Conflicts:
#	backend/ee-repo-ref.txt
2026-09-17 11:34:28 +02:00
Diego Imbert 92e51f19aa Merge remote-tracking branch 'origin/datatable-roles-redesign-part-2' into datatable-roles-redesign-part-4
# Conflicts:
#	backend/ee-repo-ref.txt
#	backend/windmill-api-integration-tests/tests/datatable_roles.rs
#	backend/windmill-api-workspaces/src/datatable_permissions_oss.rs
#	backend/windmill-api-workspaces/src/workspaces.rs
#	backend/windmill-common/src/datatable_roles_oss.rs
#	backend/windmill-common/src/workspaces.rs
#	frontend/src/lib/components/datatableAcl/PgAclEditor.svelte
#	frontend/src/lib/components/workspaceSettings/DataTablePermissionsButton.svelte
#	frontend/src/lib/components/workspaceSettings/DataTableRolesSection.svelte
#	frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte
#	frontend/src/lib/components/workspaceSettings/InstanceRolesButton.svelte
2026-09-17 10:03:05 +02:00
Diego ImbertandClaude Opus 5 36532c35d8 fix: check ACL read reach against the entry it connects from
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb
2026-09-17 10:01:19 +02:00
Diego ImbertandClaude Opus 5 79481a2f02 fix: build the ACL connection from the authorized data table entry
Resolving the settings again could land on a resource with the same
database name on another server, which the later entry checks never see.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb
2026-09-17 10:01:19 +02:00
Diego ImbertandClaude Opus 5 c884597e56 fix: hold the ACL connection to the database that was authorized
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb
2026-09-17 10:01:19 +02:00
Diego ImbertandClaude Opus 5 115448d3d1 fix: run one data table ACL apply at a time per server before it connects
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb
2026-09-17 10:01:19 +02:00
Diego ImbertandClaude Opus 5 1ece278fd2 fix: add only missing grant options before an ACL apply, never default privileges
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb
2026-09-17 10:01:19 +02:00
Diego ImbertandClaude Opus 5 7819d48df9 fix: refresh grant options only after the ACL apply validates its plan
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb
2026-09-17 10:01:19 +02:00
Diego ImbertandClaude Opus 5 ce1f720d85 fix: take every pooled connection before the ACL apply locks
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb
2026-09-17 10:01:19 +02:00
Diego ImbertandClaude Opus 5 93d346e54b feat(datatables): add an ACL editor for data table roles
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 10:01:19 +02:00
Diego ImbertandClaude Opus 5 9378828472 docs(datatables): note that legacy names with ? cannot be migrated
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 79bb761c5d fix(datatables): serialize roles going on with aliases saved from other workspaces
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 d191cfe4eb fix(datatables): let CE migrations connect as an explicitly named admin
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 31b5b0a062 fix(datatables): refuse roles while a same-workspace alias reaches the database
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 10:01:15 +02:00
windmill-internal-app[bot] d1c2f801f4 chore: update ee-repo-ref to a1873dbb67f2302b85ff5362f8387b48eccdb607
This commit updates the EE repository reference after PR #783 was merged in windmill-ee-private.

Previous ee-repo-ref: 5c853e2c20eca6b748415fc0d6862a6ebfb5fec4

New ee-repo-ref: a1873dbb67f2302b85ff5362f8387b48eccdb607

Automated by sync-ee-ref workflow.
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 d3ee67c9bc test(datatables): run the tenant matcher test on the enterprise edition
The matcher it covers is enterprise code now, so without the enterprise
edition the test hit the stub and failed the default windmill-common run. It
runs with private and enterprise, and a counterpart without them asserts that
no tenant list covers anyone, the wildcard and a workspace admin included.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 eafc551651 feat(datatables): gate the roles UI mount sites on an enterprise license
Both mount sites are still commented out; the gate travels with them.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 db05c8b7c9 test(datatables): run the roles tests on the enterprise edition, refusals without it
Each test that exercises roles runs with private and enterprise. Two tests run
without them: every roles route answers the Enterprise refusal, and a data
table saved under roles, or a named role, is refused a connection while one
not under roles resolves as before.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 11ba3cd477 feat(datatables): move the data table role catalog endpoints to the enterprise edition
The superadmin list, create, update and delete handlers move to
windmill-ee-private. The routes stay registered and, without the enterprise
edition, refuse after authentication.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 7ae3335f03 feat(datatables): move the data table permissions endpoints to the enterprise edition
The permissions read, save and usable-roles handlers move to
windmill-ee-private; the routes stay registered and, without the enterprise
edition, answer that data table roles are an Enterprise Edition feature.
ensure_governs_datatable and ensure_reaches_datatable keep their paths: the
first refuses, the second passes a data table not under roles.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 c7269e975f feat(datatables): move data table role catalog and resolution to the enterprise edition
Roles are an Enterprise Edition feature. The catalog, the Postgres logins,
CONNECT convergence, tenant evaluation and the role half of connection
resolution move to windmill-ee-private. Every public function keeps its path
and signature and forwards through datatable_roles_oss, which re-exports the
enterprise implementation or, without it, refuses.

Without the enterprise edition a data table under roles, or a caller naming a
role, is refused a connection rather than resolved as admin, and the reach and
admin-access checks refuse one under roles. A data table not under roles
resolves as before in every edition, and an instance database keeps the
CONNECT grants it was created with. The catalog lock, the stream lock, the
tenant cascades and the permissions stripping stay in OSS: they only restrict.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 8a0babd7a6 fix(datatables): no entry without roles may newly reach a database under roles
The previous guard only caught a new name replacing an entry under roles.
A whole-map save could also repoint an existing entry without roles at
that database, or another workspace could point one there, and every
caller of that entry would connect as admin. The rule is now stated on
the saved entries: one that carries no roles and newly points at an
instance database any entry under roles uses, in this workspace or
another, is refused. A declared rename carries its roles and passes.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 6d9ef5f6b7 fix(datatables): refuse a save that drops a data table's roles through an undeclared rename
A data table's roles follow its entry only through a declared rename. A
settings sync sends the whole map and never declares one, so renaming a
data table under roles there read as a delete and a new entry on the same
database: the new entry carried no roles, and every caller connected as
admin. Such a save is now refused, naming both entries.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 1dbc1e7759 fix(datatables): warn when a settings sync strands fork pointers
A settings save reported the fork pointers left resolving to nothing only
for the names in `deleted_datatables`, which `wmill sync push` never sends.
The save now works out what it removed from the locked entries, and the
CLI prints the stranded pointers it returns.

Also correct the replication helper's contract: no role or admin check
makes a replication connection safe, so a data table under roles is
refused outright rather than gated as an admin operation.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 df471e37d8 fix(datatables): wait out live listeners, and resolve stored names containing ?
Turning roles on counted a trigger as gone once disabled, and a capture
once its client stopped pinging, but the listener keeps its replication
connection until its next heartbeat notices. A trigger or capture whose
listener pinged in the last 15 seconds, the window a server holds a
listener for, now still counts as streaming.

Data table names could contain `?` before they were restricted, and such
entries are still stored. Splitting `?role=` off a reference misread them:
`a?b` became `a` with an unknown parameter, and the clone checks looked at
a different entry than the one copied. An entry stored under the whole
reference is now looked up first, in the Postgres executor, DuckDB ATTACH
and the clone checks. Agent workers cannot read the workspace and keep
the strict parse, which refuses such a name rather than misreading it.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 122a3bd164 fix(datatables): serialize roles going on with a stream starting
Turning roles on looked for enabled triggers and live captures once,
without a lock anything starting a stream also took. A trigger enabled in
that window could have its listener connect before roles committed, and a
healthy listener never checks again. Both transitions now serialize on one
advisory lock: roles going on hold it exclusive while they look, and
trigger create, edit and enable, and capture setup and ping hold it shared
while they commit. Either the look sees the stream, or the listener
connects after roles are committed and refuses.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 2f95c22242 fix(datatables): disable a data table role before deleting it
Deleting a role reassigns and drops what it owns in each registered
database on its own connection, and each of those passes commits as it
goes. A database failing part-way left the role enabled in the catalog and
able to log in, but already stripped in the databases reached before it.
The role is now disabled in its own commit first, so a failed delete
leaves a disabled role to retry.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 7279ee15e2 fix(datatables): refuse a Postgres trigger on a data table under roles when it is saved
Creating or editing a trigger that points at a data table under roles was
accepted, and its listener then retried the refused connection every 30
seconds forever. The save is now refused, and a trigger that reaches such a
data table anyway (re-enabled, or cloned into a fork) is disabled by its
listener with the reason, as a missing replication slot is.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 fa1e8bd3aa refactor(datatables): keep Postgres triggers and data table roles apart
A replication stream reads every row of every table whatever the data table's
roles grant, and its listener checks access only when it connects. Rather than
chase every way access can change and bounce the streams each one affects, a
data table now carries one or the other:

- a Postgres trigger or capture cannot be created on, or connect to, a data
  table under roles;
- roles cannot be turned on while an enabled trigger or a live capture reads
  the data table, its own or a fork's through its pointer. The refusal names
  each one to disable.

This removes the stream bounces on roles edits and on data table and workspace
deletion, and the trigger gate that admitted admins. The fork schema baseline
fix from the same review round is kept.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 dc3ebfbe74 fix(datatables): keep the fork schema baseline, and bounce streams on every removal
Three fixes from review.

`edit_datatable_config` took `forked_from` wholesale from the stored entry, so
the fork schema diff's save of an advanced baseline was silently discarded and
an applied change was offered again. Whether an entry carries a clone stamp is
still carried from the store, since that is what marks its database droppable,
but the baseline inside it is now taken from the request.

The stranded-pointer warning and the stream bounce ran over the optional
`deleted_datatables` hint, which the settings-sync CLI never sends, so removing
a governing data table through `wmill` bounced nothing. Removals are now derived
from the stored configuration against the saved one.

`delete_workspace` read the pointers to bounce before its transaction, so a fork
committing a pointer during the deletion was missed. The read now happens inside
the transaction, after the workspace row is deleted: a fork's insert key-share
locks that row through its parent foreign key, so it is either seen or fails on
the missing parent.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 8255f2ad2b fix(datatables): bounce the streams reading a data table when it is deleted
Deleting a governing data table, or the workspace that holds it, only collected
the fork pointers it stranded, for the warning. A Postgres trigger or capture
already streaming through one of those pointers kept the replication connection
it opened while the pointer still resolved, so it went on dispatching the
governing database's rows after the fork lost access — until its connection
happened to restart. The governing workspace's own streams on a deleted entry
did the same.

Both deletion paths now bounce the affected listeners inside their own
transaction, through the helper a permission change already uses, so a
listener that reconnects re-resolves the entry and finds it gone. The helper is
split so a caller can pass the (workspace, local name) pairs it already holds.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 ebe076cbe6 docs(datatables): record the stale clone database as a known limitation
A clone is three requests and `CREATE DATABASE` is not transactional, so a
failure after the first leaves a registered `wm_fork_*` behind, as it did
before data table roles. Accepted for this PR: it is harmless to data and goes
away once the clone is a single server-side operation.

The comment also records why the obvious fix is wrong: reclaiming the leftover
on retry, without durable clone ownership, can drop another workspace's fully
copied database between its import and its final fork request.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 643bc1b51a Revert "fix(datatables): let a retried clone reclaim its own leftover database"
This reverts commit 7dd3275a10.

The reclaim tied the caller to the source they administer, but not to the
database it dropped. Between another workspace's import and its final fork
request, that workspace's target is full, registered, unnamed and has no open
connection, so an admin of any instance data table could name it and have it
dropped and recreated empty. The victim's fork would then commit pointing at
the empty copy. Safe reclaim needs durable clone ownership and serialization
with the request that names the database; until then the leftover stays, as it
did before this PR.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 29fccb0978 fix(datatables): let a retried clone reclaim its own leftover database
A clone creates its target database one request before it copies into it, and
the fork that would name it is written a request after that. Any failure in
between — a pg_dump error, a bad restore, a dropped connection, the source's
roles changing mid-flow — left a registered `wm_fork_*` that no entry names,
and every retry then failed on its name. This predates data table roles.

`create_pg_database` now reclaims such a leftover before creating: only a
`wm_fork_*` database Windmill registered as a data table database and that no
data table or ducklake entry names, in any workspace, archived ones included.
The drop never terminates connections, so a clone still copying into it makes
the reclaim fail instead of being cut off. It is limited to callers who
administer the source — reaching it is not enough, since on a data table
without roles every member reaches it — and anyone else gets the refusal an
existing database always got.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 432ccc0fbd refactor(datatables): settle clonability in one place, before anything is created
A clone is three stages a workspace apart — `create_pg_database`, then
`import_pg_database`, then `apply_forked_datatable` inside the fork transaction.
Only the third can roll back, and `CREATE DATABASE` is not transactional, so any
refusal that lives there strands a registered `wm_fork_*` that no entry names
and whose name blocks the retry.

That orphan has now been fixed three times, most recently reintroduced by a
guard added one commit ago. Patching each new refusal into the first endpoint is
not the fix; having two places that can refuse is. `ensure_datatable_is_clonable`
now answers every reason a copy can be refused and returns what it resolved, and
the stage that writes the entry only does the work.

Also takes an ACCESS EXCLUSIVE lock before the rollback guard counts, so a role
created concurrently cannot slip between the check and the drop.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 e6794984d2 fix(datatables): refuse to roll back the catalog while roles exist
The down migration dropped the table and left every role behind: live Postgres
logins whose passwords only that table carried, so after a revert Windmill could
neither use, disable nor delete them, and re-applying could not recreate them
because the names were taken. Cleaning up here is not possible either — dropping
a role means reassigning what it owns in every instance database, and a
migration runs in one — so it now refuses while the catalog is non-empty and
says to delete the roles through instance settings, which does the cluster work.

Also enforces the instance-only invariant the resolved-pointer clone relies on
rather than only asserting it in a comment.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 2abcd122a8 fix(datatables): clone a fork's pointer instead of failing after the copy
Forking a fork with cloning left an orphan database. The preflight resolves the
pointer and sees the governing entry, so both endpoints ran and filled the new
database; `apply_forked_datatable` then refused the inherited pointer and rolled
the fork back, stranding a registered `wm_fork_*` that no entry names and whose
name blocks the retry.

Refusing earlier would have been the smaller change, but forking a fork and
cloning worked before pointers existed, so it would trade an orphan for a
regression. Resolve what the pointer names and write the terminal entry the
clone needs: the whole `database` object rather than a patch of its
`resource_path`, since a pointer has none, and `reference` removed with it.

Also accepts `-- role=x` and `-- Role = x`, two more spellings that fell through
to the default role.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 30f039d292 fix(datatables): honour -- role: x, and fix the DuckDB attach test
Two review findings, both real.

`attach_datatable_parses_name_and_role` never compiled: `parse_attach_datatable`
returns `Result<Option<_>>` now and one call site kept a single `unwrap`. Its
`?Role=analytics` case also asserted a refusal, contradicting the parser in the
same commit, which matches the key case-insensitively. Replaced with the cases
that are genuinely malformed, and a positive one for the cased key.

`-- role: analytics` fell through to the default role — the silent fallback the
strict parser exists to remove, for the spelling most likely to be typed. The
keyword now accepts an optional colon, attached or spaced, while a word that
merely starts with it (`rolebased`) is still not an attempt.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 5fd9ab24b8 chore(datatables): don't mount the roles UI until the ACL editor lands
Enforcement ships first. The permissions drawer is what turns roles on, and the
catalog section is what creates them — both are only useful once there is a way
to grant a role the privileges it needs, which arrives with the ACL editor. Left
mounted they would offer a feature whose other half does not exist.

The two components are complete and reviewed; only their call sites here are
commented out, with a note pointing the follow-up PRs at them.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 f7812187cc fix(datatables): carry the role annotation into the row_to_json retry
The retry rebuilds its SQL from `pruneComments(code)`, so the leading comment
block never reached the second attempt — and with it the `-- role <name>` line
that decides which login the query runs as. The retry connected as the data
table's default role instead, so a query the first attempt was denied could
succeed on the second, reported as "recovered with the row_to_json fix".

Carry the leading comment block over. The retry itself is unchanged.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 b5785e8e7a fix(datatables): refuse a malformed role query string instead of ignoring it
`?Role=analytics`, `?role=` and `?x=1&role=…` all fell through the reference
parser's exact-match rule, so the connection resolved to the data table's default
role and ran under a login the caller never asked for — the URI half of the same
trap as a malformed `-- role` annotation.

The key now matches case-insensitively, and anything else in the query string is
an error naming it; `role` is the only parameter a reference takes. Callers that
only need the entry keep a lenient `datatable_ref_name`, since they never act on
the role. The DuckDB `ATTACH` parser propagates it rather than attaching under
the default.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 ac8587e452 fix(datatables): refuse a malformed role annotation instead of ignoring it
`-- Role operator`, `-- role operator;` and `-- role operator -- why` all failed
the annotation parser's exact-match rule, so the query fell through to the data
table's default role and ran, silently, under a login the author did not choose.
Naming a role exists precisely to not do that.

A leading comment whose first word is `role` is now an annotation attempt: the
keyword matches case-insensitively, one trailing `;` is tolerated, and anything
else is an error naming the line. Only callers that already know the target is a
`datatable://` reference ever run this, so ordinary SQL keeps its comments.

Also bumps the dev shell's postgres client to 18 — it trailed the server the dev
database runs, which takes out every data table export, clone and fork-with-data.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-17 10:01:15 +02:00
Diego Imbert 3ae1688c44 fix pg_dump stuck on version 17 on nix 2026-09-17 10:01:15 +02:00
Diego Imbert d69da0bbd6 nit worker error msg 2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 eeed332591 fix(datatables): refuse the clone's database too, not only its data
A clone is two endpoints: `create_pg_database` then `import_pg_database`. Only
the second refused a data table under roles, so a fork asking to clone one
created and registered an empty `wm_fork_…` instance database and then failed —
and nothing collects it, since `drop_forked_datatable_databases` only drops
entries carrying `forked_from` and no entry names this one.

Refuse in both, so the clone stops before a database exists.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 78a12fdfcc fix(datatables): refuse to copy a data table that is under roles
pg_dump carries no roles and the import runs with --no-privileges, so a copied
data table arrives owned by the admin connection with no GRANT for any role.
The settings clone brings `permissions` across, so the fork's tenants pass
Windmill's check, connect as the role they were given, and are denied by
Postgres on everything: an entry that reads as configured and answers nothing.

Refuse the copy — in the import endpoint before any data moves, and in the fork
path the CLI takes. Replaying the source's owners and ACLs into the clone is
what lifts this, and is a change of its own. Dropping `permissions` from the
copy instead would be the unsafe half, since the copy holds the parent's rows.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 189bc3297b fix(datatables): cascade on the leave route that is used, gate migrations before the admin connection, and drop a role atomically
The tenant cascade on leaving went onto `/users/leave`. The UI and the generated client call
`/workspaces/leave` — a different handler in a different crate with the same name — which
deleted the membership and left `u/<username>` in the tenant lists. Leaving and rejoining
therefore restored the access the leave was supposed to end, and a later account taking the
username would have inherited it. The regression test drives the route the client actually
calls; without the fix it fails with "leaving kept the tenant".

The migration endpoints authorized too late. `run_datatable_migrations` opened the data table's
admin connection, created `_wm_migrations` and read it before reaching the per-migration role
check — so with nothing pending, nothing was checked at all. Rollback returned before its check
when nothing was applied, and the status endpoint had none. All three now ask, before any
connection is opened, whether the caller can reach the data table as any role at all; which role
a given migration runs as is still decided per migration, and by the executor after that.

Deleting a role committed the cluster drop and the catalog row, then swept the tenant lists in
separate transactions. A sweep failing part-way left workspaces naming a role nothing can connect
as, while the retry answered `NotFound` because the catalog entry was already gone. The sweep now
runs in the same transaction, so the drop, the row and every tenant list commit together.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 0efddb94cf fix(datatables): validate a rename against the save it describes, and re-check under the locks
Three from the round, all about deciding on state that could already have moved.

A permission save resolved the data table and checked it was instance-backed before taking any
lock, then wrote under one. A config save committing in between could move the table onto a
PostgreSQL resource — recreating exactly what the transition guard refuses — or rename it, in
which case the write targeted a key that no longer existed and reported success having changed
nothing. It now re-resolves and re-checks on the locked state.

Rename validation checked that the source existed before and the target existed after, which
still accepts `main -> decoy` against a save that keeps both: every fork of `main` then follows
onto a different data table, silently, because it keeps resolving. The rule is now the actual
old-to-new key transition — a source may only survive if another rename took its name, and a
target may only pre-exist if another rename freed it. That also stops two sources sharing one
target, and it admits a swap, which the previous guard refused: `datatables` is keyed by name, so
a swap cannot be done one save at a time, and refusing it was a regression against main. The
pointer cascade now runs in two passes through a temporary name, the way the migration cascade
one layer down already handles the same shape, so `A -> B` with `B -> C` moves each pointer once
from what it named before the save.

The tenant mutators say what they are for: they write an access decision for any workspace named,
with an arbitrary mutation, and exist for the transaction that frees or renames a principal.
Editing a decision on purpose belongs in the permissions endpoint.

Carried in the same change: the stranded-fork list is a field rather than a phrase to grep out of
a success string; the pointer cascade matches with `EXISTS` instead of a `LIKE` over the whole
document, so a workspace whose pointers name something else is not rewritten to a byte-identical
value under an exclusive lock; and `InstanceDatatableRole` drops the serde derives left over from
the JSON document, one of which would emit `pwd`.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 1db77dda87 refactor(datatables): put the role catalog in its own table, not in global_settings
Five findings across three rounds were all the same choice. A set of live Postgres credentials
was living in `global_settings`, which has generic read, list, write, config-export and CLI
round-trip paths that know nothing about what they carry: the passwords reached the instance
config and its YAML editor, a full-row upsert of a neighbouring key erased the catalog,
`GET /settings/global/{key}` and the settings listing returned them raw, and this round the
redaction that fixed the last two turned `wmill instance push` into something that wipes every
password — a fix breaking the assumption the previous fix made. `POST /settings/global/datatable_roles`
could also empty it outside the lock.

The approved plan offered a table or `global_settings`, so this is the other option it already
allowed rather than a new design. `datatable_role` is a table: no generic settings path can read
it, list it, export it, write it or round-trip it, so none of the five needs a guard. The
redaction, the hidden/protected/agent-denylist entries and the JSON document all go with it.

One row per role also removes the read-modify-write the concurrency work was about: two
concurrent creates are two inserts, and the unique index on `name` is what settles a collision.
The advisory lock stays for the one window rows do not cover — `CREATE ROLE` is invisible to
another transaction until commit, so without it both creates pass their `pg_roles` check.

Also from this round: rename mappings are checked against the configuration they claim to
describe, since fork pointers are rewritten from them — a caller could otherwise submit
`main -> missing` against an unchanged config and repoint every fork of `main` at a name nothing
has, and `A -> B` plus `B -> C` moved what pointed at `A` all the way to `C`. And the warning
naming forks a delete stranded reached the response but not the screen: both the data table
settings save and the workspace delete now show it.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 d0ea1ce598 fix(datatables): close the last ways a role or a pointer can be left pointing at nothing
The raw settings readers hand back whatever is in the row, so moving the catalog into its own
`global_settings` key protected the config machinery and left `GET /settings/global/datatable_roles`
and the settings listing returning every live password. Both now filter that one key. The
neighbouring `custom_instance_replication_pwd` has the same shape and is not touched here: it
predates this and widening the fix to it is a decision about an operator workflow, not a
consequence of this change.

Three ways a save could leave something resolving to nothing:

A permissioned data table could be moved to a PostgreSQL resource. The block was carried across
as a server-owned field, the runtime refuses roles on a resource-backed table, so the save
succeeded and every job afterwards failed. Refused instead — turning roles off first is one step,
and it keeps discarding an access decision something somebody chose.

Renaming a governing data table left every fork pointing at the old name: the data table
disappears from their pickers and their jobs stop, with nothing in the renaming workspace to
suggest why. The rename now follows into the pointers in the same transaction.

Deleting one cannot be followed the same way, so it is reported instead — the response names what
it stranded, the way deleting a workspace does, and the fork's own error already says which
workspace is gone.

Also: `ensure_instance_db_grant_options_unchecked` claimed superadmin while the permissions
handler reaches it as a workspace admin (the same class fixed last commit, one instance missed);
the role entry kept an `instance_config_schema` derive it no longer needs; `write_role_catalog`
was the one writer of that table not stamping `updated_at`; and the concurrency test dropped its
roles only on success — a failing run is exactly the one that creates them without recording them.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 c400469e67 fix(datatables): make the concurrency test pin the handlers, and the contracts describe what is enforced
The concurrency test reimplemented the read-modify-write inline, so deleting the lock from all
three handlers left it green — it pinned Postgres, not the code it was written for. It now
drives `create_datatable_role` twice concurrently and asserts the catalog kept both names.
Checked the way the last one should have been: removing the lock from the handler makes it
fail with "wmtest_a_… is a live cluster login the catalog forgot".

The contracts added last commit were stricter than this PR's own callers, which is worse than
none — the next reader sees a rule already broken and learns to ignore it.
`read_role_catalog` said superadmin-only while two of its four callers are open to any
workspace member, and `converge_connect_grants` said superadmin while
`set_datatable_permissions` reaches it as a workspace admin. Both were fine on substance: the
rule that actually holds is about the credential never reaching a response, log, audit record
or export, not about who may call. They now say that. `read_datatable_entry` gets the same
treatment rather than the one the earlier message claimed for it: it is the primitive every
resolution goes through, so it is deliberately open, and what must not escape is `permissions`
— it names the governing workspace's users, groups and folders.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 a74f73ccb1 fix(datatables): give the role catalog its own row, out of reach of the config machinery
Putting it inside `custom_instance_pg_databases` was the wrong call, and it cost two ways.
The catalog serializes a generated Postgres password per role, and that row is the
operator-facing instance config, so the passwords reached `get_instance_config` and its YAML
editor — a live cluster credential in a response body, a UI field and any log of either.
Worse in the other direction: `to_settings_map` strips the catalog, so a full-row upsert of
that key writes the row back without it and the catalog is gone, while the cluster keeps every
login it described.

`custom_instance_replication_pwd` is the precedent and says exactly why — a generated secret,
written only by the server, never operator-authored, hidden so the config machinery cannot
read, rewrite or drop it. The catalog is the same thing, so it now has the same shape:
`datatable_roles`, in `HIDDEN_SETTINGS`, `PROTECTED_SETTINGS` and the agent-worker denylist.
No redaction to keep in step with three code paths, and no way for a neighbouring write to
take it out.

Two races on the same shared documents. `edit_datatable_config` read the stored data tables
outside its transaction and then wrote the whole `datatable` document, so a permissions save
committing in between was silently rolled back; it now reads under `FOR UPDATE`. And
`set_datatable_permissions` validated role ids against the catalog before opening its
transaction, so a deletion in between let it write a deleted role back — including as the
default, which every later job then fails on; it now holds the catalog lock and the settings
row across validation and write.

Completes the authorization contracts the previous commit claimed but did not finish:
`read_datatable_entry` (which it named and missed), `resolve_governing_datatable`, whose whole
job is to answer for a workspace the caller may not belong to, and
`converge_connect_grants_with`, which had not inherited its wrapper's.

Also the generic Python SDK reference: `_format_py_params` learned the bare `*` last time, but
`extract_py_functions` is a second formatter and still rendered `datatable(name, role)`, so
code written from that page passed a keyword-only argument positionally.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 a7bff8de97 fix(datatables): serialize role catalog mutations, and state each helper's authorization contract
The catalog is one JSON document, so create, rename, enable and delete are all
read-modify-write. Two concurrent creates read the same snapshot, both succeed in the
cluster, and the second write drops the first — leaving a live Postgres login with a password
nobody recorded, which is the exact state the delete path exists to prevent. Every mutation
now runs in one transaction holding an advisory lock across the read, the cluster DDL and the
write, so a lost update cannot happen and a failure rolls the whole thing back. The DDL
helpers take that transaction rather than the pool, which is what makes the lock cover them.

Their statements moved off `sqlx::raw_sql`: the simple protocol is only needed for genuinely
multi-statement SQL, and its future is not `Send`, which an axum handler holding the
transaction requires. Each of these is one statement anyway.

The new cross-crate surface now says what callers must do. `read_role_catalog` returns
plaintext credentials; `create`/`rename`/`set_login`/`drop_instance_role` and
`converge_connect_grants` mutate cluster-wide state; `read_datatable_entry` reads a workspace's
raw config. All of them are superadmin-gated by their current handlers, but nothing said so at
the definition, which is where the next caller looks.

Also: the roles table reloads after a failed login toggle instead of leaving it claiming a flip
that did not land; the rename affordance is the design-system `Button`, not a raw one; and
`resolve_datatable_pg_as_caller` drops a `role` parameter no caller ever filled — browsing
resolves as the data table's default until the database manager grows a picker.

Why role passwords stay a plain `String` while the instance user's password beside them is a
`StringOrSecretRef`, asked three times across reviews: that one is a secret ref because an
operator supplies it and may want it from their own backend, while these are minted here and
never entered by anyone, so there is nothing for a ref to point at. Encrypting generated
secrets at rest is a separate change that would take the replication password with it. Now
said at the field.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 0d5106f618 fix(datatables): unbreak two operator messages and two comments that described other code
The two strings this branch added for states an operator hits once — the catalog write that
matched nothing, and the delete that stranded a pointer — were collapsed from their multi-line
form with the indentation left in, so both rendered with a fourteen-space gap mid-sentence.

`list_datatables` claimed to report a chain it cannot follow and then dropped it; it does drop
it, and the comment now says why that is the right place to stay quiet. The non-superadmin
check in `edit_datatable_config` was introduced as also covering references, which it does not
and need not: `reference` is overwritten from the stored entry for every caller before the
check runs.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 b8d6b3c953 fix(datatables): fail loudly where a role or a pointer can be left half-recorded
Three ways the feature could end up in a state nobody could see or undo.

Creating a role writes the cluster first and the catalog second, but the catalog write was an
`UPDATE` that matched nothing when the instance Postgres settings row was absent — leaving a
live login with a password nobody recorded: invisible to the catalog, un-recreatable because
the name is taken, and un-deletable because there is no entry to delete. It now errors, so
the operation is retryable once the row is restored.

Deleting a workspace only nulls the fork lineage; the data table entries pointing at it are
left resolving to nothing. Sweeping them is not an option — turning a pointer back into a copy
would hand each fork the database outright — so the delete now names the data tables it
stranded, and resolving one says which workspace is missing rather than reporting a data table
this workspace never had.

`InstanceDatatableRole` derived `Debug` while holding a Postgres password; it is now
hand-written so `{:?}` on the catalog cannot put a live credential in a log line.

Adds the two branches the reviews found unpinned: a caller who is not a member of the
governing workspace at all, and `NoIdentity` — the compatibility path for an agent worker that
predates this and sends no job id.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 c2c24598a2 fix(datatables): confine roles to the instance database, and stop a fork reaching the parent's bookkeeping
A data table role is a login on Windmill's own Postgres. Nothing stopped a workspace admin
putting a *resource-backed* data table under roles, at which point the executor dialled the
host that resource names — one the admin chose — with the role's real cluster password, and
`CONNECT` is granted to every registered instance database. Both ends now refuse: the
permissions endpoint rejects the save, and the chokepoint refuses to substitute credentials
on a non-instance entry rather than trusting the record it read.

Two more places reached the governing database without answering to it. The initial-migration
generator returned a `pg_dump` of the whole schema to any member. And the migration
rename/delete cascade followed a fork's pointer into the parent, so a fork admin renaming or
removing their own local entry relabelled or wiped the parent's `_wm_migrations` — after
which the parent re-runs every migration from zero. The remote half is now skipped when the
entry resolves into another workspace, which is also just correct: a fork renaming what it
calls a data table changes nothing about the data table.

Also: revoking a tenant now bounces the replication streams of every workspace holding an
entry that resolves here, not only the governing one, so a fork's trigger stops rather than
living on inside its open connection; the instance role catalog and the governing workspace's
tenant lists are no longer returned to someone who cannot edit them; and the tenant rename
dedup collapses non-adjacent duplicates, per role rather than once any role changed.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 21d97f3a4c fix(datatables): gate the paths that reach a whole database as admin
Auditing what still resolved through the unchecked resolver turned up three that act for a
caller and hand back the admin connection: `resolve_pg_source_checked` (behind schema
export, the full-schema read, database creation, import and the forked-database drop), the
connection test, and the schema snapshot a fork clone takes of its parent. On a data table
under roles each let any workspace member — or a fork admin who is nobody in the governing
workspace — read or copy the whole database whatever its roles grant.

All three now require admin reach on the governing workspace. A dump taken under a
restricted role would be a silently truncated copy rather than an error, so refusing is the
only right answer for the copy paths.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-17 10:01:15 +02:00
Diego ImbertandClaude Opus 5 34ed0c4230 feat(datatables): put a data table's connection under Postgres roles
A data table backed by the instance database resolved to exactly one Postgres connection,
`custom_instance_user`, for everyone who could reach it at all. There was no way to say
this job reads, that one writes, this one never sees the salaries table.

A data table role is now a real Postgres login on the cluster, defined once for the
instance by a superadmin and named exactly as they named it. A script that declares
`-- role analytics` connects as `analytics`, and Postgres decides what it may touch —
grants are ordinary SQL. Windmill answers only "may this caller ask for this role", from
the tenant lists on the data table entry: `u/alice`, `g/analysts`, `f/finance` or `*`.
A data table with no `permissions` block behaves exactly as before.

Everything that opens a connection on someone's behalf goes through one chokepoint,
`get_datatable_resource_from_db`, which takes the identity explicitly and fails closed when
there is none. The role logs in as itself — never `SET ROLE`, which a script could
`RESET ROLE` its way out of.

A fork's data table entry becomes a pointer at the workspace that governs it rather than a
copy of it. The settings clone used to hand a fork a byte-identical entry naming the
parent's database, which a fork admin could edit to grant themselves `admin` there; a
pointer has nothing local to edit, and its tenants are evaluated as a member of the
governing workspace, by email. `permissions` is stripped from the workspace export and
ignored on import: tenants name principals of one workspace, and a settings push is not
where an access decision should be made.

Operations that see the whole database whatever the roles grant stay with the governing
workspace's admins: editing the roles, a migration that declares none, and opening a
replication stream for a Postgres trigger or capture.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-17 10:01:15 +02:00
AlexRV12andClaude Opus 5 5bb37ca338 feat(ai-chat): merge get_job_logs and get_flow_run_details into get_run (#11172)
* feat(ai-chat): merge get_job_logs and get_flow_run_details into get_run

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01N1arUSfbbVQSq2DsjamxKF

* fix(ai-chat): tell an unreadable log apart from an empty one in get_run

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01N1arUSfbbVQSq2DsjamxKF

* fix(ai-chat): keep get_run's step tree optional when the read resolves undefined

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01N1arUSfbbVQSq2DsjamxKF

* fix(ai-chat): match each WINDMILL_TOO_BIG marker to the field it elides

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01N1arUSfbbVQSq2DsjamxKF

---------

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 10:00:41 +02:00
GuilhemandClaude Fable 5.1 189793c2e4 feat: flow chat model picker on a shared model-settings component (#11187)
* refactor: render the session chat model menu from a shared ChatModelSettings config

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

* feat: pick the flow chat's model and thinking from the provider fields the flow exposes

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

* fix: name only the thinking level the flow run will send on the model button

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix: let the flow chat take a typed model id and keep a shared thinking input editable

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix: promote a flow input to the model button only where its control can edit it

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix: drop any reasoning token the chosen model rejects before a flow chat run

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-17 09:59:05 +02:00
Diego Imbert 13162656bf Merge remote-tracking branch 'origin/datatable-roles-redesign-part-2' into datatable-roles-redesign-part-4
# Conflicts:
#	backend/ee-repo-ref.txt
#	backend/windmill-api-workspaces/src/datatable_permissions_oss.rs
2026-09-17 09:50:52 +02:00
GuilhemandClaude Fable 5.1 68f2248018 feat: keep flow inputs and seed the agent when chat mode is enabled (#11177)
* feat: keep flow inputs and seed the agent when chat mode is enabled

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

* fix: leave a linked agent's memory and streaming to the agent when enabling chat mode

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-17 09:29:05 +02:00
Guilhem 23c24a9688 feat: collect flow conversations and agent memory once their last message goes (#11178)
* feat: collect flow conversations and agent memory when their last message goes

* fix: lock the conversation lookup so a new turn orders against its cleanup

* fix: let concurrent turns recreate a collected conversation without conflicting
2026-09-17 09:27:12 +02:00
Diego ImbertandClaude Opus 5 e79bc14903 fix: check ACL read reach against the entry it connects from
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb
2026-09-17 09:00:04 +02:00
Diego ImbertandClaude Opus 5 b7623f12eb fix: build the ACL connection from the authorized data table entry
Resolving the settings again could land on a resource with the same
database name on another server, which the later entry checks never see.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb
2026-09-17 00:09:29 +02:00
Diego ImbertandClaude Opus 5 955aadbf10 fix(datatables): only drop external databases Windmill marked, and check use under the lock
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 00:08:15 +02:00
Diego Imbert a41644c795 Merge remote-tracking branch 'origin/datatable-external-instance' into datatable-external-instance-2
# Conflicts:
#	backend/ee-repo-ref.txt
2026-09-17 00:08:14 +02:00
Diego ImbertandClaude Opus 5 631c01d20d fix(datatables): compare the external cluster settings under a row lock before storing setup
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 00:07:50 +02:00
Diego ImbertandClaude Opus 5 0a2ebfb3db fix: hold the ACL connection to the database that was authorized
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb
2026-09-16 23:55:43 +02:00
Diego Imbert 0489cb1630 Merge remote-tracking branch 'origin/datatable-roles-redesign-part-2' into datatable-roles-redesign-part-4
# Conflicts:
#	backend/ee-repo-ref.txt
#	backend/windmill-api-integration-tests/tests/datatable_roles.rs
2026-09-16 23:45:48 +02:00
Diego ImbertandClaude Opus 5 455e1b8d23 [ee] feat(datatables): external_instance data tables on the external cluster
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 23:43:10 +02:00
Diego ImbertandClaude Opus 5 d7417f5cc2 Merge branch 'datatable-roles-redesign' into datatable-roles-redesign-part-2
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb
2026-09-16 23:40:04 +02:00
Diego ImbertandClaude Opus 5 5ef5f58eeb fix(datatables): scope external cluster credential readers to the crate
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 23:38:06 +02:00
Diego ImbertandClaude Opus 5 4533ec460e fix(datatables): send external cluster passwords as SCRAM verifiers
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 22:44:11 +02:00
Diego ImbertandClaude Opus 5 b3cda2f548 feat(datatables): set up an external instance cluster for data tables
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 22:35:00 +02:00
Diego ImbertandClaude Opus 5 8d0d51e99d fix: run one data table ACL apply at a time per server before it connects
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb
2026-09-16 22:25:51 +02:00
Diego ImbertandClaude Opus 5 110876d0c9 docs(datatables): note that legacy names with ? cannot be migrated
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 22:25:27 +02:00
Diego ImbertandClaude Opus 5 e5ec5217d4 fix(datatables): serialize roles going on with aliases saved from other workspaces
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 18:09:07 +02:00
GuilhemandClaude Fable 5.1 a9ec0aec3a feat: render the flow chat through the shared session chat components (#11175)
* feat: render the flow chat through the shared session chat components

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

* fix: settle the flow chat queue on the turn's outcome and guard IME enter

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

* fix: keep copilot free-tier banners and deploy-locked sends out of the flow chat

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-16 17:45:01 +02:00
Diego ImbertandClaude Opus 5 6a89ba0179 fix: add only missing grant options before an ACL apply, never default privileges
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb
2026-09-16 17:31:06 +02:00
hugocasaandClaude Opus 5 9d348f84c7 fix: skip expiry notifications for app embed and SDK tokens (#11169)
* fix: skip expiry notifications for app embed and SDK tokens

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

* refactor: share app token label prefixes between mint sites and the check

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

* fix: skip expiry alerts for impersonation and test-connection tokens

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
2026-09-16 17:28:53 +02:00
hugocasaandClaude Opus 5 02e47de8b4 feat: allow git sync auto-pull, promotion and PRs on Pro licenses (#11173)
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
2026-09-16 17:28:40 +02:00
Diego ImbertandClaude Opus 5 707b0dc8ee fix(datatables): let CE migrations connect as an explicitly named admin
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 17:25:42 +02:00
Diego ImbertandClaude Opus 5 dcc37bf6f2 fix: declare the default role in migrations written for a data table whose name contains '?'
Such a data table connects as its default role without naming it, so the
migrations the manager wrote for it declared no role and ran as admin.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 17:20:19 +02:00
Diego ImbertandClaude Opus 5 10861bb529 fix(datatables): refuse a reference naming both a legacy data table and a role
When a workspace stores both `sales` and a legacy `sales?role=analytics`, the
reference resolved to the legacy entry without a role, so browsing `sales` as
`analytics` reached another data table.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 17:03:32 +02:00
Diego ImbertandClaude Opus 5 d459b371d8 fix: refresh grant options only after the ACL apply validates its plan
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb
2026-09-16 16:53:13 +02:00
Diego ImbertandClaude Opus 5 41e1101a2d fix: take every pooled connection before the ACL apply locks
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb
2026-09-16 16:39:14 +02:00
Diego ImbertandClaude Opus 5 4416283b17 fix: read the roles of a data table whose name contains '?'
The generated client leaves a '?' in a path param unencoded, so the lookup
404'd and the raw-app picker blocked Start on such a data table.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 16:32:36 +02:00
Guilhem 64dffe6106 feat: show flow step detail inside the graph tab on narrow detail layouts (#11168)
* feat: show flow step detail as a page inside the graph tab on narrow detail layouts

* fix: pad the detail step tab, wrapped header row and raise the tabbed layout breakpoint

* fix: keep the step header pinned and switch to triggers on every trigger node tap
2026-09-16 16:12:21 +02:00
Diego ImbertandClaude Opus 5 e0360bce5c fix: never add a role to the reference of a data table whose name contains '?'
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 16:07:00 +02:00
Diego ImbertandClaude Opus 5 f36aa69fc3 feat(datatables): data table roles in the DB manager and raw apps
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 15:23:20 +02:00
Diego ImbertandClaude Opus 5 a1b91690fd feat(datatables): add an ACL editor for data table roles
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 15:22:05 +02:00
Diego ImbertandClaude Opus 5 0676f55856 fix(datatables): refuse roles while a same-workspace alias reaches the database
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 15:14:29 +02:00
windmill-internal-app[bot] aa6a52271b chore: update ee-repo-ref to a1873dbb67f2302b85ff5362f8387b48eccdb607
This commit updates the EE repository reference after PR #783 was merged in windmill-ee-private.

Previous ee-repo-ref: 5c853e2c20eca6b748415fc0d6862a6ebfb5fec4

New ee-repo-ref: a1873dbb67f2302b85ff5362f8387b48eccdb607

Automated by sync-ee-ref workflow.
2026-09-16 15:14:29 +02:00
Diego ImbertandClaude Opus 5 64d5f762dc test(datatables): run the tenant matcher test on the enterprise edition
The matcher it covers is enterprise code now, so without the enterprise
edition the test hit the stub and failed the default windmill-common run. It
runs with private and enterprise, and a counterpart without them asserts that
no tenant list covers anyone, the wildcard and a workspace admin included.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-16 15:14:29 +02:00
Diego ImbertandClaude Opus 5 4c2f1a7b60 feat(datatables): gate the roles UI mount sites on an enterprise license
Both mount sites are still commented out; the gate travels with them.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-16 15:14:29 +02:00
Diego ImbertandClaude Opus 5 16fdba4d82 test(datatables): run the roles tests on the enterprise edition, refusals without it
Each test that exercises roles runs with private and enterprise. Two tests run
without them: every roles route answers the Enterprise refusal, and a data
table saved under roles, or a named role, is refused a connection while one
not under roles resolves as before.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-16 15:14:29 +02:00
Diego ImbertandClaude Opus 5 1986ec2535 feat(datatables): move the data table role catalog endpoints to the enterprise edition
The superadmin list, create, update and delete handlers move to
windmill-ee-private. The routes stay registered and, without the enterprise
edition, refuse after authentication.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-16 15:14:29 +02:00
Diego ImbertandClaude Opus 5 a7bac22c13 feat(datatables): move the data table permissions endpoints to the enterprise edition
The permissions read, save and usable-roles handlers move to
windmill-ee-private; the routes stay registered and, without the enterprise
edition, answer that data table roles are an Enterprise Edition feature.
ensure_governs_datatable and ensure_reaches_datatable keep their paths: the
first refuses, the second passes a data table not under roles.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-16 15:14:29 +02:00
Diego ImbertandClaude Opus 5 1d6da0084e feat(datatables): move data table role catalog and resolution to the enterprise edition
Roles are an Enterprise Edition feature. The catalog, the Postgres logins,
CONNECT convergence, tenant evaluation and the role half of connection
resolution move to windmill-ee-private. Every public function keeps its path
and signature and forwards through datatable_roles_oss, which re-exports the
enterprise implementation or, without it, refuses.

Without the enterprise edition a data table under roles, or a caller naming a
role, is refused a connection rather than resolved as admin, and the reach and
admin-access checks refuse one under roles. A data table not under roles
resolves as before in every edition, and an instance database keeps the
CONNECT grants it was created with. The catalog lock, the stream lock, the
tenant cascades and the permissions stripping stay in OSS: they only restrict.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-16 15:14:29 +02:00
Diego ImbertandClaude Opus 5 3d4a2133ed fix(datatables): no entry without roles may newly reach a database under roles
The previous guard only caught a new name replacing an entry under roles.
A whole-map save could also repoint an existing entry without roles at
that database, or another workspace could point one there, and every
caller of that entry would connect as admin. The rule is now stated on
the saved entries: one that carries no roles and newly points at an
instance database any entry under roles uses, in this workspace or
another, is refused. A declared rename carries its roles and passes.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-16 15:14:29 +02:00
Diego ImbertandClaude Opus 5 f8a6347052 fix(datatables): refuse a save that drops a data table's roles through an undeclared rename
A data table's roles follow its entry only through a declared rename. A
settings sync sends the whole map and never declares one, so renaming a
data table under roles there read as a delete and a new entry on the same
database: the new entry carried no roles, and every caller connected as
admin. Such a save is now refused, naming both entries.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-16 15:14:29 +02:00
Diego ImbertandClaude Opus 5 46dfcc5c49 fix(datatables): warn when a settings sync strands fork pointers
A settings save reported the fork pointers left resolving to nothing only
for the names in `deleted_datatables`, which `wmill sync push` never sends.
The save now works out what it removed from the locked entries, and the
CLI prints the stranded pointers it returns.

Also correct the replication helper's contract: no role or admin check
makes a replication connection safe, so a data table under roles is
refused outright rather than gated as an admin operation.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-16 15:14:29 +02:00
Diego ImbertandClaude Opus 5 e9c27d63db fix(datatables): wait out live listeners, and resolve stored names containing ?
Turning roles on counted a trigger as gone once disabled, and a capture
once its client stopped pinging, but the listener keeps its replication
connection until its next heartbeat notices. A trigger or capture whose
listener pinged in the last 15 seconds, the window a server holds a
listener for, now still counts as streaming.

Data table names could contain `?` before they were restricted, and such
entries are still stored. Splitting `?role=` off a reference misread them:
`a?b` became `a` with an unknown parameter, and the clone checks looked at
a different entry than the one copied. An entry stored under the whole
reference is now looked up first, in the Postgres executor, DuckDB ATTACH
and the clone checks. Agent workers cannot read the workspace and keep
the strict parse, which refuses such a name rather than misreading it.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 15:14:29 +02:00
Diego ImbertandClaude Opus 5 65a6cc3eb5 fix(datatables): serialize roles going on with a stream starting
Turning roles on looked for enabled triggers and live captures once,
without a lock anything starting a stream also took. A trigger enabled in
that window could have its listener connect before roles committed, and a
healthy listener never checks again. Both transitions now serialize on one
advisory lock: roles going on hold it exclusive while they look, and
trigger create, edit and enable, and capture setup and ping hold it shared
while they commit. Either the look sees the stream, or the listener
connects after roles are committed and refuses.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 15:14:29 +02:00
Diego ImbertandClaude Opus 5 dd2ca5513e fix(datatables): disable a data table role before deleting it
Deleting a role reassigns and drops what it owns in each registered
database on its own connection, and each of those passes commits as it
goes. A database failing part-way left the role enabled in the catalog and
able to log in, but already stripped in the databases reached before it.
The role is now disabled in its own commit first, so a failed delete
leaves a disabled role to retry.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 15:14:29 +02:00
Diego ImbertandClaude Opus 5 6a3a2ed13c fix(datatables): refuse a Postgres trigger on a data table under roles when it is saved
Creating or editing a trigger that points at a data table under roles was
accepted, and its listener then retried the refused connection every 30
seconds forever. The save is now refused, and a trigger that reaches such a
data table anyway (re-enabled, or cloned into a fork) is disabled by its
listener with the reason, as a missing replication slot is.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 15:14:29 +02:00
Diego ImbertandClaude Opus 5 791bb8d843 refactor(datatables): keep Postgres triggers and data table roles apart
A replication stream reads every row of every table whatever the data table's
roles grant, and its listener checks access only when it connects. Rather than
chase every way access can change and bounce the streams each one affects, a
data table now carries one or the other:

- a Postgres trigger or capture cannot be created on, or connect to, a data
  table under roles;
- roles cannot be turned on while an enabled trigger or a live capture reads
  the data table, its own or a fork's through its pointer. The refusal names
  each one to disable.

This removes the stream bounces on roles edits and on data table and workspace
deletion, and the trigger gate that admitted admins. The fork schema baseline
fix from the same review round is kept.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 15:14:29 +02:00
Diego ImbertandClaude Opus 5 b68b97f5ea fix(datatables): keep the fork schema baseline, and bounce streams on every removal
Three fixes from review.

`edit_datatable_config` took `forked_from` wholesale from the stored entry, so
the fork schema diff's save of an advanced baseline was silently discarded and
an applied change was offered again. Whether an entry carries a clone stamp is
still carried from the store, since that is what marks its database droppable,
but the baseline inside it is now taken from the request.

The stranded-pointer warning and the stream bounce ran over the optional
`deleted_datatables` hint, which the settings-sync CLI never sends, so removing
a governing data table through `wmill` bounced nothing. Removals are now derived
from the stored configuration against the saved one.

`delete_workspace` read the pointers to bounce before its transaction, so a fork
committing a pointer during the deletion was missed. The read now happens inside
the transaction, after the workspace row is deleted: a fork's insert key-share
locks that row through its parent foreign key, so it is either seen or fails on
the missing parent.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 15:14:29 +02:00
Diego ImbertandClaude Opus 5 92c723f735 fix(datatables): bounce the streams reading a data table when it is deleted
Deleting a governing data table, or the workspace that holds it, only collected
the fork pointers it stranded, for the warning. A Postgres trigger or capture
already streaming through one of those pointers kept the replication connection
it opened while the pointer still resolved, so it went on dispatching the
governing database's rows after the fork lost access — until its connection
happened to restart. The governing workspace's own streams on a deleted entry
did the same.

Both deletion paths now bounce the affected listeners inside their own
transaction, through the helper a permission change already uses, so a
listener that reconnects re-resolves the entry and finds it gone. The helper is
split so a caller can pass the (workspace, local name) pairs it already holds.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 15:14:29 +02:00
Diego ImbertandClaude Opus 5 c7f28ba8ed docs(datatables): record the stale clone database as a known limitation
A clone is three requests and `CREATE DATABASE` is not transactional, so a
failure after the first leaves a registered `wm_fork_*` behind, as it did
before data table roles. Accepted for this PR: it is harmless to data and goes
away once the clone is a single server-side operation.

The comment also records why the obvious fix is wrong: reclaiming the leftover
on retry, without durable clone ownership, can drop another workspace's fully
copied database between its import and its final fork request.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 15:14:29 +02:00
Diego ImbertandClaude Opus 5 bea9667516 Revert "fix(datatables): let a retried clone reclaim its own leftover database"
This reverts commit 7dd3275a10.

The reclaim tied the caller to the source they administer, but not to the
database it dropped. Between another workspace's import and its final fork
request, that workspace's target is full, registered, unnamed and has no open
connection, so an admin of any instance data table could name it and have it
dropped and recreated empty. The victim's fork would then commit pointing at
the empty copy. Safe reclaim needs durable clone ownership and serialization
with the request that names the database; until then the leftover stays, as it
did before this PR.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 15:14:29 +02:00
Diego ImbertandClaude Opus 5 bb8c8ed735 fix(datatables): let a retried clone reclaim its own leftover database
A clone creates its target database one request before it copies into it, and
the fork that would name it is written a request after that. Any failure in
between — a pg_dump error, a bad restore, a dropped connection, the source's
roles changing mid-flow — left a registered `wm_fork_*` that no entry names,
and every retry then failed on its name. This predates data table roles.

`create_pg_database` now reclaims such a leftover before creating: only a
`wm_fork_*` database Windmill registered as a data table database and that no
data table or ducklake entry names, in any workspace, archived ones included.
The drop never terminates connections, so a clone still copying into it makes
the reclaim fail instead of being cut off. It is limited to callers who
administer the source — reaching it is not enough, since on a data table
without roles every member reaches it — and anyone else gets the refusal an
existing database always got.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 15:14:29 +02:00
Diego ImbertandClaude Opus 5 4610f8a35c refactor(datatables): settle clonability in one place, before anything is created
A clone is three stages a workspace apart — `create_pg_database`, then
`import_pg_database`, then `apply_forked_datatable` inside the fork transaction.
Only the third can roll back, and `CREATE DATABASE` is not transactional, so any
refusal that lives there strands a registered `wm_fork_*` that no entry names
and whose name blocks the retry.

That orphan has now been fixed three times, most recently reintroduced by a
guard added one commit ago. Patching each new refusal into the first endpoint is
not the fix; having two places that can refuse is. `ensure_datatable_is_clonable`
now answers every reason a copy can be refused and returns what it resolved, and
the stage that writes the entry only does the work.

Also takes an ACCESS EXCLUSIVE lock before the rollback guard counts, so a role
created concurrently cannot slip between the check and the drop.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-16 15:14:29 +02:00
Diego ImbertandClaude Opus 5 284d1ecb43 fix(datatables): refuse to roll back the catalog while roles exist
The down migration dropped the table and left every role behind: live Postgres
logins whose passwords only that table carried, so after a revert Windmill could
neither use, disable nor delete them, and re-applying could not recreate them
because the names were taken. Cleaning up here is not possible either — dropping
a role means reassigning what it owns in every instance database, and a
migration runs in one — so it now refuses while the catalog is non-empty and
says to delete the roles through instance settings, which does the cluster work.

Also enforces the instance-only invariant the resolved-pointer clone relies on
rather than only asserting it in a comment.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-16 15:14:29 +02:00
Diego ImbertandClaude Opus 5 306a35e3b0 fix(datatables): clone a fork's pointer instead of failing after the copy
Forking a fork with cloning left an orphan database. The preflight resolves the
pointer and sees the governing entry, so both endpoints ran and filled the new
database; `apply_forked_datatable` then refused the inherited pointer and rolled
the fork back, stranding a registered `wm_fork_*` that no entry names and whose
name blocks the retry.

Refusing earlier would have been the smaller change, but forking a fork and
cloning worked before pointers existed, so it would trade an orphan for a
regression. Resolve what the pointer names and write the terminal entry the
clone needs: the whole `database` object rather than a patch of its
`resource_path`, since a pointer has none, and `reference` removed with it.

Also accepts `-- role=x` and `-- Role = x`, two more spellings that fell through
to the default role.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-16 15:14:28 +02:00
Diego ImbertandClaude Opus 5 5a98030fb7 fix(datatables): honour -- role: x, and fix the DuckDB attach test
Two review findings, both real.

`attach_datatable_parses_name_and_role` never compiled: `parse_attach_datatable`
returns `Result<Option<_>>` now and one call site kept a single `unwrap`. Its
`?Role=analytics` case also asserted a refusal, contradicting the parser in the
same commit, which matches the key case-insensitively. Replaced with the cases
that are genuinely malformed, and a positive one for the cased key.

`-- role: analytics` fell through to the default role — the silent fallback the
strict parser exists to remove, for the spelling most likely to be typed. The
keyword now accepts an optional colon, attached or spaced, while a word that
merely starts with it (`rolebased`) is still not an attempt.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-16 15:14:28 +02:00
Diego ImbertandClaude Opus 5 d558b6508a chore(datatables): don't mount the roles UI until the ACL editor lands
Enforcement ships first. The permissions drawer is what turns roles on, and the
catalog section is what creates them — both are only useful once there is a way
to grant a role the privileges it needs, which arrives with the ACL editor. Left
mounted they would offer a feature whose other half does not exist.

The two components are complete and reviewed; only their call sites here are
commented out, with a note pointing the follow-up PRs at them.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-16 15:14:28 +02:00
Diego ImbertandClaude Opus 5 fce20faeae fix(datatables): carry the role annotation into the row_to_json retry
The retry rebuilds its SQL from `pruneComments(code)`, so the leading comment
block never reached the second attempt — and with it the `-- role <name>` line
that decides which login the query runs as. The retry connected as the data
table's default role instead, so a query the first attempt was denied could
succeed on the second, reported as "recovered with the row_to_json fix".

Carry the leading comment block over. The retry itself is unchanged.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb
2026-09-16 15:14:28 +02:00
Diego ImbertandClaude Opus 5 da3babccc7 fix(datatables): refuse a malformed role query string instead of ignoring it
`?Role=analytics`, `?role=` and `?x=1&role=…` all fell through the reference
parser's exact-match rule, so the connection resolved to the data table's default
role and ran under a login the caller never asked for — the URI half of the same
trap as a malformed `-- role` annotation.

The key now matches case-insensitively, and anything else in the query string is
an error naming it; `role` is the only parameter a reference takes. Callers that
only need the entry keep a lenient `datatable_ref_name`, since they never act on
the role. The DuckDB `ATTACH` parser propagates it rather than attaching under
the default.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-16 15:14:28 +02:00
Diego ImbertandClaude Opus 5 69742c0b48 fix(datatables): refuse a malformed role annotation instead of ignoring it
`-- Role operator`, `-- role operator;` and `-- role operator -- why` all failed
the annotation parser's exact-match rule, so the query fell through to the data
table's default role and ran, silently, under a login the author did not choose.
Naming a role exists precisely to not do that.

A leading comment whose first word is `role` is now an annotation attempt: the
keyword matches case-insensitively, one trailing `;` is tolerated, and anything
else is an error naming the line. Only callers that already know the target is a
`datatable://` reference ever run this, so ordinary SQL keeps its comments.

Also bumps the dev shell's postgres client to 18 — it trailed the server the dev
database runs, which takes out every data table export, clone and fork-with-data.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-16 15:14:28 +02:00
Diego Imbert 3dfbf64d9d fix pg_dump stuck on version 17 on nix 2026-09-16 15:14:28 +02:00
Diego Imbert d32ff92eac nit worker error msg 2026-09-16 15:14:28 +02:00
Diego ImbertandClaude Opus 5 01eba26582 fix(datatables): refuse the clone's database too, not only its data
A clone is two endpoints: `create_pg_database` then `import_pg_database`. Only
the second refused a data table under roles, so a fork asking to clone one
created and registered an empty `wm_fork_…` instance database and then failed —
and nothing collects it, since `drop_forked_datatable_databases` only drops
entries carrying `forked_from` and no entry names this one.

Refuse in both, so the clone stops before a database exists.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-16 15:14:28 +02:00
Diego ImbertandClaude Opus 5 cafcf3afc3 fix(datatables): refuse to copy a data table that is under roles
pg_dump carries no roles and the import runs with --no-privileges, so a copied
data table arrives owned by the admin connection with no GRANT for any role.
The settings clone brings `permissions` across, so the fork's tenants pass
Windmill's check, connect as the role they were given, and are denied by
Postgres on everything: an entry that reads as configured and answers nothing.

Refuse the copy — in the import endpoint before any data moves, and in the fork
path the CLI takes. Replaying the source's owners and ACLs into the clone is
what lifts this, and is a change of its own. Dropping `permissions` from the
copy instead would be the unsafe half, since the copy holds the parent's rows.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-16 15:14:28 +02:00
Diego ImbertandClaude Opus 5 030254af47 fix(datatables): cascade on the leave route that is used, gate migrations before the admin connection, and drop a role atomically
The tenant cascade on leaving went onto `/users/leave`. The UI and the generated client call
`/workspaces/leave` — a different handler in a different crate with the same name — which
deleted the membership and left `u/<username>` in the tenant lists. Leaving and rejoining
therefore restored the access the leave was supposed to end, and a later account taking the
username would have inherited it. The regression test drives the route the client actually
calls; without the fix it fails with "leaving kept the tenant".

The migration endpoints authorized too late. `run_datatable_migrations` opened the data table's
admin connection, created `_wm_migrations` and read it before reaching the per-migration role
check — so with nothing pending, nothing was checked at all. Rollback returned before its check
when nothing was applied, and the status endpoint had none. All three now ask, before any
connection is opened, whether the caller can reach the data table as any role at all; which role
a given migration runs as is still decided per migration, and by the executor after that.

Deleting a role committed the cluster drop and the catalog row, then swept the tenant lists in
separate transactions. A sweep failing part-way left workspaces naming a role nothing can connect
as, while the retry answered `NotFound` because the catalog entry was already gone. The sweep now
runs in the same transaction, so the drop, the row and every tenant list commit together.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-16 15:14:28 +02:00
Diego ImbertandClaude Opus 5 9872bff69f fix(datatables): validate a rename against the save it describes, and re-check under the locks
Three from the round, all about deciding on state that could already have moved.

A permission save resolved the data table and checked it was instance-backed before taking any
lock, then wrote under one. A config save committing in between could move the table onto a
PostgreSQL resource — recreating exactly what the transition guard refuses — or rename it, in
which case the write targeted a key that no longer existed and reported success having changed
nothing. It now re-resolves and re-checks on the locked state.

Rename validation checked that the source existed before and the target existed after, which
still accepts `main -> decoy` against a save that keeps both: every fork of `main` then follows
onto a different data table, silently, because it keeps resolving. The rule is now the actual
old-to-new key transition — a source may only survive if another rename took its name, and a
target may only pre-exist if another rename freed it. That also stops two sources sharing one
target, and it admits a swap, which the previous guard refused: `datatables` is keyed by name, so
a swap cannot be done one save at a time, and refusing it was a regression against main. The
pointer cascade now runs in two passes through a temporary name, the way the migration cascade
one layer down already handles the same shape, so `A -> B` with `B -> C` moves each pointer once
from what it named before the save.

The tenant mutators say what they are for: they write an access decision for any workspace named,
with an arbitrary mutation, and exist for the transaction that frees or renames a principal.
Editing a decision on purpose belongs in the permissions endpoint.

Carried in the same change: the stranded-fork list is a field rather than a phrase to grep out of
a success string; the pointer cascade matches with `EXISTS` instead of a `LIKE` over the whole
document, so a workspace whose pointers name something else is not rewritten to a byte-identical
value under an exclusive lock; and `InstanceDatatableRole` drops the serde derives left over from
the JSON document, one of which would emit `pwd`.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-16 15:14:28 +02:00
Diego ImbertandClaude Opus 5 e400daabb8 refactor(datatables): put the role catalog in its own table, not in global_settings
Five findings across three rounds were all the same choice. A set of live Postgres credentials
was living in `global_settings`, which has generic read, list, write, config-export and CLI
round-trip paths that know nothing about what they carry: the passwords reached the instance
config and its YAML editor, a full-row upsert of a neighbouring key erased the catalog,
`GET /settings/global/{key}` and the settings listing returned them raw, and this round the
redaction that fixed the last two turned `wmill instance push` into something that wipes every
password — a fix breaking the assumption the previous fix made. `POST /settings/global/datatable_roles`
could also empty it outside the lock.

The approved plan offered a table or `global_settings`, so this is the other option it already
allowed rather than a new design. `datatable_role` is a table: no generic settings path can read
it, list it, export it, write it or round-trip it, so none of the five needs a guard. The
redaction, the hidden/protected/agent-denylist entries and the JSON document all go with it.

One row per role also removes the read-modify-write the concurrency work was about: two
concurrent creates are two inserts, and the unique index on `name` is what settles a collision.
The advisory lock stays for the one window rows do not cover — `CREATE ROLE` is invisible to
another transaction until commit, so without it both creates pass their `pg_roles` check.

Also from this round: rename mappings are checked against the configuration they claim to
describe, since fork pointers are rewritten from them — a caller could otherwise submit
`main -> missing` against an unchanged config and repoint every fork of `main` at a name nothing
has, and `A -> B` plus `B -> C` moved what pointed at `A` all the way to `C`. And the warning
naming forks a delete stranded reached the response but not the screen: both the data table
settings save and the workspace delete now show it.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-16 15:14:28 +02:00
Diego ImbertandClaude Opus 5 7f3c7a19af fix(datatables): close the last ways a role or a pointer can be left pointing at nothing
The raw settings readers hand back whatever is in the row, so moving the catalog into its own
`global_settings` key protected the config machinery and left `GET /settings/global/datatable_roles`
and the settings listing returning every live password. Both now filter that one key. The
neighbouring `custom_instance_replication_pwd` has the same shape and is not touched here: it
predates this and widening the fix to it is a decision about an operator workflow, not a
consequence of this change.

Three ways a save could leave something resolving to nothing:

A permissioned data table could be moved to a PostgreSQL resource. The block was carried across
as a server-owned field, the runtime refuses roles on a resource-backed table, so the save
succeeded and every job afterwards failed. Refused instead — turning roles off first is one step,
and it keeps discarding an access decision something somebody chose.

Renaming a governing data table left every fork pointing at the old name: the data table
disappears from their pickers and their jobs stop, with nothing in the renaming workspace to
suggest why. The rename now follows into the pointers in the same transaction.

Deleting one cannot be followed the same way, so it is reported instead — the response names what
it stranded, the way deleting a workspace does, and the fork's own error already says which
workspace is gone.

Also: `ensure_instance_db_grant_options_unchecked` claimed superadmin while the permissions
handler reaches it as a workspace admin (the same class fixed last commit, one instance missed);
the role entry kept an `instance_config_schema` derive it no longer needs; `write_role_catalog`
was the one writer of that table not stamping `updated_at`; and the concurrency test dropped its
roles only on success — a failing run is exactly the one that creates them without recording them.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-16 15:14:28 +02:00
Diego ImbertandClaude Opus 5 1d9ee09b31 fix(datatables): make the concurrency test pin the handlers, and the contracts describe what is enforced
The concurrency test reimplemented the read-modify-write inline, so deleting the lock from all
three handlers left it green — it pinned Postgres, not the code it was written for. It now
drives `create_datatable_role` twice concurrently and asserts the catalog kept both names.
Checked the way the last one should have been: removing the lock from the handler makes it
fail with "wmtest_a_… is a live cluster login the catalog forgot".

The contracts added last commit were stricter than this PR's own callers, which is worse than
none — the next reader sees a rule already broken and learns to ignore it.
`read_role_catalog` said superadmin-only while two of its four callers are open to any
workspace member, and `converge_connect_grants` said superadmin while
`set_datatable_permissions` reaches it as a workspace admin. Both were fine on substance: the
rule that actually holds is about the credential never reaching a response, log, audit record
or export, not about who may call. They now say that. `read_datatable_entry` gets the same
treatment rather than the one the earlier message claimed for it: it is the primitive every
resolution goes through, so it is deliberately open, and what must not escape is `permissions`
— it names the governing workspace's users, groups and folders.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-16 15:14:28 +02:00
Diego ImbertandClaude Opus 5 715d8a0e6d fix(datatables): give the role catalog its own row, out of reach of the config machinery
Putting it inside `custom_instance_pg_databases` was the wrong call, and it cost two ways.
The catalog serializes a generated Postgres password per role, and that row is the
operator-facing instance config, so the passwords reached `get_instance_config` and its YAML
editor — a live cluster credential in a response body, a UI field and any log of either.
Worse in the other direction: `to_settings_map` strips the catalog, so a full-row upsert of
that key writes the row back without it and the catalog is gone, while the cluster keeps every
login it described.

`custom_instance_replication_pwd` is the precedent and says exactly why — a generated secret,
written only by the server, never operator-authored, hidden so the config machinery cannot
read, rewrite or drop it. The catalog is the same thing, so it now has the same shape:
`datatable_roles`, in `HIDDEN_SETTINGS`, `PROTECTED_SETTINGS` and the agent-worker denylist.
No redaction to keep in step with three code paths, and no way for a neighbouring write to
take it out.

Two races on the same shared documents. `edit_datatable_config` read the stored data tables
outside its transaction and then wrote the whole `datatable` document, so a permissions save
committing in between was silently rolled back; it now reads under `FOR UPDATE`. And
`set_datatable_permissions` validated role ids against the catalog before opening its
transaction, so a deletion in between let it write a deleted role back — including as the
default, which every later job then fails on; it now holds the catalog lock and the settings
row across validation and write.

Completes the authorization contracts the previous commit claimed but did not finish:
`read_datatable_entry` (which it named and missed), `resolve_governing_datatable`, whose whole
job is to answer for a workspace the caller may not belong to, and
`converge_connect_grants_with`, which had not inherited its wrapper's.

Also the generic Python SDK reference: `_format_py_params` learned the bare `*` last time, but
`extract_py_functions` is a second formatter and still rendered `datatable(name, role)`, so
code written from that page passed a keyword-only argument positionally.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-16 15:14:28 +02:00
Diego ImbertandClaude Opus 5 ea14c01a3c fix(datatables): serialize role catalog mutations, and state each helper's authorization contract
The catalog is one JSON document, so create, rename, enable and delete are all
read-modify-write. Two concurrent creates read the same snapshot, both succeed in the
cluster, and the second write drops the first — leaving a live Postgres login with a password
nobody recorded, which is the exact state the delete path exists to prevent. Every mutation
now runs in one transaction holding an advisory lock across the read, the cluster DDL and the
write, so a lost update cannot happen and a failure rolls the whole thing back. The DDL
helpers take that transaction rather than the pool, which is what makes the lock cover them.

Their statements moved off `sqlx::raw_sql`: the simple protocol is only needed for genuinely
multi-statement SQL, and its future is not `Send`, which an axum handler holding the
transaction requires. Each of these is one statement anyway.

The new cross-crate surface now says what callers must do. `read_role_catalog` returns
plaintext credentials; `create`/`rename`/`set_login`/`drop_instance_role` and
`converge_connect_grants` mutate cluster-wide state; `read_datatable_entry` reads a workspace's
raw config. All of them are superadmin-gated by their current handlers, but nothing said so at
the definition, which is where the next caller looks.

Also: the roles table reloads after a failed login toggle instead of leaving it claiming a flip
that did not land; the rename affordance is the design-system `Button`, not a raw one; and
`resolve_datatable_pg_as_caller` drops a `role` parameter no caller ever filled — browsing
resolves as the data table's default until the database manager grows a picker.

Why role passwords stay a plain `String` while the instance user's password beside them is a
`StringOrSecretRef`, asked three times across reviews: that one is a secret ref because an
operator supplies it and may want it from their own backend, while these are minted here and
never entered by anyone, so there is nothing for a ref to point at. Encrypting generated
secrets at rest is a separate change that would take the replication password with it. Now
said at the field.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-16 15:14:28 +02:00
Diego ImbertandClaude Opus 5 54f4467027 fix(datatables): unbreak two operator messages and two comments that described other code
The two strings this branch added for states an operator hits once — the catalog write that
matched nothing, and the delete that stranded a pointer — were collapsed from their multi-line
form with the indentation left in, so both rendered with a fourteen-space gap mid-sentence.

`list_datatables` claimed to report a chain it cannot follow and then dropped it; it does drop
it, and the comment now says why that is the right place to stay quiet. The non-superadmin
check in `edit_datatable_config` was introduced as also covering references, which it does not
and need not: `reference` is overwritten from the stored entry for every caller before the
check runs.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-16 15:14:28 +02:00
Diego ImbertandClaude Opus 5 8276ae09fa fix(datatables): fail loudly where a role or a pointer can be left half-recorded
Three ways the feature could end up in a state nobody could see or undo.

Creating a role writes the cluster first and the catalog second, but the catalog write was an
`UPDATE` that matched nothing when the instance Postgres settings row was absent — leaving a
live login with a password nobody recorded: invisible to the catalog, un-recreatable because
the name is taken, and un-deletable because there is no entry to delete. It now errors, so
the operation is retryable once the row is restored.

Deleting a workspace only nulls the fork lineage; the data table entries pointing at it are
left resolving to nothing. Sweeping them is not an option — turning a pointer back into a copy
would hand each fork the database outright — so the delete now names the data tables it
stranded, and resolving one says which workspace is missing rather than reporting a data table
this workspace never had.

`InstanceDatatableRole` derived `Debug` while holding a Postgres password; it is now
hand-written so `{:?}` on the catalog cannot put a live credential in a log line.

Adds the two branches the reviews found unpinned: a caller who is not a member of the
governing workspace at all, and `NoIdentity` — the compatibility path for an agent worker that
predates this and sends no job id.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-16 15:14:28 +02:00
Diego ImbertandClaude Opus 5 6f9457fc8e fix(datatables): confine roles to the instance database, and stop a fork reaching the parent's bookkeeping
A data table role is a login on Windmill's own Postgres. Nothing stopped a workspace admin
putting a *resource-backed* data table under roles, at which point the executor dialled the
host that resource names — one the admin chose — with the role's real cluster password, and
`CONNECT` is granted to every registered instance database. Both ends now refuse: the
permissions endpoint rejects the save, and the chokepoint refuses to substitute credentials
on a non-instance entry rather than trusting the record it read.

Two more places reached the governing database without answering to it. The initial-migration
generator returned a `pg_dump` of the whole schema to any member. And the migration
rename/delete cascade followed a fork's pointer into the parent, so a fork admin renaming or
removing their own local entry relabelled or wiped the parent's `_wm_migrations` — after
which the parent re-runs every migration from zero. The remote half is now skipped when the
entry resolves into another workspace, which is also just correct: a fork renaming what it
calls a data table changes nothing about the data table.

Also: revoking a tenant now bounces the replication streams of every workspace holding an
entry that resolves here, not only the governing one, so a fork's trigger stops rather than
living on inside its open connection; the instance role catalog and the governing workspace's
tenant lists are no longer returned to someone who cannot edit them; and the tenant rename
dedup collapses non-adjacent duplicates, per role rather than once any role changed.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-16 15:14:28 +02:00
Diego ImbertandClaude Opus 5 6ea682d741 fix(datatables): gate the paths that reach a whole database as admin
Auditing what still resolved through the unchecked resolver turned up three that act for a
caller and hand back the admin connection: `resolve_pg_source_checked` (behind schema
export, the full-schema read, database creation, import and the forked-database drop), the
connection test, and the schema snapshot a fork clone takes of its parent. On a data table
under roles each let any workspace member — or a fork admin who is nobody in the governing
workspace — read or copy the whole database whatever its roles grant.

All three now require admin reach on the governing workspace. A dump taken under a
restricted role would be a silently truncated copy rather than an error, so refusing is the
only right answer for the copy paths.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-16 15:14:28 +02:00
Diego ImbertandClaude Opus 5 22b3e69c07 feat(datatables): put a data table's connection under Postgres roles
A data table backed by the instance database resolved to exactly one Postgres connection,
`custom_instance_user`, for everyone who could reach it at all. There was no way to say
this job reads, that one writes, this one never sees the salaries table.

A data table role is now a real Postgres login on the cluster, defined once for the
instance by a superadmin and named exactly as they named it. A script that declares
`-- role analytics` connects as `analytics`, and Postgres decides what it may touch —
grants are ordinary SQL. Windmill answers only "may this caller ask for this role", from
the tenant lists on the data table entry: `u/alice`, `g/analysts`, `f/finance` or `*`.
A data table with no `permissions` block behaves exactly as before.

Everything that opens a connection on someone's behalf goes through one chokepoint,
`get_datatable_resource_from_db`, which takes the identity explicitly and fails closed when
there is none. The role logs in as itself — never `SET ROLE`, which a script could
`RESET ROLE` its way out of.

A fork's data table entry becomes a pointer at the workspace that governs it rather than a
copy of it. The settings clone used to hand a fork a byte-identical entry naming the
parent's database, which a fork admin could edit to grant themselves `admin` there; a
pointer has nothing local to edit, and its tenants are evaluated as a member of the
governing workspace, by email. `permissions` is stripped from the workspace export and
ignored on import: tenants name principals of one workspace, and a settings push is not
where an access decision should be made.

Operations that see the whole database whatever the roles grant stay with the governing
workspace's admins: editing the roles, a migration that declares none, and opening a
replication stream for a Postgres trigger or capture.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR
2026-09-16 15:14:28 +02:00
GuilhemandClaude Fable 5.1 3d08197182 feat: badge chat-input flows on the home list (#11164)
* feat: badge chat-input flows on the home list

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

* fix: keep a malformed draft value from aborting the runnables list

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

* fix: only a JSON boolean marks a draft flow as chat-enabled

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-16 14:46:49 +02:00
Ruben Fiszelandrubenfiszel c3e11bd223 chore(main): release 1.813.0 (#11141)
* chore(main): release 1.813.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com>
2026-09-16 12:12:22 +02:00
GuilhemandClaude Fable 5.1 6885226c26 collapse detail page header actions into menus below lg (#11163)
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-16 12:05:22 +02:00
Ruben FiszelandClaude Opus 5 49d0310ecc fix(apps): re-check access in place after a password sign-in (#11166)
* fix(apps): re-check access in place after a password sign-in

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* style: drop redundant comment in the password sign-in hand-back

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 12:03:32 +02:00
265 changed files with 20826 additions and 2670 deletions
+1 -1
View File
@@ -1,3 +1,3 @@
{
".": "1.812.0"
".": "1.813.0"
}
+3 -2
View File
@@ -31,8 +31,9 @@ Open-source platform for internal tools, workflows, API integrations, background
`cargo run`; a normal build cannot start one at all.
- **Enterprise**: `docs/enterprise.md` — EE file conventions and PR workflow
- **Auth surface**: `docs/auth-surface.md` — credential precedence, session/cache invalidation
scope, how OAuth login matches `login_type`, and that every superadmin route refuses `$WM_TOKEN`.
Read before designing anything that creates users, tokens or sessions.
scope, which token labels email their owner at expiry, how OAuth login matches `login_type`, and
that every superadmin route refuses `$WM_TOKEN`. Read before designing anything that creates
users, tokens or sessions.
- **Product telemetry**: `docs/feature-telemetry.md` — when to instrument a new feature with
`feature_usage`, and the four-step recipe. An unregistered `(feature, kind)` pair is dropped
silently, so frontend-only instrumentation records nothing.
+26
View File
@@ -1,5 +1,31 @@
# Changelog
## [1.813.0](https://github.com/windmill-labs/windmill/compare/v1.812.0...v1.813.0) (2026-09-16)
### Features
* back AI sessions up to the workspace object storage ([#11116](https://github.com/windmill-labs/windmill/issues/11116)) ([796b6e5](https://github.com/windmill-labs/windmill/commit/796b6e5297d8cceb842ec097f33ec1c3115058bd))
* delete a browser's copy of an AI session past its workspace retention ([#11156](https://github.com/windmill-labs/windmill/issues/11156)) ([a48ae65](https://github.com/windmill-labs/windmill/commit/a48ae656ae59d600311f81ef357d08df5226515a))
* rename saved agents from the agent editor and flag broken links ([#11147](https://github.com/windmill-labs/windmill/issues/11147)) ([57a99f6](https://github.com/windmill-labs/windmill/commit/57a99f66a88f195cac8f583b59d69d627cc1ec1d))
* retention for AI sessions on the object store and in the browser ([#11152](https://github.com/windmill-labs/windmill/issues/11152)) ([ee6d317](https://github.com/windmill-labs/windmill/commit/ee6d317e318fa8a1506fb18d51c45b627070627a))
* return an ai agent step's thinking in its job result ([#11140](https://github.com/windmill-labs/windmill/issues/11140)) ([c4e878e](https://github.com/windmill-labs/windmill/commit/c4e878e8313a72a16bfbe81fbb3935ee7728ec6f))
* stream reasoning summaries in AI agent Responses API steps ([#11124](https://github.com/windmill-labs/windmill/issues/11124)) ([b51c0ea](https://github.com/windmill-labs/windmill/commit/b51c0eabbe774c78a9cbf824a3df6528d970b8f6))
### Bug Fixes
* **apps:** re-check access in place after a password sign-in ([#11166](https://github.com/windmill-labs/windmill/issues/11166)) ([49d0310](https://github.com/windmill-labs/windmill/commit/49d0310ecc08040b6c6fa4f402584543d679e2e8))
* **apps:** run-mode inline app component uses only pinned content ([#11135](https://github.com/windmill-labs/windmill/issues/11135)) ([781b5a5](https://github.com/windmill-labs/windmill/commit/781b5a57e81eb721d97d7b87e23dd84f23895400))
* **cli:** keep the workspace color when settings are synced from git ([#11144](https://github.com/windmill-labs/windmill/issues/11144)) ([129c045](https://github.com/windmill-labs/windmill/commit/129c04559548cd1bcf67758ec416fb2a48e7b928))
* **cli:** resolve lockgen imports through modules a push leaves alone ([#11160](https://github.com/windmill-labs/windmill/issues/11160)) ([54553b2](https://github.com/windmill-labs/windmill/commit/54553b2add6941395f03ca34ee24cf335a3b23c2))
* dispatch workflow-as-code tasks from a deployed flow's inline step ([#11146](https://github.com/windmill-labs/windmill/issues/11146)) ([e8078f2](https://github.com/windmill-labs/windmill/commit/e8078f2a963166b09849650424583f5dcfd28a84))
* keep sidebar confirmation dialogs from being confined to the rail ([#11158](https://github.com/windmill-labs/windmill/issues/11158)) ([b9b5988](https://github.com/windmill-labs/windmill/commit/b9b5988ebdf3edd75add445282977c516ef5ed51))
* keep the instance users table's actions and header in view ([#11145](https://github.com/windmill-labs/windmill/issues/11145)) ([a9a9335](https://github.com/windmill-labs/windmill/commit/a9a9335a34a13ffd8cd2699adc92087b679548ca))
* stop reading an array job result as wm_failure or http response ([#11154](https://github.com/windmill-labs/windmill/issues/11154)) ([9a8a9c4](https://github.com/windmill-labs/windmill/commit/9a8a9c480cf008761ec6ceaff694085a252a32ae))
* walk the whole fork ancestry for app installations and fork conflicts ([#11151](https://github.com/windmill-labs/windmill/issues/11151)) ([73dc892](https://github.com/windmill-labs/windmill/commit/73dc892f9c9c840a5f0fb12fcbb28bbe0f38795f))
* **worker:** bound cache transfers and import fetches in bun jobs ([#11138](https://github.com/windmill-labs/windmill/issues/11138)) ([31c4325](https://github.com/windmill-labs/windmill/commit/31c43255fdcc827c3fdf65e40238f8f3a83201cd))
## [1.812.0](https://github.com/windmill-labs/windmill/compare/v1.811.1...v1.812.0) (2026-09-15)
+19
View File
@@ -37,6 +37,25 @@ _Avoid_: argument field, param
Any other place a property can be picked into: the loop iterator, skip and early-stop predicates, the retry condition, a branch predicate, timeout. Its prop picker opens in a popover from the connect button rather than taking a pane.
_Avoid_: JS field, code input
### Flow chat
**Conversation**:
One thread of messages against one chat-enabled flow, with its own agent memory. A flow has
many; the chat shows one at a time.
_Avoid_: thread, session (that names an AI session, a different thing), chat (that names the surface)
**Turn**:
One question and the answer to it: the run the question started, the handle that stops it,
and the rows it is writing. At most one per conversation, and the chat is held for its whole
length — from the moment the question takes the chat, before it has a job, until it is ended.
_Avoid_: request, exchange, message round
**Transcript**:
The rows a conversation's chat holds. Not the conversation: it is the newest page plus
whatever older pages the reader has scrolled back through, so a question it cannot answer
from what it holds is one to ask the server rather than to guess at.
_Avoid_: history, messages (too easily read as "all of them")
### Permissions
**Member**:
+34 -3
View File
@@ -92,7 +92,7 @@ export interface BenchmarkWorkspaceResource {
}
export interface BenchmarkWorkspaceJob {
/** Stable id so a case prompt can reference a specific run (e.g. for get_job_logs). */
/** Stable id so a case prompt can reference a specific run (e.g. for get_run). */
id?: string
jobKind?: CompletedJob['job_kind']
scriptPath?: string
@@ -100,6 +100,8 @@ export interface BenchmarkWorkspaceJob {
label?: string
success?: boolean
logs?: string
args?: Record<string, unknown>
result?: unknown
}
export interface BenchmarkWorkspaceRunnables {
@@ -156,7 +158,7 @@ export function registerBenchmarkWorkspaceRunnables(
...runnables,
datatables: runnables.datatables ? structuredClone(runnables.datatables) : undefined
})
// Seed any fixture jobs so list_runs / get_job_logs have data to return.
// Seed any fixture jobs so list_runs / get_run have data to return.
for (const seed of runnables.jobs ?? []) {
createBenchmarkCompletedJob({
workspace,
@@ -166,7 +168,9 @@ export function registerBenchmarkWorkspaceRunnables(
scriptPath: seed.scriptPath,
createdBy: seed.createdBy,
label: seed.label,
logs: seed.logs
logs: seed.logs,
args: seed.args,
result: seed.result
})
}
}
@@ -481,6 +485,33 @@ export function getBenchmarkJobLogs(workspace: string, jobId: string): string {
return job.logs ?? ''
}
/**
* Mirror `JobService.getFlowAllResults`, which get_run calls for the execution
* tree. Fixture jobs are single runs with no steps, so only the root entry.
*/
export function getBenchmarkFlowAllResults(workspace: string, jobId: string) {
const job = getBenchmarkCompletedJob(workspace, jobId)
if (!job) {
throw new Error(`Job "${jobId}" not found in benchmark workspace`)
}
return {
entries: [
{
job_id: jobId,
label: 'Flow',
kind: job.job_kind ?? 'script',
depth: 0,
sibling_index: 1,
sibling_count: 1,
status: job.success ? 'success' : 'failure',
success: job.success
}
],
truncated: false,
scope_filtered: false
}
}
// ============= Drafts (per-user, DB-backed in production) =============
/**
@@ -62,6 +62,7 @@ vi.mock('$lib/gen', async () => {
getBenchmarkDatatableSchema,
getBenchmarkDraftForUser,
getBenchmarkFlowByPath,
getBenchmarkFlowAllResults,
getBenchmarkJobLogs,
getBenchmarkOwnDraft,
getBenchmarkScriptByHash,
@@ -325,7 +326,11 @@ vi.mock('$lib/gen', async () => {
getJobLogs: async (data: { workspace: string; id: string }) =>
hasBenchmarkWorkspace(data.workspace)
? getBenchmarkJobLogs(data.workspace, data.id)
: actual.JobService.getJobLogs(data)
: actual.JobService.getJobLogs(data),
getFlowAllResults: async (data: { workspace: string; id: string }) =>
hasBenchmarkWorkspace(data.workspace)
? getBenchmarkFlowAllResults(data.workspace, data.id)
: actual.JobService.getFlowAllResults(data)
}),
WorkspaceService: wrapService(actual.WorkspaceService, {
getCopilotInfo: async (data: { workspace: string }) =>
+30 -2
View File
@@ -889,13 +889,13 @@
draftCountExactly: 0
toolExpect:
requiredToolsUsed:
- get_job_logs
- get_run
forbiddenToolsUsed:
- deploy_workspace_item
- delete_workspace_item
- write_script
toolCallArgs:
- tool: get_job_logs
- tool: get_run
field: id
stringIncludesAnyOf:
- 01920000-0000-7000-8000-0000000000f1
@@ -906,6 +906,34 @@
- fetches the logs for the requested job id
- explains the failure from the returned logs (connection refused to the upstream API)
- id: global-run-args-and-result
prompt: |-
What was the run 01920000-0000-7000-8000-0000000000f2 called with, and what did it return?
initial: ai_evals/fixtures/frontend/global/initial/jobs_seed.json
runtime:
maxTurns: 6
validate:
draftCountExactly: 0
toolExpect:
requiredToolsUsed:
- get_run
forbiddenToolsUsed:
- test_run_script
- run_script
- deploy_workspace_item
toolCallArgs:
- tool: get_run
field: id
stringIncludesAnyOf:
- 01920000-0000-7000-8000-0000000000f2
# Read-only, so no draft for the global judge to score — validated on tool use
# and the deterministic argument check, like the neighbouring run cases.
skipJudge: true
judgeChecklist:
- reports the arguments the run was called with (region emea, 12 recipients)
- reports what the run returned (12 sent, 3 skipped)
- does not start a new run to find out
# --- Page navigation (open_page) ---
# The assistant should take the user to a Windmill page (Runs/Schedules) with the
# right filters via open_page, rather than describing where to click or dumping the
@@ -15,6 +15,8 @@
"jobKind": "script",
"createdBy": "bob",
"success": true,
"args": { "region": "emea", "dry_run": false, "recipients": 12 },
"result": { "sent": 12, "skipped": 3, "digest_url": "https://reports.example.com/d/2026-06-09" },
"logs": "Generating daily digest...\nDigest emailed to 12 recipients\nDone in 1.2s"
},
{
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT 1 AS one FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "one",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null
]
},
"hash": "06abbf945bee93349ff88f64906b96ea1e853ef202510281427cfa9beeff81b3"
}
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace_settings ws\n SET datatable = (\n SELECT jsonb_set(ws.datatable, '{datatables}', jsonb_object_agg(\n dt.key,\n CASE WHEN dt.value->'reference'->>'workspace_id' = $1\n AND dt.value->'reference'->>'datatable' = $2\n THEN jsonb_set(dt.value, '{reference,datatable}', to_jsonb($3::text))\n ELSE dt.value END\n ))\n FROM jsonb_each(ws.datatable->'datatables') dt\n )\n WHERE EXISTS (\n SELECT 1 FROM jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) d\n WHERE d.value->'reference'->>'workspace_id' = $1\n AND d.value->'reference'->>'datatable' = $2\n )",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "06ce02cd7ce2f5a57355153edb573c242f9ba758db66e9a5e16f30e3e1494201"
}
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM flow_conversation c\n WHERE c.id = ANY($1)\n AND c.workspace_id = $2\n AND NOT EXISTS (\n SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id\n )",
"describe": {
"columns": [],
"parameters": {
"Left": [
"UuidArray",
"Text"
]
},
"nullable": []
},
"hash": "07a005f0f9e80a156cd2a5a0ae39a1fabeaa167818206a25abfe31d5582f942a"
}
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "SELECT permissioned_as, permissioned_as_email FROM v2_job\n WHERE id = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "permissioned_as",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "permissioned_as_email",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
false,
false
]
},
"hash": "0d10e0fa5cf4033c7d93c9ed56be8209046007917f44da954eccf2188e5bff1f"
}
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace_settings ws\n SET datatable = (\n SELECT jsonb_set(ws.datatable, '{datatables}', jsonb_object_agg(\n dt.key,\n CASE WHEN dt.value->'reference'->>'workspace_id' = $2\n THEN jsonb_set(dt.value, '{reference,workspace_id}', to_jsonb($1::text))\n ELSE dt.value END\n ))\n FROM jsonb_each(ws.datatable->'datatables') dt\n )\n WHERE jsonb_typeof(ws.datatable->'datatables') = 'object'\n AND ws.datatable::text LIKE '%\"reference\"%'",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "297c7a40dfce729d44aa37bc7c65560517bd25e40c0752a00467829191e2eb98"
}
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE capture_config SET server_id = NULL, last_server_ping = NULL\n WHERE workspace_id = $1 AND trigger_kind = 'postgres'\n AND (trigger_config->>'postgres_resource_path' = $2\n OR trigger_config->>'postgres_resource_path' LIKE $3)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "2a391cc1bfcd2f75b46144a394c01237e09c3060da88170f1f6e06468309d213"
}
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = $1)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Name"
]
},
"nullable": [
null
]
},
"hash": "334dbcd48fb59c96c62c2705ab2d1ce716cd52417f487cc1a8dd376017b2db7d"
}
@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM flow_conversation_message m\n USING flow_conversation c\n WHERE m.conversation_id = c.id AND c.workspace_id = $1 AND m.job_id = ANY($2)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"UuidArray"
]
},
"nullable": []
},
"hash": "462d2b2822b185a6f51fafcfa957cb3b31ee6b69abae79a214dddba0dee4425c"
}
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE postgres_trigger SET server_id = NULL, last_server_ping = NULL\n WHERE workspace_id = $1\n AND (postgres_resource_path = $2 OR postgres_resource_path LIKE $3)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "4692810d2be817bbb5de9b476d68d695941bd4fb5ccef393e4da522ed479d601"
}
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id FROM flow_conversation WHERE id = ANY($1) AND workspace_id = $2 ORDER BY id FOR UPDATE",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"UuidArray",
"Text"
]
},
"nullable": [
false
]
},
"hash": "4f52bf546579f26a1d22c239b8b0054b753cfbeb5dad1e8120fd5e8a672d50ef"
}
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "SELECT ws.workspace_id AS \"workspace_id!\", dt.key AS \"datatable!\"\n FROM workspace_settings ws\n CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt\n WHERE dt.value->'reference'->>'workspace_id' = $1\n AND dt.value->'reference'->>'datatable' = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id!",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "datatable!",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
null
]
},
"hash": "5048e21546f9710697100100e1255ab103979433bc386d7c89d0e30db12bfd57"
}
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "SELECT ws.workspace_id AS \"workspace_id!\", dt.key AS \"datatable!\"\n FROM workspace_settings ws\n CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt\n WHERE dt.value->'reference'->>'workspace_id' = $1\n AND dt.value->'reference'->>'datatable' = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id!",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "datatable!",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
null
]
},
"hash": "538dd1779874e4003932d7f17750239c625f85e25b3364bf2edf566f518c8ee2"
}
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT username FROM usr WHERE workspace_id = $1 AND email = $2 AND disabled = false",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "username",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "58e5cfe9eb87bda9f7de87c403861b6e7b9d35a41594681e2a92a87359e6a018"
}
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM flow_conversation_message m\n USING flow_conversation c\n WHERE m.conversation_id = c.id AND c.workspace_id = $1 AND m.job_id = ANY($2)\n RETURNING m.conversation_id",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "conversation_id",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"Text",
"UuidArray"
]
},
"nullable": [
false
]
},
"hash": "69bfbe9b39414b724488532cc3b3659915d9fcb3d58f16532aaffe06c44ec976"
}
@@ -0,0 +1,59 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id, workspace_id, flow_path, title, created_at, updated_at, created_by\n FROM flow_conversation\n WHERE id = $1 AND workspace_id = $2\n FOR UPDATE",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "flow_path",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "title",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 5,
"name": "updated_at",
"type_info": "Timestamptz"
},
{
"ordinal": 6,
"name": "created_by",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
false,
false,
false,
true,
false,
false,
false
]
},
"hash": "6f32c1feed096ff706ae359ad6a3ca33b3f82ca38289dfa4a69aa95041027d57"
}
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO global_settings (name, value) VALUES ($1, $2)\n ON CONFLICT (name) DO UPDATE SET value = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Jsonb"
]
},
"nullable": []
},
"hash": "6f9fb5d72f486358fa25d6887bd69b93910e028f140c07048f2c1c8d63ee6909"
}
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM ai_agent_memory a\n USING flow_conversation c\n WHERE c.id = ANY($1)\n AND c.workspace_id = $2\n AND a.conversation_id = c.id\n AND a.workspace_id = c.workspace_id\n AND NOT EXISTS (\n SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id\n )",
"describe": {
"columns": [],
"parameters": {
"Left": [
"UuidArray",
"Text"
]
},
"nullable": []
},
"hash": "89ea81b765550cf665e30533efc9672f8c98d2579fc72c07752361cc5fd683dc"
}
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM flow_conversation c\n WHERE c.id = ANY($1)\n AND NOT EXISTS (\n SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id\n )",
"describe": {
"columns": [],
"parameters": {
"Left": [
"UuidArray"
]
},
"nullable": []
},
"hash": "90b910da8d00a7c7bcf29c167e38e44eb1c0062a8241dc8fe0ed3dd95b65f89a"
}
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM flow_conversation_message WHERE job_id = ANY($1) RETURNING conversation_id",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "conversation_id",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"UuidArray"
]
},
"nullable": [
false
]
},
"hash": "967f52005f4a044b3a2e9f02ceadf90dad5246681bde6caaa633b85a5e8b2352"
}
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO global_settings (name, value) VALUES ($1, $2)\n ON CONFLICT (name) DO UPDATE SET value = $2, updated_at = now()",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Jsonb"
]
},
"nullable": []
},
"hash": "975e404ac3a6258bb8220e122e3de094c7ab23330fdbc74d6e4ad472ddd3c820"
}
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "SELECT pg_advisory_xact_lock(hashtext('datatable_role_catalog'))",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "pg_advisory_xact_lock",
"type_info": "Void"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "9f663180166f53d117e794f3f3a5723a0a43db163ecca7d5a63d4e74ab1d3be1"
}
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM ai_agent_memory a\n USING flow_conversation c\n WHERE c.id = ANY($1)\n AND a.conversation_id = c.id\n AND a.workspace_id = c.workspace_id\n AND NOT EXISTS (\n SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id\n )",
"describe": {
"columns": [],
"parameters": {
"Left": [
"UuidArray"
]
},
"nullable": []
},
"hash": "a1b23f3e62c6433d95cdac58215741a51ca0ce66bf1c674097705cf2e1b72eff"
}
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE capture_config SET server_id = NULL, last_server_ping = NULL\n WHERE workspace_id = $1 AND trigger_kind = 'postgres'\n AND (trigger_config->>'postgres_resource_path' = $2\n OR trigger_config->>'postgres_resource_path' LIKE $3)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "a57de2bb0442a5ee8a607cd63cfcf675de175796184f620cb4b09670c8b0b19f"
}
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT ws.datatable->'datatables' FROM workspace_settings ws\n WHERE ws.workspace_id = $1 FOR UPDATE",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null
]
},
"hash": "b42af37fb474bea4c5419b0a46d9eadfe384013ab970ccf9c5effd1c78321b7c"
}
@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM flow_conversation_message WHERE job_id = ANY($1)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"UuidArray"
]
},
"nullable": []
},
"hash": "bfdd60b42e32bd81e2d20b327462893147b4e5ff078531de36147d908132d636"
}
@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM usr WHERE email = $1 RETURNING username, workspace_id",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "username",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "workspace_id",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false
]
},
"hash": "c1d026c886799dabc39ce73e1fe09ccb175c7271df75d67aa9c72ad6f825a992"
}
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by, title)\n VALUES ($1, $2, $3, $4, $5)\n RETURNING id, workspace_id, flow_path, title, created_at, updated_at, created_by",
"query": "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by, title)\n VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (id) DO NOTHING\n RETURNING id, workspace_id, flow_path, title, created_at, updated_at, created_by",
"describe": {
"columns": [
{
@@ -58,5 +58,5 @@
false
]
},
"hash": "6bd23a98838e3eec309e6b696edc776bd56fc9dae1238b3272557d1562400dbe"
"hash": "c1e3ed3ecc3bcb98f60ba8196d33fee4a74f61b061e5025ecb75882208b3ba8f"
}
@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "SELECT ws.workspace_id AS \"workspace_id!\", dt.key AS \"datatable!\"\n FROM workspace_settings ws\n CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt\n WHERE dt.value->'reference'->>'workspace_id' = $1\n ORDER BY ws.workspace_id, dt.key",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id!",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "datatable!",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
null
]
},
"hash": "c5451ea9d9fa5146af242d1ee8c19ebd65b80e7ed9f29b9fb2e03767c2aa94ba"
}
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM datatable_role WHERE id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "c85d362fe2e652d4ac01a35bf470e80b993020a2ff5dcb5849dc570d52798587"
}
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT datatable FROM workspace_settings WHERE workspace_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "datatable",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
true
]
},
"hash": "d5fb5dde6300862f978739a3d9249fc2b3e7697c0da7d3195398933d3d81aadf"
}
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE token SET label = $1\n WHERE email = $2 AND token_prefix = $3\n AND (label IS NULL OR (\n label <> 'session'\n AND label <> 'guest_session'\n AND lower(label) NOT LIKE 'ephemeral%'\n AND label <> 'debugger-token'\n AND label NOT LIKE 'mcp-oauth-%'\n ))\n RETURNING token_prefix",
"query": "UPDATE token SET label = $1\n WHERE email = $2 AND token_prefix = $3\n AND (label IS NULL OR (\n label <> 'session'\n AND label <> 'guest_session'\n AND lower(label) NOT LIKE 'ephemeral%'\n AND label <> 'debugger-token'\n AND label NOT LIKE 'mcp-oauth-%'\n AND NOT starts_with(label, 'embed_app:')\n AND NOT starts_with(label, 'sdk_app:')\n AND NOT starts_with(label, 'impersonation:')\n ))\n RETURNING token_prefix",
"describe": {
"columns": [
{
@@ -20,5 +20,5 @@
false
]
},
"hash": "31ed2fb85c0c726e3cf6392be2a73c62bae004b2842a4828c6807232570f83a1"
"hash": "d631a26e5531589ff37e677f91a4f1d9f850e3e46c17130dd580426cda7a9f65"
}
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace_settings ws\n SET datatable = (\n SELECT jsonb_set(ws.datatable, '{datatables}', jsonb_object_agg(\n dt.key,\n CASE WHEN dt.value->'reference'->>'workspace_id' = $1\n AND dt.value->'reference'->>'datatable' = $2\n THEN jsonb_set(dt.value, '{reference,datatable}', to_jsonb($3::text))\n ELSE dt.value END\n ))\n FROM jsonb_each(ws.datatable->'datatables') dt\n )\n WHERE jsonb_typeof(ws.datatable->'datatables') = 'object'\n AND ws.datatable::text LIKE '%\"reference\"%'",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "da5057c86607327bafc2942c218025ca9181a0c396405984d87e422e129521c1"
}
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "SELECT value->'roles' FROM global_settings WHERE name = 'custom_instance_pg_databases'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "dc8dfc37559e9b6713bde48155f48b5a2c7b8199eace1508e102b60d1ff40c04"
}
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE postgres_trigger SET server_id = NULL, last_server_ping = NULL\n WHERE workspace_id = $1\n AND (postgres_resource_path = $2 OR postgres_resource_path LIKE $3)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "e159b2ff15633f85e839ee4fe1ec2ecd11caf228ea8d0f52ad66def595644250"
}
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT datatable FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "datatable",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
true
]
},
"hash": "e2061df65ffd4a72146c4ca316829265289c8d6f625ac272655c88e1ad0b1745"
}
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace_settings\n SET datatable = jsonb_set(\n jsonb_set(\n datatable #- ARRAY['datatables', $2, 'reference'],\n ARRAY['datatables', $2, 'database'], $3::jsonb),\n ARRAY['datatables', $2, 'forked_from'], $4::jsonb\n )\n WHERE workspace_id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Jsonb",
"Jsonb"
]
},
"nullable": []
},
"hash": "ebaf3ed3097621da59dd201b5a4b9d1f440692f183c7c378f59e4b73f1c6e241"
}
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id FROM flow_conversation WHERE id = ANY($1) ORDER BY id FOR UPDATE",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"UuidArray"
]
},
"nullable": [
false
]
},
"hash": "ec295b3890a0018475ec0a3774c7a30d71a5689efe72daf58bd1e8f6cf90c410"
}
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT workspace_id FROM workspace_settings WHERE datatable::text LIKE $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false
]
},
"hash": "f3ee09fb17955ca8d886f446d397063c4094546a7807343b570b823796372cef"
}
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace_settings\n SET datatable = CASE WHEN $3::jsonb = 'null'::jsonb\n THEN datatable #- ARRAY['datatables', $2, 'permissions']\n ELSE jsonb_set(datatable, ARRAY['datatables', $2, 'permissions'], $3::jsonb)\n END\n WHERE workspace_id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Jsonb"
]
},
"nullable": []
},
"hash": "f4adc9e26ebfebce18a29fb2c21bf06394cacb8a9699a608327b097e0ac1363e"
}
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE datatable_role SET name = $2, enabled = $3, pwd = $4 WHERE id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Varchar",
"Bool",
"Text"
]
},
"nullable": []
},
"hash": "fcb34e643b888122766e115a01394ab31ac856252aaa76c75ea27a447009c363"
}
+106 -93
View File
@@ -549,7 +549,7 @@ dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
"synstructure",
"synstructure 0.13.2",
]
[[package]]
@@ -561,7 +561,7 @@ dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
"synstructure",
"synstructure 0.13.2",
]
[[package]]
@@ -10572,9 +10572,9 @@ dependencies = [
[[package]]
name = "rmcp-macros"
version = "3.3.0"
version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "873b730df6f0a9b74b13eb514e0dca4c2db0d8b68b74af98a2e9bf3f9d436585"
checksum = "cd740c45d66ceb87e5579082abc27bd771665e464e9660a17a048c721b2a6025"
dependencies = [
"darling 0.24.1",
"proc-macro2",
@@ -12737,6 +12737,17 @@ dependencies = [
"syn 2.0.119",
]
[[package]]
name = "synstructure"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.5",
]
[[package]]
name = "sys-locale"
version = "0.3.2"
@@ -14783,7 +14794,7 @@ dependencies = [
[[package]]
name = "windmill"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"async-nats",
@@ -14871,7 +14882,7 @@ dependencies = [
[[package]]
name = "windmill-ai"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"async-stream",
"async-trait",
@@ -14904,7 +14915,7 @@ dependencies = [
[[package]]
name = "windmill-alerting"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14917,7 +14928,7 @@ dependencies = [
[[package]]
name = "windmill-api"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"argon2",
@@ -15057,7 +15068,7 @@ dependencies = [
[[package]]
name = "windmill-api-agent-workers"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15080,7 +15091,7 @@ dependencies = [
[[package]]
name = "windmill-api-assets"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15097,7 +15108,7 @@ dependencies = [
[[package]]
name = "windmill-api-auth"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15123,7 +15134,7 @@ dependencies = [
[[package]]
name = "windmill-api-client"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"reqwest 0.12.28",
"serde",
@@ -15133,7 +15144,7 @@ dependencies = [
[[package]]
name = "windmill-api-configs"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15150,7 +15161,7 @@ dependencies = [
[[package]]
name = "windmill-api-debug"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"axum 0.8.9",
"base64 0.22.1",
@@ -15172,7 +15183,7 @@ dependencies = [
[[package]]
name = "windmill-api-embeddings"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15195,7 +15206,7 @@ dependencies = [
[[package]]
name = "windmill-api-flow-conversations"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15211,7 +15222,7 @@ dependencies = [
[[package]]
name = "windmill-api-flows"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15233,7 +15244,7 @@ dependencies = [
[[package]]
name = "windmill-api-groups"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15255,7 +15266,7 @@ dependencies = [
[[package]]
name = "windmill-api-inputs"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15269,7 +15280,7 @@ dependencies = [
[[package]]
name = "windmill-api-integration-tests"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"async-nats",
@@ -15304,7 +15315,7 @@ dependencies = [
[[package]]
name = "windmill-api-jobs"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15329,7 +15340,7 @@ dependencies = [
[[package]]
name = "windmill-api-npm-proxy"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15357,7 +15368,7 @@ dependencies = [
[[package]]
name = "windmill-api-openapi"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15379,7 +15390,7 @@ dependencies = [
[[package]]
name = "windmill-api-schedule"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15399,7 +15410,7 @@ dependencies = [
[[package]]
name = "windmill-api-scripts"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15437,7 +15448,7 @@ dependencies = [
[[package]]
name = "windmill-api-settings"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15459,13 +15470,14 @@ dependencies = [
"windmill-ai",
"windmill-alerting",
"windmill-api-auth",
"windmill-audit",
"windmill-common",
"windmill-object-store",
]
[[package]]
name = "windmill-api-sse"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"lazy_static",
"serde",
@@ -15477,7 +15489,7 @@ dependencies = [
[[package]]
name = "windmill-api-users"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"argon2",
"axum 0.8.9",
@@ -15501,7 +15513,7 @@ dependencies = [
[[package]]
name = "windmill-api-workers"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15515,7 +15527,7 @@ dependencies = [
[[package]]
name = "windmill-api-workspaces"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15550,7 +15562,7 @@ dependencies = [
[[package]]
name = "windmill-audit"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"chrono",
"lazy_static",
@@ -15564,7 +15576,7 @@ dependencies = [
[[package]]
name = "windmill-autoscaling"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15583,7 +15595,7 @@ dependencies = [
[[package]]
name = "windmill-common"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"aes-gcm",
"aho-corasick",
@@ -15641,6 +15653,7 @@ dependencies = [
"pin-project-lite",
"pkcs1",
"postgres-native-tls 0.5.3",
"postgres-protocol",
"prometheus",
"quick_cache",
"rand 0.9.0",
@@ -15689,7 +15702,7 @@ dependencies = [
[[package]]
name = "windmill-dep-map"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"chrono",
"futures",
@@ -15709,7 +15722,7 @@ dependencies = [
[[package]]
name = "windmill-git-sync"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"regex",
"serde",
@@ -15726,7 +15739,7 @@ dependencies = [
[[package]]
name = "windmill-indexer"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"astral-tokio-tar",
@@ -15753,7 +15766,7 @@ dependencies = [
[[package]]
name = "windmill-jseval"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"futures",
@@ -15770,7 +15783,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"itertools 0.14.0",
"lazy_static",
@@ -15786,7 +15799,7 @@ dependencies = [
[[package]]
name = "windmill-mcp"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15807,7 +15820,7 @@ dependencies = [
[[package]]
name = "windmill-native-triggers"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15838,7 +15851,7 @@ dependencies = [
[[package]]
name = "windmill-oauth"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"arc-swap",
@@ -15863,7 +15876,7 @@ dependencies = [
[[package]]
name = "windmill-object-store"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"async-stream",
@@ -15898,7 +15911,7 @@ dependencies = [
[[package]]
name = "windmill-operator"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"futures",
@@ -15916,7 +15929,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"convert_case 0.6.0",
"serde",
@@ -15925,7 +15938,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -15937,7 +15950,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"serde_json",
@@ -15949,7 +15962,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"gosyn",
@@ -15961,7 +15974,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -15973,7 +15986,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"serde_json",
@@ -15985,7 +15998,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"nu-parser",
@@ -15996,7 +16009,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -16007,7 +16020,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -16019,7 +16032,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-asset"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -16030,7 +16043,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -16052,7 +16065,7 @@ dependencies = [
[[package]]
name = "windmill-parser-r"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"serde_json",
@@ -16064,7 +16077,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ruby"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16078,7 +16091,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"convert_case 0.6.0",
@@ -16095,7 +16108,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16108,7 +16121,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql-asset"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"serde",
@@ -16120,7 +16133,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16138,7 +16151,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts-asset"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"serde-wasm-bindgen",
@@ -16154,7 +16167,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wac"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -16170,7 +16183,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16184,7 +16197,7 @@ dependencies = [
[[package]]
name = "windmill-queue"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -16223,7 +16236,7 @@ dependencies = [
[[package]]
name = "windmill-runtime-nativets"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"const_format",
@@ -16263,7 +16276,7 @@ dependencies = [
[[package]]
name = "windmill-sql-datatype-parser-wasm"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"getrandom 0.3.4",
"wasm-bindgen",
@@ -16274,7 +16287,7 @@ dependencies = [
[[package]]
name = "windmill-store"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -16309,7 +16322,7 @@ dependencies = [
[[package]]
name = "windmill-test-utils"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16333,7 +16346,7 @@ dependencies = [
[[package]]
name = "windmill-trigger"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16366,7 +16379,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-amqp"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16393,7 +16406,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-azure"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16426,7 +16439,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-email"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16446,7 +16459,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-gcp"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16480,7 +16493,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-http"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16516,7 +16529,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-kafka"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16539,7 +16552,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-mqtt"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16563,7 +16576,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-nats"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"async-nats",
@@ -16587,7 +16600,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-postgres"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16622,7 +16635,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-sqs"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16650,7 +16663,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-websocket"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16675,7 +16688,7 @@ dependencies = [
[[package]]
name = "windmill-types"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"bitflags 2.13.2",
@@ -16694,7 +16707,7 @@ dependencies = [
[[package]]
name = "windmill-worker"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"async-once-cell",
@@ -16812,7 +16825,7 @@ dependencies = [
[[package]]
name = "windmill-worker-volumes"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"bytes",
"futures",
@@ -17500,14 +17513,14 @@ dependencies = [
[[package]]
name = "yoke-derive"
version = "0.8.2"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
"synstructure",
"syn 3.0.5",
"synstructure 0.14.0",
]
[[package]]
@@ -17541,14 +17554,14 @@ dependencies = [
[[package]]
name = "zerofrom-derive"
version = "0.1.7"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
"synstructure",
"syn 3.0.5",
"synstructure 0.14.0",
]
[[package]]
@@ -17620,9 +17633,9 @@ dependencies = [
[[package]]
name = "zlib-rs"
version = "0.6.7"
version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12"
checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112"
[[package]]
name = "zmij"
+3 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.812.0"
version = "1.813.0"
authors.workspace = true
edition.workspace = true
@@ -88,7 +88,7 @@ members = [
exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"]
[workspace.package]
version = "1.812.0"
version = "1.813.0"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -624,6 +624,7 @@ wasm-bindgen-test = "^0"
convert_case = "0.6.0"
getrandom = "0.2"
tokio-postgres = {version = "^0.7", features = ["array-impls", "with-serde_json-1", "with-chrono-0_4", "with-uuid-1", "with-bit-vec-0_6"]}
postgres-protocol = "0.6"
rust-postgres = { package = "tokio-postgres", git = "https://github.com/MaterializeInc/rust-postgres", rev = "78c1222577bb091d69bc22b1bc7ad01c14675abe"}
rust-postgres-native-tls = { package = "postgres-native-tls", git = "https://github.com/MaterializeInc/rust-postgres", features = ["runtime"], rev = "78c1222577bb091d69bc22b1bc7ad01c14675abe" }
bit-vec = "=0.6.3"
+1 -1
View File
@@ -1 +1 @@
d252afcc80e77fcc4f9a2a346b80908c8605a6c0
151be033ea2bf2f768f381a8cd3b888bdd366cf9
@@ -0,0 +1,18 @@
-- Refuse while the catalog holds anything. Each row is a live Postgres login with a password
-- only this table carries, so dropping it would leave credentials on the cluster that Windmill can
-- no longer disable, delete or even name — and re-applying could not recreate them, because the
-- role names would already be taken. Cleaning them up here is not an option either: dropping a
-- role means reassigning what it owns in *every* instance database, and a migration runs in one.
--
-- Delete the roles through instance settings first; that path does the cluster work.
LOCK TABLE datatable_role IN ACCESS EXCLUSIVE MODE;
DO $$
BEGIN
IF EXISTS (SELECT 1 FROM datatable_role) THEN
RAISE EXCEPTION 'Cannot roll back: % data table role(s) still exist as Postgres logins. Delete them in instance settings first, which drops them from the cluster.',
(SELECT count(*) FROM datatable_role);
END IF;
END $$;
DROP TABLE IF EXISTS datatable_role;
@@ -0,0 +1,21 @@
-- The instance's data table role catalog: one row per Postgres login Windmill created for data
-- table access.
--
-- A table rather than a `global_settings` key, because the value is a set of live cluster
-- credentials and that table has generic read, list, write and CLI round-trip paths that know
-- nothing about what they are carrying. Every one of them is a way to leak the passwords or to
-- overwrite the catalog with a copy that has none, and a row nothing generic touches has none of
-- those. One row per role also makes two concurrent creates two inserts rather than a
-- read-modify-write over one document.
CREATE TABLE datatable_role (
id VARCHAR(50) PRIMARY KEY,
-- The Postgres role name, verbatim. Unique because it is the cluster's own key.
name VARCHAR(63) NOT NULL UNIQUE,
enabled BOOLEAN NOT NULL DEFAULT true,
-- Generated by Windmill, never entered by anyone, and never leaves the server.
pwd TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
GRANT ALL ON datatable_role TO windmill_user;
GRANT ALL ON datatable_role TO windmill_admin;
@@ -0,0 +1,22 @@
-- Roles on the external cluster are live logins there; dropping the column would forget them.
LOCK TABLE datatable_role;
DO $$
BEGIN
IF EXISTS (SELECT 1 FROM datatable_role WHERE cluster <> 'instance') THEN
RAISE EXCEPTION 'datatable_role holds roles on the external instance cluster. Delete them in instance settings first.';
END IF;
-- Before this, only data tables on Windmill's own cluster could be under roles, and a role
-- block left with just `admin` survives deleting every external role.
IF EXISTS (
SELECT 1 FROM workspace_settings ws,
jsonb_each(CASE WHEN jsonb_typeof(ws.datatable->'datatables') = 'object'
THEN ws.datatable->'datatables' ELSE '{}'::jsonb END) dt
WHERE dt.value->'database'->>'resource_type' = 'external_instance'
AND dt.value ? 'permissions'
) THEN
RAISE EXCEPTION 'external instance data tables are still under roles. Turn their roles off first.';
END IF;
END $$;
ALTER TABLE datatable_role DROP CONSTRAINT datatable_role_cluster_name_key;
ALTER TABLE datatable_role ADD CONSTRAINT datatable_role_name_key UNIQUE (name);
ALTER TABLE datatable_role DROP COLUMN cluster;
@@ -0,0 +1,8 @@
-- A data table role is a Postgres login on one cluster: Windmill's own ('instance'), or the external
-- instance cluster ('external_instance'). Role names are the cluster's own key, so they are unique
-- per cluster rather than across the instance.
ALTER TABLE datatable_role
ADD COLUMN cluster VARCHAR(20) NOT NULL DEFAULT 'instance'
CHECK (cluster IN ('instance', 'external_instance'));
ALTER TABLE datatable_role DROP CONSTRAINT datatable_role_name_key;
ALTER TABLE datatable_role ADD CONSTRAINT datatable_role_cluster_name_key UNIQUE (cluster, name);
+24 -24
View File
@@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windmill-common"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"aho-corasick",
"anyhow",
@@ -6274,7 +6274,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"proc-macro2",
"quote",
@@ -6286,7 +6286,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"convert_case",
"serde",
@@ -6295,7 +6295,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6307,7 +6307,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"serde_json",
@@ -6319,7 +6319,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"gosyn",
@@ -6331,7 +6331,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6343,7 +6343,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"serde_json",
@@ -6355,7 +6355,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"nu-parser",
@@ -6366,7 +6366,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -6377,7 +6377,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -6389,7 +6389,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-asset"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -6400,7 +6400,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -6422,7 +6422,7 @@ dependencies = [
[[package]]
name = "windmill-parser-r"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"serde_json",
@@ -6434,7 +6434,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ruby"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6448,7 +6448,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"convert_case",
@@ -6465,7 +6465,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6478,7 +6478,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql-asset"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"serde",
@@ -6490,7 +6490,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6508,7 +6508,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts-asset"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"serde-wasm-bindgen",
@@ -6524,7 +6524,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wac"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -6540,7 +6540,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wasm"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"getrandom 0.2.17",
@@ -6572,7 +6572,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6586,7 +6586,7 @@ dependencies = [
[[package]]
name = "windmill-types"
version = "1.812.0"
version = "1.813.0"
dependencies = [
"anyhow",
"bitflags",
@@ -12,7 +12,7 @@ resolver = "2"
members = ["."]
[workspace.package]
version = "1.812.0"
version = "1.813.0"
edition = "2021"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
@@ -730,7 +730,12 @@ pub fn parse_asset_syntax(
s: &str,
enable_default_syntax: bool,
) -> Option<(AssetKind, Cow<'_, str>)> {
if enable_default_syntax && s == "datatable" {
// `datatable` and `datatable?role=analyst` both name the default data table: the role picks
// which Postgres login the connection is made as, not which data table is read.
if enable_default_syntax
&& s.strip_prefix("datatable")
.is_some_and(|rest| rest.is_empty() || rest.starts_with('?'))
{
return Some((AssetKind::DataTable, Cow::Borrowed("main")));
} else if enable_default_syntax && s == "ducklake" {
return Some((AssetKind::Ducklake, Cow::Borrowed("main")));
@@ -741,6 +746,14 @@ pub fn parse_asset_syntax(
if *kind == AssetKind::Dbt {
return Some((*kind, Cow::Owned(canonicalize_table_asset_path(suffix))));
}
// Same reasoning as above, for the explicit form. Specific to data tables: a
// `Resource`'s `?table=` is part of what it names, and stripping it would merge two
// different assets.
if *kind == AssetKind::DataTable {
if let Some((path, _role)) = suffix.split_once('?') {
return Some((*kind, Cow::Borrowed(path)));
}
}
// The suffix is kept verbatim. For S3 the path encodes the storage:
// `s3://<storage>/<key>`, with an EMPTY storage segment for the
// workspace default — so `s3:///key` yields `/key` (leading slash
@@ -1692,6 +1705,25 @@ fn parse_trigger_spec(s: &str) -> Option<TriggerSpec> {
mod pipeline_annotation_tests {
use super::*;
#[test]
fn a_datatable_role_is_not_part_of_the_asset_it_names() {
// The role picks which Postgres login the connection is made as, so two references that
// differ only by role are the same asset and must land on one graph node.
assert_eq!(
parse_asset_syntax("datatable://sales?role=analytics", false),
Some((AssetKind::DataTable, Cow::Borrowed("sales")))
);
assert_eq!(
parse_asset_syntax("datatable?role=analytics", true),
Some((AssetKind::DataTable, Cow::Borrowed("main")))
);
// A resource's `?table=` is part of what it names, so it is kept.
assert_eq!(
parse_asset_syntax("$res:f/db/pg?table=users", false),
Some((AssetKind::Resource, Cow::Borrowed("f/db/pg?table=users")))
);
}
#[test]
fn s3_path_keeps_storage_distinction() {
// An S3 asset path is `<storage>/<key>` with an empty storage segment
+3 -15
View File
@@ -4712,17 +4712,11 @@ const GIT_AUTO_PULL_LOCK_ID: i64 = 737_483_921;
/// Poll every git-sync repository with auto-pull enabled and enqueue a pull when
/// the tracked branch has new commits (repo → Windmill direction).
///
/// Runs on a single replica at a time (advisory lock) and only on
/// Enterprise-licensed instances. Detection is `git ls-remote`; GitHub-App
/// repositories are skipped here and sync via webhooks instead (phase 2).
/// Runs on a single replica at a time (advisory lock). Detection is
/// `git ls-remote`; GitHub-App repositories are skipped here and sync via
/// webhooks instead (phase 2).
#[cfg(feature = "private")]
pub async fn poll_git_auto_pull(db: &Pool<Postgres>) {
use windmill_common::ee_oss::{get_license_plan, LicensePlan};
if !matches!(get_license_plan().await, LicensePlan::Enterprise) {
return;
}
let mut lock_conn = match db.acquire().await {
Ok(c) => c,
Err(e) => {
@@ -4792,12 +4786,6 @@ const GIT_CREDENTIAL_LOCK_ID: i64 = 737_483_923;
/// sync down on its expiry date.
#[cfg(all(feature = "enterprise", feature = "private"))]
async fn maintain_git_credentials(db: &Pool<Postgres>) {
use windmill_common::ee_oss::{get_license_plan, LicensePlan};
if !matches!(get_license_plan().await, LicensePlan::Enterprise) {
return;
}
// Transaction-scoped advisory lock, as for the schedule reconcile above: a
// session lock on a pooled connection would ride back into the pool still
// held if the sweep died before unlocking, and wedge the pass on every
+247 -1
View File
@@ -37,7 +37,7 @@ async fn seed_side_rows(db: &Pool<Postgres>, ws: &str, job_id: Uuid) -> anyhow::
.bind(ws)
.execute(db)
.await?;
// created_seq is assigned by a trigger; inserting a value is rejected.
// created_seq is an identity column; supplying a value is rejected.
sqlx::query(
"INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id)
VALUES ($1, 'assistant', 'hi', $2)",
@@ -121,6 +121,184 @@ async fn test_delete_jobs_removes_side_rows(db: Pool<Postgres>) -> anyhow::Resul
Ok(())
}
/// (conversation rows, agent-memory rows) for one conversation.
async fn conversation_and_memory_counts(
db: &Pool<Postgres>,
conversation_id: Uuid,
) -> anyhow::Result<(i64, i64)> {
Ok((
count(
db,
"SELECT count(*) FROM flow_conversation WHERE id = $1",
conversation_id,
)
.await?,
count(
db,
"SELECT count(*) FROM ai_agent_memory WHERE conversation_id = $1",
conversation_id,
)
.await?,
))
}
/// A conversation outlives the jobs behind its messages until the last one goes: only then
/// are the row and the agent's memory for it left with nothing, and only then are they
/// deleted. Both halves matter — the surviving half is what a single data-modifying CTE
/// would break, since its emptiness check would read the snapshot from before the delete.
#[sqlx::test(fixtures("base"))]
async fn test_delete_jobs_removes_a_conversation_once_its_last_message_goes(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
initialize_tracing().await;
let first_job = Uuid::new_v4();
let second_job = Uuid::new_v4();
insert_job(&db, WS, first_job).await?;
insert_job(&db, WS, second_job).await?;
let conv_id = Uuid::new_v4();
sqlx::query(
"INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by)
VALUES ($1, $2, 'f/flow', 'test-user')",
)
.bind(conv_id)
.bind(WS)
.execute(&db)
.await?;
for job_id in [first_job, second_job] {
sqlx::query(
"INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id)
VALUES ($1, 'assistant', 'hi', $2)",
)
.bind(conv_id)
.bind(job_id)
.execute(&db)
.await?;
}
sqlx::query(
"INSERT INTO ai_agent_memory (workspace_id, conversation_id, step_id, messages)
VALUES ($1, $2, 'a', '[]'::jsonb)",
)
.bind(WS)
.bind(conv_id)
.execute(&db)
.await?;
let mut conn = db.acquire().await?;
windmill_common::jobs::delete_jobs(&mut conn, &[first_job]).await?;
drop(conn);
assert_eq!(
conversation_and_memory_counts(&db, conv_id).await?,
(1, 1),
"a conversation with a message left must survive, memory included"
);
let mut conn = db.acquire().await?;
windmill_common::jobs::delete_jobs(&mut conn, &[second_job]).await?;
drop(conn);
assert_eq!(
conversation_and_memory_counts(&db, conv_id).await?,
(0, 0),
"the last message going should take the conversation and its memory"
);
Ok(())
}
/// Turns that start while retention is collecting their conversation must land, not fail:
/// the conversation lookup locks the row, so each turn waits for the collector's commit,
/// finds the conversation gone, and creates it again — the first insert wins and the other
/// reads its row. Without the lock a turn's message insert is what waits, on the parent
/// row's key lock, and fails its FK check afterwards.
#[sqlx::test(fixtures("base"))]
async fn test_new_turns_wait_for_conversation_cleanup_and_recreate(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
initialize_tracing().await;
let old_job = Uuid::new_v4();
let new_jobs = [Uuid::new_v4(), Uuid::new_v4()];
insert_job(&db, WS, old_job).await?;
for job in new_jobs {
insert_job(&db, WS, job).await?;
}
let conv_id = Uuid::new_v4();
sqlx::query(
"INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by)
VALUES ($1, $2, 'f/flow', 'test-user')",
)
.bind(conv_id)
.bind(WS)
.execute(&db)
.await?;
sqlx::query(
"INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id)
VALUES ($1, 'user', 'hi', $2)",
)
.bind(conv_id)
.bind(old_job)
.execute(&db)
.await?;
// The collector holds the conversation row locked and deleted, uncommitted.
let mut cleanup = db.begin().await?;
windmill_common::jobs::delete_jobs(&mut *cleanup, &[old_job]).await?;
let turns: Vec<_> = new_jobs
.into_iter()
.map(|new_job| {
let db = db.clone();
tokio::spawn(async move {
let mut tx = db.begin().await?;
windmill_common::flow_conversations::get_or_create_conversation_with_id(
&mut tx,
WS,
"f/flow",
"test-user",
"hi again",
conv_id,
)
.await?;
windmill_common::flow_conversations::add_message_to_conversation_tx(
&mut tx,
conv_id,
Some(new_job),
"hi again",
windmill_common::flow_conversations::MessageType::User,
None,
true,
)
.await?;
tx.commit().await?;
anyhow::Ok(())
})
})
.collect();
tokio::time::sleep(std::time::Duration::from_millis(300)).await;
cleanup.commit().await?;
for turn in turns {
turn.await??;
}
assert_eq!(
conversation_and_memory_counts(&db, conv_id).await?.0,
1,
"the turns must have created the conversation again, once"
);
assert_eq!(
count(
&db,
"SELECT count(*) FROM flow_conversation_message WHERE conversation_id = $1",
conv_id,
)
.await?,
2,
"both turns' messages should be there"
);
Ok(())
}
#[sqlx::test(fixtures("base"))]
async fn test_clear_schedule_removes_side_rows(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
@@ -192,6 +370,74 @@ async fn test_workspace_delete_removes_side_rows(db: Pool<Postgres>) -> anyhow::
Ok(())
}
/// The purge endpoint carries its own copy of the emptied-conversation rule, so it gets the
/// same guard: the conversation and its memory go with the last message, and not before.
#[sqlx::test(fixtures("base"))]
async fn test_jobs_export_delete_removes_a_conversation_once_its_last_message_goes(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
initialize_tracing().await;
let first_job = Uuid::new_v4();
let second_job = Uuid::new_v4();
insert_job(&db, WS, first_job).await?;
insert_job(&db, WS, second_job).await?;
let conv_id = Uuid::new_v4();
sqlx::query(
"INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by)
VALUES ($1, $2, 'f/flow', 'test-user')",
)
.bind(conv_id)
.bind(WS)
.execute(&db)
.await?;
for job_id in [first_job, second_job] {
sqlx::query(
"INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id)
VALUES ($1, 'assistant', 'hi', $2)",
)
.bind(conv_id)
.bind(job_id)
.execute(&db)
.await?;
}
sqlx::query(
"INSERT INTO ai_agent_memory (workspace_id, conversation_id, step_id, messages)
VALUES ($1, $2, 'a', '[]'::jsonb)",
)
.bind(WS)
.bind(conv_id)
.execute(&db)
.await?;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let purge = |job_id: Uuid| async move {
reqwest::Client::new()
.post(format!("http://localhost:{port}/api/w/{WS}/jobs/delete"))
.header("Authorization", "Bearer SECRET_TOKEN")
.json(&[job_id])
.send()
.await
};
assert!(purge(first_job).await?.status().is_success());
assert_eq!(
conversation_and_memory_counts(&db, conv_id).await?,
(1, 1),
"a conversation with a message left must survive the purge endpoint too"
);
assert!(purge(second_job).await?.status().is_success());
assert_eq!(
conversation_and_memory_counts(&db, conv_id).await?,
(0, 0),
"the last message going should take the conversation and its memory"
);
Ok(())
}
/// The `/jobs/delete` purge endpoint must scope every side-table delete to the path
/// workspace. A `test-workspace` admin passing a job id from another workspace must not be
/// able to delete that workspace's job or side rows (the side tables no longer cascade, so
+3
View File
@@ -1141,6 +1141,9 @@ impl NewToken {
/// [`ensure_scopes_within_caller`] first (internal narrowing mints intentionally
/// skip it, since their scopes derive from the action being authorized, not the
/// caller's token).
///
/// A token the system mints for itself with an `expiration` needs a label reserved in
/// `windmill_common::auth::is_user_token`, or its expiry alerts its owner (docs/auth-surface.md).
pub async fn create_token_internal(
tx: &mut sqlx::PgConnection,
db: &DB,
+5
View File
@@ -154,6 +154,7 @@ async fn list_flows(
"favorite.path IS NOT NULL as starred",
"ws_error_handler_muted",
"o.labels",
"(o.value->>'chat_input_enabled')::bool as chat_input_enabled",
"draft.email IS NOT NULL as is_draft",
// Per-path draft owners as a JSON array; see scripts.rs for the rationale
// (non-member superadmin identity fallback via `password`, legacy NULL-email row).
@@ -301,6 +302,10 @@ async fn list_flows(
ws_error_handler_muted: None,
deployment_msg: None,
labels: None,
chat_input_enabled: v
.get("value")
.and_then(|fv| fv.get("chat_input_enabled"))
.and_then(|b| b.as_bool()),
// No deployed row to inherit folder labels from.
inherited_labels: None,
is_draft: true,
@@ -800,6 +800,14 @@ async fn delete_folder(
not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?;
// See the same call in `delete_group`: a freed name must not stay in a tenant list.
windmill_common::workspaces::remove_datatable_tenant_in_workspace(
&mut tx,
&w_id,
&format!("f/{name}"),
)
.await?;
let del = sqlx::query_scalar!(
"DELETE FROM folder WHERE name = $1 AND workspace_id = $2 RETURNING 1",
name,
@@ -797,6 +797,15 @@ async fn delete_group(
}
not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?;
// A tenant list names a principal, so a freed name must not linger in one: a later group
// reusing it would silently inherit the data table access this one had.
windmill_common::workspaces::remove_datatable_tenant_in_workspace(
&mut tx,
&w_id,
&format!("g/{name}"),
)
.await?;
sqlx::query!(
"DELETE FROM usr_to_group WHERE group_ = $1 AND workspace_id = $2",
name,
@@ -0,0 +1,108 @@
//! Who may read and change a data table's grants and owners. On the Enterprise Edition: its
//! administrators, from the workspace that governs it. Without it: nobody. Each refusal is decided
//! before anything connects to the data table, so the fixture's database never has to exist.
use serde_json::{json, Value};
use sqlx::{Pool, Postgres};
use windmill_test_utils::*;
fn grant_select_on_public() -> Value {
json!({
"target": {"kind": "schema", "schema": "public"},
"change": {"type": "grant", "role": "analytics", "privileges": ["SELECT"],
"scope": "all_tables"},
"statements": [r#"GRANT SELECT ON ALL TABLES IN SCHEMA "public" TO "analytics""#]
})
}
async fn post_acl(
port: u16,
w_id: &str,
action: &str,
token: &str,
) -> anyhow::Result<reqwest::Response> {
Ok(reqwest::Client::new()
.post(format!(
"http://localhost:{port}/api/w/{w_id}/workspaces/datatable_acl/main/{action}"
))
.header("Authorization", format!("Bearer {token}"))
.json(&grant_select_on_public())
.send()
.await?)
}
/// A fork reaches the data table through a pointer: it may use it, never change what each role may
/// touch on it — not even as an admin of the fork.
#[cfg(all(feature = "private", feature = "enterprise"))]
#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))]
async fn a_fork_cannot_change_access_on_the_data_table_it_points_at(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
for action in ["plan", "apply"] {
let resp = post_acl(port, "wm-fork-dt", action, "SECRET_TOKEN_2").await?;
assert_eq!(resp.status(), 401, "{action}: {}", resp.text().await?);
}
Ok(())
}
#[cfg(all(feature = "private", feature = "enterprise"))]
#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))]
async fn a_member_who_is_not_an_admin_cannot_change_access(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
for action in ["plan", "apply"] {
let resp = post_acl(port, "test-workspace", action, "SECRET_TOKEN_2").await?;
assert_eq!(resp.status(), 401, "{action}: {}", resp.text().await?);
}
Ok(())
}
/// Not even reading, and not even on a data table that is not under roles — which any member
/// reaches, so only the edition stands between them and the instance's credentials.
#[cfg(not(all(feature = "private", feature = "enterprise")))]
#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))]
async fn only_the_enterprise_edition_has_the_access_editor(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
initialize_tracing().await;
sqlx::query(
"UPDATE workspace_settings
SET datatable = datatable #- '{datatables,main,permissions}'
WHERE workspace_id = 'test-workspace'",
)
.execute(&db)
.await?;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let read = reqwest::Client::new()
.get(format!(
"http://localhost:{port}/api/w/test-workspace/workspaces/datatable_acl/main?kind=database"
))
.header("Authorization", "Bearer SECRET_TOKEN")
.send()
.await?;
let mut responses = vec![("read", read)];
for action in ["plan", "apply"] {
responses.push((
action,
post_acl(port, "test-workspace", action, "SECRET_TOKEN").await?,
));
}
for (action, resp) in responses {
assert_eq!(resp.status(), 400, "{action}");
let body = resp.text().await?;
assert!(
body.contains("Data table roles are a Windmill Enterprise Edition feature"),
"{action}: {body}"
);
}
Ok(())
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,45 @@
-- A data table under roles in `test-workspace`, and a fork whose entry points at it rather than
-- carrying a copy. `test-user-2` is a non-admin of the parent and an admin of the fork: the shape
-- the pointer exists for.
-- Empty registry: role provisioning grants CONNECT on every database named here, and the data
-- table's `dt_main` is a name in workspace settings, not a database that exists.
INSERT INTO global_settings (name, value) VALUES
('custom_instance_pg_databases', '{"user_pwd": "pw", "databases": {}}'::jsonb)
ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value;
INSERT INTO datatable_role (id, name, enabled, pwd) VALUES ('role1', 'analytics', true, 'pw');
UPDATE workspace_settings SET datatable = '{
"datatables": {
"main": {
"database": {"resource_type": "instance", "resource_path": "dt_main"},
"permissions": {
"default_role": "role1",
"roles": {
"admin": {"tenants": []},
"role1": {"tenants": ["u/test-user-2", "g/analysts", "f/finance"]}
}
}
}
}
}'::jsonb WHERE workspace_id = 'test-workspace';
INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES
('test-workspace', 'analysts', 'Analysts', '{}');
INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) VALUES
('test-workspace', 'finance', 'finance', '{}', '{}');
INSERT INTO workspace (id, name, owner, parent_workspace_id) VALUES
('wm-fork-dt', 'fork of test-workspace', 'test2@windmill.dev', 'test-workspace');
INSERT INTO workspace_key (workspace_id, kind, key) VALUES ('wm-fork-dt', 'cloud', 'test-key');
INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES
('wm-fork-dt', 'all', 'All users', '{}');
INSERT INTO usr (workspace_id, email, username, is_admin, role) VALUES
('wm-fork-dt', 'test2@windmill.dev', 'test-user-2', true, 'Admin');
INSERT INTO workspace_settings (workspace_id, datatable) VALUES ('wm-fork-dt', '{
"datatables": {
"main": {"reference": {"workspace_id": "test-workspace", "datatable": "main"}}
}
}'::jsonb);
@@ -179,9 +179,11 @@ async fn test_trigger_token_labels_still_creatable(db: Pool<Postgres>) -> anyhow
"http-test-user-2-cd34",
"email-test-user-2-ef56",
"my-ci-token",
// Minted client-side by the editor (every TypeScript editor load) and the debugger.
// Minted client-side by the editor (every TypeScript editor load), the debugger and
// the object-storage "Test from a worker" button.
"Ephemeral lsp token",
"debugger-token",
"ephemeral-test-connection: s3_bucket",
] {
let resp = create_token_with_label(port, "SECRET_TOKEN_2", label).await;
assert_eq!(
+12 -2
View File
@@ -668,10 +668,16 @@ pub async fn handle_chat_conversation_messages(
flow_path: &str,
run_query: &RunJobQuery,
user_message_raw: Option<&Box<serde_json::value::RawValue>>,
job_id: Uuid,
) -> error::Result<()> {
// Names the query parameter rather than the field: it is not a flow argument, and
// supplying it as one is the first thing tried on reading `memory_id is required`.
let memory_id = run_query.memory_id.ok_or_else(|| {
windmill_common::error::Error::BadRequest(
"memory_id is required for chat-enabled flows".to_string(),
"memory_id is required for chat-enabled flows. Pass it as the `memory_id` query \
parameter, not as a flow argument: it names the conversation the turn belongs to, \
so a fresh UUID starts one and reusing a UUID continues it."
.to_string(),
)
})?;
@@ -698,10 +704,13 @@ pub async fn handle_chat_conversation_messages(
)
.await?;
// The run this message started. Its args are the only record of what the message
// carried besides its text — attachments and every other flow input — and nothing
// written later points at them: an assistant row holds the AI agent step's job.
add_message_to_conversation_tx(
tx,
memory_id,
None,
Some(job_id),
&user_message,
MessageType::User,
None,
@@ -826,6 +835,7 @@ pub async fn run_flow<'c>(
&flow_path.to_string(),
&run_query,
args.args.get("user_message"),
uuid,
)
.await?;
}
+55 -5
View File
@@ -692,16 +692,64 @@ pub async fn delete_jobs(
.await?
.rows_affected();
let conversation_message_deleted = sqlx::query!(
// One row per message deleted, so the conversation of a chat losing several appears
// several times: the count is taken before the dedup below.
let mut conversation_ids: Vec<Uuid> = sqlx::query_scalar!(
"DELETE FROM flow_conversation_message m
USING flow_conversation c
WHERE m.conversation_id = c.id AND c.workspace_id = $1 AND m.job_id = ANY($2)",
WHERE m.conversation_id = c.id AND c.workspace_id = $1 AND m.job_id = ANY($2)
RETURNING m.conversation_id",
&w_id,
&job_ids
)
.execute(&mut *tx)
.await?
.rows_affected();
.fetch_all(&mut *tx)
.await?;
let conversation_message_deleted = conversation_ids.len() as u64;
// Same rule, lock and statement order as retention (windmill_common::jobs::delete_jobs,
// which says why): a conversation with no messages left goes, and its memory with it.
conversation_ids.sort_unstable();
conversation_ids.dedup();
let mut memory_deleted = 0;
let mut conversation_deleted = 0;
if !conversation_ids.is_empty() {
sqlx::query_scalar!(
"SELECT id FROM flow_conversation WHERE id = ANY($1) AND workspace_id = $2 ORDER BY id FOR UPDATE",
&conversation_ids,
&w_id
)
.fetch_all(&mut *tx)
.await?;
memory_deleted = sqlx::query!(
"DELETE FROM ai_agent_memory a
USING flow_conversation c
WHERE c.id = ANY($1)
AND c.workspace_id = $2
AND a.conversation_id = c.id
AND a.workspace_id = c.workspace_id
AND NOT EXISTS (
SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id
)",
&conversation_ids,
&w_id
)
.execute(&mut *tx)
.await?
.rows_affected();
conversation_deleted = sqlx::query!(
"DELETE FROM flow_conversation c
WHERE c.id = ANY($1)
AND c.workspace_id = $2
AND NOT EXISTS (
SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id
)",
&conversation_ids,
&w_id
)
.execute(&mut *tx)
.await?
.rows_affected();
}
// Resolutions are not exported, so a delete-then-reimport of the same UUID would
// otherwise resurrect the old annotation on a job that never carried one.
@@ -737,6 +785,8 @@ pub async fn delete_jobs(
+ zombie_deleted
+ dispatch_event_deleted
+ conversation_message_deleted
+ memory_deleted
+ conversation_deleted
+ resolution_deleted
+ jobs_deleted;
+2 -1
View File
@@ -11,7 +11,7 @@ path = "src/lib.rs"
[features]
default = []
enterprise = ["license"]
private = ["windmill-common/private"]
private = ["windmill-common/private", "windmill-audit/private"]
parquet = ["windmill-common/parquet", "windmill-object-store/parquet"]
license = ["dep:rsa"]
@@ -19,6 +19,7 @@ license = ["dep:rsa"]
windmill-ai = { workspace = true, default-features = false }
windmill-alerting.workspace = true
windmill-api-auth.workspace = true
windmill-audit.workspace = true
windmill-common = { workspace = true, default-features = false }
axum.workspace = true
anyhow.workspace = true
@@ -0,0 +1,44 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! Where the data table role catalog endpoints come from: the enterprise implementation, or a
//! refusal. Roles are an Enterprise Edition feature; see `windmill_common::datatable_roles_oss`.
#[cfg(all(feature = "private", feature = "enterprise"))]
pub(crate) use crate::datatable_roles_ee::{
create_datatable_role, delete_datatable_role, list_datatable_roles, update_datatable_role,
};
#[cfg(not(all(feature = "private", feature = "enterprise")))]
pub(crate) use ce::*;
// The routes stay registered so the API has one shape; each answers after authentication, before
// anything is read.
#[cfg(not(all(feature = "private", feature = "enterprise")))]
mod ce {
use windmill_api_auth::ApiAuthed;
use windmill_common::{
datatable_roles_oss::datatable_roles_unavailable as unavailable, error::Result,
};
pub(crate) async fn list_datatable_roles(_authed: ApiAuthed) -> Result<String> {
Err(unavailable())
}
pub(crate) async fn create_datatable_role(_authed: ApiAuthed) -> Result<String> {
Err(unavailable())
}
pub(crate) async fn update_datatable_role(_authed: ApiAuthed) -> Result<String> {
Err(unavailable())
}
pub(crate) async fn delete_datatable_role(_authed: ApiAuthed) -> Result<String> {
Err(unavailable())
}
}
+199 -5
View File
@@ -17,6 +17,9 @@ mod audit_logs_s3;
mod audit_logs_s3_backfill;
#[cfg(feature = "parquet")]
mod background_task;
#[cfg(all(feature = "private", feature = "enterprise"))]
mod datatable_roles_ee;
mod datatable_roles_oss;
#[cfg(feature = "private")]
mod ee;
pub mod ee_oss;
@@ -57,7 +60,7 @@ use windmill_common::{
global_settings::{
AI_CONFIG_SETTING, APP_WORKSPACED_ROUTE_SETTING, AUTOMATE_USERNAME_CREATION_SETTING,
CRITICAL_ALERT_MUTE_UI_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING,
DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS,
DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, EXTERNAL_INSTANCE_PG_SETTING,
GITHUB_APP_WEBHOOK_BASE_URL_SETTING, HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING,
HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING,
INSTANCE_BANNER_SETTING, MAX_RETENTION_OVERRIDE_WORKSPACES,
@@ -151,10 +154,36 @@ pub fn global_service() -> Router {
"/list_custom_instance_pg_databases",
post(list_custom_instance_pg_databases),
)
.route(
"/datatable_roles",
get(datatable_roles_oss::list_datatable_roles)
.post(datatable_roles_oss::create_datatable_role),
)
.route(
"/datatable_roles/{id}",
post(datatable_roles_oss::update_datatable_role)
.delete(datatable_roles_oss::delete_datatable_role),
)
.route(
"/refresh_custom_instance_user_pwd",
post(refresh_custom_instance_user_pwd),
)
.route(
"/external_instance_pg/status",
get(get_external_instance_pg_status),
)
.route(
"/external_instance_pg/setup",
post(setup_external_instance_pg),
)
.route(
"/external_instance_pg/databases",
get(list_external_instance_pg_databases),
)
.route(
"/external_instance_pg/databases/{name}",
post(create_external_instance_pg_database).delete(drop_external_instance_pg_database),
)
.route(
"/setup_custom_instance_pg_database/{name}",
post(setup_custom_instance_pg_database),
@@ -864,6 +893,14 @@ pub async fn set_global_setting_internal(
)));
}
if key == EXTERNAL_INSTANCE_PG_SETTING {
return windmill_common::external_instance_pg::write_external_instance_pg_setting(
db,
Some(&value),
)
.await;
}
run_setting_pre_write_hook(db, &key, &value).await?;
match value {
@@ -1245,7 +1282,7 @@ async fn set_instance_config(
let desired_map = desired.global_settings.to_settings_map();
if !desired_map.is_empty() {
let current_map = current.global_settings.to_settings_map();
let settings_diff =
let mut settings_diff =
instance_config::diff_global_settings(&current_map, &desired_map, ApplyMode::Merge);
let ai_config_changed = settings_diff
.upserts
@@ -1274,8 +1311,15 @@ async fn set_instance_config(
}
for (key, value) in &settings_diff.upserts {
run_setting_pre_write_hook(&db, key, value).await?;
if key != EXTERNAL_INSTANCE_PG_SETTING {
run_setting_pre_write_hook(&db, key, value).await?;
}
}
windmill_common::external_instance_pg::write_external_instance_pg_from_diff(
&db,
&mut settings_diff,
)
.await?;
instance_config::apply_settings_diff(&db, &settings_diff)
.await
@@ -1634,6 +1678,8 @@ struct CustomInstanceDb {
tag: Option<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
used_by_workspaces: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
workspace_id: Option<String>,
}
#[derive(Deserialize, Debug, Serialize, Default)]
@@ -1730,6 +1776,135 @@ async fn refresh_custom_instance_user_pwd(
Ok(Json(()))
}
async fn get_external_instance_pg_status(
authed: ApiAuthed,
Extension(db): Extension<DB>,
) -> JsonResult<windmill_common::external_instance_pg::ExternalInstancePgStatus> {
require_super_admin(&db, &authed).await?;
Ok(Json(
windmill_common::external_instance_pg::external_instance_pg_status(&db).await?,
))
}
#[derive(Deserialize)]
struct SetupExternalInstancePgBody {
#[serde(default)]
rotate_passwords: bool,
}
async fn setup_external_instance_pg(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Json(body): Json<SetupExternalInstancePgBody>,
) -> JsonResult<windmill_common::external_instance_pg::ExternalInstancePgSetupReport> {
require_super_admin(&db, &authed).await?;
let report = windmill_common::external_instance_pg::setup_external_instance_pg_unchecked(
&db,
body.rotate_passwords,
)
.await?;
let rotated = body.rotate_passwords.to_string();
let success = report.success.to_string();
windmill_audit::audit_oss::audit_log(
&db,
&authed,
"settings.setup_external_instance_pg",
windmill_audit::ActionKind::Update,
"global",
Some(&authed.email),
Some(
[
("rotate_passwords", rotated.as_str()),
("success", success.as_str()),
]
.into(),
),
)
.await?;
Ok(Json(report))
}
#[derive(Serialize)]
struct ExternalInstancePgDatabase {
#[serde(flatten)]
status: windmill_common::instance_config::CustomInstanceDb,
used_by_workspaces: Vec<String>,
}
async fn list_external_instance_pg_databases(
authed: ApiAuthed,
Extension(db): Extension<DB>,
) -> JsonResult<std::collections::BTreeMap<String, ExternalInstancePgDatabase>> {
require_super_admin(&db, &authed).await?;
let databases = windmill_common::external_instance_pg::external_instance_databases(&db).await?;
let mut usages =
windmill_common::external_instance_pg::external_instance_database_usages(&db).await?;
Ok(Json(
databases
.into_iter()
.map(|(name, status)| {
let used_by_workspaces = usages.remove(&name).unwrap_or_default();
(
name,
ExternalInstancePgDatabase {
status,
used_by_workspaces: used_by_workspaces.into_iter().collect(),
},
)
})
.collect(),
))
}
async fn create_external_instance_pg_database(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path(dbname): Path<String>,
Json(body): Json<SetupCustomInstanceDbBody>,
) -> JsonResult<()> {
require_super_admin(&db, &authed).await?;
let tag = body.tag.as_deref().unwrap_or("datatable");
windmill_common::external_instance_pg::create_external_instance_database_unchecked(
&db, &dbname, tag, None,
)
.await?;
windmill_audit::audit_oss::audit_log(
&db,
&authed,
"settings.create_external_instance_pg_database",
windmill_audit::ActionKind::Create,
"global",
Some(&authed.email),
Some([("dbname", dbname.as_str()), ("tag", tag)].into()),
)
.await?;
Ok(Json(()))
}
async fn drop_external_instance_pg_database(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path(dbname): Path<String>,
) -> JsonResult<()> {
require_super_admin(&db, &authed).await?;
// A data table naming a dropped database fails on every job, far from the drop that caused it.
windmill_common::external_instance_pg::drop_external_instance_database_unchecked(
&db, &dbname, None,
)
.await?;
windmill_audit::audit_oss::audit_log(
&db,
&authed,
"settings.drop_external_instance_pg_database",
windmill_audit::ActionKind::Delete,
"global",
Some(&authed.email),
Some([("dbname", dbname.as_str())].into()),
)
.await?;
Ok(Json(()))
}
#[derive(Deserialize)]
struct SetupCustomInstanceDbBody {
tag: Option<String>,
@@ -1741,12 +1916,31 @@ async fn setup_custom_instance_pg_database(
Path(dbname): Path<String>,
Json(body): Json<SetupCustomInstanceDbBody>,
) -> JsonResult<CustomInstanceDb> {
// Before anything is recorded: the status written below replaces the registry entry, and with it
// the workspace a fork copy is reserved for.
require_super_admin(&db, &authed).await?;
// A re-run keeps the fork reservation: without it, the workspace the copy was made for could no
// longer import into it or finish its fork.
let workspace_id = sqlx::query_scalar::<_, Option<String>>(
"SELECT value->'databases'->$1->>'workspace_id' FROM global_settings
WHERE name = 'custom_instance_pg_databases'",
)
.bind(&dbname)
.fetch_optional(&db)
.await?
.flatten();
let mut logs = CustomInstanceDbLogs::default();
let result = setup_custom_instance_pg_database_inner(authed, &db, &dbname, &mut logs).await;
let success = result.is_ok();
let error = result.err().map(|e| e.to_string());
let status =
CustomInstanceDb { logs, success, error, tag: body.tag, used_by_workspaces: vec![] };
let status = CustomInstanceDb {
logs,
success,
error,
tag: body.tag,
used_by_workspaces: vec![],
workspace_id,
};
let status_json = serde_json::to_value(&status).map_err(to_anyhow)?;
// Save that the database was setup successfully
sqlx::query!(
+34 -5
View File
@@ -1703,14 +1703,25 @@ async fn delete_user(
.await?;
windmill_common::user_drafts::delete_drafts_of_email(&mut *tx, &email_to_delete).await?;
let usernames = sqlx::query_scalar!(
"DELETE FROM usr WHERE email = $1 RETURNING username",
let memberships = sqlx::query!(
"DELETE FROM usr WHERE email = $1 RETURNING username, workspace_id",
&email_to_delete
)
.fetch_all(&mut *tx)
.await?;
for username in usernames {
for row in memberships {
let username = row.username;
// A tenant list names a principal of its workspace, so the name has to be freed in every
// workspace this account belonged to: a later account taking the username would otherwise
// inherit the data table access it had.
windmill_common::workspaces::remove_datatable_tenant_in_workspace(
&mut tx,
&row.workspace_id,
&format!("u/{username}"),
)
.await?;
sqlx::query!("DELETE FROM password WHERE email = $1", &email_to_delete)
.execute(&mut *tx)
.await?;
@@ -2456,6 +2467,15 @@ pub async fn delete_workspace_user_internal(
tx: &mut Transaction<'_, Postgres>,
authed: Option<&ApiAuthed>, // None for system operations
) -> Result<()> {
// Same reasoning as the `extra_perms` sweep below: a freed username must not stay named
// anywhere that grants access, tenant lists included.
windmill_common::workspaces::remove_datatable_tenant_in_workspace(
tx,
w_id,
&format!("u/{username_to_delete}"),
)
.await?;
// ---- Clean up extra_perms referencing this user ----
let extra_perms_tables = [
"script",
@@ -3882,8 +3902,8 @@ async fn update_token_label(
Path(token_prefix): Path<String>,
Json(req): Json<UpdateTokenLabelRequest>,
) -> Result<String> {
// The new label must not collide with a system-token namespace (`session`,
// `ephemeral*`, `debugger-token`, `mcp-oauth-*`): those labels are
// The new label must not collide with a system-token namespace (see
// `windmill_common::auth::is_user_token`): those labels are
// load-bearing, and a user-set collision would orphan the token — hidden
// from the UI (`isUserToken`) and rejected by the editability guard below —
// while it still authenticates. (`is_user_token(None)` is true, so clearing
@@ -3922,6 +3942,9 @@ async fn update_token_label(
AND lower(label) NOT LIKE 'ephemeral%'
AND label <> 'debugger-token'
AND label NOT LIKE 'mcp-oauth-%'
AND NOT starts_with(label, 'embed_app:')
AND NOT starts_with(label, 'sdk_app:')
AND NOT starts_with(label, 'impersonation:')
))
RETURNING token_prefix",
req.label.as_deref(),
@@ -3962,6 +3985,12 @@ async fn leave_workspace(
) -> Result<String> {
forbid_job_token_account_destruction(&authed)?;
let mut tx = db.begin().await?;
windmill_common::workspaces::remove_datatable_tenant_in_workspace(
&mut tx,
&w_id,
&format!("u/{}", authed.username),
)
.await?;
sqlx::query!(
"DELETE FROM usr WHERE workspace_id = $1 AND username = $2",
&w_id,
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,44 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! Where the ACL planner comes from: the enterprise one, or a refusal.
//!
//! Data table roles are an Enterprise Edition feature, and so is everything here — reading who
//! owns what included. `private` alone is not that edition — community builds carry it — so the
//! planner is behind `enterprise` as well.
#[cfg(all(feature = "private", feature = "enterprise"))]
pub(crate) use crate::datatable_acl_ee::plan_statements;
#[cfg(all(feature = "private", feature = "enterprise"))]
pub(crate) fn ensure_datatable_acl_available() -> windmill_common::error::Result<()> {
Ok(())
}
#[cfg(not(all(feature = "private", feature = "enterprise")))]
use {
crate::datatable_acl::{AclChange, AclPlan, AclTarget, CatalogFacts},
windmill_common::{datatable_roles_oss::datatable_roles_unavailable, error::Result},
};
/// Checked first by every ACL route, before anything is read or connected to.
#[cfg(not(all(feature = "private", feature = "enterprise")))]
pub(crate) fn ensure_datatable_acl_available() -> Result<()> {
Err(datatable_roles_unavailable())
}
#[cfg(not(all(feature = "private", feature = "enterprise")))]
pub(crate) fn plan_statements(
_target: &AclTarget,
_change: &AclChange,
_dbname: &str,
_pg_role: &str,
_facts: &CatalogFacts,
) -> Result<AclPlan> {
Err(datatable_roles_unavailable())
}
@@ -11,7 +11,7 @@
//! to keep that file focused on core workspace configuration.
use crate::workspaces::{
is_instance_datatable, pg_dump_database, strip_unreplayable_dump_lines, ItemComparison,
managed_datatable_kind, pg_dump_database, strip_unreplayable_dump_lines, ItemComparison,
PgDumpOptions,
};
@@ -30,6 +30,7 @@ use windmill_api_auth::{require_super_admin, ApiAuthed};
use windmill_api_jobs::run_wait_result_internal;
use windmill_audit::audit_oss::audit_log;
use windmill_audit::ActionKind;
use windmill_common::datatable_roles::ADMIN_DATATABLE_ROLE;
use windmill_common::db::UserDB;
use windmill_common::error::{pg_error_message, Error, JsonResult, Result};
use windmill_common::jobs::{JobPayload, RawCode};
@@ -38,7 +39,11 @@ use windmill_common::runnable_settings::{ConcurrencySettingsWithCustom, Debounci
use windmill_common::scripts::ScriptLang;
use windmill_common::users::username_to_permissioned_as;
use windmill_common::worker::to_raw_value;
use windmill_common::workspaces::get_datatable_resource_from_db_unchecked;
use windmill_common::worker::SqlAnnotations;
use windmill_common::workspaces::{
ensure_can_use_datatable_role, ensure_datatable_admin_access,
get_datatable_resource_from_db_unchecked, resolve_governing_datatable, DatatableAccess,
};
use windmill_common::{PgDatabase, DB};
use windmill_git_sync::{
handle_deployment_metadata, handle_deployment_metadata_batch, DeployedObject,
@@ -86,6 +91,42 @@ pub(crate) fn routes() -> Router {
)
}
/// Refuse a migration whose role this caller may not use, before a job is pushed or a version
/// recorded.
///
/// A migration that declares `-- role <name>` runs as that role, so the caller has to be one of its
/// tenants. One that declares none runs as `admin` and reaches every object in the database
/// whatever the roles grant, so it is for the admins of the workspace that governs the data table
/// — a fork can run a migration under a role it holds, never a migration under `admin`.
///
/// The executor re-checks the role when it resolves the connection, so this is not the boundary. It
/// is what makes the refusal legible: which migration, and which role.
async fn ensure_migration_role_allowed(
db: &DB,
w_id: &str,
datatable_name: &str,
authed: &ApiAuthed,
sql: &str,
timestamp: i64,
name: &str,
) -> Result<()> {
let context = format!("Migration {timestamp} ({name})");
let access = DatatableAccess::Authed(authed.to_authed_ref());
match SqlAnnotations::datatable_role(sql)? {
Some(role) => {
ensure_can_use_datatable_role(db, w_id, datatable_name, Some(&role), &access, &context)
.await
}
None => ensure_datatable_admin_access(db, w_id, datatable_name, &access)
.await
.map_err(|e| {
Error::NotAuthorized(format!(
"{context} declares no role, so it would run as admin. {e}"
))
}),
}
}
#[derive(Serialize)]
struct AppliedMigration {
version: i64,
@@ -128,7 +169,18 @@ async fn datatable_database_arg(
.await?
.ok_or_else(|| Error::internal_err(format!("datatable {datatable_name} not found")))?;
Ok(to_raw_value(&format!("datatable://{datatable_name}")))
// `?role=admin` rather than a bare reference, so a migration that declares no `-- role` runs
// as the connection that owns the schema instead of falling through to the data table's
// default role — which is what `ensure_migration_role_allowed` gated it as, and which is the
// only role a DDL statement can be expected to succeed under. A migration that does declare a
// role overrides this: the annotation wins over the reference.
//
// A legacy name containing `?` cannot be migrated through this reference: the appended query
// makes it neither an exact name nor a parseable one. Accepted on purpose, since such names can
// no longer be created and none are expected to carry migrations.
Ok(to_raw_value(&format!(
"datatable://{datatable_name}?role={ADMIN_DATATABLE_ROLE}"
)))
}
/// Run a migration's SQL as a normal Windmill `postgresql` job, permissioned as
@@ -384,6 +436,11 @@ async fn run_datatable_migrations(
Path((w_id, datatable_name)): Path<(String, String)>,
Query(query): Query<RunDatatableMigrationsQuery>,
) -> JsonResult<RunDatatableMigrationsResult> {
// Before the admin connection is opened at all: the bookkeeping below is created and read
// through it, so a caller no role covers must be refused here rather than after the fact.
crate::datatable_permissions::ensure_reaches_datatable(&db, &w_id, &datatable_name, &authed)
.await?;
audit_log(
&db,
&authed,
@@ -440,6 +497,16 @@ async fn run_datatable_migrations(
if applied_versions.contains(&m.timestamp) {
continue;
}
ensure_migration_role_allowed(
&db,
&w_id,
&datatable_name,
&authed,
&m.code_up,
m.timestamp,
&m.name,
)
.await?;
run_datatable_migration_job(&db, &user_db, &authed, &w_id, &database_arg, &m.code_up)
.await
.map_err(|e| {
@@ -506,6 +573,11 @@ async fn rollback_datatable_migrations(
Path((w_id, datatable_name)): Path<(String, String)>,
Query(query): Query<RollbackDatatableMigrationsQuery>,
) -> JsonResult<RollbackDatatableMigrationsResult> {
// Before the admin connection is opened at all: the bookkeeping below is created and read
// through it, so a caller no role covers must be refused here rather than after the fact.
crate::datatable_permissions::ensure_reaches_datatable(&db, &w_id, &datatable_name, &authed)
.await?;
audit_log(
&db,
&authed,
@@ -588,6 +660,17 @@ async fn rollback_datatable_migrations(
))
})?;
ensure_migration_role_allowed(
&db,
&w_id,
&datatable_name,
&authed,
&code_down,
version,
&definition.name,
)
.await?;
let database_arg = datatable_database_arg(&db, &w_id, &datatable_name).await?;
run_datatable_migration_job(&db, &user_db, &authed, &w_id, &database_arg, &code_down)
.await
@@ -748,10 +831,15 @@ async fn read_applied_datatable_versions(
/// List a data table's migrations annotated with whether each has been applied.
async fn datatable_migrations_status(
_authed: ApiAuthed,
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path((w_id, datatable_name)): Path<(String, String)>,
) -> JsonResult<DatatableMigrationsStatusResult> {
// Reads `_wm_migrations` through the data table's admin connection, so it answers to the same
// question as running one: may you reach this data table at all.
crate::datatable_permissions::ensure_reaches_datatable(&db, &w_id, &datatable_name, &authed)
.await?;
let enabled = datatable_migrations_enabled(&db, &w_id, &datatable_name).await?;
if !enabled {
return Ok(Json(DatatableMigrationsStatusResult {
@@ -1431,6 +1519,15 @@ async fn generate_initial_datatable_migration(
Extension(db): Extension<DB>,
Path((w_id, datatable_name)): Path<(String, String)>,
) -> JsonResult<DatatableMigration> {
// Returns a `pg_dump` of the whole schema and writes into the data table's own bookkeeping, so
// it answers to the workspace that governs it rather than to whoever is asking.
ensure_datatable_admin_access(
&db,
&w_id,
&datatable_name,
&DatatableAccess::Authed(authed.to_authed_ref()),
)
.await?;
validate_datatable_path_segment(&datatable_name)?;
ensure_datatable_migrations_enabled(&db, &w_id, &datatable_name).await?;
@@ -1459,7 +1556,9 @@ async fn generate_initial_datatable_migration(
// without what a replay elsewhere cannot run: the replaying user owns none of this
// database's objects, and the grants Windmill plants in an instance database (`ALTER
// DEFAULT PRIVILEGES FOR ROLE ...`) fail even replaying onto the same server.
let no_acl = is_instance_datatable(&db, &w_id, &datatable_name).await?;
let no_acl = managed_datatable_kind(&db, &w_id, &datatable_name)
.await?
.is_some();
let dump_file = pg_dump_database(
&pg_db,
PgDumpOptions {
@@ -1601,9 +1700,21 @@ pub(crate) struct DatatableRename {
pub(crate) to: String,
}
async fn resolve_datatable_pg(db: &DB, w_id: &str, datatable: &str) -> Result<PgDatabase> {
/// The database whose `_wm_migrations` a rename or delete of `datatable` in `w_id` should touch —
/// `None` when that is somebody else's.
///
/// A fork's entry points at the workspace that governs the data table, so renaming or removing it
/// changes what the fork calls the data table and nothing more. Following the pointer here would
/// let a fork admin relabel or wipe the *governing* workspace's migration bookkeeping through
/// their own settings form, and the parent would then re-run every migration from zero.
async fn resolve_datatable_pg(db: &DB, w_id: &str, datatable: &str) -> Result<Option<PgDatabase>> {
let governing = resolve_governing_datatable(db, w_id, datatable).await?;
if governing.workspace_id != w_id {
return Ok(None);
}
let db_resource = get_datatable_resource_from_db_unchecked(db, w_id, datatable).await?;
serde_json::from_value(db_resource)
.map(Some)
.map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))
}
@@ -1621,7 +1732,9 @@ fn ignore_missing_wm_migrations(e: tokio_postgres::Error) -> Result<()> {
/// Drop a data table's rows from its own database's `_wm_migrations`.
async fn remote_forget_datatable_migrations(db: &DB, w_id: &str, datatable: &str) -> Result<()> {
let pg_db = resolve_datatable_pg(db, w_id, datatable).await?;
let Some(pg_db) = resolve_datatable_pg(db, w_id, datatable).await? else {
return Ok(());
};
let (client, connection) = pg_db.connect(Some(db)).await?;
tokio::spawn(async move {
let _ = connection.await;
@@ -1646,7 +1759,9 @@ async fn remote_rename_datatable_migrations(
from: &str,
to: &str,
) -> Result<()> {
let pg_db = resolve_datatable_pg(db, w_id, resolve_by).await?;
let Some(pg_db) = resolve_datatable_pg(db, w_id, resolve_by).await? else {
return Ok(());
};
let (client, connection) = pg_db.connect(Some(db)).await?;
tokio::spawn(async move {
let _ = connection.await;
@@ -0,0 +1,79 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! Who may connect to a data table as which role.
//!
//! The decision lives on the data table entry of the workspace that governs it, which is not
//! necessarily the workspace asking: a fork's entry points at its parent's, and everything here
//! resolves through that pointer first. Nothing in this module runs SQL against the data table —
//! a save is tenant lists and a default, and the Postgres roles themselves are the instance
//! catalog's business.
use axum::{routing::get, Router};
use windmill_api_auth::ApiAuthed;
use windmill_common::error::Result;
use windmill_common::workspaces::GoverningDatatable;
use windmill_common::DB;
use crate::datatable_permissions_oss as roles;
pub(crate) fn routes() -> Router {
Router::new()
.route(
"/datatable_permissions/{datatable_name}",
get(roles::get_datatable_permissions).post(roles::set_datatable_permissions),
)
.route(
"/datatable_usable_roles/{datatable_name}",
get(roles::list_usable_datatable_roles),
)
}
/// Administering a data table — its permissions, its migrations that declare no role, its exports
/// — is for the admins of the workspace that governs it. A fork can use the data table; it never
/// administers it.
// The gate for whatever administers a data table under roles, which the routes of this module alone
// do not always reach.
#[allow(dead_code)]
pub(crate) async fn ensure_governs_datatable(
db: &DB,
authed: &ApiAuthed,
w_id: &str,
governing: &GoverningDatatable,
) -> Result<()> {
roles::ensure_governs_datatable(db, authed, w_id, governing).await
}
/// Refuse a caller that no tenant of this data table covers.
///
/// The bookkeeping endpoints below open the data table's `admin` connection to read or create
/// `_wm_migrations` before they know which migration will run — so without this, someone covered
/// by no role at all can still force admin-backed reads and writes on a database they may not
/// touch. It asks only "may you reach this data table as anything"; which role a given migration
/// runs as is still decided per migration, and by the executor after that.
pub(crate) async fn ensure_reaches_datatable(
db: &DB,
w_id: &str,
datatable_name: &str,
authed: &ApiAuthed,
) -> Result<()> {
roles::ensure_reaches_datatable(db, w_id, datatable_name, authed).await
}
/// [`ensure_reaches_datatable`] against an entry already resolved, for a caller that goes on to
/// connect from that same entry.
pub(crate) async fn ensure_reaches_governing_datatable(
db: &DB,
w_id: &str,
datatable_name: &str,
governing: &GoverningDatatable,
authed: &ApiAuthed,
) -> Result<()> {
roles::ensure_reaches_governing_datatable(db, w_id, datatable_name, governing, authed).await
}
@@ -0,0 +1,112 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! Where the data table permissions endpoints and their gates come from: the enterprise
//! implementation, or a refusal. Roles are an Enterprise Edition feature; see
//! `windmill_common::datatable_roles_oss`.
#[cfg(all(feature = "private", feature = "enterprise"))]
pub(crate) use crate::datatable_permissions_ee::{
ensure_governs_datatable, ensure_reaches_datatable, ensure_reaches_governing_datatable,
get_datatable_permissions, list_usable_datatable_roles, set_datatable_permissions,
usable_datatable_roles,
};
#[cfg(not(all(feature = "private", feature = "enterprise")))]
pub(crate) use ce::*;
#[cfg(not(all(feature = "private", feature = "enterprise")))]
mod ce {
use windmill_api_auth::ApiAuthed;
use windmill_common::{
datatable_roles_oss::datatable_roles_unavailable as unavailable,
error::Result,
workspaces::{resolve_governing_datatable, GoverningDatatable},
DB,
};
/// Nobody administers a data table's roles without them.
#[allow(dead_code)]
pub(crate) async fn ensure_governs_datatable(
_db: &DB,
_authed: &ApiAuthed,
_w_id: &str,
_governing: &GoverningDatatable,
) -> Result<()> {
Err(unavailable())
}
/// A data table not under roles is reached as it was before roles existed. One under roles is
/// refused: no role of it can be connected as.
pub(crate) async fn ensure_reaches_datatable(
db: &DB,
w_id: &str,
datatable_name: &str,
_authed: &ApiAuthed,
) -> Result<()> {
let governing = resolve_governing_datatable(db, w_id, datatable_name).await?;
if governing.datatable.permissions.is_none() {
Ok(())
} else {
Err(unavailable())
}
}
pub(crate) async fn ensure_reaches_governing_datatable(
_db: &DB,
_w_id: &str,
_datatable_name: &str,
governing: &GoverningDatatable,
_authed: &ApiAuthed,
) -> Result<()> {
if governing.datatable.permissions.is_none() {
Ok(())
} else {
Err(unavailable())
}
}
// The routes stay registered so the API has one shape; each answers after authentication,
// before anything is read.
pub(crate) async fn get_datatable_permissions(_authed: ApiAuthed) -> Result<String> {
Err(unavailable())
}
pub(crate) async fn set_datatable_permissions(_authed: ApiAuthed) -> Result<String> {
Err(unavailable())
}
pub(crate) async fn list_usable_datatable_roles(_authed: ApiAuthed) -> Result<String> {
Err(unavailable())
}
pub(crate) struct UsableDatatableRoles {
pub(crate) permissioned: bool,
pub(crate) roles: Vec<String>,
pub(crate) default_role: String,
}
/// A data table not under roles is used as `admin`, as before roles existed. One under roles
/// is refused: no role of it can be connected as.
pub(crate) async fn usable_datatable_roles(
_db: &DB,
_authed: &ApiAuthed,
_w_id: &str,
governing: &GoverningDatatable,
) -> Result<UsableDatatableRoles> {
if governing.datatable.permissions.is_some() {
return Err(unavailable());
}
Ok(UsableDatatableRoles {
permissioned: false,
roles: vec![],
default_role: windmill_common::datatable_roles::ADMIN_DATATABLE_ROLE.to_string(),
})
}
}
@@ -1,7 +1,11 @@
#[cfg(feature = "parquet")]
pub mod ai_session_backups;
pub mod data_metrics;
pub mod datatable_acl;
pub mod datatable_acl_oss;
pub mod datatable_migrations;
pub mod datatable_permissions;
pub mod datatable_permissions_oss;
pub mod deployment_requests;
pub mod workspaces;
pub mod workspaces_extra;
@@ -9,3 +13,9 @@ pub mod workspaces_oss;
#[cfg(feature = "private")]
pub mod workspaces_ee;
#[cfg(all(feature = "private", feature = "enterprise"))]
pub mod datatable_acl_ee;
#[cfg(all(feature = "private", feature = "enterprise"))]
pub mod datatable_permissions_ee;
File diff suppressed because it is too large Load Diff
@@ -110,6 +110,21 @@ pub(crate) async fn change_workspace_id(
.execute(&mut *tx)
.await?;
// A fork copy reserved for the old id would otherwise be unreachable: its creator cannot
// import into it or finish its fork under the new id, and nothing else would ever drop it.
sqlx::query(
r#"UPDATE global_settings SET value = jsonb_set(value, '{databases}', (
SELECT COALESCE(jsonb_object_agg(k, CASE WHEN v->>'workspace_id' = $1
THEN jsonb_set(v, '{workspace_id}', to_jsonb($2::text)) ELSE v END), '{}'::jsonb)
FROM jsonb_each(COALESCE(value->'databases', '{}'::jsonb)) AS e(k, v)
))
WHERE name = 'custom_instance_pg_databases'"#,
)
.bind(&old_id)
.bind(&rw.new_id)
.execute(&mut *tx)
.await?;
// Duplicate workspace settings (keep copy in old workspace for reference)
info!("Duplicating workspace_settings table");
sqlx::query!(
@@ -492,6 +507,30 @@ pub(crate) async fn change_workspace_id(
.fetch_all(&mut *tx)
.await?;
// A fork's data table entry names the workspace that governs it by id, so the rename has to
// follow there too — anywhere, not just in the reparented children: a detached workspace can
// point at this one without being its fork. Left behind, the pointer resolves to the archived
// shell and every job through it stops.
info!("Re-pointing data table references to the new workspace id");
sqlx::query!(
r#"UPDATE workspace_settings ws
SET datatable = (
SELECT jsonb_set(ws.datatable, '{datatables}', jsonb_object_agg(
dt.key,
CASE WHEN dt.value->'reference'->>'workspace_id' = $2
THEN jsonb_set(dt.value, '{reference,workspace_id}', to_jsonb($1::text))
ELSE dt.value END
))
FROM jsonb_each(ws.datatable->'datatables') dt
)
WHERE jsonb_typeof(ws.datatable->'datatables') = 'object'
AND ws.datatable::text LIKE '%"reference"%'"#,
&rw.new_id,
&old_id,
)
.execute(&mut *tx)
.await?;
info!("Updating workspace_protection_rule table");
sqlx::query!(
"UPDATE workspace_protection_rule SET workspace_id = $1 WHERE workspace_id = $2",
@@ -971,6 +1010,22 @@ pub(crate) async fn delete_workspace(
// but the destructive cleanup itself runs only after the commit below: a delete that
// fails mid-way must never leave a live workspace with its fork data destroyed and no
// registry row to retry from. Read-only: nothing is dropped here.
// Read before the delete: another workspace's data table entry can point at one of this
// workspace's, and deleting the workspace it names leaves that pointer resolving to nothing.
// Nothing sweeps them — turning them back into copies would hand each fork the database
// outright — so the deleter is told which data tables they just stranded.
let stranded_pointers = sqlx::query!(
r#"SELECT ws.workspace_id AS "workspace_id!", dt.key AS "datatable!"
FROM workspace_settings ws
CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt
WHERE dt.value->'reference'->>'workspace_id' = $1
ORDER BY ws.workspace_id, dt.key"#,
&w_id,
)
.fetch_all(&db)
.await
.unwrap_or_default();
let fork_ducklake_cleanups = prepare_fork_ducklake_cleanups(&db, &w_id, None)
.await
.unwrap_or_else(|e| {
@@ -1289,7 +1344,23 @@ pub(crate) async fn delete_workspace(
tracing::warn!("failed to broadcast fork lineage change: {e:#}");
}
Ok(format!("Deleted workspace {}", &w_id))
if stranded_pointers.is_empty() {
Ok(format!("Deleted workspace {}", &w_id))
} else {
let stranded = stranded_pointers
.iter()
.map(|r| format!("{}/{}", r.workspace_id, r.datatable))
.collect::<Vec<_>>()
.join(", ");
Ok(format!(
concat!(
"Deleted workspace {}. These data tables were governed by it and no longer ",
"resolve: {}. Their databases still exist; a superadmin can point them at ",
"another workspace's data table."
),
&w_id, stranded
))
}
}
#[derive(Deserialize)]
@@ -1343,15 +1414,18 @@ pub async fn drop_forked_datatable_databases(
let mut errors: Vec<String> = Vec::new();
for dt_name in &req.datatable_names {
let dt = match datatables.get(dt_name) {
Some(dt) if dt.forked_from.is_some() => dt,
// Only a clone is droppable, and a clone is terminal by construction: a kept data table is
// a pointer at the parent's database, which this fork does not own.
let database = match datatables.get(dt_name) {
Some(dt) if dt.forked_from.is_some() => match dt.database.as_ref() {
Some(database) => database,
None => continue,
},
_ => continue,
};
if dt.database.resource_type
== windmill_common::workspaces::DataTableCatalogResourceType::Instance
{
let db_to_drop = &dt.database.resource_path;
if database.resource_type.is_windmill_managed() {
let db_to_drop = &database.resource_path;
if !db_to_drop.starts_with("wm_fork_") {
errors.push(format!(
"Refusing to drop instance database '{}' for datatable://{}: name does not start with 'wm_fork_'",
@@ -1359,7 +1433,42 @@ pub async fn drop_forked_datatable_databases(
));
continue;
}
if let Err(e) = windmill_common::drop_custom_instance_database(&db, db_to_drop).await {
// The fork's own entry is what is going away; anything else still reaching the copy,
// a child fork's pointer at this entry included, keeps it. The lock keeps a child fork
// from gaining such a pointer before the drop.
let dropped = async {
let mut tx = db.begin().await?;
windmill_common::workspaces::lock_fork_datatables(&mut tx, &w_id).await?;
if database.resource_type
== windmill_common::workspaces::DataTableCatalogResourceType::ExternalInstance
{
windmill_common::external_instance_pg::drop_external_instance_database_unchecked(
&db,
db_to_drop,
Some((&w_id, dt_name)),
)
.await?;
} else {
let uses = windmill_common::workspaces::managed_database_uses(
&mut tx,
windmill_common::workspaces::DataTableCatalogResourceType::Instance,
db_to_drop,
Some((&w_id, dt_name)),
)
.await?;
if !uses.is_empty() {
return Err(Error::BadRequest(format!(
"it is still used by {}",
uses.join(", ")
)));
}
windmill_common::drop_custom_instance_database(&db, db_to_drop).await?;
}
tx.commit().await?;
Ok::<_, Error>(())
}
.await;
if let Err(e) = dropped {
errors.push(format!(
"Could not drop instance database '{}' for datatable://{}: {}",
db_to_drop, dt_name, e
@@ -1726,7 +1835,17 @@ async fn resolve_fork_catalog_pg(
"ducklake://{ducklake_name}: malformed registry catalog identity `{catalog}`"
))
})?;
let catalog_resource = if resource_type == "instance" {
let catalog_resource = if resource_type == "external_instance" {
serde_json::to_value(
windmill_common::external_instance_pg::external_instance_connection_unchecked(
db,
resource_path,
false,
)
.await?,
)
.map_err(|e| Error::internal_err(format!("serializing pg creds: {e}")))?
} else if resource_type == "instance" {
let mut pg_creds = windmill_common::PgDatabase::parse_uri(
&windmill_common::get_database_url().await?.as_str().await,
)?;
+832 -6
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.812.0
version: 1.813.0
title: Windmill API
contact:
@@ -1572,6 +1572,104 @@ paths:
schema:
type: object
/settings/external_instance_pg/status:
get:
summary: Returns whether the external instance cluster is configured and how its last setup went
operationId: getExternalInstancePgStatus
tags:
- setting
responses:
"200":
description: external instance cluster status
content:
application/json:
schema:
$ref: "#/components/schemas/ExternalInstancePgStatus"
/settings/external_instance_pg/setup:
post:
summary: Sets up the external instance cluster with its saved admin login, optionally rotating the passwords Windmill manages on it (enterprise edition only)
operationId: setupExternalInstancePg
tags:
- setting
requestBody:
required: true
content:
application/json:
schema:
type: object
properties:
rotate_passwords:
type: boolean
responses:
"200":
description: the setup report, also stored as the last setup
content:
application/json:
schema:
$ref: "#/components/schemas/ExternalInstancePgSetupReport"
/settings/external_instance_pg/databases:
get:
summary: Lists the databases Windmill created on the external instance cluster, with the workspaces whose data tables, Ducklake catalogs or pending fork cleanups use each
operationId: listExternalInstancePgDatabases
tags:
- setting
responses:
"200":
description: databases by name
content:
application/json:
schema:
type: object
additionalProperties:
$ref: "#/components/schemas/CustomInstanceDb"
/settings/external_instance_pg/databases/{name}:
post:
summary: Creates a database on the external instance cluster (enterprise edition only)
operationId: createExternalInstancePgDatabase
tags:
- setting
parameters:
- name: name
in: path
required: true
schema:
type: string
requestBody:
required: true
content:
application/json:
schema:
type: object
properties:
tag:
$ref: "#/components/schemas/CustomInstanceDbTag"
responses:
"200":
description: database created
content:
application/json:
schema: {}
delete:
summary: Drops a database Windmill created on the external instance cluster, refused while a data table, Ducklake catalog or pending fork cleanup uses it
operationId: dropExternalInstancePgDatabase
tags:
- setting
parameters:
- name: name
in: path
required: true
schema:
type: string
responses:
"200":
description: database dropped
content:
application/json:
schema: {}
/settings/list_custom_instance_pg_databases:
post:
summary: Returns the set-up statuses of custom instance pg databases
@@ -1588,6 +1686,97 @@ paths:
additionalProperties:
$ref: "#/components/schemas/CustomInstanceDb"
/settings/datatable_roles:
get:
summary: list the data table roles of one Windmill-managed Postgres cluster
operationId: listInstanceDatatableRoles
tags:
- setting
parameters:
- in: query
name: cluster
required: false
schema:
$ref: "#/components/schemas/DatatableRoleCluster"
responses:
"200":
description: the cluster's role catalog
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/InstanceDatatableRole"
post:
summary: create a data table role on a Windmill-managed Postgres cluster
operationId: createInstanceDatatableRole
tags:
- setting
requestBody:
required: true
content:
application/json:
schema:
type: object
required: [name]
properties:
name:
type: string
cluster:
$ref: "#/components/schemas/DatatableRoleCluster"
responses:
"200":
description: the created role
content:
application/json:
schema:
$ref: "#/components/schemas/InstanceDatatableRole"
/settings/datatable_roles/{id}:
post:
summary: rename a data table role or turn its login on and off
operationId: updateInstanceDatatableRole
tags:
- setting
parameters:
- in: path
name: id
required: true
schema:
type: string
requestBody:
required: true
content:
application/json:
schema:
type: object
properties:
name:
type: string
enabled:
type: boolean
responses:
"200":
description: the updated role
content:
application/json:
schema:
$ref: "#/components/schemas/InstanceDatatableRole"
delete:
summary: drop a data table role from the cluster and from every workspace that named it
operationId: deleteInstanceDatatableRole
tags:
- setting
parameters:
- in: path
name: id
required: true
schema:
type: string
responses:
"200":
description: deleted
/settings/setup_custom_instance_pg_database/{name}:
post:
summary: Runs CREATE DATABASE on the Windmill Postgres and grants access to the custom_instance_user
@@ -5141,15 +5330,197 @@ paths:
type: array
items:
type: object
required: [name, resource_type, resource_path]
required: [name, resource_type, resource_path, permissioned]
properties:
name:
type: string
resource_type:
type: string
enum: [postgres, instance]
enum: [postgres, instance, external_instance]
resource_path:
type: string
governing_workspace_id:
type: string
permissioned:
type: boolean
/w/{workspace}/workspaces/datatable_permissions/{datatable_name}:
get:
summary: get who may connect to a data table as which role
operationId: getDatatablePermissions
tags:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: datatable_name
in: path
required: true
schema:
type: string
responses:
"200":
description: the data table's roles and their tenants
content:
application/json:
schema:
$ref: "#/components/schemas/DatatablePermissions"
post:
summary: set who may connect to a data table as which role
operationId: setDatatablePermissions
tags:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: datatable_name
in: path
required: true
schema:
type: string
requestBody:
required: true
content:
application/json:
schema:
type: object
required: [permissioned]
properties:
permissioned:
type: boolean
default_role:
type: string
roles:
type: array
items:
$ref: "#/components/schemas/DatatableRoleTenants"
responses:
"200":
description: status
content:
text/plain:
schema:
type: string
/w/{workspace}/workspaces/datatable_acl/{datatable_name}:
get:
summary: read the owner and grants of an instance data table's database, schema or table
operationId: getDatatableAcl
tags:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: datatable_name
in: path
required: true
schema:
type: string
- name: kind
in: query
required: true
schema:
type: string
enum: [database, schema, table]
- name: schema
in: query
required: false
schema:
type: string
- name: table
in: query
required: false
schema:
type: string
responses:
"200":
description: owner and grants
content:
application/json:
schema:
$ref: "#/components/schemas/DatatableAclInfo"
/w/{workspace}/workspaces/datatable_acl/{datatable_name}/plan:
post:
summary: preview the SQL an ownership or grant change would run (data table administrators only)
operationId: planDatatableAcl
tags:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: datatable_name
in: path
required: true
schema:
type: string
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/AclChangeRequest"
responses:
"200":
description: statements that would run, in a single transaction
content:
application/json:
schema:
$ref: "#/components/schemas/AclPlan"
/w/{workspace}/workspaces/datatable_acl/{datatable_name}/apply:
post:
summary: run an ownership or grant change exactly as planned (data table administrators only)
operationId: applyDatatableAcl
tags:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: datatable_name
in: path
required: true
schema:
type: string
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/AclChangeRequest"
responses:
"200":
description: change applied
content:
text/plain:
schema:
type: string
/w/{workspace}/workspaces/datatable_usable_roles/{datatable_name}:
get:
summary: list the data table roles the caller may connect as
operationId: listUsableDatatableRoles
tags:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: datatable_name
in: path
required: true
schema:
type: string
responses:
"200":
description: usable roles
content:
application/json:
schema:
type: object
required: [permissioned, roles, default_role]
properties:
permissioned:
type: boolean
roles:
type: array
items:
type: string
default_role:
type: string
/w/{workspace}/workspaces/list_datatable_schemas:
get:
@@ -5223,6 +5594,21 @@ paths:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: datatable_name
in: query
description: list only this data table; each listed data table opens a connection to its database
schema:
type: string
- name: role_for
in: query
description: the data table `role` applies to; every other one is listed as its default role
schema:
type: string
- name: role
in: query
description: the role to list `role_for` as; refused, in that entry's `error`, if the caller may not use it
schema:
type: string
responses:
"200":
description: table metadata of all datatables
@@ -5256,6 +5642,11 @@ paths:
required: true
schema:
type: string
- name: role
in: query
description: the data table role to read the table as; defaults to the data table's default role
schema:
type: string
responses:
"200":
description: schema of one datatable table
@@ -5331,7 +5722,22 @@ paths:
description: status
content:
application/json:
schema: {}
schema:
type: object
properties:
stranded_references:
description: >-
Data tables in other workspaces that were governed by one this save deleted
and no longer resolve.
type: array
items:
type: object
required: [workspace_id, datatable]
properties:
workspace_id:
type: string
datatable:
type: string
/w/{workspace}/workspaces/run_datatable_migrations/{datatable_name}:
post:
@@ -12010,6 +12416,13 @@ paths:
properties:
draft_only:
type: boolean
chat_input_enabled:
type: boolean
description: |
`chat_input_enabled` of the flow's value,
projected so the list can mark flows that open
as a chat. Omitted when the value has no such
field.
is_draft:
type: boolean
description: |
@@ -28951,6 +29364,12 @@ components:
type: boolean
has_deploy_errors:
type: boolean
chat_input_enabled:
type: boolean
description: >-
flow-only. `chat_input_enabled` of the flow's value, projected so
the list can mark flows that open as a chat. Omitted when the
value has no such field.
raw_app:
type: boolean
execution_mode:
@@ -33308,6 +33727,365 @@ components:
- ducklake
- datatable
ExternalInstancePgSetupStep:
type: object
required: [name, status, message]
properties:
name:
type: string
status:
type: string
enum: [ok, warning, error]
message:
type: string
ExternalInstancePgSetupReport:
type: object
required: [success, finished_at, steps]
properties:
success:
type: boolean
description: no step failed; warnings leave it true
finished_at:
type: string
format: date-time
steps:
type: array
items:
$ref: "#/components/schemas/ExternalInstancePgSetupStep"
ExternalInstancePgStatus:
type: object
required: [configured, database_count]
properties:
configured:
type: boolean
database_count:
type: integer
last_setup:
$ref: "#/components/schemas/ExternalInstancePgSetupReport"
DatatableRoleCluster:
type: string
description: >-
The Windmill-managed Postgres cluster a data table role is a login on: Windmill's own
(behind `instance` data tables) or the external instance cluster (behind
`external_instance` ones). Defaults to `instance`.
enum: [instance, external_instance]
InstanceDatatableRole:
type: object
required: [id, name, enabled, cluster]
properties:
id:
type: string
name:
type: string
enabled:
type: boolean
cluster:
$ref: "#/components/schemas/DatatableRoleCluster"
DatatableRoleTenants:
type: object
required: [id, tenants]
properties:
id:
type: string
name:
type: string
tenants:
type: array
items:
type: string
DatatablePermissions:
type: object
required: [supported, permissioned, default_role, roles, editable, available_roles]
properties:
supported:
type: boolean
description: >-
Whether this data table can be put under roles at all. Only one on a database Windmill
manages can: a role is a login on that database's cluster.
cluster:
$ref: "#/components/schemas/DatatableRoleCluster"
permissioned:
type: boolean
default_role:
type: string
roles:
type: array
items:
$ref: "#/components/schemas/DatatableRoleTenants"
governing_workspace_id:
type: string
editable:
type: boolean
available_roles:
type: array
items:
$ref: "#/components/schemas/InstanceDatatableRole"
ungoverned_reachers:
type: array
items:
type: object
required: [workspace_id, datatable]
properties:
workspace_id:
type: string
datatable:
type: string
AclTarget:
description: what access is read or changed on
oneOf:
- $ref: "#/components/schemas/AclTargetDatabase"
- $ref: "#/components/schemas/AclTargetSchema"
- $ref: "#/components/schemas/AclTargetTable"
discriminator:
propertyName: kind
mapping:
database: "#/components/schemas/AclTargetDatabase"
schema: "#/components/schemas/AclTargetSchema"
table: "#/components/schemas/AclTargetTable"
AclTargetDatabase:
type: object
required: [kind]
properties:
kind:
type: string
enum: [database]
AclTargetSchema:
type: object
required: [kind, schema]
properties:
kind:
type: string
enum: [schema]
schema:
type: string
AclTargetTable:
type: object
required: [kind, schema, table]
properties:
kind:
type: string
enum: [table]
schema:
type: string
table:
type: string
AclChange:
description: one change to plan or apply
oneOf:
- $ref: "#/components/schemas/AclChangeSetOwner"
- $ref: "#/components/schemas/AclChangeGrant"
- $ref: "#/components/schemas/AclChangeRevoke"
discriminator:
propertyName: type
mapping:
set_owner: "#/components/schemas/AclChangeSetOwner"
grant: "#/components/schemas/AclChangeGrant"
revoke: "#/components/schemas/AclChangeRevoke"
AclChangeSetOwner:
type: object
description: >-
hands the target to role — for a schema, with everything already in it but an extension's
members, which stay with the extension
required: [type, role]
properties:
type:
type: string
enum: [set_owner]
role:
type: string
description: a data table role of the instance, or admin
AclChangeGrant:
type: object
required: [type, role, privileges, scope]
properties:
type:
type: string
enum: [grant]
role:
type: string
description: a data table role of the instance, or admin
privileges:
type: array
items:
type: string
scope:
$ref: "#/components/schemas/AclGrantScope"
AclChangeRevoke:
type: object
required: [type, role, privileges, scope]
properties:
type:
type: string
enum: [revoke]
role:
type: string
description: a data table role of the instance, other than admin
privileges:
type: array
items:
type: string
scope:
$ref: "#/components/schemas/AclGrantScope"
objects:
type: array
description: >-
objects inside the target the revoke covers, empty for the target itself. Only with the
target scope; a revoke on all objects of a kind is refused, since it cannot say which
grants it takes back.
items:
$ref: "#/components/schemas/AclObject"
AclGrantScope:
type: string
enum:
[
target,
all_tables,
all_sequences,
all_functions,
future_tables,
future_sequences,
future_functions,
]
AclChangeRequest:
type: object
required: [target, change]
properties:
target:
$ref: "#/components/schemas/AclTarget"
change:
$ref: "#/components/schemas/AclChange"
statements:
type: array
description: >-
The statements the plan showed. Required to apply, which plans again and refuses if
the result differs.
items:
type: string
AclPlan:
type: object
required: [statements, warnings]
properties:
statements:
type: array
items:
type: string
warnings:
type: array
items:
type: string
AclObject:
type: object
required: [name, kind]
properties:
name:
type: string
kind:
type: string
description: >-
TABLE, SEQUENCE, FUNCTION, PROCEDURE or TYPE — what the object is. A revoke turns it
into the keyword it takes, ROUTINE for both routine kinds; a type's grants are read
only.
args:
type: string
description: identity arguments of a routine, which is what tells two of the same name apart
AclGrant:
type: object
required: [grantee, privileges, sources]
properties:
grantee:
type: string
privileges:
type: array
items:
type: string
object:
$ref: "#/components/schemas/AclObject"
future:
type: string
description: >-
set for a default privilege, naming the kind of object it covers (TABLES, SEQUENCES,
FUNCTIONS, TYPES, or SCHEMAS). On a schema, the defaults set in that schema; on the
database, the ones set database-wide, which apply in every schema and which no schema's
own defaults take back.
sources:
type: array
description: >-
the roles the grant comes from, each once — who granted it, or for a default privilege
the role whose future objects it covers. A revoke of some of the grant's privileges
takes them back from every source that gave them.
items:
$ref: "#/components/schemas/AclSource"
AclSource:
type: object
required: [role, privileges, reachable]
properties:
role:
type: string
privileges:
type: array
description: >-
what role gave of the grant's privileges. A revoke is held back only by a source out of
reach that gave some of what it takes back.
items:
type: string
reachable:
type: boolean
description: >-
whether the data table's connection can take back what role gave. On an object that is
the owner, when the connection acts for the owner, and otherwise the connection itself;
for a default privilege, a creating role the connection acts for. What a source out of
reach gave is not revocable from here; privileges only other sources gave still are.
DatatableAclInfo:
type: object
required: [owner, roles, editable, supports_maintain, dbname, grants, children]
properties:
owner:
type: string
roles:
type: array
description: the roles a change may name; empty unless the caller may change anything
items:
type: string
editable:
type: boolean
description: whether the caller may plan and apply changes
supports_maintain:
type: boolean
description: whether the server is Postgres 17+, which added the MAINTAIN table privilege
dbname:
type: string
description: the database the target lives in
grants:
type: array
items:
$ref: "#/components/schemas/AclGrant"
children:
type: array
description: a database's schemas, or a schema's tables
items:
type: string
CustomInstanceDb:
type: object
required:
@@ -33332,6 +34110,9 @@ components:
items:
type: string
description: Workspaces that reference this database via a ducklake catalog or datatable database with resource_type 'instance'. Computed at request time, not persisted.
workspace_id:
type: string
description: The workspace a member created this database for as a fork copy. Only that workspace can import into it or point a fork at it.
NewSqsTrigger:
type: object
@@ -35305,6 +36086,7 @@ components:
- postgresql
- mysql
- instance
- external_instance
resource_path:
type: string
required:
@@ -35356,9 +36138,11 @@ components:
type: object
additionalProperties:
type: object
required: [database]
properties:
database:
description: >-
Set on an entry that owns its database. Absent on a fork's entry, which points at
another workspace's data table instead.
type: object
properties:
resource_type:
@@ -35366,10 +36150,22 @@ components:
enum:
- postgresql
- instance
- external_instance
resource_path:
type: string
required:
- resource_type
reference:
description: >-
The workspace and data table that govern this one. Server-owned: written by fork
creation, and carried across a settings save whatever the request says.
type: object
required: [workspace_id, datatable]
properties:
workspace_id:
type: string
datatable:
type: string
migrations_enabled:
type: boolean
description: Whether the SQL migrations feature is opted in for this data table
@@ -35438,7 +36234,17 @@ components:
DataTableTables:
type: object
required: [datatable_name, schemas]
required:
[
datatable_name,
schemas,
instance,
permissioned,
usable_roles,
default_role,
can_create_schema,
creatable_schemas,
]
properties:
datatable_name:
type: string
@@ -35451,6 +36257,26 @@ components:
type: string
error:
type: string
instance:
type: boolean
description: on the instance database, the only kind that can be under roles or have its access edited
permissioned:
type: boolean
usable_roles:
type: array
description: the roles the caller may connect as, by name; empty when not under roles
items:
type: string
default_role:
type: string
can_create_schema:
type: boolean
description: whether the role the listing connected as may create schemas
creatable_schemas:
type: array
description: the schemas the role the listing connected as may create in
items:
type: string
DataTableTableSchema:
type: object
+9 -3
View File
@@ -57,7 +57,7 @@ use windmill_audit::audit_oss::{audit_log, AuditAuthorable};
use windmill_audit::ActionKind;
use windmill_common::{
apps::{AppScriptId, ListAppQuery, APP_WORKSPACED_ROUTE},
auth::TOKEN_PREFIX_LEN,
auth::{APP_EMBED_TOKEN_LABEL_PREFIX, RAW_APP_SDK_TOKEN_LABEL_PREFIX, TOKEN_PREFIX_LEN},
cache::{self, future::FutureCachedExt},
db::{DbWithOptAuthed, UserDB},
error::{to_anyhow, Error, JsonResult, Result},
@@ -1522,7 +1522,10 @@ async fn mint_raw_app_sdk_token(
scopes.push(windmill_api_auth::scopes::GUEST_SENTINEL.to_string());
(label, exp)
}
None => (format!("sdk_app:{app_path}"), requested_exp),
None => (
format!("{RAW_APP_SDK_TOKEN_LABEL_PREFIX}{app_path}"),
requested_exp,
),
};
let token_config = NewToken::new(
Some(label),
@@ -1804,7 +1807,10 @@ pub async fn mint_app_embed_token(
scopes.push(windmill_api_auth::scopes::GUEST_SENTINEL.to_string());
(label, exp)
}
None => (format!("embed_app:{app_path}"), requested_exp),
None => (
format!("{APP_EMBED_TOKEN_LABEL_PREFIX}{app_path}"),
requested_exp,
),
};
let token_config = NewToken::new(
Some(label),
+6
View File
@@ -567,6 +567,9 @@ async fn set_config(
};
let mut tx = user_db.begin(&authed).await?;
if matches!(nc.trigger_kind, TriggerKind::Postgres) {
windmill_common::datatable_roles::lock_datatable_streams(&mut *tx, false).await?;
}
sqlx::query!(
r#"
@@ -614,6 +617,9 @@ async fn ping_config(
)>,
) -> Result<()> {
let mut tx = user_db.begin(&authed).await?;
if matches!(trigger_kind, TriggerKind::Postgres) {
windmill_common::datatable_roles::lock_datatable_streams(&mut *tx, false).await?;
}
sqlx::query!(
r#"
+11 -4
View File
@@ -8343,8 +8343,9 @@ pub async fn run_wait_result_flow_by_version(
/// job lives, in particular DuckDB, which runs in-process in the worker.
///
/// What it does permit is any statement against the workspace's data tables, writes and DDL
/// included: the helper's body is an unrestricted SQL template and data tables carry no
/// per-user ACL. Narrowing that is a separate decision from this exemption.
/// included: the helper's body is an unrestricted SQL template. What that reaches is the
/// operator's own data table role — the preview job is permissioned as them, so the executor
/// resolves it under their tenancy like any other job.
///
/// The database argument is only half the target: the executor honors a `-- database`
/// directive in the SQL over it, and `-- s3` redirects the result set, so both are refused.
@@ -8576,7 +8577,7 @@ async fn run_inline_preview_script(
#[cfg(not(feature = "run_inline"))]
async fn run_inline_preview_script() -> error::Result<Response> {
Err(error::Error::InternalErr(
"inline preview requires the worker feature".to_string(),
"inline preview requires the run_inline feature on the worker".to_string(),
))
}
@@ -8700,7 +8701,12 @@ fn register_potential_assets_on_inline_execution(
.as_ref()
.and_then(|args| args.get("database"))
.map(|v| v.get().trim_matches('"'))
.and_then(|dt| dt.strip_prefix("datatable://"));
.and_then(|dt| dt.strip_prefix("datatable://"))
// `?role=` picks the connection, not the data table. Anything else after a `?` may be
// part of a name stored before names were restricted, so it stays.
.map(|dt| {
windmill_common::workspaces::parse_datatable_ref(dt).map_or(dt, |(name, _)| name)
});
if let Some(datatable) = datatable {
let re = regex::Regex::new(r#"SET search_path TO "([^"]+)";"#).unwrap();
let (schema, content) = if let Some(captures) = re.captures(&preview.content) {
@@ -9553,6 +9559,7 @@ async fn run_preview_flow_job(
&flow_path,
&run_query,
user_message.as_ref(),
uuid,
)
.await?;
}
+20 -5
View File
@@ -116,6 +116,10 @@ struct RunnableItem {
use_codebase: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
has_deploy_errors: Option<bool>,
// flow-only: projected from the value so the home list can badge flows that
// open as a chat without fetching every flow's value.
#[serde(skip_serializing_if = "Option::is_none")]
chat_input_enabled: Option<bool>,
// app-only
#[serde(skip_serializing_if = "Option::is_none")]
raw_app: Option<bool>,
@@ -274,7 +278,7 @@ fn branch_sqls() -> Branches {
o.ws_error_handler_muted, o.created_at as edited_at, \
o.hash, o.language::text as language, o.kind::text as script_kind, o.auto_kind, \
o.codebase IS NOT NULL as use_codebase, \
(o.lock_error_logs IS NOT NULL) as has_deploy_errors, \
(o.lock_error_logs IS NOT NULL) as has_deploy_errors, NULL::bool as chat_input_enabled, \
NULL::bool as raw_app, NULL::text as execution_mode, NULL::bigint as id, NULL::bigint as version, \
o.created_at as sort_time, lower(COALESCE(NULLIF(o.summary, ''), o.path)) as sort_name, o.hash as tiebreak \
FROM script o \
@@ -291,6 +295,7 @@ fn branch_sqls() -> Branches {
o.ws_error_handler_muted, o.edited_at, \
NULL::bigint as hash, NULL::text as language, NULL::text as script_kind, NULL::text as auto_kind, \
NULL::bool as use_codebase, NULL::bool as has_deploy_errors, \
(o.value->>'chat_input_enabled')::bool as chat_input_enabled, \
NULL::bool as raw_app, NULL::text as execution_mode, NULL::bigint as id, NULL::bigint as version, \
o.edited_at as sort_time, lower(COALESCE(NULLIF(o.summary, ''), o.path)) as sort_name, 0::bigint as tiebreak \
FROM flow o \
@@ -306,7 +311,7 @@ fn branch_sqls() -> Branches {
{draft_users}, o.labels, folder_labels(o.workspace_id, o.path) as inherited_labels, \
NULL::bool as ws_error_handler_muted, av.created_at as edited_at, \
NULL::bigint as hash, NULL::text as language, NULL::text as script_kind, NULL::text as auto_kind, \
NULL::bool as use_codebase, NULL::bool as has_deploy_errors, \
NULL::bool as use_codebase, NULL::bool as has_deploy_errors, NULL::bool as chat_input_enabled, \
av.raw_app, o.policy->>'execution_mode' as execution_mode, o.id, \
o.versions[array_upper(o.versions, 1)] as version, \
COALESCE(av.created_at, 'epoch'::timestamptz) as sort_time, lower(COALESCE(NULLIF(o.summary, ''), o.path)) as sort_name, 0::bigint as tiebreak \
@@ -344,11 +349,21 @@ fn draft_branch_sql(kind: &str) -> String {
let kind_cols = match kind {
"script" => {
"d.value->>'language' as language, d.value->>'kind' as script_kind, \
d.value->>'auto_kind' as auto_kind, false as raw_app"
d.value->>'auto_kind' as auto_kind, false as raw_app, \
NULL::bool as chat_input_enabled"
}
// Type-guarded rather than a bare `::bool` cast: draft JSON is stored
// unvalidated, so a malformed value must yield NULL, not abort the list,
// and the string "true" must not count as enabled.
"flow" => {
"NULL::text as language, NULL::text as script_kind, NULL::text as auto_kind, \
false as raw_app, \
CASE WHEN json_typeof(d.value->'value'->'chat_input_enabled') = 'boolean' \
THEN (d.value->'value'->>'chat_input_enabled')::bool END as chat_input_enabled"
}
_ => {
"NULL::text as language, NULL::text as script_kind, NULL::text as auto_kind, \
(d.typ = 'raw_app') as raw_app"
(d.typ = 'raw_app') as raw_app, NULL::bool as chat_input_enabled"
}
};
format!(
@@ -359,7 +374,7 @@ fn draft_branch_sql(kind: &str) -> String {
NULL::text[] as labels, NULL::text[] as inherited_labels, \
NULL::bool as ws_error_handler_muted, o.created_at as edited_at, \
NULL::bigint as hash, o.language, o.script_kind, o.auto_kind, \
NULL::bool as use_codebase, NULL::bool as has_deploy_errors, \
NULL::bool as use_codebase, NULL::bool as has_deploy_errors, o.chat_input_enabled, \
o.raw_app, NULL::text as execution_mode, NULL::bigint as id, NULL::bigint as version, \
o.created_at as sort_time, lower(COALESCE(NULLIF(o.summary, ''), o.draft_path, o.path)) as sort_name, 0::bigint as tiebreak \
FROM ( \
+11
View File
@@ -352,6 +352,17 @@ async fn update_username_in_workpsace<'c>(
new_username: &str,
w_id: &str,
) -> error::Result<()> {
// ---- data table tenants ----
// Tenants name the user, so the rename has to follow here too; a list left naming the old
// username silently drops the access instead of moving it.
windmill_common::workspaces::rename_datatable_tenant_in_workspace(
tx,
w_id,
&format!("u/{old_username}"),
&format!("u/{new_username}"),
)
.await?;
// ---- instance and workspace users ----
sqlx::query!(
"UPDATE usr SET username = $1 WHERE email = $2",
@@ -1639,7 +1639,7 @@ pub(crate) async fn tarball_workspace(
mute_critical_alerts: row.mute_critical_alerts,
color: row.color.clone(),
operator_settings: row.operator_settings.clone(),
datatable: row.datatable.clone(),
datatable: windmill_common::workspaces::strip_datatable_permissions(row.datatable.clone()),
slack_team_id: row.slack_team_id.clone(),
slack_name: row.slack_name.clone(),
slack_command_script: row.slack_command_script.clone(),
@@ -1703,7 +1703,7 @@ pub(crate) async fn tarball_workspace(
mute_critical_alerts: row.mute_critical_alerts,
color: row.color,
operator_settings: row.operator_settings,
datatable: row.datatable,
datatable: windmill_common::workspaces::strip_datatable_permissions(row.datatable),
slack_team_id: row.slack_team_id,
slack_name: row.slack_name,
slack_command_script: row.slack_command_script,
+1
View File
@@ -75,6 +75,7 @@ bitflags.workspace = true
once_cell.workspace = true
phf.workspace = true
tokio-postgres.workspace = true
postgres-protocol.workspace = true
postgres-native-tls.workspace = true
native-tls.workspace = true
+26 -7
View File
@@ -19,10 +19,11 @@ use crate::{
};
/// Whether `label` denotes a user-created token rather than a system token
/// (`session`, `guest_session`, `ephemeral*`, `debugger-token`, `mcp-oauth-*`). System-token
/// labels are load-bearing — session cleanup, super_admin propagation, expiry
/// notifications and username overrides all key off them — so they must not be
/// user-editable. `None` (no label) is treated as a user token.
/// (`session`, `guest_session`, `ephemeral*`, `debugger-token`, `mcp-oauth-*`,
/// `embed_app:*`, `sdk_app:*`, `impersonation:*`). System-token labels are load-bearing —
/// session cleanup, super_admin propagation, expiry notifications and username overrides
/// all key off them — so they must not be user-editable. `None` (no label) is treated as
/// a user token.
///
/// This is the canonical copy. When updating it, also update its mirrors:
/// - the `update_token_label` editability guard (SQL `WHERE`) in
@@ -40,15 +41,30 @@ pub fn is_user_token(label: Option<&str>) -> bool {
&& !l.to_lowercase().starts_with("ephemeral")
&& l != "debugger-token"
&& !l.starts_with("mcp-oauth-")
// Short-lived tokens the server mints per app open or per service-account
// impersonation (EE `users_ee.rs`) and nobody manages, so an expiry warning
// for one is noise.
&& !l.starts_with(APP_EMBED_TOKEN_LABEL_PREFIX)
&& !l.starts_with(RAW_APP_SDK_TOKEN_LABEL_PREFIX)
&& !l.starts_with("impersonation:")
}
}
}
/// Label prefix, followed by the app path, of the token an app viewer's sandboxed iframe
/// runs with. Reserved in [`is_user_token`], whose SQL and frontend mirrors spell it out.
pub const APP_EMBED_TOKEN_LABEL_PREFIX: &str = "embed_app:";
/// Label prefix, followed by the app path, of the token a raw app's bundle uses for the
/// frontend SDK. Reserved in [`is_user_token`], whose SQL and frontend mirrors spell it out.
pub const RAW_APP_SDK_TOKEN_LABEL_PREFIX: &str = "sdk_app:";
/// Whether `label` belongs to a namespace only the server mints, and which therefore must be
/// rejected by `create_token`. Narrower than [`is_user_token`], which also drives label
/// editability and expiry notifications and can afford to reserve more: `Ephemeral lsp token`
/// and `debugger-token` are minted by the editor and the debugger through that same handler,
/// so reserving them would break those features.
/// editability and expiry notifications and can afford to reserve more: `Ephemeral lsp token`,
/// `debugger-token` and `ephemeral-test-connection: *` are minted by the editor, the debugger
/// and object-storage connection tests through that same handler, so reserving them would
/// break those features.
///
/// `username_override_from_label` trusts a label to name the entity acting only if it is in
/// here, so anything added must be unmintable by a member.
@@ -961,6 +977,9 @@ mod tests {
assert!(!is_user_token(Some("Ephemeral lsp token")));
assert!(!is_user_token(Some("debugger-token")));
assert!(!is_user_token(Some("mcp-oauth-client")));
assert!(!is_user_token(Some("embed_app:f/team/dashboard")));
assert!(!is_user_token(Some("sdk_app:u/admin/raw app")));
assert!(!is_user_token(Some("impersonation:admin@windmill.dev")));
}
#[test]
@@ -0,0 +1,445 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! The instance's data table role catalogs.
//!
//! A data table role is a real Postgres login role on one cluster — Windmill's own, or the external
//! instance cluster — named exactly as the user named it, shared by every database Windmill manages
//! on that cluster. Each cluster has its own catalog: a role exists where it was created and nowhere
//! else. Windmill decides who may ask for a role (the per-data-table tenant lists in
//! [`crate::workspaces`]); Postgres decides what the role may then touch. The catalog here is only
//! the first half's vocabulary plus the cluster provisioning.
//!
//! Entries are keyed by a generated id so a rename moves nothing else: tenants name the id.
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use crate::{
error::{Error, Result},
workspaces::DataTableCatalogResourceType,
DB,
};
/// The cluster a role catalog belongs to.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum DatatableRoleCluster {
/// Windmill's own Postgres, behind `instance` data tables.
#[default]
Instance,
/// The external instance cluster, behind `external_instance` data tables.
ExternalInstance,
}
impl DatatableRoleCluster {
pub fn as_str(self) -> &'static str {
match self {
Self::Instance => "instance",
Self::ExternalInstance => "external_instance",
}
}
pub fn parse(value: &str) -> Result<Self> {
match value {
"instance" => Ok(Self::Instance),
"external_instance" => Ok(Self::ExternalInstance),
other => Err(Error::BadRequest(format!(
"Unknown data table role cluster '{other}': expected instance or external_instance"
))),
}
}
/// The cluster whose roles a data table on `kind` can use. `None` for a resource-backed one,
/// which is never under roles.
pub fn of(kind: DataTableCatalogResourceType) -> Option<Self> {
match kind {
DataTableCatalogResourceType::Instance => Some(Self::Instance),
DataTableCatalogResourceType::ExternalInstance => Some(Self::ExternalInstance),
DataTableCatalogResourceType::Postgresql => None,
}
}
}
/// The connection every data table resolved to before roles existed (`custom_instance_user`). It
/// owns every pre-existing object, so it is a reserved name rather than a catalog entry: never
/// created, renamed or dropped.
pub const ADMIN_DATATABLE_ROLE: &str = "admin";
/// The login the admin connection uses, and the role every created role is granted to — that
/// membership is what later lets it `ALTER ... OWNER TO` a role and drop it.
pub const CUSTOM_INSTANCE_USER: &str = "custom_instance_user";
/// One catalog entry, as stored in `datatable_role`. The password is per role and instance-wide;
/// it belongs to the instance, not to any workspace's settings.
/// No `Serialize`/`Deserialize`: the catalog is rows now, and a derived `Serialize` would emit
/// `pwd` — the same way out for a credential that the hand-written `Debug` below closes on the log
/// side.
#[derive(Clone)]
pub struct InstanceDatatableRole {
/// The Postgres role name, verbatim.
pub name: String,
pub enabled: bool,
/// Absent only for a role whose provisioning did not finish; resolving as it then errors
/// rather than falling back to admin.
///
/// A plain string rather than a `StringOrSecretRef` like the instance user's password: that
/// one is a secret ref because an operator supplies it and may want it to come from their own
/// backend, while this one is minted here and never entered by anyone, so there is nothing for
/// a ref to point at. Encrypting generated secrets at rest is a separate change that would
/// take the replication password with it.
pub pwd: Option<String>,
}
/// Hand-written so `{:?}` on a catalog cannot put a live Postgres password in a log line or an
/// audit record. Everything else about the entry is safe to print.
impl std::fmt::Debug for InstanceDatatableRole {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("InstanceDatatableRole")
.field("name", &self.name)
.field("enabled", &self.enabled)
.field("pwd", &self.pwd.as_ref().map(|_| "<redacted>"))
.finish()
}
}
pub type DatatableRoleCatalog = BTreeMap<String, InstanceDatatableRole>;
/// Names Postgres or Windmill already owns. `admin` is excluded because it never reaches the
/// cluster as a role name at all — it resolves to `custom_instance_user`.
fn is_reserved_role_name(name: &str) -> bool {
let lower = name.to_ascii_lowercase();
lower == ADMIN_DATATABLE_ROLE
|| lower == "postgres"
|| lower == "public"
|| lower.starts_with("pg_")
|| lower.starts_with("windmill_")
|| lower.starts_with("custom_instance_")
}
/// The charset is what makes every downstream interpolation safe: the name reaches Postgres as a
/// quoted identifier, a `-- role <name>` annotation, and a `?role=` query parameter.
pub fn validate_role_name(name: &str) -> Result<()> {
if name.is_empty() || name.len() > 63 {
return Err(Error::BadRequest(format!(
"Invalid data table role name '{name}': it must be between 1 and 63 characters"
)));
}
if !name
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-')
{
return Err(Error::BadRequest(format!(
"Invalid data table role name '{name}': only letters, digits, '_' and '-' are allowed"
)));
}
if is_reserved_role_name(name) {
return Err(Error::BadRequest(format!(
"'{name}' is reserved and cannot be used as a data table role name"
)));
}
Ok(())
}
/// A double-quoted Postgres identifier. Doubling `"` is Postgres's own escaping inside one, so this
/// quotes any name — schema, table or role. Role names are validated as well
/// ([`validate_role_name`]) because they also travel unquoted, in `-- role <name>` and `?role=`.
pub fn quote_ident(name: &str) -> String {
format!("\"{}\"", name.replace('"', "\"\""))
}
/// Serialize the mutations that are not already serialized by the row itself.
///
/// A create is an insert and a delete is a delete, which Postgres orders for us — the unique index
/// on `name` is what makes two concurrent creates of the same name one winner and one error. What
/// still needs it is the window between the cluster DDL and the row: `CREATE ROLE` is not visible
/// to another transaction's `pg_roles` check until commit, so without this two creates of the same
/// name both pass their existence check and one fails on the index having already made the login.
/// Held for the transaction, so the DDL has to run on that same transaction to be covered.
pub async fn lock_role_catalog(tx: &mut sqlx::Transaction<'_, sqlx::Postgres>) -> Result<()> {
sqlx::query!("SELECT pg_advisory_xact_lock(hashtext('datatable_role_catalog'))")
.execute(&mut **tx)
.await?;
Ok(())
}
/// A replication stream reads every row whatever a data table's roles grant. Turning roles on looks
/// for streams holding this exclusive; whatever can start a Postgres trigger or capture streaming
/// holds it shared on the transaction that commits it. So either the look sees the stream, or the
/// stream's listener connects after roles are committed and refuses. Held for the transaction.
pub async fn lock_datatable_streams(conn: &mut sqlx::PgConnection, exclusive: bool) -> Result<()> {
let lock = if exclusive {
"pg_advisory_xact_lock"
} else {
"pg_advisory_xact_lock_shared"
};
sqlx::query(&format!("SELECT {lock}(hashtext('datatable_streams'))"))
.execute(conn)
.await?;
Ok(())
}
/// Whether an instance database is reached only through entries under roles is decided by two
/// writes that lock different workspaces' settings rows: turning roles on for one entry, and a
/// settings save pointing an entry without roles at the database. Each holds this for every
/// database it decides on, so neither reads past the other's uncommitted write. Held for the
/// transaction; the names are locked in sorted order so two holders cannot deadlock.
pub async fn lock_instance_databases_governance<'a>(
conn: &mut sqlx::PgConnection,
dbnames: impl IntoIterator<Item = &'a str>,
) -> Result<()> {
let dbnames: std::collections::BTreeSet<&str> = dbnames.into_iter().collect();
for dbname in dbnames {
sqlx::query("SELECT pg_advisory_xact_lock(hashtext('datatable_instance_database:' || $1))")
.bind(dbname)
.execute(&mut *conn)
.await?;
}
Ok(())
}
/// Disclosure: returns every role's stored Postgres password in plaintext. Any server path that
/// has to resolve or name a role may call it — including handlers open to a workspace member, who
/// need the names — but callers MUST NOT let `pwd` reach a response, a log line, an audit record
/// or an export. Nothing about who may call it: the credential is the whole risk, and `Debug` is
/// hand-written to redact it for the same reason.
pub async fn read_role_catalog(
db: &DB,
cluster: DatatableRoleCluster,
) -> Result<DatatableRoleCatalog> {
crate::datatable_roles_oss::read_role_catalog(db, cluster).await
}
/// As [`read_role_catalog`], reading inside the caller's transaction so the value is the one
/// [`lock_role_catalog`] is protecting. Same disclosure contract.
pub async fn read_role_catalog_tx(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
cluster: DatatableRoleCluster,
) -> Result<DatatableRoleCatalog> {
crate::datatable_roles_oss::read_role_catalog_tx(tx, cluster).await
}
/// The cluster a role belongs to, or `None` if no role has this id.
pub async fn role_cluster(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
id: &str,
) -> Result<Option<DatatableRoleCluster>> {
crate::datatable_roles_oss::role_cluster(tx, id).await
}
/// Record a role, in the caller's transaction. On Windmill's own cluster that commits it with the
/// `CREATE ROLE` it describes; on the external cluster the role already exists by then.
///
/// Authorization: writes a generated Postgres credential. Callers MUST restrict this to superadmin
/// paths and MUST hold [`lock_role_catalog`] on `tx`.
pub async fn insert_role_catalog_entry(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
id: &str,
cluster: DatatableRoleCluster,
role: &InstanceDatatableRole,
) -> Result<()> {
crate::datatable_roles_oss::insert_role_catalog_entry(tx, id, cluster, role).await
}
/// Update a role's recorded name, login flag and password. Same contract as
/// [`insert_role_catalog_entry`].
pub async fn update_role_catalog_entry(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
id: &str,
role: &InstanceDatatableRole,
) -> Result<()> {
crate::datatable_roles_oss::update_role_catalog_entry(tx, id, role).await
}
/// Forget a role. Same contract as [`insert_role_catalog_entry`]; run it in the transaction that
/// drops the cluster login, so the two cannot disagree.
pub async fn delete_role_catalog_entry(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
id: &str,
) -> Result<()> {
crate::datatable_roles_oss::delete_role_catalog_entry(tx, id).await
}
/// Resolve the role a caller named to its catalog id. A disabled role is an error rather than a
/// silent fallback: the caller asked for something the instance deliberately turned off.
pub fn role_id_by_name<'a>(catalog: &'a DatatableRoleCatalog, name: &str) -> Result<&'a str> {
let entry = catalog
.iter()
.find(|(_, role)| role.name == name)
.ok_or_else(|| {
Error::NotFound(format!(
"'{name}' is not a data table role of this database's cluster. Defined roles: {}.",
catalog
.values()
.map(|r| r.name.as_str())
.collect::<Vec<_>>()
.join(", ")
))
})?;
if !entry.1.enabled {
return Err(Error::BadRequest(format!(
"Data table role '{name}' is disabled on this instance"
)));
}
Ok(entry.0.as_str())
}
/// Every database Windmill manages on `cluster`. Role provisioning has to reach all of them: a role
/// that cannot `CONNECT` to a database is refused by Postgres before any grant matters.
///
/// Authorization: checks nothing, and names every managed database across all workspaces. Callers
/// MUST be superadmin-gated or keep the names server-side; never return them to a workspace caller.
pub async fn registered_instance_databases(
db: &DB,
cluster: DatatableRoleCluster,
) -> Result<Vec<String>> {
crate::datatable_roles_oss::registered_instance_databases(db, cluster).await
}
/// `CONNECT` on `dbname` for every enabled role of `cluster`, and none for `PUBLIC`. Run at role
/// creation, at database creation, and lazily whenever a managed data table is administered, so a
/// database provisioned before a role existed is repaired rather than left silently unreachable.
///
/// Authorization: rewrites a database's ACL with the server's own credentials and checks nothing.
/// Callers MUST have authorized administration of `dbname` — superadmin, or an admin of the
/// workspace governing a data table on it.
pub async fn converge_connect_grants(
db: &DB,
cluster: DatatableRoleCluster,
dbname: &str,
) -> Result<()> {
crate::datatable_roles_oss::converge_connect_grants(db, cluster, dbname).await
}
/// As [`converge_connect_grants`], with the catalog of `cluster` the caller already read. Same
/// contract.
pub async fn converge_connect_grants_with(
db: &DB,
cluster: DatatableRoleCluster,
dbname: &str,
catalog: &DatatableRoleCatalog,
) -> Result<()> {
crate::datatable_roles_oss::converge_connect_grants_with(db, cluster, dbname, catalog).await
}
/// `CREATE ROLE <name> LOGIN PASSWORD ...; GRANT <name> TO custom_instance_user` on `cluster`. No
/// privileges beyond that — an admin grants them through SQL or the ACL editor.
///
/// On Windmill's own cluster the DDL runs on `tx`, so it commits with the catalog row. The external
/// cluster is another server: the role is created there before `tx` commits, and callers MUST drop
/// it again ([`drop_datatable_role`]) if `tx` then fails to commit.
///
/// Authorization: creates a cluster-wide Postgres login. Callers MUST restrict this to superadmin
/// paths, and MUST hold [`lock_role_catalog`] on `tx`.
pub async fn create_datatable_role(
db: &DB,
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
cluster: DatatableRoleCluster,
name: &str,
password: &str,
) -> Result<()> {
crate::datatable_roles_oss::create_datatable_role(db, tx, cluster, name, password).await
}
/// Authorization: alters a cluster-wide Postgres login. Callers MUST restrict this to superadmin
/// paths, and MUST hold [`lock_role_catalog`] on `tx`.
pub async fn set_datatable_role_login(
db: &DB,
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
cluster: DatatableRoleCluster,
name: &str,
enabled: bool,
) -> Result<()> {
crate::datatable_roles_oss::set_datatable_role_login(db, tx, cluster, name, enabled).await
}
/// A rename discards an md5-hashed password, so the caller has to hand over a fresh one. On the
/// external cluster the rename lands before `tx` commits, and callers MUST rename it back if `tx`
/// then fails to commit.
///
/// Authorization: renames a cluster-wide Postgres login. Callers MUST restrict this to superadmin
/// paths, and MUST hold [`lock_role_catalog`] on `tx`.
pub async fn rename_datatable_role(
db: &DB,
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
cluster: DatatableRoleCluster,
from: &str,
to: &str,
password: &str,
) -> Result<()> {
crate::datatable_roles_oss::rename_datatable_role(db, tx, cluster, from, to, password).await
}
/// A role owning anything in any database blocks its own `DROP ROLE`, and both its objects and the
/// privileges granted to it are only visible from inside each database — hence the pass over the
/// registry. An unreachable database aborts the whole delete: dropping the role while one database
/// still holds objects owned by it leaves those objects owned by a numeric OID nobody can name.
///
/// Each pass runs as the cluster's administrator rather than `custom_instance_user`: on Windmill's
/// own cluster the instance's Postgres user, on the external one its configured admin login. Both
/// own the databases and can therefore revoke a grant whoever made it. `custom_instance_user`
/// could only undo what it granted itself, so a privilege planted by an operator in psql — the
/// ordinary way privileges reach a role — would survive and block the drop.
///
/// Authorization: drops a cluster-wide Postgres login and reassigns everything it owns. Callers
/// MUST restrict this to superadmin paths, and MUST hold [`lock_role_catalog`] on `tx`.
///
/// The per-database passes open their own connections and cannot join `tx`; the lock is what keeps
/// a concurrent mutation out while they run. On Windmill's own cluster only the final `DROP ROLE`
/// is on `tx`, so it commits or rolls back with the catalog write that forgets the role; on the
/// external cluster it runs there, and tolerates a role already gone so a retry after a failed
/// commit can finish. The passes commit as they go, so callers MUST have disabled the role in an
/// earlier committed transaction: a failure part-way then leaves a disabled role to retry, not an
/// enabled one already stripped in some databases.
pub async fn drop_datatable_role(
db: &DB,
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
cluster: DatatableRoleCluster,
name: &str,
) -> Result<()> {
crate::datatable_roles_oss::drop_datatable_role(db, tx, cluster, name).await
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn role_names_are_validated() {
assert!(validate_role_name("analytics").is_ok());
assert!(validate_role_name("read-only_2").is_ok());
assert!(validate_role_name("").is_err());
assert!(validate_role_name(&"a".repeat(64)).is_err());
assert!(validate_role_name("has space").is_err());
assert!(validate_role_name("quote\"injection").is_err());
// Reserved, case-insensitively.
assert!(validate_role_name("admin").is_err());
assert!(validate_role_name("Postgres").is_err());
assert!(validate_role_name("pg_read_all_data").is_err());
assert!(validate_role_name("windmill_user").is_err());
assert!(validate_role_name("custom_instance_user").is_err());
}
#[test]
fn a_disabled_role_is_an_error_not_a_fallback() {
let mut catalog = DatatableRoleCatalog::new();
catalog.insert(
"id1".to_string(),
InstanceDatatableRole {
name: "analytics".to_string(),
enabled: false,
pwd: Some("x".to_string()),
},
);
assert!(role_id_by_name(&catalog, "analytics").is_err());
assert!(role_id_by_name(&catalog, "nope").is_err());
catalog.get_mut("id1").unwrap().enabled = true;
assert_eq!(role_id_by_name(&catalog, "analytics").unwrap(), "id1");
}
}
@@ -0,0 +1,243 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! Where data table roles come from: the enterprise implementation, or a refusal.
//!
//! Roles are an Enterprise Edition feature. An edition without them creates, grants and connects
//! as none, and a data table saved under roles — by an enterprise build, before a downgrade — is
//! refused rather than resolved as `admin`. A data table not under roles, asked for no role,
//! resolves as it always has. `private` alone is not that edition: community builds carry it.
use crate::error::Error;
/// What every roles path answers without the Enterprise Edition. The frontend matches this exact
/// sentence (`datatableUsableRoles.ts`) to read the refusal as "not under roles": reword both.
pub fn datatable_roles_unavailable() -> Error {
Error::BadRequest("Data table roles are a Windmill Enterprise Edition feature".to_string())
}
#[cfg(all(feature = "private", feature = "enterprise"))]
pub(crate) use crate::datatable_roles_ee::{
can_use_datatable_role, can_use_datatable_role_in_governing_workspace, converge_connect_grants,
converge_connect_grants_with, create_datatable_role, delete_role_catalog_entry,
drop_datatable_role, ensure_can_use_datatable_role, ensure_datatable_admin_access,
ensure_instance_db_grant_options_unchecked, forget_datatable_role_everywhere,
insert_role_catalog_entry, read_role_catalog, read_role_catalog_tx,
registered_instance_databases, rename_datatable_role, resolve_datatable_role_connection,
role_cluster, set_datatable_role_login, update_role_catalog_entry,
};
#[cfg(not(all(feature = "private", feature = "enterprise")))]
pub(crate) use ce::*;
#[cfg(not(all(feature = "private", feature = "enterprise")))]
mod ce {
use super::datatable_roles_unavailable as unavailable;
use crate::{
datatable_roles::{DatatableRoleCatalog, DatatableRoleCluster, InstanceDatatableRole},
db::AuthedRef,
error::Result,
workspaces::{
resolve_governing_datatable, DataTableRoleTenants, DatatableAccess, GoverningDatatable,
},
DB,
};
type Tx<'a> = sqlx::Transaction<'a, sqlx::Postgres>;
pub(crate) async fn read_role_catalog(
_db: &DB,
_cluster: DatatableRoleCluster,
) -> Result<DatatableRoleCatalog> {
Err(unavailable())
}
pub(crate) async fn read_role_catalog_tx(
_tx: &mut Tx<'_>,
_cluster: DatatableRoleCluster,
) -> Result<DatatableRoleCatalog> {
Err(unavailable())
}
pub(crate) async fn role_cluster(
_tx: &mut Tx<'_>,
_id: &str,
) -> Result<Option<DatatableRoleCluster>> {
Err(unavailable())
}
pub(crate) async fn insert_role_catalog_entry(
_tx: &mut Tx<'_>,
_id: &str,
_cluster: DatatableRoleCluster,
_role: &InstanceDatatableRole,
) -> Result<()> {
Err(unavailable())
}
pub(crate) async fn update_role_catalog_entry(
_tx: &mut Tx<'_>,
_id: &str,
_role: &InstanceDatatableRole,
) -> Result<()> {
Err(unavailable())
}
pub(crate) async fn delete_role_catalog_entry(_tx: &mut Tx<'_>, _id: &str) -> Result<()> {
Err(unavailable())
}
pub(crate) async fn registered_instance_databases(
_db: &DB,
_cluster: DatatableRoleCluster,
) -> Result<Vec<String>> {
Err(unavailable())
}
/// Nothing to converge: with no roles to admit, a managed database keeps the `CONNECT` grants
/// it was created with, as it did before roles existed.
pub(crate) async fn converge_connect_grants(
_db: &DB,
_cluster: DatatableRoleCluster,
_dbname: &str,
) -> Result<()> {
Ok(())
}
/// As [`converge_connect_grants`].
pub(crate) async fn converge_connect_grants_with(
_db: &DB,
_cluster: DatatableRoleCluster,
_dbname: &str,
_catalog: &DatatableRoleCatalog,
) -> Result<()> {
Ok(())
}
pub(crate) async fn create_datatable_role(
_db: &DB,
_tx: &mut Tx<'_>,
_cluster: DatatableRoleCluster,
_name: &str,
_password: &str,
) -> Result<()> {
Err(unavailable())
}
pub(crate) async fn set_datatable_role_login(
_db: &DB,
_tx: &mut Tx<'_>,
_cluster: DatatableRoleCluster,
_name: &str,
_enabled: bool,
) -> Result<()> {
Err(unavailable())
}
pub(crate) async fn rename_datatable_role(
_db: &DB,
_tx: &mut Tx<'_>,
_cluster: DatatableRoleCluster,
_from: &str,
_to: &str,
_password: &str,
) -> Result<()> {
Err(unavailable())
}
pub(crate) async fn drop_datatable_role(
_db: &DB,
_tx: &mut Tx<'_>,
_cluster: DatatableRoleCluster,
_name: &str,
) -> Result<()> {
Err(unavailable())
}
pub(crate) async fn ensure_instance_db_grant_options_unchecked(
_db: &DB,
_cluster: DatatableRoleCluster,
_dbname: &str,
) -> Result<()> {
Err(unavailable())
}
/// No tenant list covers anyone: there is no role to connect as.
pub(crate) fn can_use_datatable_role(
_tenants: &DataTableRoleTenants,
_authed: &AuthedRef<'_>,
) -> bool {
false
}
pub(crate) async fn can_use_datatable_role_in_governing_workspace(
_db: &DB,
_governing_w_id: &str,
_w_id: &str,
_tenants: &DataTableRoleTenants,
_access: &DatatableAccess<'_>,
) -> Result<bool> {
Err(unavailable())
}
/// Reached only for a data table under roles or a caller naming a role: both are refused.
pub(crate) async fn resolve_datatable_role_connection(
_db: &DB,
_w_id: &str,
_name: &str,
_governing: &GoverningDatatable,
_db_resource: serde_json::Value,
_role: Option<&str>,
_access: DatatableAccess<'_>,
) -> Result<serde_json::Value> {
Err(unavailable())
}
/// A data table not under roles, asked for no role or for `admin`, is not a role decision and
/// passes, as it did before roles existed. Anything else is refused.
pub(crate) async fn ensure_can_use_datatable_role(
db: &DB,
w_id: &str,
name: &str,
role: Option<&str>,
_access: &DatatableAccess<'_>,
_context: &str,
) -> Result<()> {
let governing = resolve_governing_datatable(db, w_id, name).await?;
if governing.datatable.permissions.is_none()
&& role.is_none_or(|r| r == crate::datatable_roles::ADMIN_DATATABLE_ROLE)
{
Ok(())
} else {
Err(unavailable())
}
}
/// A data table not under roles is the `admin` connection for anyone who reaches it, as before
/// roles existed. One under roles is refused.
pub(crate) async fn ensure_datatable_admin_access(
db: &DB,
w_id: &str,
name: &str,
_access: &DatatableAccess<'_>,
) -> Result<()> {
let governing = resolve_governing_datatable(db, w_id, name).await?;
if governing.datatable.permissions.is_none() {
Ok(())
} else {
Err(unavailable())
}
}
pub(crate) async fn forget_datatable_role_everywhere(
_tx: &mut Tx<'_>,
_role_id: &str,
) -> Result<()> {
Err(unavailable())
}
}
@@ -0,0 +1,439 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! The external Postgres cluster behind `external_instance` data tables and Ducklake catalogs.
//!
//! Windmill administers that cluster itself, logged in as the user in
//! [`EXTERNAL_INSTANCE_PG_SETTING`]. It creates `custom_instance_user` and
//! `custom_instance_replication_user` there, with passwords it generates and keeps in
//! [`EXTERNAL_INSTANCE_PG_STATE_SETTING`]. They share their names with the roles on Windmill's own
//! cluster, but they are different roles with different passwords.
//!
//! The cluster may hold data Windmill did not create. Two Windmill instances sharing one is not
//! supported: each would keep resetting the passwords the other depends on.
use std::collections::{BTreeMap, BTreeSet};
use serde::{Deserialize, Serialize};
use crate::{
error::{Error, Result},
global_settings::{EXTERNAL_INSTANCE_PG_SETTING, EXTERNAL_INSTANCE_PG_STATE_SETTING},
instance_config::{CustomInstanceDb, ExternalInstancePg},
DB,
};
/// What Windmill keeps about the external cluster. Server-managed and hidden: never part of the
/// instance config, never readable by an agent worker. No `Debug`: it carries live passwords.
#[derive(Serialize, Deserialize, Clone, Default)]
pub struct ExternalInstancePgState {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub user_pwd: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub replication_pwd: Option<String>,
/// The databases Windmill created on the cluster. It only ever drops one of these.
#[serde(default)]
pub databases: BTreeMap<String, CustomInstanceDb>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub last_setup: Option<ExternalInstancePgSetupReport>,
/// The cluster ([`external_instance_pg_address`]) the last successful setup converged. Databases
/// are only created on a cluster setup succeeded on: the passwords above exist as soon as setup
/// first runs, whether or not the cluster accepted them.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub set_up_for: Option<String>,
}
/// What identifies the cluster a configuration points at. Other fields (admin login, sslmode) can
/// change without it becoming another cluster.
pub fn external_instance_pg_address(config: &ExternalInstancePg) -> String {
format!(
"{}:{}",
config.host.trim().to_lowercase(),
config.port.unwrap_or(5432)
)
}
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct ExternalInstancePgSetupReport {
/// No step failed. Warnings leave it true.
pub success: bool,
pub finished_at: chrono::DateTime<chrono::Utc>,
pub steps: Vec<ExternalInstancePgSetupStep>,
}
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct ExternalInstancePgSetupStep {
pub name: String,
pub status: SetupStepStatus,
pub message: String,
}
#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum SetupStepStatus {
Ok,
Warning,
Error,
}
/// The status the settings page shows without running anything.
#[derive(Serialize, Debug)]
pub struct ExternalInstancePgStatus {
pub configured: bool,
pub database_count: usize,
#[serde(skip_serializing_if = "Option::is_none")]
pub last_setup: Option<ExternalInstancePgSetupReport>,
}
/// Authorization: returns the cluster's admin password and checks nothing. Callers MUST be
/// superadmin or an internal server path.
pub(crate) async fn read_external_instance_pg_config<'c>(
executor: impl sqlx::PgExecutor<'c>,
) -> Result<Option<ExternalInstancePg>> {
let value = sqlx::query_scalar!(
"SELECT value FROM global_settings WHERE name = $1",
EXTERNAL_INSTANCE_PG_SETTING
)
.fetch_optional(executor)
.await?;
value
.map(|v| {
serde_json::from_value(v).map_err(|e| {
Error::internal_err(format!("reading {EXTERNAL_INSTANCE_PG_SETTING}: {e}"))
})
})
.transpose()
}
/// Authorization: returns the passwords Windmill generated on the cluster and checks nothing.
/// Callers MUST be superadmin or an internal server path.
pub(crate) async fn read_external_instance_pg_state<'c>(
executor: impl sqlx::PgExecutor<'c>,
) -> Result<ExternalInstancePgState> {
let value = sqlx::query_scalar!(
"SELECT value FROM global_settings WHERE name = $1",
EXTERNAL_INSTANCE_PG_STATE_SETTING
)
.fetch_optional(executor)
.await?;
match value {
None => Ok(ExternalInstancePgState::default()),
Some(v) => serde_json::from_value(v).map_err(|e| {
Error::internal_err(format!("reading {EXTERNAL_INSTANCE_PG_STATE_SETTING}: {e}"))
}),
}
}
pub async fn external_instance_pg_status(db: &DB) -> Result<ExternalInstancePgStatus> {
let configured = read_external_instance_pg_config(db).await?.is_some();
let state = read_external_instance_pg_state(db).await?;
Ok(ExternalInstancePgStatus {
configured,
database_count: state.databases.len(),
last_setup: state.last_setup,
})
}
/// The databases Windmill created on the external cluster, without the passwords kept beside them.
///
/// Authorization: names every database across all workspaces, and the workspace each fork copy is
/// reserved for, and checks nothing. Callers MUST be superadmin or an internal authorization or
/// lifecycle path that does not return the names to a workspace caller.
pub async fn external_instance_databases(db: &DB) -> Result<BTreeMap<String, CustomInstanceDb>> {
Ok(read_external_instance_pg_state(db).await?.databases)
}
/// The workspaces whose data tables or Ducklake catalogs name each database on the external cluster,
/// and the forks whose Ducklake metadata schemas there are still waiting to be dropped: those rows
/// outlive a settings change, and cleanup cannot drop a schema in a database that is gone. A row
/// whose schema is already dropped only waits on object storage, which needs no database.
///
/// Authorization: reads every workspace's settings and checks nothing. Callers MUST be superadmin
/// or an internal lifecycle path.
pub async fn external_instance_database_usages<'c>(
db: impl sqlx::PgExecutor<'c>,
) -> Result<BTreeMap<String, BTreeSet<String>>> {
let rows = sqlx::query_as::<_, (String, String)>(
"SELECT ws.workspace_id, entry->'database'->>'resource_path'
FROM workspace_settings ws
CROSS JOIN LATERAL jsonb_each(
CASE WHEN jsonb_typeof(ws.datatable->'datatables') = 'object'
THEN ws.datatable->'datatables'
ELSE '{}'::jsonb END
) AS dt(k, entry)
WHERE entry->'database'->>'resource_type' = 'external_instance'
AND entry->'database'->>'resource_path' IS NOT NULL
UNION ALL
SELECT ws.workspace_id, entry->'catalog'->>'resource_path'
FROM workspace_settings ws
CROSS JOIN LATERAL jsonb_each(
CASE WHEN jsonb_typeof(ws.ducklake->'ducklakes') = 'object'
THEN ws.ducklake->'ducklakes'
ELSE '{}'::jsonb END
) AS dl(k, entry)
WHERE entry->'catalog'->>'resource_type' = 'external_instance'
AND entry->'catalog'->>'resource_path' IS NOT NULL
UNION ALL
SELECT workspace_id, substring(catalog FROM length('external_instance:') + 1)
FROM fork_ducklake_namespace
WHERE catalog LIKE 'external\\_instance:%' AND NOT schema_dropped",
)
.fetch_all(db)
.await?;
let mut usages: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
for (workspace_id, dbname) in rows {
usages.entry(dbname).or_default().insert(workspace_id);
}
Ok(usages)
}
/// Refuse to unset the cluster while Windmill still has databases or data table roles on it, or a
/// workspace still points at one: every data table there would stop resolving, and every role
/// would be a login nothing can drop any more. Allowed on every edition, so a
/// downgraded instance can still clear a setting it no longer uses.
pub async fn ensure_external_instance_pg_removable(db: &DB) -> Result<()> {
ensure_external_instance_pg_unused(db, &format!("removing {EXTERNAL_INSTANCE_PG_SETTING}"))
.await
}
/// Refuse while Windmill has databases or data table roles on the cluster, or a workspace points
/// at one of its databases. `before` finishes the sentence saying what to do first.
async fn ensure_external_instance_pg_unused(db: &DB, before: &str) -> Result<()> {
let state = read_external_instance_pg_state(db).await?;
let usages = external_instance_database_usages(db).await?;
let roles = sqlx::query_scalar::<_, String>(
"SELECT name FROM datatable_role WHERE cluster = 'external_instance' ORDER BY name",
)
.fetch_all(db)
.await?;
if state.databases.is_empty() && usages.is_empty() && roles.is_empty() {
return Ok(());
}
let mut held = vec![];
if !(state.databases.is_empty() && usages.is_empty()) {
let names = state
.databases
.keys()
.chain(usages.keys())
.collect::<BTreeSet<_>>()
.into_iter()
.cloned()
.collect::<Vec<_>>()
.join(", ");
held.push(format!("databases in use ({names})"));
}
if !roles.is_empty() {
held.push(format!("data table roles ({})", roles.join(", ")));
}
Err(Error::BadRequest(format!(
"The external instance cluster still holds {}. Drop them and repoint the data tables and \
Ducklake catalogs using them before {before}.",
held.join(" and ")
)))
}
/// Refuse a workspace setting that newly names an `external_instance` database on an edition
/// without them.
pub fn ensure_external_instance_available() -> Result<()> {
crate::external_instance_pg_oss::ensure_external_instance_available()
}
/// The connection an `external_instance` database resolves to: `custom_instance_user`, or the
/// replication user, on the external cluster.
///
/// Authorization: returns live credentials and checks nothing. Callers MUST have authorized access
/// to the data table that names `dbname`.
pub async fn external_instance_connection_unchecked(
db: &DB,
dbname: &str,
replication: bool,
) -> Result<crate::PgDatabase> {
crate::external_instance_pg_oss::external_instance_connection_unchecked(db, dbname, replication)
.await
}
/// Create `dbname` on the external cluster and register it. Refuses a name already taken there,
/// whoever took it.
///
/// Authorization: checks nothing. Callers MUST be superadmin, or be cloning a data table they may
/// fork into a `wm_fork_` database.
pub async fn create_external_instance_database_unchecked(
db: &DB,
dbname: &str,
tag: &str,
for_workspace: Option<&str>,
) -> Result<()> {
crate::external_instance_pg_oss::create_external_instance_database_unchecked(
db,
dbname,
tag,
for_workspace,
)
.await
}
/// Drop `dbname` from the external cluster: only a database Windmill registered creating, and still
/// carries the mark it set there. Refused while anything uses it
/// ([`crate::workspaces::managed_database_uses`]), except the `exempt` data table entry: the fork
/// copy being cleaned up.
///
/// Authorization: checks nothing. Callers MUST be superadmin, or be deleting the fork that owns
/// this `wm_fork_` database.
pub async fn drop_external_instance_database_unchecked(
db: &DB,
dbname: &str,
exempt: Option<(&str, &str)>,
) -> Result<()> {
crate::external_instance_pg_oss::drop_external_instance_database_unchecked(db, dbname, exempt)
.await
}
/// Serializes everything that changes which databases exist on the external cluster, or which data
/// tables name them: setup, creates, drops, and data table saves. Held until `tx` ends.
pub async fn lock_external_instance_pg_state(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
) -> Result<()> {
sqlx::query("SELECT pg_advisory_xact_lock(hashtext($1))")
.bind(EXTERNAL_INSTANCE_PG_STATE_SETTING)
.execute(&mut **tx)
.await?;
Ok(())
}
/// Refuse a data table naming `dbname` unless Windmill created it on the external cluster. Takes
/// the lock drops take, so none can remove the database before `tx`, which saves the data table,
/// commits.
pub async fn ensure_external_instance_database_registered(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
dbname: &str,
) -> Result<()> {
lock_external_instance_pg_state(tx).await?;
if read_external_instance_pg_state(&mut **tx)
.await?
.databases
.contains_key(dbname)
{
return Ok(());
}
Err(Error::BadRequest(format!(
"Windmill did not create a database named '{dbname}' on the external instance cluster. \
Create it from the instance settings first."
)))
}
/// Write [`EXTERNAL_INSTANCE_PG_SETTING`]: `None`, null or an empty string unsets it. Every writer
/// of global settings goes through this for that key — the per-key and bulk endpoints as well as
/// the declarative sync — instead of writing the row itself.
///
/// The checks and the write share one transaction holding [`lock_external_instance_pg_state`]. A
/// check taken outside it could pass while a database create still reads the old cluster, which
/// would then register a database there after the setting names another one.
///
/// Authorization: checks nothing. Callers MUST be superadmin, or the declarative instance config
/// sync, which applies what the operator deployed.
pub async fn write_external_instance_pg_setting(
db: &DB,
value: Option<&serde_json::Value>,
) -> Result<()> {
let value = match value {
None | Some(serde_json::Value::Null) => None,
Some(serde_json::Value::String(s)) if s.trim().is_empty() => None,
Some(value) => Some(value),
};
let mut tx = db.begin().await?;
lock_external_instance_pg_state(&mut tx).await?;
match value {
None => {
ensure_external_instance_pg_removable(db).await?;
sqlx::query("DELETE FROM global_settings WHERE name = $1")
.bind(EXTERNAL_INSTANCE_PG_SETTING)
.execute(&mut *tx)
.await?;
}
Some(value) => {
crate::external_instance_pg_oss::validate_external_instance_pg_setting(value)?;
ensure_external_instance_pg_not_repointed(db, value).await?;
sqlx::query(
"INSERT INTO global_settings (name, value) VALUES ($1, $2)
ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value, updated_at = now()",
)
.bind(EXTERNAL_INSTANCE_PG_SETTING)
.bind(value)
.execute(&mut *tx)
.await?;
}
}
tx.commit().await?;
tracing::info!(
"{} global setting {EXTERNAL_INSTANCE_PG_SETTING}",
if value.is_some() { "Set" } else { "Unset" }
);
Ok(())
}
/// [`write_external_instance_pg_setting`] for a settings diff: writes the key if the diff touches
/// it, and takes it out of the diff so the generic apply does not write it again.
///
/// Authorization: checks nothing. Callers MUST be superadmin, or the declarative instance config
/// sync, which applies what the operator deployed.
pub async fn write_external_instance_pg_from_diff(
db: &DB,
diff: &mut crate::instance_config::SettingsDiff,
) -> Result<()> {
if let Some(value) = diff.upserts.remove(EXTERNAL_INSTANCE_PG_SETTING) {
write_external_instance_pg_setting(db, Some(&value)).await?;
}
if let Some(i) = diff
.deletes
.iter()
.position(|k| k == EXTERNAL_INSTANCE_PG_SETTING)
{
diff.deletes.remove(i);
write_external_instance_pg_setting(db, None).await?;
}
Ok(())
}
/// Refuse pointing the setting at another host or port while databases or data table roles live on
/// the current one. Data tables name databases, and the role catalog names logins, not clusters, so
/// both would silently resolve to whatever the new cluster holds under the same names. Other fields
/// (admin login, sslmode) may change freely.
async fn ensure_external_instance_pg_not_repointed(
db: &DB,
value: &serde_json::Value,
) -> Result<()> {
let Some(current) = read_external_instance_pg_config(db).await? else {
return Ok(());
};
let Ok(desired) = serde_json::from_value::<ExternalInstancePg>(value.clone()) else {
return Ok(());
};
if external_instance_pg_address(&current) == external_instance_pg_address(&desired) {
return Ok(());
}
ensure_external_instance_pg_unused(
db,
&format!("pointing {EXTERNAL_INSTANCE_PG_SETTING} at another cluster"),
)
.await
}
/// Converge the external cluster on the configured login: check what it can do, create or update
/// Windmill's two roles with the stored passwords, and report anything that would get in the way.
/// With `rotate_passwords`, generate new passwords first. Safe to run again; running it again is
/// how a failed rotation is repaired.
///
/// Authorization: administers the external cluster with its admin credentials and checks nothing.
/// Callers MUST be superadmin.
pub async fn setup_external_instance_pg_unchecked(
db: &DB,
rotate_passwords: bool,
) -> Result<ExternalInstancePgSetupReport> {
crate::external_instance_pg_oss::setup_external_instance_pg_unchecked(db, rotate_passwords)
.await
}
@@ -0,0 +1,81 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! Where the external instance cluster comes from: the enterprise implementation, or a refusal.
//! `private` alone is not that edition: community builds carry it.
use crate::error::Error;
pub fn external_instance_pg_unavailable() -> Error {
Error::BadRequest(
"External instance databases are a Windmill Enterprise Edition feature".to_string(),
)
}
#[cfg(all(feature = "private", feature = "enterprise"))]
pub(crate) use crate::external_instance_pg_ee::{
create_external_instance_database_unchecked, drop_external_instance_database_unchecked,
external_instance_connection_unchecked, setup_external_instance_pg_unchecked,
validate_external_instance_pg_setting,
};
#[cfg(all(feature = "private", feature = "enterprise"))]
pub(crate) fn ensure_external_instance_available() -> crate::error::Result<()> {
Ok(())
}
#[cfg(not(all(feature = "private", feature = "enterprise")))]
pub(crate) use ce::*;
#[cfg(not(all(feature = "private", feature = "enterprise")))]
mod ce {
use super::external_instance_pg_unavailable as unavailable;
use crate::{
error::Result, external_instance_pg::ExternalInstancePgSetupReport, PgDatabase, DB,
};
pub(crate) fn validate_external_instance_pg_setting(_value: &serde_json::Value) -> Result<()> {
Err(unavailable())
}
pub(crate) fn ensure_external_instance_available() -> Result<()> {
Err(unavailable())
}
pub(crate) async fn setup_external_instance_pg_unchecked(
_db: &DB,
_rotate_passwords: bool,
) -> Result<ExternalInstancePgSetupReport> {
Err(unavailable())
}
pub(crate) async fn external_instance_connection_unchecked(
_db: &DB,
_dbname: &str,
_replication: bool,
) -> Result<PgDatabase> {
Err(unavailable())
}
pub(crate) async fn create_external_instance_database_unchecked(
_db: &DB,
_dbname: &str,
_tag: &str,
_for_workspace: Option<&str>,
) -> Result<()> {
Err(unavailable())
}
pub(crate) async fn drop_external_instance_database_unchecked(
_db: &DB,
_dbname: &str,
_exempt: Option<(&str, &str)>,
) -> Result<()> {
Err(unavailable())
}
}
@@ -36,30 +36,20 @@ pub async fn get_or_create_conversation_with_id(
title: &str,
conversation_id: Uuid,
) -> Result<FlowConversation> {
// Check if conversation already exists
let existing_conversation = sqlx::query_as!(
FlowConversation,
"SELECT id, workspace_id, flow_path, title, created_at, updated_at, created_by
FROM flow_conversation
WHERE id = $1 AND workspace_id = $2",
conversation_id,
w_id
)
.fetch_optional(&mut **tx)
.await?;
if let Some(existing) = existing_conversation {
if let Some(existing) = lock_conversation(tx, w_id, conversation_id).await? {
return Ok(existing);
}
// Truncate title to 25 characters max
let title = truncate_with_ellipsis(title, 25);
// Create new conversation with provided ID
let conversation = sqlx::query_as!(
// Every turn released by the same collector's commit finds no row: the first insert
// wins, the others wait on it, do nothing, and read the row it created.
let created = sqlx::query_as!(
FlowConversation,
"INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by, title)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (id) DO NOTHING
RETURNING id, workspace_id, flow_path, title, created_at, updated_at, created_by",
conversation_id,
w_id,
@@ -67,10 +57,41 @@ pub async fn get_or_create_conversation_with_id(
username,
title
)
.fetch_one(&mut **tx)
.fetch_optional(&mut **tx)
.await?;
if let Some(conversation) = created {
return Ok(conversation);
}
Ok(conversation)
lock_conversation(tx, w_id, conversation_id)
.await?
.ok_or_else(|| {
crate::error::Error::BadRequest(format!(
"conversation {conversation_id} belongs to another workspace"
))
})
}
/// Locked, so a turn orders against retention collecting the conversation
/// (windmill_common::jobs::delete_jobs): either the turn goes first and the collector then
/// sees its message, or it waits and finds the row gone and creates it again. Unlocked, the
/// message insert would wait on the parent row's lock instead and then fail its FK check.
async fn lock_conversation(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
w_id: &str,
conversation_id: Uuid,
) -> Result<Option<FlowConversation>> {
Ok(sqlx::query_as!(
FlowConversation,
"SELECT id, workspace_id, flow_path, title, created_at, updated_at, created_by
FROM flow_conversation
WHERE id = $1 AND workspace_id = $2
FOR UPDATE",
conversation_id,
w_id
)
.fetch_optional(&mut **tx)
.await?)
}
/// Add a message to a conversation using an existing transaction
@@ -57,6 +57,8 @@ pub const SAML_METADATA_SETTING: &str = "saml_metadata";
pub const SMTP_SETTING: &str = "smtp_settings";
pub const TEAMS_SETTING: &str = "teams";
pub const INDEXER_SETTING: &str = "indexer_settings";
pub const EXTERNAL_INSTANCE_PG_SETTING: &str = "external_instance_pg";
pub const EXTERNAL_INSTANCE_PG_STATE_SETTING: &str = "external_instance_pg_state";
pub const TIMEOUT_WAIT_RESULT_SETTING: &str = "timeout_wait_result";
pub const UNIQUE_ID_SETTING: &str = "uid";
@@ -355,6 +357,9 @@ pub const AGENT_WORKER_BLOCKED_SETTINGS: &[&str] = &[
// resolve datatable connections through the dedicated datatable endpoints, never these.
"custom_instance_pg_databases",
"custom_instance_replication_pwd",
// The external cluster's admin login, and the passwords Windmill generated on it.
EXTERNAL_INSTANCE_PG_SETTING,
EXTERNAL_INSTANCE_PG_STATE_SETTING,
];
/// Whether an agent worker may read the given global setting over HTTP.
+71 -32
View File
@@ -350,6 +350,8 @@ pub struct GlobalSettings {
pub ducklake_settings: Option<DucklakeSettings>,
#[serde(skip_serializing_if = "Option::is_none")]
pub custom_instance_pg_databases: Option<CustomInstancePgDatabases>,
#[serde(skip_serializing_if = "Option::is_none")]
pub external_instance_pg: Option<ExternalInstancePg>,
// Opaque settings (EE-private structs or no clear schema)
#[serde(skip_serializing_if = "Option::is_none")]
@@ -809,6 +811,9 @@ pub struct CustomInstanceDb {
pub error: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub tag: Option<String>,
/// The workspace a member created this fork copy for. Absent when a superadmin created it.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub workspace_id: Option<String>,
}
/// Setup log entries for a custom instance database.
@@ -833,6 +838,36 @@ pub struct CustomInstanceDbLogs {
pub replication_user_error: Option<String>,
}
// ---------------------------------------------------------------------------
// External instance PG cluster
// ---------------------------------------------------------------------------
/// The external Postgres cluster Windmill manages for `external_instance` data tables and Ducklake
/// catalogs. `user` logs in as the cluster's administrator: it needs `CREATEDB` and `CREATEROLE`.
/// `dbname` is only where that login connects to run cluster-wide statements.
///
/// Every field defaults rather than being required: this deserializes as part of the whole
/// instance config, and one malformed row must not make every other setting unreadable. The
/// write path and every use reject an incomplete value instead.
#[derive(Deserialize, Serialize, Clone, Debug, Default)]
#[cfg_attr(feature = "instance_config_schema", derive(schemars::JsonSchema))]
pub struct ExternalInstancePg {
#[serde(default)]
pub host: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub port: Option<u16>,
#[serde(default)]
pub user: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub password: Option<StringOrSecretRef>,
#[serde(skip_serializing_if = "Option::is_none")]
pub dbname: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub sslmode: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub root_certificate_pem: Option<String>,
}
// ---------------------------------------------------------------------------
// Autoscaling (worker config)
// ---------------------------------------------------------------------------
@@ -969,6 +1004,7 @@ pub const PROTECTED_SETTINGS: &[&str] = &[
"ducklake_settings",
"custom_instance_pg_databases",
"custom_instance_replication_pwd",
"external_instance_pg_state",
"uid",
"rsa_keys",
"jwt_secret",
@@ -994,6 +1030,8 @@ pub const HIDDEN_SETTINGS: &[&str] = &[
// Server-only (written by setup/refresh via direct SQL), never operator-authored —
// hidden so the config machinery can't read, rewrite, or drop it.
"custom_instance_replication_pwd",
// Same for the passwords and database registry Windmill keeps for the external cluster.
"external_instance_pg_state",
];
/// Top-level settings whose entire value is sensitive and must be fully redacted in logs.
@@ -1005,6 +1043,7 @@ const SENSITIVE_SETTINGS: &[&str] = &[
"license_key",
"ducklake_user_pg_pwd",
"custom_instance_replication_pwd",
"external_instance_pg_state",
"pip_index_url",
"pip_extra_index_url",
"npm_config_registry",
@@ -1030,6 +1069,7 @@ const NESTED_SENSITIVE_FIELDS: &[(&str, &[&str])] = &[
&["secret_key", "serviceAccountKey"],
),
("custom_instance_pg_databases", &["user_pwd"]),
("external_instance_pg", &["password"]),
];
fn redact_json_value(value: &serde_json::Value) -> serde_json::Value {
@@ -1358,7 +1398,8 @@ pub async fn sync_global_settings_declarative(
crate::global_settings::parse_allowed_origins_setting(desired.get(origins_key))
.map_err(|e| anyhow::anyhow!("{origins_key}: {e}"))?;
let diff = diff_global_settings(current, desired, ApplyMode::Replace);
let mut diff = diff_global_settings(current, desired, ApplyMode::Replace);
crate::external_instance_pg::write_external_instance_pg_from_diff(db, &mut diff).await?;
apply_settings_diff(db, &diff).await?;
Ok(())
@@ -1491,6 +1532,10 @@ pub fn resolve_env_refs(settings: &mut GlobalSettings) -> Result<(), String> {
resolve_env_option(&mut pg.user_pwd)?;
}
if let Some(pg) = &mut settings.external_instance_pg {
resolve_env_option(&mut pg.password)?;
}
Ok(())
}
@@ -2460,39 +2505,33 @@ mod tests {
}
#[test]
fn custom_instance_replication_pwd_is_isolated_from_config() {
// The replication-role password is server-only: written by setup/refresh via direct
// SQL, never operator-authored. It must stay out of the declarative config surface
// (hidden on read) and be undeletable, so config sync can't read, rewrite, or drop it.
assert!(HIDDEN_SETTINGS.contains(&"custom_instance_replication_pwd"));
assert!(PROTECTED_SETTINGS.contains(&"custom_instance_replication_pwd"));
assert!(SENSITIVE_SETTINGS.contains(&"custom_instance_replication_pwd"));
fn server_generated_db_passwords_are_isolated_from_config() {
// These hold passwords the server generates: written by setup/refresh via direct SQL,
// never operator-authored. They must stay out of the declarative config surface
// (hidden on read) and be undeletable, so config sync can't read, rewrite, or drop them.
for key in [
"custom_instance_replication_pwd",
"external_instance_pg_state",
] {
assert!(HIDDEN_SETTINGS.contains(&key), "{key}");
assert!(PROTECTED_SETTINGS.contains(&key), "{key}");
assert!(SENSITIVE_SETTINGS.contains(&key), "{key}");
// A stray desired value (e.g. flattened into `extra`) is ignored, not upserted.
let mut desired = BTreeMap::new();
desired.insert(
"custom_instance_replication_pwd".to_string(),
serde_json::json!("attacker-set"),
);
let diff = diff_global_settings(&BTreeMap::new(), &desired, ApplyMode::Merge);
assert!(
diff.upserts.is_empty(),
"hidden setting must not be upserted"
);
// A stray desired value (e.g. flattened into `extra`) is ignored, not upserted.
let mut desired = BTreeMap::new();
desired.insert(key.to_string(), serde_json::json!("attacker-set"));
let diff = diff_global_settings(&BTreeMap::new(), &desired, ApplyMode::Merge);
assert!(diff.upserts.is_empty(), "{key} must not be upserted");
// A current value is never deleted by a Replace that omits it.
let mut current = BTreeMap::new();
current.insert(
"custom_instance_replication_pwd".to_string(),
serde_json::json!("live"),
);
let diff = diff_global_settings(&current, &BTreeMap::new(), ApplyMode::Replace);
assert!(
!diff
.deletes
.contains(&"custom_instance_replication_pwd".to_string()),
"hidden setting must not be deleted"
);
// A current value is never deleted by a Replace that omits it.
let mut current = BTreeMap::new();
current.insert(key.to_string(), serde_json::json!("live"));
let diff = diff_global_settings(&current, &BTreeMap::new(), ApplyMode::Replace);
assert!(
!diff.deletes.contains(&key.to_string()),
"{key} must not be deleted"
);
}
}
#[test]
+52 -3
View File
@@ -478,6 +478,12 @@ pub static WORKER_INTERNAL_SERVER_INLINE_UTILS: OnceCell<WorkerInternalServerInl
/// set-based deletes below cost one scan per table per call instead. Because the cascade no
/// longer fires, every code path that deletes from `v2_job` by id must go through this helper
/// (or delete these tables itself) or it will leave orphan rows behind.
/// **Transaction contract:** call this inside a transaction. The conversation cleanup below
/// locks rows to serialise itself against a concurrent delete, and on an autocommit
/// connection that lock is released at statement end, silently restoring the race.
/// A conversation is collected only once every message row of it has gone with a job; a
/// row written with no job id (an MCP tool call, persisted under no job of its own) keeps
/// its conversation and the agent's memory for it alive for as long as it exists.
pub async fn delete_jobs(conn: &mut sqlx::PgConnection, ids: &[uuid::Uuid]) -> error::Result<()> {
sqlx::query!(
"DELETE FROM dispatch_event WHERE producer_job_id = ANY($1)",
@@ -485,12 +491,55 @@ pub async fn delete_jobs(conn: &mut sqlx::PgConnection, ids: &[uuid::Uuid]) -> e
)
.execute(&mut *conn)
.await?;
sqlx::query!(
"DELETE FROM flow_conversation_message WHERE job_id = ANY($1)",
let mut conversation_ids: Vec<uuid::Uuid> = sqlx::query_scalar!(
"DELETE FROM flow_conversation_message WHERE job_id = ANY($1) RETURNING conversation_id",
ids
)
.execute(&mut *conn)
.fetch_all(&mut *conn)
.await?;
conversation_ids.sort_unstable();
conversation_ids.dedup();
if !conversation_ids.is_empty() {
// A conversation is a view over its messages: once the last one goes with its job,
// the row and the agent's memory for it are all that is left, and nothing else
// collects them — `ai_agent_memory` carries no job id for retention to match on.
// Two statements rather than one CTE: a data-modifying CTE reads the snapshot from
// before the delete above, so every conversation would still look non-empty.
// Two calls each deleting one of a conversation's last messages would each still see
// the other's row — uncommitted deletes are invisible across transactions — so
// neither would collect it and nothing would try again. Taking the conversation row
// first serialises them: the second reads the first's delete and finds it empty.
sqlx::query_scalar!(
"SELECT id FROM flow_conversation WHERE id = ANY($1) ORDER BY id FOR UPDATE",
&conversation_ids
)
.fetch_all(&mut *conn)
.await?;
// Memory first, since it reads the conversation row for its workspace.
sqlx::query!(
"DELETE FROM ai_agent_memory a
USING flow_conversation c
WHERE c.id = ANY($1)
AND a.conversation_id = c.id
AND a.workspace_id = c.workspace_id
AND NOT EXISTS (
SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id
)",
&conversation_ids
)
.execute(&mut *conn)
.await?;
sqlx::query!(
"DELETE FROM flow_conversation c
WHERE c.id = ANY($1)
AND NOT EXISTS (
SELECT 1 FROM flow_conversation_message m WHERE m.conversation_id = c.id
)",
&conversation_ids
)
.execute(&mut *conn)
.await?;
}
sqlx::query!("DELETE FROM zombie_job_counter WHERE job_id = ANY($1)", ids)
.execute(&mut *conn)
.await?;
+150 -14
View File
@@ -37,6 +37,10 @@ pub mod bench;
pub mod cache;
pub mod client;
pub mod data_metrics;
pub mod datatable_roles;
#[cfg(all(feature = "private", feature = "enterprise"))]
mod datatable_roles_ee;
pub mod datatable_roles_oss;
pub mod db;
#[cfg(all(feature = "enterprise", feature = "private"))]
mod db_entra_ee;
@@ -54,6 +58,10 @@ pub mod ee_oss;
pub mod email_ee;
pub mod email_oss;
pub mod error;
pub mod external_instance_pg;
#[cfg(all(feature = "private", feature = "enterprise"))]
mod external_instance_pg_ee;
pub mod external_instance_pg_oss;
pub mod external_ip;
#[cfg(feature = "private")]
pub mod feature_usage_ee;
@@ -1020,6 +1028,20 @@ impl Future for TokioPgConnection {
}
}
impl TokioPgConnection {
/// Drive the connection and hand back what the server sends outside of a query's response —
/// notices above all, which driving it as a future silently discards.
pub fn poll_message(
&mut self,
cx: &mut core::task::Context<'_>,
) -> core::task::Poll<Option<Result<tokio_postgres::AsyncMessage, tokio_postgres::Error>>> {
match self {
TokioPgConnection::Tls(conn) => conn.poll_message(cx),
TokioPgConnection::NoTls(conn) => conn.poll_message(cx),
}
}
}
impl PgDatabase {
/// The role the connection logs in as, whichever way it authenticates.
pub fn login_name(&self) -> &str {
@@ -1064,7 +1086,13 @@ impl PgDatabase {
if err_str.contains("password authentication failed for user")
&& err_str.contains("custom_instance_user")
{
if let Some(db) = main_db {
// The external instance cluster has a `custom_instance_user` of its own, whose
// password setup manages. Rotating the local one would break every instance
// data table and fix nothing.
let local = PgDatabase::parse_uri(&get_database_url().await?.as_str().await)?;
let on_local_cluster = local.host == self.host
&& local.port.unwrap_or(5432) == self.port.unwrap_or(5432);
if let Some(db) = main_db.filter(|_| on_local_cluster) {
tracing::warn!(
"custom_instance_user password auth failed, refreshing and retrying..."
);
@@ -1514,12 +1542,51 @@ pub async fn drop_custom_instance_database(db: &DB, dbname: &str) -> error::Resu
Ok(())
}
/// What `custom_instance_user` holds on an instance database.
///
/// `WITH GRANT OPTION` throughout: this is the connection every data table resolves to as `admin`,
/// and it is the one that hands privileges to data table roles. Postgres refuses to let a role pass
/// on a privilege it does not itself hold with grant option, so without these an admin could own
/// the database and still be unable to grant `SELECT` on it to `analytics`.
pub(crate) fn instance_db_grants(dbname: &str) -> String {
format!(
"GRANT CONNECT ON DATABASE \"{dbname}\" TO custom_instance_user WITH GRANT OPTION;
GRANT CREATE ON DATABASE \"{dbname}\" TO custom_instance_user WITH GRANT OPTION;
DO $$ BEGIN
IF EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = 'public') THEN
GRANT USAGE ON SCHEMA public TO custom_instance_user WITH GRANT OPTION;
GRANT CREATE ON SCHEMA public TO custom_instance_user WITH GRANT OPTION;
ALTER DEFAULT PRIVILEGES IN SCHEMA public
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO custom_instance_user;
END IF;
END $$;"
)
}
/// Re-apply [`instance_db_grants`] to a managed database provisioned before data table roles
/// existed, whose grants carry no grant option. Connects as the cluster's administrator — the
/// database and `public` schema owner — since only it can hand out an option it holds.
///
/// Authorization: reaches a managed database with the server's own credentials and checks
/// nothing. Callers MUST have authorized administration of `dbname` — superadmin, or an admin of
/// the workspace governing a data table on it.
pub async fn ensure_instance_db_grant_options_unchecked(
db: &DB,
cluster: crate::datatable_roles::DatatableRoleCluster,
dbname: &str,
) -> error::Result<()> {
crate::datatable_roles_oss::ensure_instance_db_grant_options_unchecked(db, cluster, dbname)
.await
}
/// Create a custom instance database: CREATE DATABASE, grant permissions, register in global_settings.
/// The `tag` is stored in global_settings metadata (e.g. "datatable" or "ducklake").
/// The `tag` is stored in global_settings metadata (e.g. "datatable" or "ducklake"). `for_workspace`
/// is the workspace a member creates a fork copy for; see [`ensure_fork_database_available_to`].
pub async fn create_custom_instance_database(
db: &DB,
dbname: &str,
tag: &str,
for_workspace: Option<&str>,
) -> error::Result<()> {
let dbname = dbname.trim();
validate_dbname(dbname)?;
@@ -1553,17 +1620,7 @@ pub async fn create_custom_instance_database(
let (client, connection) = new_pg_creds.connect(Some(db)).await?;
let join_handle = tokio::spawn(async move { connection.await });
if let Err(e) = client
.batch_execute(&format!(
"GRANT CONNECT ON DATABASE \"{dbname}\" TO custom_instance_user;
GRANT USAGE ON SCHEMA public TO custom_instance_user;
GRANT CREATE ON SCHEMA public TO custom_instance_user;
GRANT CREATE ON DATABASE \"{dbname}\" TO custom_instance_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA public
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO custom_instance_user;"
))
.await
{
if let Err(e) = client.batch_execute(&instance_db_grants(dbname)).await {
tracing::warn!(
"Failed to grant permissions on '{}': {}. Continuing.",
dbname,
@@ -1583,7 +1640,8 @@ pub async fn create_custom_instance_database(
},
"success": true,
"error": null,
"tag": tag
"tag": tag,
"workspace_id": for_workspace,
});
sqlx::query!(
r#"UPDATE global_settings SET value = jsonb_set(value, '{databases}', (COALESCE(value->'databases', '{}'::jsonb) || to_jsonb($1::json))) WHERE name = 'custom_instance_pg_databases'"#,
@@ -1592,10 +1650,88 @@ pub async fn create_custom_instance_database(
.execute(db)
.await?;
// A data table role can only reach a database it may CONNECT to, and PUBLIC's default CONNECT
// would otherwise let every role in regardless of what this instance defines. Best-effort: a
// failure here leaves the database usable as `admin`, and the next role change repairs it.
if let Err(e) = crate::datatable_roles::converge_connect_grants(
db,
crate::datatable_roles::DatatableRoleCluster::Instance,
dbname,
)
.await
{
tracing::warn!("Could not set CONNECT grants on instance database '{dbname}': {e}");
}
tracing::info!("Created custom instance database '{}'", dbname);
Ok(())
}
/// The system's CA bundle file, for libpq clients that cannot take `sslrootcert=system`: that value
/// needs libpq 16, and verify-full only.
pub fn system_ca_bundle() -> Option<std::path::PathBuf> {
std::env::var_os("SSL_CERT_FILE")
.map(std::path::PathBuf::from)
.into_iter()
.chain(
[
"/etc/ssl/certs/ca-certificates.crt",
"/etc/pki/tls/certs/ca-bundle.crt",
"/etc/ssl/cert.pem",
"/etc/ssl/ca-bundle.pem",
]
.map(std::path::PathBuf::from),
)
.find(|path| path.is_file())
}
/// Refuse a workspace member writing a fork copy into, or pointing a fork at, the managed database
/// `dbname` of `kind`, unless `w_id` created it for that ([`create_custom_instance_database`], or
/// its external instance counterpart) and nothing uses it yet. The `wm_fork_` prefix is no
/// authorization: every database of a cluster answers to the same `custom_instance_user`, so a name
/// is all it takes to reach another workspace's copy.
///
/// Authorization: reads the global registries and every workspace's settings, and names other
/// workspaces in its refusal. Callers MUST have authorized `w_id` for the caller first — a member
/// of it forking or importing there — and MUST NOT call it on a workspace the caller is not in.
pub async fn ensure_fork_database_available_to(
db: &DB,
kind: workspaces::DataTableCatalogResourceType,
dbname: &str,
w_id: &str,
) -> error::Result<()> {
let created_for = match kind {
workspaces::DataTableCatalogResourceType::ExternalInstance => {
external_instance_pg::external_instance_databases(db)
.await?
.remove(dbname)
.and_then(|entry| entry.workspace_id)
}
_ => sqlx::query_scalar::<_, Option<String>>(
"SELECT value->'databases'->$1->>'workspace_id' FROM global_settings
WHERE name = 'custom_instance_pg_databases'",
)
.bind(dbname)
.fetch_optional(db)
.await?
.flatten(),
};
if created_for.as_deref() != Some(w_id) {
return Err(Error::BadRequest(format!(
"Database '{dbname}' was not created for a fork of workspace '{w_id}'"
)));
}
let uses =
workspaces::managed_database_uses(&mut *db.acquire().await?, kind, dbname, None).await?;
if !uses.is_empty() {
return Err(Error::BadRequest(format!(
"Database '{dbname}' is already in use: {}",
uses.join(", ")
)));
}
Ok(())
}
/// Connection options parsed from a database URL.
///
/// The only place a database URL becomes `PgConnectOptions`. Providers that mint the password

Some files were not shown because too many files have changed in this diff Show More