Merge remote-tracking branch 'origin/main' into glm/session-deployed-runs

The run page and runs list now mount in process (#11721, #11724), which gives
`/run/<id>` a slot of its own instead of resolving to an iframe. The tab tests
here read better for it: a tab showing a run is `kind: 'run'`, so it cannot
match the viewer test that routes a proposed run to an item's page, and the
card keeps the form as intended.

`Run again` arrives with it, seeding a deployed page's form from the tab url's
fragment. The detail pages now read both that and a chat call's proposal at
init, the proposal last — re-pointing a tab rebuilds its url without the
fragment, so in practice only one of them seeds — and `ItemViewerView`'s load
key takes both, since either changing has to rebuild the form.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Guilhem Lemouel
2026-10-13 05:26:14 -07:00
co-authored by Claude Opus 5
73 changed files with 3692 additions and 1586 deletions
+3
View File
@@ -33,6 +33,9 @@ Open-source platform for internal tools, workflows, API integrations, background
administers behind `external_instance` data tables and Ducklake catalogs: its invariants (one
lifecycle lock, managed-object markers, the setup gate, per-cluster roles, fork copy ownership)
and how to run one locally
- **Data table viewer roles**: `docs/datatable-viewer-roles.md`, on what a connection naming no
role holds: the default role as a floor, where the viewer comes from, why an admin's roles are
not the viewer's, the per-role-set logins and the order their rows and logins are written in
- **Enterprise**: `docs/enterprise.md` — EE file conventions and PR workflow
- **Operator write rights**: `docs/operator-write-rights.md` — which `operator_settings` flags are
enforced rather than cosmetic, and why a right that is granted-unless-withdrawn needs `Option`
+4 -1
View File
@@ -196,7 +196,10 @@ not have read is itself the failure — e.g. `write_variable.value` on a secret
Global (and flow) initial fixtures can seed `workspace.datatables` so the
`list_datatables`, `get_datatable_table_schema`, and `exec_datatable_sql` tools
return seeded data during evals. Each entry is
`{ datatable_name, schemas: { <schema>: { <table>: { columns, rows? } } } }`.
`{ datatable_name, schemas: { <schema>: { <table>: { columns, rows? } } } }`,
plus `migrations_enabled: true` and `migrations: [{ timestamp, name, code_up, code_down?, applied? }]`
for a datatable that keeps its schema as migrations, which the migration tools then
read and write in memory.
SQL runs through a small in-memory engine (`datatableSqlEngine.ts`), not a real
database. Writes are **stateful within a case**: `CREATE`/`DROP`/`INSERT`/`UPDATE`/
`DELETE` mutate the seeded datatable in place, so a later `list_datatables`,
@@ -30,6 +30,17 @@ export interface BenchmarkDatatableSeed {
[table: string]: BenchmarkDatatableTableSeed
}
}
/** Whether the datatable keeps its schema as migrations. */
migrations_enabled?: boolean
migrations?: BenchmarkDatatableMigrationSeed[]
}
export interface BenchmarkDatatableMigrationSeed {
timestamp: number
name: string
code_up: string
code_down?: string
applied?: boolean
}
export interface DatatableSqlResult {
+129 -1
View File
@@ -692,10 +692,138 @@ export function listBenchmarkDatatables(workspace: string): DataTableTables[] |
datatable_name: datatable.datatable_name,
schemas: Object.fromEntries(
Object.entries(datatable.schemas).map(([schema, tables]) => [schema, Object.keys(tables)])
)
),
...(datatable.migrations_enabled && { migrations_enabled: true })
}))
}
// ============= Datatable migrations =============
function benchmarkDatatable(workspace: string, datatableName: string): BenchmarkDatatableSeed {
const datatable = (benchmarkWorkspaceRunnables.get(workspace)?.datatables ?? []).find(
(entry) => entry.datatable_name === datatableName
)
if (!datatable) {
throw new Error(`datatable "${datatableName}" not found`)
}
return datatable
}
/** Run a migration body through the in-memory engine, one statement at a time and without the
* transaction frame, so the tables it creates show up in later reads. */
function applyBenchmarkMigrationSql(datatable: BenchmarkDatatableSeed, sql: string): void {
// Split on the semicolons outside single-quoted literals.
for (const statement of sql.match(/(?:'(?:[^']|'')*'|[^;'])+/g) ?? []) {
const trimmed = statement.replace(/^\s*--.*$/gm, '').trim()
if (trimmed !== '' && !/^(begin|end|commit)$/i.test(trimmed)) {
applyDatatableSql(datatable, trimmed)
}
}
}
export function getBenchmarkDatatableMigrationsStatus(input: {
workspace: string
datatableName: string
}) {
const datatable = benchmarkDatatable(input.workspace, input.datatableName)
if (!datatable.migrations_enabled) {
return { enabled: false, migrations: [] }
}
return {
enabled: true,
migrations: (datatable.migrations ?? []).map(({ applied, ...migration }) => ({
...migration,
status: applied ? ('ran' as const) : ('not_run' as const)
}))
}
}
export function createBenchmarkDatatableMigration(input: {
workspace: string
datatableName: string
requestBody: { name: string; code_up: string; code_down?: string }
}) {
const datatable = benchmarkDatatable(input.workspace, input.datatableName)
if (!datatable.migrations_enabled) {
throw new Error(
`Migrations are not enabled for data table '${input.datatableName}'. Enable them first.`
)
}
const migrations = (datatable.migrations ??= [])
const timestamp = Math.max(20260101000000, ...migrations.map((m) => m.timestamp)) + 1
migrations.push({ timestamp, ...input.requestBody })
return { datatable: input.datatableName, timestamp, ...input.requestBody }
}
export function upsertBenchmarkDatatableMigration(input: {
workspace: string
datatableName: string
requestBody: { timestamp: number; name: string; code_up: string; code_down?: string }
}): string {
const datatable = benchmarkDatatable(input.workspace, input.datatableName)
const migrations = (datatable.migrations ??= [])
const existing = migrations.find((m) => m.timestamp === input.requestBody.timestamp)
if (existing?.applied) {
throw new Error(
`Migration ${existing.timestamp} on data table '${input.datatableName}' has already been applied and cannot be modified`
)
}
if (existing) {
// Replaced whole, as the server does: a down left out is stored as none.
delete existing.code_down
Object.assign(existing, input.requestBody)
} else {
migrations.push({ ...input.requestBody })
migrations.sort((a, b) => a.timestamp - b.timestamp)
}
return `Upserted migration ${input.requestBody.timestamp} in ${input.datatableName}`
}
export function runBenchmarkDatatableMigrations(input: {
workspace: string
datatableName: string
upTo?: number
only?: number
}) {
const datatable = benchmarkDatatable(input.workspace, input.datatableName)
const applied: { version: number; name: string }[] = []
for (const migration of datatable.migrations ?? []) {
if (
migration.applied ||
(input.only !== undefined && migration.timestamp !== input.only) ||
(input.only === undefined && input.upTo !== undefined && migration.timestamp > input.upTo)
) {
continue
}
applyBenchmarkMigrationSql(datatable, migration.code_up)
migration.applied = true
applied.push({ version: migration.timestamp, name: migration.name })
}
return { applied }
}
export function rollbackBenchmarkDatatableMigration(input: {
workspace: string
datatableName: string
only?: number
}) {
const datatable = benchmarkDatatable(input.workspace, input.datatableName)
const last = (datatable.migrations ?? [])
.filter((m) => m.applied && (input.only === undefined || m.timestamp === input.only))
.at(-1)
if (!last) {
return { rolled_back: [] }
}
if (last.code_down === undefined) {
throw new Error(
`Cannot roll back migration ${last.timestamp} (${last.name}): it has no down migration`
)
}
applyBenchmarkMigrationSql(datatable, last.code_down)
last.applied = false
return { rolled_back: [{ version: last.timestamp, name: last.name }] }
}
// ============= DuckLake catalogs and declared metrics =============
/** Seeded DuckLake names, or `null` for a non-benchmark workspace. */
@@ -92,6 +92,11 @@ vi.mock('$lib/gen', async () => {
getBenchmarkCompletedJob,
getBenchmarkCompletedJobResultMaybe,
getBenchmarkDatatableSchema,
getBenchmarkDatatableMigrationsStatus,
createBenchmarkDatatableMigration,
upsertBenchmarkDatatableMigration,
runBenchmarkDatatableMigrations,
rollbackBenchmarkDatatableMigration,
getBenchmarkDraftForUser,
getBenchmarkFlowByPath,
getBenchmarkFlowAllResults,
@@ -400,7 +405,27 @@ vi.mock('$lib/gen', async () => {
schemaName: data.schemaName,
tableName: data.tableName
})
: actual.WorkspaceService.getDataTableTableSchema(data)
: actual.WorkspaceService.getDataTableTableSchema(data),
getDatatableMigrationsStatus: async (data: any) =>
hasBenchmarkWorkspace(data.workspace)
? getBenchmarkDatatableMigrationsStatus(data)
: actual.WorkspaceService.getDatatableMigrationsStatus(data),
createDatatableMigration: async (data: any) =>
hasBenchmarkWorkspace(data.workspace)
? createBenchmarkDatatableMigration(data)
: actual.WorkspaceService.createDatatableMigration(data),
upsertDatatableMigration: async (data: any) =>
hasBenchmarkWorkspace(data.workspace)
? upsertBenchmarkDatatableMigration(data)
: actual.WorkspaceService.upsertDatatableMigration(data),
runDatatableMigrations: async (data: any) =>
hasBenchmarkWorkspace(data.workspace)
? runBenchmarkDatatableMigrations(data)
: actual.WorkspaceService.runDatatableMigrations(data),
rollbackDatatableMigrations: async (data: any) =>
hasBenchmarkWorkspace(data.workspace)
? rollbackBenchmarkDatatableMigration(data)
: actual.WorkspaceService.rollbackDatatableMigrations(data)
}),
DataMetricService: wrapService(actual.DataMetricService, {
listDataMetrics: async (data: { workspace: string }) =>
+51
View File
@@ -901,6 +901,57 @@
- treats creating the table as a normal SQL statement and does not claim a separate registration step is needed
- does not write a script to create the table
# The same request on a data table that keeps its schema as migrations: the table is
# created by a migration, since plain SQL would leave it out of the history.
- id: global-datatable-schema-change-uses-migration
prompt: |-
Add a new table called refunds to our workspace data table, with an order id and a refund amount.
initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_migrations_seed.json
runtime:
maxTurns: 8
validate:
draftCountExactly: 0
toolExpect:
requiredToolsUsed:
- create_datatable_migration
forbiddenToolsUsed:
- write_script
- deploy_workspace_item
- delete_workspace_item
toolCallArgs:
- tool: create_datatable_migration
field: code_up
stringIncludesAnyOf:
- create table
skipJudge: true
judgeChecklist:
- notices that the data table keeps its schema as migrations
- creates the refunds table through a migration rather than a plain CREATE TABLE statement
- includes an order id and a refund amount column
- gives the migration a down that drops the table
# Row data is not schema: it is written with plain SQL even where migrations are enabled.
- id: global-datatable-row-change-skips-migration
prompt: |-
Mark order number 2 as shipped in our workspace data table.
initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_migrations_seed.json
runtime:
maxTurns: 12
validate:
draftCountExactly: 0
toolExpect:
requiredToolsUsed:
- exec_datatable_sql
forbiddenToolsUsed:
- create_datatable_migration
- write_script
- deploy_workspace_item
- delete_workspace_item
skipJudge: true
judgeChecklist:
- runs an UPDATE on the orders table setting the status of order id 2 to shipped
- does not create a migration for a change to row data
- id: global-test25-datatable-mutate-rows
prompt: |-
Mark order number 2 as shipped in our workspace data table.
@@ -0,0 +1,36 @@
{
"workspace": {
"datatables": [
{
"datatable_name": "main",
"migrations_enabled": true,
"migrations": [
{
"timestamp": 20260501090000,
"name": "create_orders",
"code_up": "BEGIN;\n\nCREATE TABLE orders (id serial PRIMARY KEY, customer_id int, total numeric, status text, created_at timestamptz DEFAULT now());\n\nEND;",
"code_down": "BEGIN;\n\nDROP TABLE orders;\n\nEND;",
"applied": true
}
],
"schemas": {
"public": {
"orders": {
"columns": {
"id": "int4",
"customer_id": "int4",
"total": "numeric",
"status": "text",
"created_at": "timestamptz"
},
"rows": [
{ "id": 1, "customer_id": 1, "total": 42.5, "status": "shipped", "created_at": "2026-05-01T10:00:00Z" },
{ "id": 2, "customer_id": 2, "total": 19.99, "status": "pending", "created_at": "2026-05-02T11:30:00Z" }
]
}
}
}
}
]
}
}
+1 -1
View File
@@ -1 +1 @@
1e538620a9b2f08b044997d032e71d8720ecbd5c
40ab35596e6fcb82a4dfc56e186befb807f8e2af
@@ -0,0 +1 @@
DROP TABLE IF EXISTS datatable_role_set;
@@ -0,0 +1,21 @@
-- The logins behind a data table connection made as several roles at once: one Postgres login per
-- distinct set of data table roles, a member of each. Which roles a login holds is its
-- memberships on the cluster, and its name is derived from their catalog ids, so a row only has to
-- carry the credential.
--
-- A row is written after its login exists and removed before the login is dropped, so a row always
-- names a login that can authenticate with `pwd`.
CREATE TABLE datatable_role_set (
cluster VARCHAR(20) NOT NULL CHECK (cluster IN ('instance', 'external_instance')),
name VARCHAR(63) NOT NULL,
-- Generated by Windmill, never entered by anyone, and never leaves the server.
pwd TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
-- Recorded at most once a day. A login unused for long enough is dropped, and made again if
-- its set of roles is asked for later.
last_used_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (cluster, name)
);
GRANT ALL ON datatable_role_set TO windmill_user;
GRANT ALL ON datatable_role_set TO windmill_admin;
+107 -59
View File
@@ -2,21 +2,22 @@
# It is not intended for manual editing.
version = 4
[[package]]
name = "addr2line"
version = "0.25.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b"
dependencies = [
"gimli",
]
[[package]]
name = "adler2"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
[[package]]
name = "aead"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0"
dependencies = [
"crypto-common 0.1.7",
"generic-array",
]
[[package]]
name = "aes"
version = "0.7.5"
@@ -24,11 +25,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8"
dependencies = [
"cfg-if",
"cipher",
"cipher 0.3.0",
"cpufeatures 0.2.17",
"opaque-debug",
]
[[package]]
name = "aes"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
dependencies = [
"cfg-if",
"cipher 0.4.4",
"cpufeatures 0.2.17",
]
[[package]]
name = "aes-gcm"
version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1"
dependencies = [
"aead",
"aes 0.8.4",
"cipher 0.4.4",
"ctr",
"ghash",
"subtle",
]
[[package]]
name = "ahash"
version = "0.7.8"
@@ -389,21 +415,6 @@ dependencies = [
"tokio",
]
[[package]]
name = "backtrace"
version = "0.3.76"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6"
dependencies = [
"addr2line",
"cfg-if",
"libc",
"miniz_oxide",
"object",
"rustc-demangle",
"windows-link",
]
[[package]]
name = "base64"
version = "0.21.7"
@@ -539,7 +550,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2cb03d1bed155d89dce0f845b7899b18a9a163e148fd004e1c28421a783e2d8e"
dependencies = [
"block-padding",
"cipher",
"cipher 0.3.0",
]
[[package]]
@@ -763,6 +774,16 @@ dependencies = [
"generic-array",
]
[[package]]
name = "cipher"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
dependencies = [
"crypto-common 0.1.7",
"inout",
]
[[package]]
name = "clang-sys"
version = "1.8.1"
@@ -1095,6 +1116,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
dependencies = [
"generic-array",
"rand_core 0.6.4",
"typenum",
]
@@ -1107,6 +1129,15 @@ dependencies = [
"hybrid-array",
]
[[package]]
name = "ctr"
version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835"
dependencies = [
"cipher 0.4.4",
]
[[package]]
name = "ctutils"
version = "0.4.2"
@@ -1163,7 +1194,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac41dd49fb554432020d52c875fc290e110113f864c6b1b525cd62c7e7747a5d"
dependencies = [
"byteorder",
"cipher",
"cipher 0.3.0",
"opaque-debug",
]
@@ -1636,10 +1667,14 @@ dependencies = [
]
[[package]]
name = "gimli"
version = "0.32.3"
name = "ghash"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7"
checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1"
dependencies = [
"opaque-debug",
"polyval",
]
[[package]]
name = "git-version"
@@ -2101,6 +2136,15 @@ dependencies = [
"serde_core",
]
[[package]]
name = "inout"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
dependencies = [
"generic-array",
]
[[package]]
name = "interpolator"
version = "0.5.0"
@@ -2116,17 +2160,6 @@ dependencies = [
"rustversion",
]
[[package]]
name = "io-uring"
version = "0.7.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62"
dependencies = [
"bitflags",
"cfg-if",
"libc",
]
[[package]]
name = "ipnet"
version = "2.12.0"
@@ -2455,7 +2488,7 @@ version = "3.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c42f95f9d296f2dcb50665f507ed5a68a171453142663ce44d77a4eb217b053"
dependencies = [
"aes",
"aes 0.7.5",
"base64 0.21.7",
"block-modes",
"crc-any",
@@ -2619,9 +2652,9 @@ dependencies = [
[[package]]
name = "mio"
version = "1.1.1"
version = "1.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc"
checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee"
dependencies = [
"libc",
"wasi 0.11.1+wasi-snapshot-preview1",
@@ -3220,6 +3253,18 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6"
[[package]]
name = "polyval"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25"
dependencies = [
"cfg-if",
"cpufeatures 0.2.17",
"opaque-debug",
"universal-hash",
]
[[package]]
name = "postgres-native-tls"
version = "0.5.1"
@@ -4031,12 +4076,6 @@ dependencies = [
"serde_json",
]
[[package]]
name = "rustc-demangle"
version = "0.1.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d"
[[package]]
name = "rustc-hash"
version = "2.1.1"
@@ -5350,34 +5389,31 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
version = "1.47.1"
version = "1.53.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038"
checksum = "e95f91fcc7a621e8b030f6aa23c71fe9838ae2fb4d8118b75602a328f5144044"
dependencies = [
"backtrace",
"bytes",
"io-uring",
"libc",
"mio",
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"slab",
"socket2",
"tokio-macros",
"tracing",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
name = "tokio-macros"
version = "2.5.0"
version = "2.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"
checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.117",
"syn 3.0.7",
]
[[package]]
@@ -5875,6 +5911,16 @@ version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
[[package]]
name = "universal-hash"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea"
dependencies = [
"crypto-common 0.1.7",
"subtle",
]
[[package]]
name = "unscanny"
version = "0.1.0"
@@ -6296,6 +6342,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
name = "windmill-common"
version = "1.830.0"
dependencies = [
"aes-gcm",
"aho-corasick",
"anyhow",
"arc-swap",
@@ -6323,6 +6370,7 @@ dependencies = [
"git-version",
"globset",
"hex",
"hkdf",
"hmac 0.12.1",
"hyper",
"indexmap",
+19
View File
@@ -4449,6 +4449,24 @@ pub async fn monitor_db(
}
};
// Drop the data table role set logins nobody has connected with for a month. Every ~40 min,
// spawned for the same reason as the credential maintenance above: it opens a connection per
// managed database for each login it drops.
let datatable_role_set_sweep_f = async {
if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(240) {
if let Some(db) = conn.as_sql() {
let db = db.clone();
tokio::spawn(async move {
if let Err(e) =
windmill_common::datatable_roles::sweep_unused_role_set_logins(&db).await
{
tracing::error!("Error sweeping unused data table role set logins: {e}");
}
});
}
}
};
// Re-encrypt the legacy-format secret variables once new values are written in AES-256-GCM.
// Every ~5 min until one pass has gone through every workspace; spawned so a large instance
// does not hold up the join below.
@@ -4538,6 +4556,7 @@ pub async fn monitor_db(
cleanup_scheduled_job_deletions_f,
git_auto_pull_f,
git_credential_maintenance_f,
datatable_role_set_sweep_f,
ai_session_retention_f,
legacy_secret_upgrade_f,
pipeline_freshness_watchdog_f,
@@ -1847,3 +1847,222 @@ async fn cleanup_waits_out_a_save_racing_it_for_an_instance_database(
assert_eq!(users, vec!["test-workspace".to_string()]);
Ok(())
}
#[cfg(all(feature = "private", feature = "enterprise"))]
#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))]
async fn a_connection_naming_no_role_holds_the_default_one_and_the_viewers(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
use windmill_common::workspaces::{get_datatable_resource_from_db, DatatableAccess};
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let suffix: String = uuid::Uuid::new_v4().simple().to_string()[..8].to_string();
let names = [
format!("wmtest_floor_{suffix}"),
format!("wmtest_emea_{suffix}"),
format!("wmtest_amer_{suffix}"),
];
let roles_url = format!("http://localhost:{port}/api/settings/datatable_roles");
const MEMBER_LOGINS: &str = "SELECT m.rolname::text FROM pg_auth_members am
JOIN pg_roles r ON r.oid = am.roleid JOIN pg_roles m ON m.oid = am.member
WHERE r.rolname = $1 AND m.rolname LIKE 'wm\\_set\\_%'";
// The job is the app publisher's in every case: only the viewer the server recorded differs.
let connect = |role: Option<&'static str>, viewer: Option<&'static str>| {
get_datatable_resource_from_db(
&db,
"test-workspace",
"main",
role,
DatatableAccess::PermissionedAs {
permissioned_as: "u/test-user",
email: "test@windmill.dev",
end_user_email: viewer,
},
)
};
let set_permissions = |default_role: String, roles: Value| {
sqlx::query(
"UPDATE workspace_settings
SET datatable = jsonb_set(datatable, '{datatables,main,permissions}', $1)
WHERE workspace_id = 'test-workspace'",
)
.bind(json!({ "default_role": default_role, "roles": roles }))
.execute(&db)
};
let outcome = async {
let mut ids = vec![];
for name in &names {
let created: Value = authed(client().post(&roles_url), "SECRET_TOKEN")
.json(&json!({ "name": name }))
.send()
.await?
.error_for_status()?
.json()
.await?;
ids.push(created["id"].as_str().unwrap().to_string());
}
// `test-user-2` is named by both regional roles, one of them through a group, and by
// `admin`; `test-user-3` by one; the workspace admin `test-user` by neither.
let roles = json!({
"admin": {"tenants": ["u/test-user-2"]},
(ids[0].as_str()): {"tenants": ["*"]},
(ids[1].as_str()): {"tenants": ["u/test-user-2", "u/test-user-3"]},
(ids[2].as_str()): {"tenants": ["g/analysts"]},
});
set_permissions(ids[0].clone(), roles.clone()).await?;
sqlx::query(
"INSERT INTO usr_to_group (workspace_id, usr, group_)
VALUES ('test-workspace', 'test-user-2', 'analysts')",
)
.execute(&db)
.await?;
let member_of = |login: String| async {
let mut roles: Vec<String> = sqlx::query_scalar(
"SELECT r.rolname::text FROM pg_auth_members am
JOIN pg_roles r ON r.oid = am.roleid JOIN pg_roles m ON m.oid = am.member
WHERE m.rolname = $1",
)
.bind(login)
.fetch_all(&db)
.await?;
roles.sort();
Ok::<_, anyhow::Error>(roles)
};
let all = connect(None, Some("test2@windmill.dev")).await?;
let login = all["user"].as_str().unwrap().to_string();
assert!(login.starts_with("wm_set_"), "{login}");
let mut expected = names.to_vec();
expected.sort();
assert_eq!(
member_of(login.clone()).await?,
expected,
"the login holds the default role and the viewer's, and not `admin`"
);
assert_eq!(connect(None, Some("test2@windmill.dev")).await?, all);
let two = connect(None, Some("test3@windmill.dev")).await?;
let mut expected = names[..2].to_vec();
expected.sort();
assert_eq!(
member_of(two["user"].as_str().unwrap().to_string()).await?,
expected
);
// An admin may use every role, the one owning the tables included. As a viewer they add
// what names them, which here is nothing; nor does a job with no viewer recorded.
for viewer in [Some("test@windmill.dev"), None] {
assert_eq!(connect(None, viewer).await?["user"], names[0].as_str());
}
// A named role is that role alone, whoever is viewing.
let named: &'static str = Box::leak(names[1].clone().into_boxed_str());
assert_eq!(
connect(Some(named), Some("test2@windmill.dev")).await?["user"],
names[1].as_str()
);
// A default of `admin` is the connection it always was: the viewer's roles are not added.
set_permissions("admin".to_string(), roles.clone()).await?;
let admin = connect(None, Some("test2@windmill.dev")).await?;
assert_eq!(admin, connect(Some("admin"), None).await?);
assert!(!admin["user"].as_str().unwrap().starts_with("wm_"));
// With no default role the viewer's roles are all there is, the one open to everyone
// included, and a connection left with none is refused rather than resolved as anything.
set_permissions("$none".to_string(), roles.clone()).await?;
assert_eq!(
connect(None, Some("test3@windmill.dev")).await?,
two,
"no default changes nothing for a viewer the roles name"
);
assert!(connect(None, None).await.is_err());
// The same holds for a default the job's own identity is not a tenant of.
let mut closed = roles.clone();
closed[ids[0].as_str()] = json!({"tenants": []});
set_permissions(ids[0].clone(), closed).await?;
let as_member = |viewer: Option<&'static str>| {
get_datatable_resource_from_db(
&db,
"test-workspace",
"main",
None,
DatatableAccess::PermissionedAs {
permissioned_as: "u/test-user-3",
email: "test3@windmill.dev",
end_user_email: viewer,
},
)
};
assert_eq!(
as_member(Some("test3@windmill.dev")).await?["user"],
names[1].as_str()
);
assert!(as_member(None).await.is_err());
set_permissions(ids[0].clone(), roles).await?;
// A set login nobody connects with any more is dropped, a recent one is kept, and a
// dropped one is made again for the next viewer who needs it.
let exists = || async {
sqlx::query_scalar::<_, bool>(
"SELECT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = $1)",
)
.bind(&login)
.fetch_one(&db)
.await
};
windmill_common::datatable_roles::sweep_unused_role_set_logins(&db).await?;
assert!(exists().await?, "a set login in use was swept");
sqlx::query("UPDATE datatable_role_set SET last_used_at = now() - interval '31 days'")
.execute(&db)
.await?;
windmill_common::datatable_roles::sweep_unused_role_set_logins(&db).await?;
assert!(!exists().await?, "an unused set login was kept");
assert_eq!(
connect(None, Some("test2@windmill.dev")).await?["user"],
login.as_str()
);
// A disabled role must not stay reachable through a login that is a member of it.
authed(
client().post(format!("{roles_url}/{}", ids[2])),
"SECRET_TOKEN",
)
.json(&json!({ "enabled": false }))
.send()
.await?
.error_for_status()?;
let can_login: bool =
sqlx::query_scalar("SELECT rolcanlogin FROM pg_roles WHERE rolname = $1")
.bind(&login)
.fetch_one(&db)
.await?;
assert!(!can_login, "{login} still logs in with a disabled role");
assert_eq!(
connect(None, Some("test2@windmill.dev")).await?["user"],
two["user"]
);
Ok::<_, anyhow::Error>(())
}
.await;
// Roles are cluster-wide, so they outlive this test's throwaway database.
for name in &names {
let logins: Vec<String> = sqlx::query_scalar(MEMBER_LOGINS)
.bind(name)
.fetch_all(&db)
.await
.unwrap_or_default();
for login in logins {
let _ = sqlx::query(&format!("DROP ROLE IF EXISTS \"{login}\""))
.execute(&db)
.await;
}
let _ = sqlx::query(&format!("DROP ROLE IF EXISTS \"{name}\""))
.execute(&db)
.await;
}
outcome
}
@@ -880,7 +880,11 @@ struct DatatableMigrationsStatusResult {
/// Whether the SQL-migrations feature is enabled for a data table. Honors the
/// explicit `migrations_enabled` flag; when unset (data tables predating the
/// feature) it is considered enabled only if migrations already exist.
async fn datatable_migrations_enabled(db: &DB, w_id: &str, datatable_name: &str) -> Result<bool> {
pub(crate) async fn datatable_migrations_enabled(
db: &DB,
w_id: &str,
datatable_name: &str,
) -> Result<bool> {
let flag: Option<bool> = sqlx::query_scalar!(
"SELECT (ws.datatable->'datatables'->$2->>'migrations_enabled')::boolean \
FROM workspace_settings ws WHERE ws.workspace_id = $1",
@@ -2325,6 +2325,9 @@ struct DataTableTables {
/// What the role the listing connected as may create.
can_create_schema: bool,
creatable_schemas: Vec<String>,
/// Whether its schema is kept as versioned migrations, which a change made with plain SQL
/// would bypass.
migrations_enabled: bool,
}
#[derive(Deserialize)]
@@ -2570,8 +2573,17 @@ async fn list_one_datatable_tables(
default_role: windmill_common::datatable_roles::ADMIN_DATATABLE_ROLE.to_string(),
can_create_schema: false,
creatable_schemas: vec![],
migrations_enabled: false,
};
let result: Result<()> = async {
// Read from the workspace settings alone, so it is known even when the database is not
// reachable.
entry.migrations_enabled = crate::datatable_migrations::datatable_migrations_enabled(
db,
w_id,
&entry.datatable_name,
)
.await?;
let governing = resolve_governing_datatable(db, w_id, &entry.datatable_name).await?;
entry.instance = governing.role_cluster().is_some();
let usable =
+8
View File
@@ -5568,6 +5568,9 @@ paths:
type: boolean
default_role:
type: string
description: >-
The id of one of `roles`, `admin` (the default), or `$none` for no default
role: a connection that names no role then holds its viewer's roles only.
roles:
type: array
items:
@@ -34782,6 +34785,7 @@ components:
type: boolean
default_role:
type: string
description: A role id, `admin`, or `$none` when the data table has no default role.
roles:
type: array
items:
@@ -37332,6 +37336,7 @@ components:
default_role,
can_create_schema,
creatable_schemas,
migrations_enabled,
]
properties:
datatable_name:
@@ -37365,6 +37370,9 @@ components:
description: the schemas the role the listing connected as may create in
items:
type: string
migrations_enabled:
type: boolean
description: whether its schema is kept as versioned migrations, which a change made with plain SQL would bypass
DataTableTableSchema:
type: object
+10 -3
View File
@@ -8982,6 +8982,7 @@ async fn run_preview_script(
check_tag_available_for_workspace(&db, &w_id, &tag, &push_args, &authed).await?;
let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?;
let end_user_email = run_end_user_email(&db, &w_id, &authed, None).await?;
let (uuid, tx) = push(
&db,
tx,
@@ -9029,7 +9030,7 @@ async fn run_preview_script(
None,
Some(&authed.clone().into()),
false,
None,
end_user_email,
authed.trigger_or_fallback(None),
None,
scope_ceiling.as_deref(),
@@ -9068,6 +9069,7 @@ async fn run_inline_preview_script(
if let Some(job_id) = job_id {
register_potential_assets_on_inline_execution(job_id, &w_id, &preview);
}
let end_user_email = run_end_user_email(&db, &w_id, &authed, None).await?;
let utils = get_worker_internal_server_inline_utils()?;
let result = utils.run_inline_preview_script.as_ref()(RunInlinePreviewScriptFnParams {
content: preview.content,
@@ -9078,6 +9080,7 @@ async fn run_inline_preview_script(
created_by: authed.display_username().to_string(),
permissioned_as: username_to_permissioned_as(&authed.username),
permissioned_as_email: authed.email.clone(),
end_user_email,
lang: preview.language,
job_dir: "".to_string(),
worker_name: "".to_string(),
@@ -9217,6 +9220,7 @@ async fn run_inline_script_inner(
.to_string(),
));
}
let end_user_email = run_end_user_email(&db, &w_id, &authed, None).await?;
let utils = get_worker_internal_server_inline_utils()?;
let authed_owned: windmill_common::db::Authed = authed.clone().into();
let result = utils.run_inline_script.as_ref()(RunInlineScriptFnParams {
@@ -9228,6 +9232,7 @@ async fn run_inline_script_inner(
created_by: authed.display_username().to_string(),
permissioned_as: username_to_permissioned_as(&authed.username),
permissioned_as_email: authed.email.clone(),
end_user_email,
job_dir: "".to_string(),
worker_name: "".to_string(),
worker_dir: "".to_string(),
@@ -9404,6 +9409,7 @@ async fn run_bundle_preview_script(
// tracing::info!("is_tar 1: {is_tar}");
// hmap.insert("")
let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?;
let end_user_email = run_end_user_email(&db, &w_id, &authed, None).await?;
let (uuid, ntx) = push(
&db,
ltx,
@@ -9447,7 +9453,7 @@ async fn run_bundle_preview_script(
None,
Some(&authed.clone().into()),
false,
None,
end_user_email,
authed.trigger_or_fallback(None),
None,
scope_ceiling.as_deref(),
@@ -10151,6 +10157,7 @@ async fn run_preview_flow_job(
.await?;
let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?;
let end_user_email = run_end_user_email(&db, &w_id, &authed, None).await?;
let (uuid, mut tx) = push(
&db,
tx,
@@ -10182,7 +10189,7 @@ async fn run_preview_flow_job(
None,
Some(&authed.clone().into()),
false,
None,
end_user_email,
authed.trigger_or_fallback(None),
None,
scope_ceiling.as_deref(),
@@ -72,6 +72,11 @@ impl DatatableRoleCluster {
/// created, renamed or dropped.
pub const ADMIN_DATATABLE_ROLE: &str = "admin";
/// Stands in a data table's `default_role` for "none": a connection that names no role then holds
/// its viewer's roles only, and is refused without any. `$` is outside both the role-name charset
/// and the catalog ids, and a server predating it refuses it as a role the data table lacks.
pub const NO_DEFAULT_DATATABLE_ROLE: &str = "$none";
/// The login the admin connection uses, and the role every created role is granted to — that
/// membership is what later lets it `ALTER ... OWNER TO` a role and drop it.
pub const CUSTOM_INSTANCE_USER: &str = "custom_instance_user";
@@ -121,8 +126,13 @@ fn is_reserved_role_name(name: &str) -> bool {
|| lower.starts_with("pg_")
|| lower.starts_with("windmill_")
|| lower.starts_with("custom_instance_")
|| lower.starts_with(ROLE_SET_LOGIN_PREFIX)
}
/// Prefix of the logins Windmill creates for a connection made as several roles at once. Reserved,
/// so no data table role can be created under, renamed to or taken over at such a name.
pub const ROLE_SET_LOGIN_PREFIX: &str = "wm_set_";
/// The charset is what makes every downstream interpolation safe: the name reaches Postgres as a
/// quoted identifier, a `-- role <name>` annotation, and a `?role=` query parameter.
pub fn validate_role_name(name: &str) -> Result<()> {
@@ -414,6 +424,13 @@ pub async fn drop_datatable_role(
crate::datatable_roles_oss::drop_datatable_role(db, tx, cluster, name).await
}
/// Drop the logins made for a connection as several roles at once that nobody has connected with
/// for a month, on every cluster. They are made again on demand, so this only bounds how many
/// exist. Safe to call from any server at any time: it serializes on the role catalog lock.
pub async fn sweep_unused_role_set_logins(db: &DB) -> Result<()> {
crate::datatable_roles_oss::sweep_unused_role_set_logins(db).await
}
#[cfg(test)]
mod tests {
use super::*;
@@ -29,7 +29,8 @@ pub(crate) use crate::datatable_roles_ee::{
ensure_instance_db_grant_options_unchecked, forget_datatable_role_everywhere,
insert_role_catalog_entry, read_role_catalog, read_role_catalog_tx,
registered_instance_databases, rename_datatable_role, resolve_datatable_role_connection,
role_cluster, set_datatable_role_login, update_role_catalog_entry,
role_cluster, set_datatable_role_login, sweep_unused_role_set_logins,
update_role_catalog_entry,
};
#[cfg(not(all(feature = "private", feature = "enterprise")))]
@@ -235,6 +236,11 @@ mod ce {
}
}
/// No roles, so no set login to sweep.
pub(crate) async fn sweep_unused_role_set_logins(_db: &DB) -> Result<()> {
Ok(())
}
pub(crate) async fn forget_datatable_role_everywhere(
_tx: &mut Tx<'_>,
_role_id: &str,
+4
View File
@@ -474,6 +474,8 @@ pub struct RunInlinePreviewScriptFnParams {
pub created_by: String,
pub permissioned_as: String,
pub permissioned_as_email: String,
/// The caller's recorded end user: `job_perms.end_user_email` of the calling job.
pub end_user_email: Option<String>,
pub base_internal_url: String,
pub worker_name: String,
pub conn: crate::worker::Connection,
@@ -496,6 +498,8 @@ pub struct RunInlineScriptFnParams {
pub created_by: String,
pub permissioned_as: String,
pub permissioned_as_email: String,
/// The caller's recorded end user: `job_perms.end_user_email` of the calling job.
pub end_user_email: Option<String>,
pub base_internal_url: String,
pub worker_name: String,
pub conn: crate::worker::Connection,
+15 -3
View File
@@ -1564,7 +1564,8 @@ pub struct DataTableReference {
/// The access decision for one data table: which role a caller gets, and who may ask for each.
#[derive(Deserialize, Serialize, Debug, Clone, Default)]
pub struct DataTablePermissions {
/// A role id from the instance catalog, or `admin`. Absent = `admin`.
/// A role id from the instance catalog, `admin`, or
/// [`crate::datatable_roles::NO_DEFAULT_DATATABLE_ROLE`]. Absent = `admin`.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub default_role: Option<String>,
/// Keyed by instance role id, plus the reserved `admin` key. A role absent from this map
@@ -2161,9 +2162,13 @@ pub enum DatatableAccess<'a> {
PermissionedAs {
permissioned_as: &'a str,
email: &'a str,
/// The job's viewer: MUST be what the server recorded in `job_perms.end_user_email`,
/// never an argument or anything else the job or its caller can set.
end_user_email: Option<&'a str>,
},
/// A job identified by id; its owner is read from `v2_job`. For agent workers and anything
/// else that authenticates as infrastructure rather than as the job's user.
/// A job identified by id; its owner is read from `v2_job` and its viewer from `job_perms`.
/// For agent workers and anything else that authenticates as infrastructure rather than as
/// the job's user.
Job(uuid::Uuid),
/// No identity established. Unpermissioned data tables resolve as before; permissioned ones
/// are refused, so a caller predating this feature fails closed.
@@ -2211,6 +2216,13 @@ pub async fn can_use_datatable_role_in_governing_workspace(
///
/// The resolved role **logs in as itself**. Never `SET ROLE`: a script could `RESET ROLE` its way
/// back to admin.
///
/// A default role other than `admin` is a floor rather than the whole connection: a resolution
/// naming no role holds it when the job's own identity may use it, plus the roles whose tenant
/// lists name the job's viewer, as one login that is a member of each. With no role at all it is
/// refused. Being an admin adds none of the viewer's and `admin` is never among them, so a viewer
/// cannot be handed the connection that owns the tables and bypasses row-level security. A named
/// role is that role alone.
pub async fn get_datatable_resource_from_db(
db: &DB,
w_id: &str,
+63
View File
@@ -1726,6 +1726,23 @@ pub async fn cached_result_path(
hasher.update(b"wac_step:");
hasher.update(step_key.as_bytes());
}
// What a run returns can depend on who it ran for (`WM_END_USER_EMAIL`, a data table used
// as the viewer), so a result cached for one viewer is never served to another. Read from
// `job_perms`: a flow job re-fetched between steps does not carry it.
let end_user: Option<Option<String>> = sqlx::query_scalar(
"SELECT end_user_email FROM job_perms WHERE job_id = $1 AND workspace_id = $2",
)
.bind(job.id)
.bind(&job.workspace_id)
.fetch_optional(db)
.await?;
// Length-prefixed: the arguments follow undelimited, and the end of one viewer's address
// must not read as the start of another run's first argument name.
if let Some(end_user) = end_user.flatten() {
hasher.update(b"end_user:");
hasher.update((end_user.len() as u64).to_le_bytes());
hasher.update(end_user.as_bytes());
}
hash_args(
db,
client,
@@ -2475,6 +2492,52 @@ mod tests {
}
}
// A cache hit returns before the job runs, so the key is all that keeps a result computed
// for one viewer (a data table used as the viewer, `WM_END_USER_EMAIL`) from another.
#[sqlx::test(migrations = "../migrations")]
async fn a_cached_result_is_keyed_by_the_recorded_viewer(pool: sqlx::PgPool) {
let client = test_client();
// The last two differ only in where the viewer's address ends and the first argument
// name starts.
let mut paths = vec![];
for (viewer, arg) in [
(Some("a@windmill.dev"), "mode"),
(Some("a@windmill.dev"), "mode"),
(Some("b@windmill.dev"), "mode"),
(None, "mode"),
(Some("a@windmill.co"), "mode"),
(Some("a@windmill.com"), "ode"),
] {
let args = HashMap::from([(arg.to_string(), to_raw_value(&"all"))]);
let job = MiniPulledJob {
id: uuid::Uuid::new_v4(),
args: Some(sqlx::types::Json(args)),
..test_job()
};
sqlx::query(
"INSERT INTO job_perms (job_id, email, username, is_admin, is_operator,
workspace_id, groups, folders, end_user_email)
VALUES ($1, 'publisher@windmill.dev', 'publisher', false, false, 'test', '{}',
'{}', $2)",
)
.bind(job.id)
.bind(viewer)
.execute(&pool)
.await
.unwrap();
paths.push(cached_result_path(&pool, &client, &job, None).await.unwrap());
}
assert_eq!(
paths[0], paths[1],
"one viewer shares their own cached result"
);
assert_ne!(paths[0], paths[2], "two viewers share a cached result");
assert_ne!(paths[0], paths[3]);
assert_ne!(paths[2], paths[3]);
assert_ne!(paths[4], paths[5], "an address runs into the argument names");
}
#[tokio::test]
async fn test_transform_array_over_1000_passthrough() {
let db_url = std::env::var("DATABASE_URL")
@@ -2758,6 +2758,7 @@ async fn transform_attach_datatable(
DatatableAccess::PermissionedAs {
permissioned_as: &job.permissioned_as,
email: &job.permissioned_as_email,
end_user_email: job.permissioned_as_end_user_email.as_deref(),
},
)
.await?
@@ -1229,6 +1229,7 @@ pub async fn do_postgresql(
DatatableAccess::PermissionedAs {
permissioned_as: &job.permissioned_as,
email: &job.permissioned_as_email,
end_user_email: job.permissioned_as_end_user_email.as_deref(),
},
)
.await?
+4 -2
View File
@@ -7253,7 +7253,7 @@ pub fn init_worker_internal_server_inline_utils(
base_internal_url,
killpill_rx: Arc::new(killpill_rx),
run_inline_preview_script: Arc::new(|params| {
let job = MiniPulledJob::new_inline(
let mut job = MiniPulledJob::new_inline(
params.workspace_id,
params.args,
params.created_by,
@@ -7265,6 +7265,7 @@ pub fn init_worker_internal_server_inline_utils(
"inline_preview".to_string(),
Some(params.lang),
);
job.permissioned_as_end_user_email = params.end_user_email;
Box::pin(async move {
let mut mem_peak: i32 = -1;
let mut canceled_by: Option<CanceledBy> = None;
@@ -7314,7 +7315,7 @@ pub fn init_worker_internal_server_inline_utils(
let content_info =
get_script_content_by_hash(&script_hash, &params.workspace_id, &params.conn)
.await?;
let job = MiniPulledJob::new_inline(
let mut job = MiniPulledJob::new_inline(
params.workspace_id,
params.args,
params.created_by,
@@ -7326,6 +7327,7 @@ pub fn init_worker_internal_server_inline_utils(
"inline_run".to_string(),
content_info.language,
);
job.permissioned_as_end_user_email = params.end_user_email;
let mut mem_peak: i32 = -1;
let mut canceled_by: Option<CanceledBy> = None;
let mut column_order: Option<Vec<String>> = None;
+6 -6
View File
@@ -19,14 +19,14 @@
"pg-gateway": "0.3.0-beta.4",
"svelte": "^5.56.8",
"tar-stream": "^3.1.7",
"windmill-parser-wasm-asset": "1.753.0",
"windmill-parser-wasm-asset": "1.830.0",
"windmill-parser-wasm-csharp": "1.510.1",
"windmill-parser-wasm-go": "1.761.0",
"windmill-parser-wasm-java": "1.510.1",
"windmill-parser-wasm-nu": "1.510.1",
"windmill-parser-wasm-php": "1.647.1",
"windmill-parser-wasm-py": "1.782.0",
"windmill-parser-wasm-py-imports": "1.693.1",
"windmill-parser-wasm-py": "1.830.0",
"windmill-parser-wasm-py-imports": "1.830.0",
"windmill-parser-wasm-r": "1.668.1",
"windmill-parser-wasm-regex": "1.764.0",
"windmill-parser-wasm-ruby": "1.526.1",
@@ -291,7 +291,7 @@
"utility-types": ["utility-types@3.11.0", "", {}, "sha512-6Z7Ma2aVEWisaL6TvBCy7P8rm2LQoPv6dJ7ecIaIixHcwfbJ0x7mWdbcwlIM5IGQxPZSFYeqRCqlOOeKoJYMkw=="],
"windmill-parser-wasm-asset": ["windmill-parser-wasm-asset@1.753.0", "", {}, "sha512-zpJhjvcU8EWRoOJzas/nRGKjGdQnvzeB9GOxP+Mdmnk8BFk3uehsmHS2Krxyjo36fUrCHqobbnffEqf0g3LIGg=="],
"windmill-parser-wasm-asset": ["windmill-parser-wasm-asset@1.830.0", "", {}, "sha512-q4HSaMOd5cF8pmfPcEGiGeQbO8vhtv89us7zbO12IlgpxpWT8ojvw66J75y4yUt1PaEuaoztSaX8Gkxxn/6MXw=="],
"windmill-parser-wasm-csharp": ["windmill-parser-wasm-csharp@1.510.1", "", {}, "sha512-qm09YmnbeYHLwYn1jUnObVzPhYO9NZKMlIO7nlo7zPJBXqksgG5fK/KCtwGw9rChrnz+DsvM9wP5FhrwRLMtwQ=="],
@@ -303,9 +303,9 @@
"windmill-parser-wasm-php": ["windmill-parser-wasm-php@1.647.1", "", {}, "sha512-u2qaMkupSdhJibxvkLh3r/y36IARvnYNTLXWvOKxcQ0G/BPUB4+yF5o/yf47vv9zUV5WZv4mrdsKDt/pZDYeDg=="],
"windmill-parser-wasm-py": ["windmill-parser-wasm-py@1.782.0", "", {}, "sha512-ZnpEvhbffaJI+Uq7HH1mVO6kUgR4uiAIIxaLa30Pqb1/oz0h4abT6W7bFOApIE5X9SDOuycoaDBEr9Ogm6ZLzQ=="],
"windmill-parser-wasm-py": ["windmill-parser-wasm-py@1.830.0", "", {}, "sha512-+P4mIyp3lXkgjsr3/20Oc31ank8orQDd4K/v8TnjTQnCyxyy+VB9rynLESnkH6OlcEVSk2qf9pDM9c367E8DoQ=="],
"windmill-parser-wasm-py-imports": ["windmill-parser-wasm-py-imports@1.693.1", "", {}, "sha512-FC0KbREe2G/sa/9kYIR930wmWw+VL6PvEIqg12J3dsJes3A+0x5JIUPT/jeD+c24DrG0ko/Ub7yDnYs56Bem7g=="],
"windmill-parser-wasm-py-imports": ["windmill-parser-wasm-py-imports@1.830.0", "", {}, "sha512-Pi9gHStNFTEqfcDrjSxK5SnI/1dTG4xKcVUD3BHlR1aQ7E/VxkTD/D0INJm7mNC2usKb2bO21NMdKmGh+RAXEw=="],
"windmill-parser-wasm-r": ["windmill-parser-wasm-r@1.668.1", "", {}, "sha512-5YNeUibxpNBvYrxCgQcz1PxGhTFx2CyEpg2udtIhq7bx0d4gF/KDZVupMeQmAObmrEtTSFGUWNRJ4zXSWNrSpQ=="],
+3 -3
View File
@@ -31,14 +31,14 @@
"pg-gateway": "0.3.0-beta.4",
"svelte": "^5.56.8",
"tar-stream": "^3.1.7",
"windmill-parser-wasm-asset": "1.753.0",
"windmill-parser-wasm-asset": "1.830.0",
"windmill-parser-wasm-csharp": "1.510.1",
"windmill-parser-wasm-go": "1.761.0",
"windmill-parser-wasm-java": "1.510.1",
"windmill-parser-wasm-nu": "1.510.1",
"windmill-parser-wasm-php": "1.647.1",
"windmill-parser-wasm-py": "1.782.0",
"windmill-parser-wasm-py-imports": "1.693.1",
"windmill-parser-wasm-py": "1.830.0",
"windmill-parser-wasm-py-imports": "1.830.0",
"windmill-parser-wasm-r": "1.668.1",
"windmill-parser-wasm-regex": "1.764.0",
"windmill-parser-wasm-ruby": "1.526.1",
+9
View File
@@ -115,6 +115,14 @@ export function replaceInlineScripts(
}
return [];
}
/** `parent_version` is the editor's stale-draft base: a version id that changes on
* every deploy and means nothing in another workspace, so it is kept out of the
* tracked file and out of what a push deploys. */
export function stripEditorState(app: any) {
if (app?.value && typeof app.value == "object") {
delete app.value.parent_version;
}
}
export function isExecutionModeAnonymous(app: any) {
return app?.["policy"]?.["execution_mode"] == "anonymous";
}
@@ -294,6 +302,7 @@ export async function pushApp(
}
const path = localPath + "app.yaml";
const localApp = (await yamlParseFile(path)) as AppFile;
stripEditorState(localApp);
const isolation = statedIsolation(localApp, false);
replaceInlineScripts(localApp.value, localPath, true);
+13 -2
View File
@@ -149,7 +149,11 @@ import {
extractCurrentMapping,
} from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts";
import { generateFlowLockInternal } from "../flow/flow_metadata.ts";
import { markAccessFromPolicy, markIsolationFromPolicy } from "../app/app.ts";
import {
markAccessFromPolicy,
markIsolationFromPolicy,
stripEditorState,
} from "../app/app.ts";
import {
APP_BACKEND_FOLDER,
generateAppLocksInternal,
@@ -1329,6 +1333,7 @@ export function ZipFSElement(
log.error(`Failed to parse app.yaml at path: ${p}`);
throw error;
}
stripEditorState(app);
let inlineScripts;
try {
inlineScripts = extractInlineScriptsForApps(
@@ -2749,12 +2754,18 @@ export async function compareDynFSElement(
}
return o;
} else {
let o: any;
try {
return yamlParseContent(k, v);
o = yamlParseContent(k, v);
} catch (error) {
log.error(`Failed to parse YAML content at path: ${k}`);
throw error;
}
// A file tracked before the field was dropped still carries it.
if (path.basename(k) == "app.yaml") {
stripEditorState(o);
}
return o;
}
}
+50 -1
View File
@@ -34,6 +34,7 @@ export const SKILLS: SkillMetadata[] = [
{ name: "resources", description: "MUST use when managing resources." },
{ name: "write-workflow-as-code", description: "MUST use when writing or modifying Windmill Workflow-as-Code scripts using workflow, task, step, sleep, approvals, taskScript, taskFlow, task_script, or task_flow." },
{ name: "write-pipeline", description: "MUST use when creating or modifying a data pipeline, a set of scripts marked `pipeline` and wired together by `on` / `materialize` annotations." },
{ name: "datatables", description: "MUST use when changing a data table's schema, writing or running its migrations, or working with its roles and permissions." },
{ name: "cli-commands", description: "MUST use when using the CLI, including debugging job failures and inspecting run history via `wmill job`." },
{ name: "preview", description: "MUST use when opening the Windmill dev page / visual preview of a flow, script, or app. Triggers on words like preview, open, navigate to, visualize, see the flow/app/script, and after writing a flow/script/app for visual verification." },
];
@@ -6557,7 +6558,7 @@ data:
- \`<datatable>/<table>\` — Specific table in public schema
- \`<datatable>/<schema>:<table>\` — Table in specific schema
**Roles:** when a datatable is under roles, its queries run as a role, which only reaches what it was granted. \`roles\` records the role the app uses each datatable through; the app's code must pass the same role: \`wmill.datatable('main', { role: 'analyst' })\` in TypeScript, \`wmill.datatable('main', role='analyst')\` in Python. A datatable without an entry is used as its default role.
**Roles:** when a datatable is under roles, its queries run as a role, which only reaches what it was granted. \`roles\` records the role the app uses each datatable through; the app's code must pass the same role: \`wmill.datatable('main', { role: 'analyst' })\` in TypeScript, \`wmill.datatable('main', role='analyst')\` in Python. A datatable without an entry is used as its default role, if it has one, plus the roles of whoever is using the app unless that default is \`admin\`.
## SQL Migrations (sql_to_apply/)
@@ -6863,6 +6864,7 @@ Data tables are PostgreSQL databases managed by Windmill. Backend runnables quer
3. **No DDL inside runnables**: runnables only read and write rows (SELECT, INSERT, UPDATE, DELETE) on existing tables. Never CREATE, ALTER or DROP a table from a runnable.
4. **Qualify table names**: an unqualified name means the \`public\` schema, so write every other table as \`schema.table\`, in table creation and queries alike. The app's \`data\` config sets the default datatable and schema its tables go in; use them consistently across runnables.
5. **Pass the role**: when the app's \`data.roles\` gives a data table a role, every \`wmill.datatable\` call on it passes that role (\`wmill.datatable('main', { role: 'analyst' })\` in TypeScript, \`wmill.datatable('main', role='analyst')\` in Python). The role only reaches what it was granted, so a query outside it fails with \`permission denied\`.
6. **Per-viewer rows**: a data table with no entry in \`data.roles\` is called with no role. Unless its default role is \`admin\`, the connection then holds the default role, if it has one, plus every role the person using the app is a tenant of, and is refused when that is none. Row-level security policies written as \`pg_has_role(current_user, '<role>', 'member')\` return that viewer's rows with nothing added to the runnable. A role pins the connection to that one role for every viewer, so where rows should differ per viewer give the data table no \`data.roles\` entry.
\`wmill datatable list\` lists the workspace's data tables. Create or change tables with a migration in \`sql_to_apply/\` (see "SQL Migrations" above), then add them to \`data.tables\` in \`raw_app.yaml\`.
@@ -8122,6 +8124,53 @@ Node \`f/sales/orders_daily\` (runs when \`orders\` is produced, writes a rollup
SELECT date_trunc('day', ts) AS day, count(*) AS n
FROM ducklake.main.orders GROUP BY 1
\`\`\`
`,
"datatables": `---
name: datatables
description: MUST use when changing a data table's schema, writing or running its migrations, or working with its roles and permissions.
---
# Data tables: schema changes, migrations and permissions
A data table is a PostgreSQL database managed by Windmill and shared by the whole workspace: scripts, flows and apps reach it by name, \`main\` being the default. It is configured in the workspace settings (Data Tables), never created by SQL.
## Changing the schema
A data table with migrations enabled keeps its schema as an ordered list of versioned SQL migrations, each an \`up\` and an optional \`down\`. That list is what git sync and a workspace merge carry, and what rebuilds the schema in another workspace. So first find out whether migrations are enabled on the data table:
- **Enabled**: make every schema change (CREATE, ALTER, DROP, anything but row data) as a new migration. A change made with plain SQL exists in that one database only, and the history no longer describes it.
- **Not enabled**: schema changes are plain SQL. Enabling migrations is a workspace admin's decision, made in the Data Tables settings, not a step to take on the way to something else.
Writing a migration:
- Wrap the statements in \`BEGIN; ... END;\` so it applies atomically. Leave the frame out only for a statement that cannot run in a transaction, such as \`CREATE INDEX CONCURRENTLY\`: such a migration can fail half applied.
- Give it a \`down\` that undoes the \`up\` whenever the change is reversible. Without one it cannot be rolled back.
- Migrations apply oldest first. One that is applied is not edited: correct it with a new migration.
- Row data is not schema. Insert, update and delete rows with plain SQL, migrations or not.
\`wmill datatable migrate new <name>\` scaffolds \`migrations/datatable/<datatable>/<timestamp>_<name>.up.sql\` and \`.down.sql\`, \`wmill datatable migrate up\` pushes the local files and applies what is pending, \`wmill datatable migrate down\` rolls back the most recent one and \`wmill datatable migrate status\` lists them. All take \`-d <datatable>\` and default to \`main\`.
## A migration is not a draft
A migration is saved to the workspace and changes the database as soon as it is applied, for everything that reads those tables. To try a schema out without touching tables already in use:
- put the new tables in a schema of their own, created by the same migration (\`CREATE SCHEMA IF NOT EXISTS\`), and point what you are building at it. A raw app names its schema in its \`data\` config, which is why a new app gets a new schema by default;
- or work in a workspace fork whose data table was cloned when the fork was created ("Clone schema only" or "Clone schema and data"): migrations applied there change that copy alone, and reach the parent workspace when the fork is merged. A fork that kept the original data table points at the parent's database, so a migration applied in it changes the parent's tables at once.
Altering or dropping a table other scripts or apps use is neither of those: say what depends on it before doing it.
## Roles and permissions
Data table roles are a self-hosted Enterprise Edition feature, for data tables on managed Postgres (the instance database or the external managed instance). Without them, and always on a data table backed by a Postgres resource, everyone who can use the data table reaches all of it: there is no per-user, per-schema or per-row access to promise.
A data table under roles is reached as a Postgres role, and Postgres enforces what that role was granted:
- roles are assigned, per data table, to users, groups or folders. \`admin\` is the connection the data table had before roles: it owns the existing objects and ignores grants;
- a connection either names a role (\`wmill.datatable('main', { role: 'analyst' })\`, \`role='analyst'\` in Python, \`datatable://main?role=analyst\`) and holds that role alone, or names none and is resolved from the data table's default role;
- \`permission denied\` means the role in use lacks a grant. Say which role and which object: the grant is a data table admin's to give, in the roles drawer of the Database Manager. Do not switch to \`admin\` to get past it;
- a migration runs as the role its SQL declares on its first line, \`-- role <name>\`, the same one in its \`down\`. One that declares none runs as \`admin\`, which only those who may use \`admin\` can do.
Read the Permissions section of https://www.windmill.dev/docs/core_concepts/persistent_storage/data_tables before designing who reaches what: it covers tenants, grants, the default role and the roles a connection that names none ends up holding, rows that differ per viewer through row-level security, and how forks and raw apps behave under roles.
`,
"cli-commands": `---
name: cli-commands
+93
View File
@@ -0,0 +1,93 @@
# Data table connections and the viewer's roles
A data table under roles (Enterprise Edition) resolves a connection that names a role to that
role alone. A connection that names none is resolved from the data table's default role:
| Default role | A connection naming no role holds |
|---|---|
| `admin` (until changed) | `admin`, as before viewer roles existed |
| another role | that role when the job's own identity may use it, plus **every role the job's viewer is a tenant of** |
| none (`NO_DEFAULT_DATATABLE_ROLE`) | the viewer's roles only |
A connection left with no role at all is refused. Postgres row-level security can then answer
per viewer from a backend runnable that runs as the app's publisher, with nothing written in the
runnable.
Policies test membership, which is true for a role's own login too, so one policy serves a named
connection and an unnamed one:
```sql
CREATE POLICY emea ON orders FOR SELECT
USING (region = 'emea' AND pg_has_role(current_user, 'sales_emea', 'member'));
```
A policy names its role as text, so renaming the role breaks it.
## Code
| Where | What |
|---|---|
| `windmill-common/src/workspaces.rs` | `DatatableAccess`, `get_datatable_resource_from_db` |
| `windmill-common/src/datatable_roles_ee.rs` | `resolve_datatable_role_connection`, `viewer_role_ids`, `ensure_role_set_login`, `retire_role_set_logins_of`, `sweep_unused_role_set_logins` |
| `windmill-api-jobs/src/execution.rs` | `run_end_user_email`, where a run's viewer is recorded |
| `frontend/.../workspaceSettings/DataTablePermissionsButton.svelte` | the roles drawer, which warns while the default is `admin` |
## What holds it together
- **A default of `admin` is not a floor.** `admin` owns the tables and bypasses row-level
security, so nothing could be added to it, and it is the default until someone changes it.
Such a data table resolves exactly as it did before viewer roles existed. Nothing ever sets a
default back to `admin`: removing or deleting the default role leaves none.
- **The default role is tenant-checked against the job's own identity**, the publisher for a
publisher-mode app, and held only when that passes. The viewer's roles are added either way.
- **No role at all is a refusal, never an empty answer.** There is no login that would read
nothing: what a role sees is up to each table's grants and policies. A job without a recorded
viewer (anonymous app viewer, schedule, trigger, a tool job started by an AI agent step), or
whose viewer is not a member of the governing workspace, therefore needs a default it may use
or a named role.
- **A set login owns what it creates.** Postgres gives a new object to `current_user`, and
default privileges go by the creating role, so a table created over a connection that holds
several roles belongs to that set's login: no other role or viewer reaches it, and it passes to
`custom_instance_user` when the login is swept. Unnamed connections are for rows. To create an
object, name a role or run `SET ROLE <role>` first, which a set login may do for any role it
holds. Migrations are not affected: one that declares no role runs as `admin`.
- **The viewer is `job_perms.end_user_email`, and nothing else.** The server records it when the
run starts: the session behind an app request, the caller of a direct run or preview, and for a
job started with another job's token, that job's viewer. No argument reaches it.
- **A cached result belongs to one viewer.** `cached_result_path` hashes the run's recorded viewer
into the key, since a cache hit returns before any connection is resolved. Runs with no viewer
recorded keep sharing one entry.
- **Only a tenant list naming the viewer counts.** A workspace admin may *use* every role,
`admin` included. The viewer's roles are those whose tenants name them (user, group, folder,
`*`), and never `admin`.
- **One role is that role's login. Several are a login per distinct set.** `wm_set_<hash>` is a
member of exactly those roles, with `INHERIT`; the hash is over their catalog ids, so a rename
moves nothing and a change in someone's groups resolves them to another login instead of
editing one. Its privileges are its memberships, so `SET ROLE` only narrows and `RESET ROLE`
comes back to it. The prefix is reserved in `validate_role_name`.
- **Resolving stays cheap where no viewer could add anything.** The viewer is only looked up when
some role other than the default and `admin` has tenants on that data table.
- **A `datatable_role_set` row with a password names a login that can authenticate with it.** The
row is written after the login exists and removed, in its own commit, before the login is
dropped or made `NOLOGIN`. A login without a row is taken back with a new password on next use,
and so is one whose row has an empty password: that is what a sweep leaves when it could
neither drop a login nor let it back in, so the next sweep still finds it.
- **Set logins are bounded by use, not by the number of role combinations.** One exists only for
a set some viewer actually connected with, so there are at most as many as (viewer, data table)
pairs. `last_used_at` is recorded at most once a day, and `sweep_unused_role_set_logins`, run
from the monitor loop, drops those unused for 30 days; a dropped one is made again on demand.
The sweep goes by this instance's rows, never by the cluster's `wm_set_*` roles: a cluster can
serve several Windmill instances.
- **Disabling or dropping a role retires every set login holding it.** `NOLOGIN` on a role does
not reach a login that is a member of it.
- **A worker or server that predates this connects as the default role alone**, so a mixed fleet
shows a viewer fewer rows, never more. It refuses a data table with no default role, whose
marker it reads as a role the data table lacks.
## Trust
A job that names no role runs its author's code with the viewer's reach into the data table, the
same trust as an app in viewer execution mode or a script someone else wrote and you run. Whoever
can deploy a publisher-mode app, or a script that others run, can therefore read through the
non-`admin` roles of the people who use it. Naming a role in the code is how an author pins the
connection to one role instead.
+12 -12
View File
@@ -95,19 +95,19 @@
"vscode-languageclient": "~9.0.1",
"vscode-uri": "~3.1.0",
"vscode-ws-jsonrpc": "~3.5.0",
"windmill-parser-wasm-asset": "1.753.0",
"windmill-parser-wasm-asset": "1.830.0",
"windmill-parser-wasm-csharp": "1.510.1",
"windmill-parser-wasm-go": "1.761.0",
"windmill-parser-wasm-java": "1.510.1",
"windmill-parser-wasm-nu": "1.510.1",
"windmill-parser-wasm-php": "1.647.1",
"windmill-parser-wasm-py": "1.782.0",
"windmill-parser-wasm-py": "1.830.0",
"windmill-parser-wasm-r": "1.668.1",
"windmill-parser-wasm-regex": "1.764.0",
"windmill-parser-wasm-ruby": "1.526.1",
"windmill-parser-wasm-rust": "1.647.1",
"windmill-parser-wasm-ts": "1.695.0",
"windmill-parser-wasm-wac": "1.668.6",
"windmill-parser-wasm-wac": "1.830.0",
"windmill-parser-wasm-yaml": "1.770.0",
"windmill-sql-datatype-parser-wasm": "1.512.0",
"windmill-utils-internal": "1.8.2",
@@ -14644,9 +14644,9 @@
}
},
"node_modules/windmill-parser-wasm-asset": {
"version": "1.753.0",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-asset/-/windmill-parser-wasm-asset-1.753.0.tgz",
"integrity": "sha512-zpJhjvcU8EWRoOJzas/nRGKjGdQnvzeB9GOxP+Mdmnk8BFk3uehsmHS2Krxyjo36fUrCHqobbnffEqf0g3LIGg=="
"version": "1.830.0",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-asset/-/windmill-parser-wasm-asset-1.830.0.tgz",
"integrity": "sha512-q4HSaMOd5cF8pmfPcEGiGeQbO8vhtv89us7zbO12IlgpxpWT8ojvw66J75y4yUt1PaEuaoztSaX8Gkxxn/6MXw=="
},
"node_modules/windmill-parser-wasm-csharp": {
"version": "1.510.1",
@@ -14674,9 +14674,9 @@
"integrity": "sha512-u2qaMkupSdhJibxvkLh3r/y36IARvnYNTLXWvOKxcQ0G/BPUB4+yF5o/yf47vv9zUV5WZv4mrdsKDt/pZDYeDg=="
},
"node_modules/windmill-parser-wasm-py": {
"version": "1.782.0",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.782.0.tgz",
"integrity": "sha512-ZnpEvhbffaJI+Uq7HH1mVO6kUgR4uiAIIxaLa30Pqb1/oz0h4abT6W7bFOApIE5X9SDOuycoaDBEr9Ogm6ZLzQ=="
"version": "1.830.0",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.830.0.tgz",
"integrity": "sha512-+P4mIyp3lXkgjsr3/20Oc31ank8orQDd4K/v8TnjTQnCyxyy+VB9rynLESnkH6OlcEVSk2qf9pDM9c367E8DoQ=="
},
"node_modules/windmill-parser-wasm-r": {
"version": "1.668.1",
@@ -14704,9 +14704,9 @@
"integrity": "sha512-9EFxeRZWmfb7EyhSlcG7dzTTKETPRYAvpRlxxLkhhtI5I219wFgI7kwrMpz4stXHJj/aqBknVv66NQHRstSJmw=="
},
"node_modules/windmill-parser-wasm-wac": {
"version": "1.668.6",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-wac/-/windmill-parser-wasm-wac-1.668.6.tgz",
"integrity": "sha512-/ovcLWlIO+TQMrnWwcWoXquJI6hTZ+Zpo4POhV8z6e+pb4cVModGysASI9mERn2e9uj8/xSRmYqk54TLG8oUbQ=="
"version": "1.830.0",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-wac/-/windmill-parser-wasm-wac-1.830.0.tgz",
"integrity": "sha512-M67qPkzDnuMNWGrZTj9V4ajDoCQrTsQJ89UMHq308t5CtCdtiX8Z3freL+8SoRAG/dVlqQ8roK6dzncHIEM0hQ=="
},
"node_modules/windmill-parser-wasm-yaml": {
"version": "1.770.0",
+3 -3
View File
@@ -161,19 +161,19 @@
"vscode-languageclient": "~9.0.1",
"vscode-uri": "~3.1.0",
"vscode-ws-jsonrpc": "~3.5.0",
"windmill-parser-wasm-asset": "1.753.0",
"windmill-parser-wasm-asset": "1.830.0",
"windmill-parser-wasm-csharp": "1.510.1",
"windmill-parser-wasm-go": "1.761.0",
"windmill-parser-wasm-java": "1.510.1",
"windmill-parser-wasm-nu": "1.510.1",
"windmill-parser-wasm-php": "1.647.1",
"windmill-parser-wasm-py": "1.782.0",
"windmill-parser-wasm-py": "1.830.0",
"windmill-parser-wasm-r": "1.668.1",
"windmill-parser-wasm-regex": "1.764.0",
"windmill-parser-wasm-ruby": "1.526.1",
"windmill-parser-wasm-rust": "1.647.1",
"windmill-parser-wasm-ts": "1.695.0",
"windmill-parser-wasm-wac": "1.668.6",
"windmill-parser-wasm-wac": "1.830.0",
"windmill-parser-wasm-yaml": "1.770.0",
"windmill-sql-datatype-parser-wasm": "1.512.0",
"windmill-utils-internal": "1.8.2",
+10 -2
View File
@@ -45,7 +45,12 @@
import { favoriteManager } from './sidebar/FavoriteMenu.svelte'
import DatatableRoleBadge from './DatatableRoleBadge.svelte'
import type { AclTarget, Asset, DataTableTables } from '$lib/gen'
import { ADMIN_DATATABLE_ROLE, datatableNameTakesRole, type DatatableRowAction } from './dbTypes'
import {
ADMIN_DATATABLE_ROLE,
datatableNameTakesRole,
namedDefaultRole,
type DatatableRowAction
} from './dbTypes'
import TextInput from './text_input/TextInput.svelte'
import Checkbox from './common/checkbox/Checkbox.svelte'
import DbSchemaDiagram from './dbdiagram/DbSchemaDiagram.svelte'
@@ -417,7 +422,10 @@
if (!entry?.permissioned || !datatableNameTakesRole(datatable)) return undefined
const roles = entry.usable_roles
if (roles.length === 0) return { kind: 'none' }
const role = (datatable === currentDatatable ? currentRole : undefined) ?? entry.default_role
const role =
(datatable === currentDatatable ? currentRole : undefined) ??
namedDefaultRole(entry.default_role) ??
roles[0]
if (roles.length === 1 && roles[0] === ADMIN_DATATABLE_ROLE && role === ADMIN_DATATABLE_ROLE) {
return undefined
}
@@ -3,7 +3,7 @@
import { type MetricDataPoint, MetricsService } from '$lib/gen'
import { displayTime } from '$lib/utils'
import { workspaceStore } from '$lib/stores'
import { useOperatingWorkspace } from './operatingWorkspace.svelte'
import {
CategoryScale,
Chart as ChartJS,
@@ -26,6 +26,8 @@
let { jobId, jobUpdateLastFetch }: Props = $props();
const operatingWorkspace = useOperatingWorkspace()
let jobMetricsLastFetch: Date | undefined = undefined
let jobMemoryStats: MetricDataPoint[] | undefined = $state(undefined)
@@ -38,7 +40,7 @@
async function loadMetricsData() {
try {
let jobStatsPromise = MetricsService.getJobMetrics({
workspace: $workspaceStore!,
workspace: $operatingWorkspace!,
id: jobId,
requestBody: {
from_timestamp: jobMetricsLastFetch?.toISOString(),
+46 -30
View File
@@ -11,14 +11,12 @@
import { sendUserToast } from '$lib/toast'
import { displayDate } from '$lib/utils'
import { userWorkspaces, superadmin, devopsRole, enterpriseLicense } from '$lib/stores'
import { useHostedPage } from '$lib/components/hostedPage'
import {
userStore,
workspaceStore,
userWorkspaces,
superadmin,
devopsRole,
enterpriseLicense
} from '$lib/stores'
useOperatingUser,
useOperatingWorkspace
} from '$lib/components/operatingWorkspace.svelte'
import { Button, Drawer, DrawerContent, Skeleton, Tab, Tabs } from '$lib/components/common'
import TextInput from '$lib/components/text_input/TextInput.svelte'
import RunChart from '$lib/components/RunChart.svelte'
@@ -81,6 +79,13 @@
let { initialPath }: Props = $props()
// Under a host — an AI session's preview tab — the document URL, the page header band and
// the navigation workspace all belong to the host: the filters live where it keeps them,
// the filter row renders in the page, and requests go to the workspace it names.
const hosted = useHostedPage()
const operatingWorkspace = useOperatingWorkspace()
const operatingUser = useOperatingUser()
let paths: string[] = $state([])
let usernames: string[] = $state([])
let folders: string[] = $state([])
@@ -91,7 +96,7 @@
folders,
jobTriggerKinds,
isSuperAdminOrDevops: !!$superadmin || !!$devopsRole,
isAdminsWorkspace: $workspaceStore === 'admins'
isAdminsWorkspace: $operatingWorkspace === 'admins'
})
)
let perPage = useLocalStorageValue('runs_per_page', 1000, 'number')
@@ -115,6 +120,8 @@
// what was seeded at mount is gone. Re-apply it. Editing a filter writes with `replaceState`,
// which never reaches `page.url`, so a filter the user clears stays cleared.
$effect(() => {
// A host remounts the page when it re-points it, and its document URL is not this page's.
if (hosted) return
page.url.href
untrack(() => filters.seed(filterSeeds()))
})
@@ -171,7 +178,9 @@
let manualTimeframe = $derived(timeframe.type === 'manual' ? timeframe : undefined)
let graph: 'RunChart' | 'ConcurrencyChart' = $state(
typeOfChart(page.url.searchParams.get('graph'))
typeOfChart(
(hosted ? new URLSearchParams(hosted.search) : page.url.searchParams).get('graph')
)
)
let innerWidth = $state(window.innerWidth)
let jobsLoader = useJobsLoader(() => ({
@@ -183,7 +192,7 @@
resultError,
perPage: perPage.val,
lookback: graph === 'RunChart' ? 0 : lookback,
currentWorkspace: $workspaceStore ?? ''
currentWorkspace: $operatingWorkspace ?? ''
}))
let batchProgress = $derived(jobsLoader.batchProgress)
let currentBatchSize = $derived(jobsLoader.currentBatchSize)
@@ -211,18 +220,18 @@
}
async function loadUsernames(): Promise<void> {
usernames = await UserService.listUsernames({ workspace: $workspaceStore! })
usernames = await UserService.listUsernames({ workspace: $operatingWorkspace! })
}
async function loadFolders(): Promise<void> {
folders = await FolderService.listFolders({
workspace: $workspaceStore!
workspace: $operatingWorkspace!
}).then((x) => x.map((y) => y.name))
}
async function loadPaths() {
const npaths_scripts = await ScriptService.listScriptPaths({ workspace: $workspaceStore ?? '' })
const npaths_flows = await FlowService.listFlowPaths({ workspace: $workspaceStore ?? '' })
const npaths_scripts = await ScriptService.listScriptPaths({ workspace: $operatingWorkspace ?? '' })
const npaths_flows = await FlowService.listFlowPaths({ workspace: $operatingWorkspace ?? '' })
paths = npaths_scripts.concat(npaths_flows).sort()
}
@@ -265,7 +274,7 @@
const resultFilter = filters.val.result && JSON.parse(filters.val.result)
const { minTs, maxTs } = timeframe.computeMinMax()
return {
workspace: $workspaceStore ?? '',
workspace: $operatingWorkspace ?? '',
startedBefore: maxTs ?? undefined,
startedAfter: minTs ?? undefined,
schedulePath: filters.val.schedule_path ?? undefined,
@@ -323,7 +332,7 @@
}
async function cancelJobs(uuidsToCancel: string[], forceCancel: boolean = false) {
const uuids = await JobService.cancelSelection({
workspace: $workspaceStore ?? '',
workspace: $operatingWorkspace ?? '',
requestBody: uuidsToCancel,
forceCancel: forceCancel,
allWorkspaces: filters.val.all_workspaces || undefined
@@ -350,7 +359,7 @@
for (const j of jobs ?? []) if (j.workspace_id) workspaceById.set(j.id, j.workspace_id)
const byWorkspace = new Map<string, string[]>()
for (const id of jobIds) {
const ws = workspaceById.get(id) ?? $workspaceStore ?? ''
const ws = workspaceById.get(id) ?? $operatingWorkspace ?? ''
const bucket = byWorkspace.get(ws)
if (bucket) bucket.push(id)
else byWorkspace.set(ws, [id])
@@ -428,7 +437,7 @@
}
async function reRunJobs(jobIdsToReRun: string[], batchReRunOptions: BatchReRunOptions) {
if (!$workspaceStore) return
if (!$operatingWorkspace) return
if (askingForConfirmation) {
askingForConfirmation.loading = true
@@ -442,7 +451,7 @@
// workaround because EventSource does not support POST requests
// https://medium.com/@david.richards.tech/sse-server-sent-events-using-a-post-request-without-eventsource-1c0bd6f14425
const response = await fetch(`${OpenAPI.BASE}/w/${$workspaceStore}/jobs/run/batch_rerun_jobs`, {
const response = await fetch(`${OpenAPI.BASE}/w/${$operatingWorkspace}/jobs/run/batch_rerun_jobs`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body)
@@ -555,7 +564,7 @@
}
$effect(() => {
if ($workspaceStore) {
if ($operatingWorkspace) {
untrack(() => {
loadUsernames()
loadFolders()
@@ -603,7 +612,7 @@
openedRunId = undefined
}
let resolutionNote = $state('')
const phone = $derived(pageHeader.barWidth > 0 && pageHeader.barWidth < PHONE_BAR)
const phone = $derived(!hosted && pageHeader.barWidth > 0 && pageHeader.barWidth < PHONE_BAR)
</script>
{#snippet runsHint()}
@@ -620,7 +629,7 @@
two or three buttons, so they take a wider gap than the band's own. `flex-1 min-w-0` passes
the band's flexibility down to the search field, the one control here that gives width
back; the rest keep theirs or drop out of the row entirely at a width. -->
<div class="flex items-center gap-3 flex-1 min-w-0">
<div class="flex items-center gap-3 flex-1 min-w-0 {hosted ? 'flex-wrap justify-end' : ''}">
<!-- Always the icon form: the bar has room for two counters, never for the two labelled
blocks. pr-2 on top of the row's own gap because the counts hang outside their icons,
so without it the badge butts against the first filter.
@@ -816,12 +825,12 @@
schema={runsFilterSearchbarSchema}
presets={buildRunsFilterPresets({
isSuperAdminOrDevops: !!$superadmin || !!$devopsRole,
isAdminsWorkspace: $workspaceStore === 'admins'
isAdminsWorkspace: $operatingWorkspace === 'admins'
})}
bind:value={filters.val}
placeholder="Filter runs..."
size="sm"
autofocus
autofocus={!hosted}
/>
<TimeframeSelect
unifiedSize="sm"
@@ -833,11 +842,11 @@
<!-- The filters are shallow-routed, so the search has to come off `window.location` at click
time — `page.url` never sees them. Always the canonical `/runs`: only that is a recognized
preview page, and the `/runs/<path>` route mirrors its path into `?path=` anyway. -->
<div class={phone ? 'hidden' : 'contents'}>
<div class={phone || hosted ? 'hidden' : 'contents'}>
<OpenInSessionButton
source={{
page: () => RUNS_PATH + window.location.search,
workspaceId: $workspaceStore ?? undefined
workspaceId: $operatingWorkspace ?? undefined
}}
btnProps={{ unifiedSize: 'sm' }}
/>
@@ -900,13 +909,14 @@
</DrawerContent>
</Drawer>
<!-- The document's history is the host's: its Back is not a move on this page. -->
<svelte:window
onpopstate={() => {
reset()
if (!hosted) reset()
}}
/>
{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find((_) => _.id === $workspaceStore)?.operator_settings?.runs}
{#if operatingUser.current?.operator && $operatingWorkspace && !$userWorkspaces.find((_) => _.id === $operatingWorkspace)?.operator_settings?.runs}
<div class="bg-red-100 border-l-4 border-red-600 text-orange-700 p-4 m-4 mt-12" role="alert">
<p class="font-bold">Unauthorized</p>
<p>Page not available for operators</p>
@@ -915,7 +925,13 @@
<div class="w-full h-full flex flex-col" bind:clientWidth={innerWidth}>
<!-- `actionsFlexible`: the filter row holds a search field that can give width back, so the
bar lets this box shrink on a phone instead of holding its content width. -->
<PageHeaderContent afterName={runsHint} actions={headerActions} actionsFlexible />
{#if hosted}
<div class="flex px-4 pt-3 pb-2">
{@render headerActions()}
</div>
{:else}
<PageHeaderContent afterName={runsHint} actions={headerActions} actionsFlexible />
{/if}
<!-- Graph -->
<div id="runs-chart" class="p-2 px-4 bg-surface-tertiary mx-4 border rounded-md">
@@ -1106,7 +1122,7 @@
batchRerunOptionsIsOpen = true
}
},
...(!$userStore?.operator
...(!operatingUser.current?.operator
? [
{
// Operators are rejected by the endpoint, so offering it would only 403.
@@ -284,13 +284,21 @@
saveDrawerOpen = false
}
// `parent_version` is the draft's stale-check base. Deployed, it would put a version
// id that changes on every deploy into git-synced files and version diffs.
function withoutDraftBase<T extends App | undefined>(a: T): T {
if (!a) return a
const { parent_version: _pv, ...rest } = a
return rest as T
}
async function createApp(path: string) {
policy = await updatePolicy($app, policy)
try {
await AppService.createApp({
workspace: $workspaceStore!,
requestBody: {
value: $app,
value: withoutDraftBase($app),
path,
summary: $summary,
policy,
@@ -349,7 +357,7 @@
orderedJsonStringify(
replaceFalseWithUndefined({
summary: $summary,
value: $app,
value: withoutDraftBase($app),
path: newEditedPath || savedApp.path,
policy,
custom_path: customPath,
@@ -392,6 +400,7 @@
// Strip off extra information
deployedValue = replaceFalseWithUndefined({
...deployedApp,
value: withoutDraftBase(deployedApp.value as App),
id: undefined,
created_at: undefined,
created_by: undefined,
@@ -406,7 +415,7 @@
workspace: $workspaceStore!,
path: $appPath!,
requestBody: {
value: $app!,
value: withoutDraftBase($app!),
summary: $summary,
policy,
path: npath,
@@ -753,7 +762,7 @@
deployed: deployedValue ?? savedApp,
current: {
summary: $summary,
value: $app,
value: withoutDraftBase($app),
path: newEditedPath || savedApp.path,
policy,
custom_path: customPath,
@@ -847,7 +856,7 @@
bind:deployedValue
currentValue={{
summary: $summary,
value: $app,
value: withoutDraftBase($app),
path: newEditedPath || savedApp?.path,
policy,
custom_path: customPath,
@@ -898,7 +907,7 @@
deployed: deployedValue ?? savedApp,
current: {
summary: $summary,
value: $app,
value: withoutDraftBase($app),
path: newEditedPath || savedApp.path,
policy,
custom_path: customPath,
@@ -0,0 +1,141 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
const { statusMock, createMock, upsertMock, runMock } = vi.hoisted(() => ({
upsertMock: vi.fn(),
statusMock: vi.fn(),
createMock: vi.fn(),
runMock: vi.fn()
}))
vi.mock('./shared', () => ({
createToolDef: (_schema: unknown, name: string, description: string) => ({
type: 'function',
function: { name, description, parameters: {} }
})
}))
vi.mock('$lib/gen', () => ({
WorkspaceService: {
getDatatableMigrationsStatus: statusMock,
createDatatableMigration: createMock,
upsertDatatableMigration: upsertMock,
runDatatableMigrations: runMock
}
}))
import { getDatatableMigrationTools } from './datatableMigrationTools'
function create(args: Record<string, unknown>) {
const tool = getDatatableMigrationTools().find(
(entry) => entry.def.function.name === 'create_datatable_migration'
)!
return tool.fn({
args: {
datatable_name: 'main',
name: 'add_refunds',
code_up: 'BEGIN;\nCREATE TABLE refunds ();\nEND;',
...args
},
workspace: 'test-workspace',
helpers: {},
toolCallbacks: { setToolStatus: vi.fn(), removeToolStatus: vi.fn() } as any,
toolId: 'tool-1'
})
}
const applied = { timestamp: 10, name: 'init', code_up: '', status: 'ran' }
const created = { timestamp: 30, name: 'add_refunds', code_up: '', status: 'not_run' }
beforeEach(() => {
for (const mock of [statusMock, createMock, upsertMock, runMock]) mock.mockReset()
createMock.mockResolvedValue({ timestamp: 30 })
statusMock.mockResolvedValue({ enabled: true, migrations: [applied, created] })
runMock.mockResolvedValue({ applied: [{ version: 30, name: 'add_refunds' }] })
})
describe('create_datatable_migration', () => {
it('applies only the migration it created', async () => {
const result = await create({})
expect(runMock).toHaveBeenCalledWith({
workspace: 'test-workspace',
datatableName: 'main',
only: 30
})
expect(result).toContain('Applied migration 30 (add_refunds)')
})
it('leaves it pending when an earlier migration is not applied', async () => {
statusMock.mockResolvedValue({
enabled: true,
migrations: [
applied,
{ timestamp: 20, name: 'earlier', code_up: '', status: 'not_run' },
created
]
})
const result = await create({})
expect(runMock).not.toHaveBeenCalled()
expect(result).toContain('20 (earlier)')
})
it('keeps a migration that fails to apply and rewrites it on request', async () => {
runMock.mockRejectedValueOnce({ body: 'relation "refunds" already exists' })
const failed = await create({})
expect(failed).toContain('relation "refunds" already exists')
expect(failed).toContain('with version 30')
statusMock.mockResolvedValue({
enabled: true,
migrations: [applied, { ...created, code_down: 'DROP TABLE refunds;' }]
})
await create({ version: 30, code_up: 'BEGIN;\nCREATE TABLE refunds2 ();\nEND;' })
expect(createMock).toHaveBeenCalledTimes(1)
expect(upsertMock).toHaveBeenCalledWith({
workspace: 'test-workspace',
datatableName: 'main',
requestBody: {
timestamp: 30,
code_down: 'DROP TABLE refunds;',
name: 'add_refunds',
code_up: 'BEGIN;\nCREATE TABLE refunds2 ();\nEND;'
}
})
expect(runMock).toHaveBeenLastCalledWith({
workspace: 'test-workspace',
datatableName: 'main',
only: 30
})
})
it('refuses to rewrite a version that is not pending', async () => {
const result = await create({ version: 10 })
expect(upsertMock).not.toHaveBeenCalled()
expect(runMock).not.toHaveBeenCalled()
expect(result).toContain('cannot be rewritten')
})
it('does not call unreadable earlier migrations pending', async () => {
statusMock.mockResolvedValue({
enabled: true,
migrations: [
{ ...applied, status: 'unknown' },
{ ...created, status: 'unknown' }
],
error: 'connection refused'
})
const result = await create({})
expect(runMock).not.toHaveBeenCalled()
expect(result).toContain('connection refused')
expect(result).not.toContain('10 (init)')
})
it('declares the role above the SQL of both directions', async () => {
await create({ role: 'app_rw', code_down: 'DROP TABLE refunds;', apply: false })
expect(createMock.mock.calls[0][0].requestBody).toEqual({
name: 'add_refunds',
code_up: '-- role app_rw\nBEGIN;\nCREATE TABLE refunds ();\nEND;',
code_down: '-- role app_rw\nDROP TABLE refunds;'
})
expect(runMock).not.toHaveBeenCalled()
})
})
@@ -0,0 +1,391 @@
import { z } from 'zod'
import { WorkspaceService } from '$lib/gen'
import { withMigrationRole } from '$lib/components/datatableMigrationRole'
import { createToolDef, type ToolCallbacks } from './shared'
import { NONE, RUN_PREVIEW, type SessionTool } from './sessionCapabilities'
/**
* Tools over a data table's versioned SQL migrations (`datatable_migrations.rs`).
*
* A data table with migrations enabled keeps its schema as an ordered list of definitions, each
* applied once per database and recorded there. That list is what git sync and a workspace merge
* carry, so a schema change made with plain SQL exists in one database only.
*/
const memo = <T>(factory: () => T): (() => T) => {
let cached: T | undefined
return () => (cached ??= factory())
}
function apiErrorMessage(e: unknown): string {
const err = e as { body?: unknown; message?: unknown } | undefined
const body = err?.body
if (typeof body === 'string' && body !== '') return body
if (typeof err?.message === 'string') return err.message
return String(e)
}
function fail(toolCallbacks: ToolCallbacks, toolId: string, message: string): string {
toolCallbacks.setToolStatus(toolId, { content: message, error: message })
return message
}
const migrationLabel = (m: { version: number; name: string }) => `${m.version} (${m.name})`
const datatableNameSchema = () =>
z.string().describe('The datatable the migrations belong to, e.g. "main".')
// ============= Tool definitions =============
const getListMigrationsSchema = memo(() =>
z.object({
datatable_name: datatableNameSchema(),
version: z
.number()
.optional()
.describe('Return the SQL of this one migration instead of the list.')
})
)
const getListMigrationsToolDef = memo(() =>
createToolDef(
getListMigrationsSchema(),
'list_datatable_migrations',
"List a datatable's migrations in order, each with its version, name and whether it is applied or pending. Pass `version` to read one migration's SQL."
)
)
const getCreateMigrationSchema = memo(() =>
z.object({
datatable_name: datatableNameSchema(),
name: z
.string()
.regex(/^[A-Za-z0-9_-]+$/)
.describe(
'Short name of the change, e.g. "add_refunds_table". Letters, digits, "_" and "-" only.'
),
code_up: z
.string()
.describe(
'The SQL that makes the change. Wrap the statements in `BEGIN; ... END;` so they apply atomically, unless one of them cannot run in a transaction.'
),
code_down: z
.string()
.optional()
.describe(
'The SQL that undoes `code_up`. Without it the migration cannot be rolled back, so provide it whenever the change is reversible.'
),
role: z
.string()
.regex(/^[A-Za-z0-9_-]{1,63}$/)
.optional()
.describe(
'The datatable role the migration runs as, on a datatable that has roles. Omit to run as admin, which only the datatable admins may do.'
),
version: z
.number()
.optional()
.describe(
'Rewrite this migration instead of adding one: only for a pending migration, typically the one that just failed to apply.'
),
apply: z
.boolean()
.optional()
.describe('Set false to save the migration as pending without running it. Defaults to true.')
})
)
const getCreateMigrationToolDef = memo(() =>
createToolDef(
getCreateMigrationSchema(),
'create_datatable_migration',
'Change the schema of a datatable that has migrations enabled: saves the change as a new migration and applies it. One that fails to apply stays saved as pending: call this again with its `version` to fix it.'
)
)
const getRunMigrationsSchema = memo(() => z.object({ datatable_name: datatableNameSchema() }))
const getRunMigrationsToolDef = memo(() =>
createToolDef(
getRunMigrationsSchema(),
'run_datatable_migrations',
"Apply a datatable's pending migrations, oldest first. It stops at the first one that fails."
)
)
const getRollbackMigrationSchema = memo(() =>
z.object({
datatable_name: datatableNameSchema(),
version: z
.number()
.describe('The applied migration to undo, normally the most recent one, as listed.')
})
)
const getRollbackMigrationToolDef = memo(() =>
createToolDef(
getRollbackMigrationSchema(),
'rollback_datatable_migration',
'Undo one applied migration of a datatable by running its down SQL. The migration stays in the list as pending.'
)
)
// ============= Tools =============
export function getDatatableMigrationTools(): SessionTool<{}>[] {
return [
{
requires: NONE,
def: getListMigrationsToolDef(),
planModeSafe: true,
fn: async ({ args, workspace, toolId, toolCallbacks }) => {
const { datatable_name, version } = getListMigrationsSchema().parse(args)
toolCallbacks.setToolStatus(toolId, {
content: `Listing migrations of "${datatable_name}"...`
})
try {
const status = await WorkspaceService.getDatatableMigrationsStatus({
workspace,
datatableName: datatable_name
})
if (!status.enabled) {
toolCallbacks.setToolStatus(toolId, {
content: `Migrations are not enabled on "${datatable_name}"`
})
return `Migrations are not enabled on datatable "${datatable_name}": its schema is changed with plain SQL and there is no history to list. Enabling them is a workspace admin's decision, made in Workspace settings → Data Tables.`
}
if (version !== undefined) {
const migration = status.migrations.find((m) => m.timestamp === version)
if (!migration) {
return fail(
toolCallbacks,
toolId,
`No migration ${version} on datatable "${datatable_name}"`
)
}
toolCallbacks.setToolStatus(toolId, {
content: `Read migration ${migrationLabel({ version, name: migration.name })}`
})
return JSON.stringify(
{
version,
name: migration.name,
status: migrationStatus(migration.status),
code_up: migration.code_up,
...(migration.code_down != null && { code_down: migration.code_down })
},
null,
2
)
}
toolCallbacks.setToolStatus(toolId, {
content: `Listed ${status.migrations.length} migration(s) of "${datatable_name}"`
})
return JSON.stringify(
{
migrations: status.migrations.map((m) => ({
version: m.timestamp,
name: m.name,
status: migrationStatus(m.status)
})),
// Set when the database could not be read, which is why every status is unknown.
...(status.error && { error: status.error })
},
null,
2
)
} catch (e) {
return fail(toolCallbacks, toolId, `Error listing migrations: ${apiErrorMessage(e)}`)
}
}
},
{
// `ensure_not_operator` in datatable_migrations.rs: the condition `run_preview` stands for.
requires: RUN_PREVIEW,
def: getCreateMigrationToolDef(),
requiresConfirmation: true,
confirmationMessage: (args) =>
`${args?.version === undefined ? 'Create' : 'Rewrite'}${args?.apply === false ? '' : ' and apply'} data table migration`,
queuedLabel: (args) => `Write migration "${args?.name ?? ''}"`,
showDetails: true,
fn: async ({ args, workspace, toolId, toolCallbacks }) => {
const parsed = getCreateMigrationSchema().parse(args)
const datatableName = parsed.datatable_name
toolCallbacks.setToolStatus(toolId, {
content: `Saving migration "${parsed.name}" on "${datatableName}"...`
})
// The role is an annotation in the SQL, read from above everything else, and a rollback
// has to run as the role that created what it drops.
const definition = {
name: parsed.name,
code_up: withRole(parsed.code_up, parsed.role),
...(parsed.code_down !== undefined && {
code_down: withRole(parsed.code_down, parsed.role)
})
}
let version: number
try {
if (parsed.version === undefined) {
const created = await WorkspaceService.createDatatableMigration({
workspace,
datatableName,
requestBody: definition
})
version = created.timestamp
} else {
version = parsed.version
// The server inserts at a version it does not know, which would wedge a new
// migration into the middle of the history, and stores a down left out as none.
const { migrations } = await WorkspaceService.getDatatableMigrationsStatus({
workspace,
datatableName
})
const existing = migrations.find((m) => m.timestamp === version)
if (existing?.status !== 'not_run') {
return fail(
toolCallbacks,
toolId,
`Migration ${version} is not a pending migration of datatable "${datatableName}", so it cannot be rewritten. Omit \`version\` to add a new migration.`
)
}
await WorkspaceService.upsertDatatableMigration({
workspace,
datatableName,
requestBody: {
timestamp: version,
...(existing.code_down != null && { code_down: existing.code_down }),
...definition
}
})
}
} catch (e) {
return fail(toolCallbacks, toolId, `Error saving migration: ${apiErrorMessage(e)}`)
}
const label = migrationLabel({ version, name: parsed.name })
if (parsed.apply === false) {
toolCallbacks.setToolStatus(toolId, { content: `Saved migration ${label} as pending` })
return `Saved migration ${label} as pending. It is not applied yet.`
}
// Applying it alone would run it ahead of migrations it may depend on, and applying
// those with it would run SQL the user was never shown.
let earlierPending: string[]
try {
const status = await WorkspaceService.getDatatableMigrationsStatus({
workspace,
datatableName
})
const earlier = status.migrations.filter((m) => m.timestamp < version)
// Unreadable is not pending: reporting it so would send the user to apply
// migrations that already ran.
if (status.error || earlier.some((m) => m.status === 'unknown')) {
throw new Error(status.error ?? 'the applied migrations could not be read')
}
earlierPending = earlier
.filter((m) => m.status === 'not_run')
.map((m) => migrationLabel({ version: m.timestamp, name: m.name }))
} catch (e) {
return fail(
toolCallbacks,
toolId,
`Saved migration ${label} as pending, but could not check what else is pending, so it was not applied: ${apiErrorMessage(e)}`
)
}
if (earlierPending.length > 0) {
toolCallbacks.setToolStatus(toolId, {
content: `Saved migration ${label}; ${earlierPending.length} earlier migration(s) still pending`
})
return `Saved migration ${label} as pending without applying it: earlier migrations are not applied yet (${earlierPending.join(', ')}) and it may depend on them. Tell the user, then apply them all in order with run_datatable_migrations.`
}
toolCallbacks.setToolStatus(toolId, { content: `Applying migration ${label}...` })
try {
await WorkspaceService.runDatatableMigrations({
workspace,
datatableName,
only: version
})
} catch (e) {
// Kept whatever failed: part of it may be applied, or all of it with only the record
// missing, and the definition with its down is then the one trace of that.
return fail(
toolCallbacks,
toolId,
`Migration ${label} is saved but failed to apply: ${apiErrorMessage(e)}. It stays pending and blocks later migrations. If nothing of it was applied, fix it by calling this tool again with version ${version}; otherwise, or to drop it, tell the user, who can edit or delete it in Workspace settings → Data Tables.`
)
}
toolCallbacks.setToolStatus(toolId, { content: `Applied migration ${label}` })
return `Applied migration ${label} on datatable "${datatableName}".`
}
},
{
requires: RUN_PREVIEW,
def: getRunMigrationsToolDef(),
requiresConfirmation: true,
confirmationMessage: 'Apply pending data table migrations',
queuedLabel: (args) => `Apply migrations on "${args?.datatable_name ?? 'datatable'}"`,
showDetails: true,
fn: async ({ args, workspace, toolId, toolCallbacks }) => {
const { datatable_name } = getRunMigrationsSchema().parse(args)
toolCallbacks.setToolStatus(toolId, {
content: `Applying migrations on "${datatable_name}"...`
})
try {
const { applied } = await WorkspaceService.runDatatableMigrations({
workspace,
datatableName: datatable_name
})
if (applied.length === 0) {
toolCallbacks.setToolStatus(toolId, { content: 'No pending migration to apply' })
return `No pending migration to apply on datatable "${datatable_name}".`
}
toolCallbacks.setToolStatus(toolId, {
content: `Applied ${applied.length} migration(s) on "${datatable_name}"`
})
return `Applied ${applied.length} migration(s) on datatable "${datatable_name}": ${applied.map(migrationLabel).join(', ')}.`
} catch (e) {
// Those before the failing one are applied: the list is what says where it stopped.
return fail(
toolCallbacks,
toolId,
`Error applying migrations: ${apiErrorMessage(e)}. Migrations before the failing one may have been applied: list them to see.`
)
}
}
},
{
requires: RUN_PREVIEW,
def: getRollbackMigrationToolDef(),
requiresConfirmation: true,
confirmationMessage: (args) => `Roll back data table migration ${args?.version ?? ''}`,
queuedLabel: (args) => `Roll back migration ${args?.version ?? ''}`,
showDetails: true,
fn: async ({ args, workspace, toolId, toolCallbacks }) => {
const { datatable_name, version } = getRollbackMigrationSchema().parse(args)
toolCallbacks.setToolStatus(toolId, {
content: `Rolling back migration ${version} on "${datatable_name}"...`
})
try {
const { rolled_back } = await WorkspaceService.rollbackDatatableMigrations({
workspace,
datatableName: datatable_name,
only: version
})
if (rolled_back.length === 0) {
toolCallbacks.setToolStatus(toolId, { content: `Migration ${version} is not applied` })
return `Migration ${version} is not applied on datatable "${datatable_name}", so there is nothing to roll back.`
}
const label = rolled_back.map(migrationLabel).join(', ')
toolCallbacks.setToolStatus(toolId, { content: `Rolled back migration ${label}` })
return `Rolled back migration ${label} on datatable "${datatable_name}". It is pending again.`
} catch (e) {
return fail(toolCallbacks, toolId, `Error rolling back: ${apiErrorMessage(e)}`)
}
}
}
]
}
function migrationStatus(status: 'ran' | 'not_run' | 'unknown'): 'applied' | 'pending' | 'unknown' {
return status === 'ran' ? 'applied' : status === 'not_run' ? 'pending' : 'unknown'
}
function withRole(sql: string, role: string | undefined): string {
return role === undefined ? sql : withMigrationRole(sql, role)
}
@@ -286,6 +286,7 @@ export function getDatatableTools(): SessionTool<{}>[] {
datatable_name: d.datatable_name,
schemas: d.schemas,
...(d.error && { error: d.error }),
...(d.migrations_enabled && { migrations_enabled: true }),
...(d.permissioned && { usable_roles: d.usable_roles, default_role: d.default_role })
})),
null,
@@ -121,6 +121,7 @@ import {
} from '../skills/skillResources'
import { MAX_SKILL_DESCRIPTION_LENGTH, MAX_SKILL_INSTRUCTIONS_LENGTH } from '../skills/skillMd'
import {
getDatatablePrompt,
getDatatableSdkReference,
getFlowPrompt,
getPipelinePrompt,
@@ -162,6 +163,7 @@ import { searchDocsTool, readDocsPageTool } from '../docs/core'
import { createDbSchemaTool } from '../script/core'
import type { ContextElement } from '../context'
import { getDatatableTools } from '../datatableTools'
import { getDatatableMigrationTools } from '../datatableMigrationTools'
import { getDucklakeTools } from '../ducklakeTools'
import { fileTools } from '../files/fileTools'
import type { AttachedFilesStore } from '../files/attachedFiles.svelte'
@@ -364,7 +366,7 @@ const scriptLangSchema = z.enum($ScriptLang.enum)
const getInstructionsSchema = z.object({
subject: instructionSubjectSchema.describe(
'What to get authoring instructions for: a workspace item type (script, flow, resource, app), "pipeline" for building a data pipeline (a DAG of annotated scripts wired by storage assets — NOT a flow), or "datatable" for the wmill.datatable() SQL SDK used inside runnables. Schedules, triggers, and variables don\'t need instructions — their tool schemas describe everything.'
'What to get authoring instructions for: a workspace item type (script, flow, resource, app), "pipeline" for building a data pipeline (a DAG of annotated scripts wired by storage assets — NOT a flow), or "datatable" for how data tables work (schema changes and migrations, roles and permissions) plus the wmill.datatable() SQL SDK used inside runnables. Schedules, triggers, and variables don\'t need instructions — their tool schemas describe everything.'
),
language: scriptLangSchema
.optional()
@@ -1454,6 +1456,16 @@ const buildGlobalSystemPrompt = (
? '\n- If the user message includes an ACTIVE PREVIEW section, that is the page the side panel is showing — resolve "this page", "here" and "it" against it, and against `open` (the item the user has open in its editor) when there is one. It already tells you what get_preview_status would, so do not call that tool to learn what is on screen; call it only to check the panel\'s *other* tabs.'
: ''
const pipelineBullet = `- A "data pipeline" is NOT a flow: it is a DAG of independent scripts in one folder, wired by storage assets (DuckLake/data tables/S3) and triggers via top-of-file \`pipeline\` / \`on <ref>\` annotation comments written in each script's comment syntax (\`--\` for SQL, \`#\` for Python/Bash, \`//\` for TS — a \`//\` line in a SQL node is a syntax error). When the user asks for a data pipeline (or to ingest/transform/materialize data across steps), call get_instructions with subject "pipeline" and build annotated script drafts — do not build a flow. list_pipelines lists the existing ones by folder.${pipelineAlphaNote}`
// Roles need a self-hosted Enterprise Edition. The edition is knowable only in the browser,
// so elsewhere the guidance stays and nothing is claimed about the instance.
const datatableRolesAvailable = !isCloudHosted() && (!BROWSER || !!get(enterpriseLicense))
const datatableRolesBullet = datatableRolesAvailable
? `
- A datatable can be under roles; list_datatables then reports its \`usable_roles\` and \`default_role\`. When working on a raw app that pins a role (\`data.roles\` in its raw_app.yaml), pass that role to the datatable tools, and to wmill.datatable in its runnables, so you see and change only what the app itself can. For anything else about roles, grants or a \`permission denied\`, call get_instructions with subject "datatable".`
: BROWSER
? `
- Datatable roles and permissions are not available on this instance (they need a self-hosted Enterprise Edition): everyone who can use a datatable reaches all of it, so do not promise per-user, per-schema or per-row access on one.`
: ''
// Hosting and edition come from the hostname and a store the app populates at init, so
// they are knowable only in the browser: a non-browser caller reads false for both and
// would be told "self-hosted Community Edition" whatever it targets. No base URL here
@@ -1624,13 +1636,9 @@ Data Tables:
- Use get_datatable_table_schema only when you need a table's column names/types; list_datatables is enough for table-list or availability summaries.${when(
canRunPreview,
`
- Use exec_datatable_sql to explore data, run queries, mutate rows, or change schema (CREATE/ALTER/DROP). Creating a table is a normal CREATE TABLE statement — it appears in list_datatables afterward, with no registration step.`
)}${
isCloudHosted()
? ''
: `
- A raw app may use a datatable through a role (\`data.roles\` in its raw_app.yaml). When working on such an app, pass that role to the datatable tools, and to wmill.datatable in its runnables, so you see and change only what the app itself can.`
}${when(
- Use exec_datatable_sql to explore data, run queries, mutate rows, or change schema (CREATE/ALTER/DROP). Creating a table is a normal CREATE TABLE statement — it appears in list_datatables afterward, with no registration step.
- A datatable that list_datatables reports with \`migrations_enabled\` keeps its schema as versioned migrations: change its schema with create_datatable_migration, never exec_datatable_sql. list_datatable_migrations reads them, run_datatable_migrations applies what is pending, rollback_datatable_migration undoes one. A migration is applied for real, not drafted, so call get_instructions with subject "datatable" before your first one.`
)}${datatableRolesBullet}${when(
canWriteDraft,
`
- When writing runnable code (inline app runnables, scripts, flow modules) that reads or writes datatable data at runtime, it accesses a datatable via wmill.datatable(). Default to TypeScript (bun) unless the user asked for another language. Call get_instructions with subject "datatable" and language "bun" for the TypeScript SQL SDK reference (or language "python3" for Python) — it returns only that language so you get just what you need.`
@@ -2526,7 +2534,9 @@ ${getResourcePrompt()}`
function getDatatableInstructions(language?: ScriptLang): string {
// Default to the TypeScript SDK so we return only what's needed, not both.
const lang = language ?? 'bun'
return `# Datatable SQL SDK reference
return `${getDatatablePrompt()}
# Datatable SQL SDK reference
Datatables are workspace-scoped managed PostgreSQL databases. In chat, explore and shape them with the datatable tools you were given. The reference below is for code you author inside runnables (inline app runnables, scripts, or flow rawscript modules) that reads or writes datatable data at runtime.
@@ -3484,7 +3494,7 @@ export const globalTools: SessionTool<{}>[] = [
def: createToolDef(
getInstructionsSchema,
'get_instructions',
'Get authoring guidance for scripts, flows, data pipelines, resources, apps, or the datatable SQL SDK (wmill.datatable()) used inside runnables.'
'Get authoring guidance for scripts, flows, data pipelines, resources, apps, or data tables: their migrations and permissions, and the SQL SDK (wmill.datatable()) used inside runnables.'
),
planModeSafe: true,
fn: async (ctx) => {
@@ -4776,6 +4786,7 @@ export const globalTools: SessionTool<{}>[] = [
},
// Workspace-scoped datatable tools (unrestricted: no whitelist, no creation policy)
...getDatatableTools(),
...getDatatableMigrationTools(),
// Workspace DuckLake: pipeline storage prerequisite, and declared measures
...getDucklakeTools(),
// Read-only tools over files the user attached to the conversation
+10 -1
View File
@@ -32,6 +32,9 @@ export const isDbType = (str?: string): str is DbType => !!str && dbTypes.includ
/** The role every data table has: the one it connects as when it is not under roles. */
export const ADMIN_DATATABLE_ROLE = 'admin'
/** What `default_role` holds for a data table with no default role. Mirrors the backend's
* `NO_DEFAULT_DATATABLE_ROLE`. */
export const NO_DEFAULT_DATATABLE_ROLE = '$none'
/** What the server accepts in `-- role <name>` and `?role=<name>`. */
export function isDatatableRoleName(name: string): boolean {
@@ -52,7 +55,13 @@ export function defaultMigrationRole(
permissioned: boolean | undefined,
defaultRole: string | undefined
): string | undefined {
return permissioned && !datatableNameTakesRole(name) ? defaultRole : undefined
return permissioned && !datatableNameTakesRole(name) ? namedDefaultRole(defaultRole) : undefined
}
/** A data table's default role, or `undefined` when it has none: the server reports that as
* `NO_DEFAULT_DATATABLE_ROLE`, which is not a role anything can connect as. */
export function namedDefaultRole(defaultRole: string | undefined): string | undefined {
return defaultRole === NO_DEFAULT_DATATABLE_ROLE ? undefined : defaultRole
}
/** `datatable://<name>`, with `?role=<role>` when a role is named. Throws rather than build a
@@ -14,6 +14,7 @@
ADMIN_DATATABLE_ROLE,
datatableNameTakesRole,
defaultMigrationRole,
namedDefaultRole,
type DbInput
} from '../dbTypes'
import type { PendingRowAction, SelectedTable } from '../DBManager.svelte'
@@ -173,15 +174,15 @@
return rolesOfCurrent?.permissioned ? selectedRole : undefined
}
const entry = datatableTree.current.find((t) => t.datatable_name === datatable)
return entry?.permissioned ? entry.default_role : undefined
return entry?.permissioned ? namedDefaultRole(entry.default_role) : undefined
}
function defaultRoleOf(datatable: string): string | undefined {
if (datatable === selectedDatatable && rolesOfCurrent?.permissioned) {
return rolesOfCurrent.default_role
return namedDefaultRole(rolesOfCurrent.default_role)
}
const entry = datatableTree.current.find((t) => t.datatable_name === datatable)
return entry?.permissioned ? entry.default_role : undefined
return entry?.permissioned ? namedDefaultRole(entry.default_role) : undefined
}
const tableDatatable = (t: SelectedTable) => t.datatable ?? selectedDatatable
@@ -13,7 +13,7 @@
import { Pane, Splitpanes } from 'svelte-splitpanes'
import PanelSection from '../apps/editor/settingsPanel/common/PanelSection.svelte'
import { JobService, type InputTransform, type ListSelectedJobGroupsResponse } from '$lib/gen'
import { workspaceStore } from '$lib/stores'
import { useOperatingWorkspace } from '../operatingWorkspace.svelte'
import type { Schema } from '$lib/common'
import InputTransformForm from '../InputTransformForm.svelte'
import type { FlowPropPickerConfig, PropPickerContext } from '../prop_picker'
@@ -43,6 +43,8 @@
onConfirm: (options: BatchReRunOptions) => void
} = $props()
const operatingWorkspace = useOperatingWorkspace()
let selected: JobGroup | undefined = $state()
let options: BatchReRunOptions = $state({ flow: {}, script: {} })
watch(
@@ -65,7 +67,7 @@
const listSelectedJobsSchemaCache = new Map<string, JobGroup>()
async function fetchJobGroups(): Promise<JobGroup[]> {
if (!$workspaceStore) return []
if (!$operatingWorkspace) return []
const cachedSelectedIds = selectedIds.filter((id) => listSelectedJobsSchemaCache.has(id))
const nonCachedSelectedIds = selectedIds.filter((id) => !listSelectedJobsSchemaCache.has(id))
@@ -75,7 +77,7 @@
)
const newJobGroups = nonCachedSelectedIds.length
? await JobService.listSelectedJobGroups({
workspace: $workspaceStore,
workspace: $operatingWorkspace,
requestBody: nonCachedSelectedIds
})
: []
@@ -7,7 +7,7 @@
import { Skeleton, Tab, Tabs } from '../common'
import HighlightCode from '../HighlightCode.svelte'
import FlowStatusViewer from '../FlowStatusViewer.svelte'
import { workspaceStore } from '$lib/stores'
import { useOperatingWorkspace } from '../operatingWorkspace.svelte'
import WorkflowTimeline from '../WorkflowTimeline.svelte'
import { isFlowPreview, isScriptPreview, type StateStore } from '$lib/utils'
import { setContext, untrack, createEventDispatcher } from 'svelte'
@@ -26,6 +26,7 @@
let { id, workspace, onResolutionChanged }: Props = $props()
const dispatch = createEventDispatcher()
const operatingWorkspace = useOperatingWorkspace()
let job: (Job & { result_stream?: string }) | undefined = $state(undefined)
@@ -185,7 +186,7 @@
<div class="text-xs text-emphasis font-semibold mb-1">Inputs</div>
<JobArgs
id={job?.id}
workspace={job?.workspace_id ?? $workspaceStore ?? 'no_w'}
workspace={job?.workspace_id ?? $operatingWorkspace ?? 'no_w'}
args={job?.args}
/>
</div>
File diff suppressed because it is too large Load Diff
@@ -21,6 +21,7 @@
import { forLater } from '$lib/forLater'
import { twMerge } from 'tailwind-merge'
import Portal from '$lib/components/Portal.svelte'
import { useHostedPage } from '../hostedPage'
import WaitTimeWarning from '../common/waitTimeWarning/WaitTimeWarning.svelte'
import DropdownV2 from '../DropdownV2.svelte'
@@ -29,6 +30,7 @@
import './runs-grid.css'
const dispatch = createEventDispatcher()
const hosted = useHostedPage()
interface Props {
job: Job
@@ -60,7 +62,10 @@
</script>
<Portal name="run-row">
<ScheduleEditor onUpdate={() => goto('/schedules')} bind:this={scheduleEditor} />
<ScheduleEditor
onUpdate={() => (hosted ? hosted.openLink('/schedules') : goto('/schedules'))}
bind:this={scheduleEditor}
/>
</Portal>
<!-- svelte-ignore a11y_click_events_have_key_events -->
<!-- svelte-ignore a11y_no_static_element_interactions -->
@@ -13,7 +13,6 @@
Wrench
} from 'lucide-svelte'
import Popover from '../Popover.svelte'
import { userStore, workspaceStore } from '$lib/stores'
import './runs-grid.css'
import { useKeyPressed } from '$lib/svelte5Utils.svelte'
import { twMerge } from 'tailwind-merge'
@@ -21,6 +20,9 @@
import DropdownMenu, { type Props as DropdownMenuProps } from '../DropdownMenu.svelte'
import { clickOutside, isJobCancelable, isJobReRunnable, isJobResolvable } from '$lib/utils'
import { goto } from '$lib/navigation'
import { useHostedPage } from '../hostedPage'
import { useOperatingUser, useOperatingWorkspace } from '../operatingWorkspace.svelte'
import { withWorkspaceParam } from '../sessions/sessionMode.svelte'
import BarsStaggered from '../icons/BarsStaggered.svelte'
interface Props {
@@ -201,6 +203,16 @@
}
const dispatch = createEventDispatcher()
const hosted = useHostedPage()
const operatingWorkspace = useOperatingWorkspace()
const operatingUser = useOperatingUser()
// Under a host the link opens where it decides, in the workspace the run belongs to: the
// document's own router would leave the host for the navigation workspace.
function open(href: string) {
if (!hosted) goto(href)
else hosted.openLink(withWorkspaceParam(href, selectedWorkspace ?? $operatingWorkspace))
}
let scrollToIndex = $state(0)
@@ -272,7 +284,7 @@
actions.push({
label: 'Show run details',
icon: ExternalLinkIcon,
onClick: () => goto(`/run/${selectedIds[0]}`)
onClick: () => open(`/run/${selectedIds[0]}`)
})
const job = flatJobs?.find(
(jobOrDate) => jobOrDate.type === 'job' && jobOrDate.job.id === selectedIds[0]
@@ -282,14 +294,20 @@
actions.push({
label: 'Go to script page',
icon: Code2Icon,
onClick: () => goto(`/scripts/get/${job.job.script_hash}`)
onClick: () =>
open(
// A host addresses an item by its path, and pins a version in the query.
hosted
? `/scripts/get/${job.job.script_path}?version=${job.job.script_hash}`
: `/scripts/get/${job.job.script_hash}`
)
})
}
if (job.job.job_kind === 'flow') {
actions.push({
label: 'Go to flow page',
icon: BarsStaggered,
onClick: () => goto(`/flows/get/${job.job.script_path}`)
onClick: () => open(`/flows/get/${job.job.script_path}`)
})
}
}
@@ -314,7 +332,7 @@
onHover: (hover) => (hoveredDropdownAction = hover ? 'cancel' : null)
})
// Operators are rejected by the endpoint, so offering the action would only 403.
const canResolve = !$userStore?.operator
const canResolve = !operatingUser.current?.operator
const resolvable = canResolve ? selectedIdsPossibleActions.resolvableJobIds.length : 0
const unresolvable = canResolve ? selectedIdsPossibleActions.unresolvableJobIds.length : 0
if (resolvable)
@@ -401,7 +419,7 @@
? jobCountString(jobs.length + externalJobs.length, lastFetchWentToEnd)
: ''}<Tooltip>{externalJobs.length} jobs obscured</Tooltip>
</div>
{:else if $workspaceStore !== 'admins' && omittedObscuredJobs}
{:else if $operatingWorkspace !== 'admins' && omittedObscuredJobs}
<div class="flex flex-row">
{jobs ? jobCountString(jobs.length, lastFetchWentToEnd) : ''}
<Popover>
@@ -6,7 +6,10 @@
import { resource } from 'runed'
import { canSkipRetryChainQuery } from './scriptRetryChain'
let { job }: { job: Job } = $props()
let {
job,
onNavigate = goto
}: { job: Job; onNavigate?: (url: string) => void | Promise<void> } = $props()
type AttemptColor = 'green' | 'red' | 'blue' | 'gray'
type Attempt = { id: string; color: AttemptColor; current: boolean }
@@ -141,7 +144,7 @@
size="xs"
color="gray"
variant="border"
onclick={() => goto(`/run/${self?.handledJob}?workspace=${ws}`)}
onclick={() => onNavigate(`/run/${self?.handledJob}?workspace=${ws}`)}
>
Handled run
</Button>
@@ -163,7 +166,7 @@
size="xs"
color={attempt.color}
variant={attempt.current ? 'contained' : 'border'}
onclick={() => goto(`/run/${attempt.id}?workspace=${ws}`)}
onclick={() => onNavigate(`/run/${attempt.id}?workspace=${ws}`)}
>
{i === 0 ? 'Original' : `Attempt ${i}`}
</Button>
@@ -180,7 +183,7 @@
size="xs"
color={handler.color}
variant="border"
onclick={() => goto(`/run/${handler.id}?workspace=${ws}`)}
onclick={() => onNavigate(`/run/${handler.id}?workspace=${ws}`)}
>
{HANDLER_LABEL[handler.kind]}
</Button>
@@ -29,6 +29,8 @@
path,
version,
pendingRunId,
locationHash = '',
dbtRetryFrom,
workspaceId,
tabId,
container,
@@ -45,6 +47,10 @@
version?: string
/** The chat tool call this tab was opened to answer, from its URL's `?pending_run=`. */
pendingRunId?: string
/** `#`-fragment of the tab's URL: the args a run's `Run again` prefills the form with. */
locationHash?: string
/** The failed dbt run that `Run again` offers to resume, from `?dbt_retry_from=`. */
dbtRetryFrom?: string
workspaceId: string
/** Whether this side is the one on screen. Hidden sides stay mounted, so this gates
* the window-level keyboard handler — several instances listen on the window. */
@@ -96,13 +102,21 @@
reloadKey++
}
const searchParams = $derived(version ? new URLSearchParams({ version }) : undefined)
const searchParams = $derived(
version || dbtRetryFrom
? new URLSearchParams({
...(version ? { version } : {}),
...(dbtRetryFrom ? { dbt_retry_from: dbtRetryFrom } : {})
})
: undefined
)
// Remounts the detail page below, which loads on mount. The call is part of it because
// the detail pages open their run form on what it proposed as they build it: a tab
// already on this item, re-pointed to a new call, has to build that form again.
// Remounts the detail page below, which loads on mount. The fragment and the call are
// part of it because the detail pages read both once, as they build their form: from the
// fragment for a `Run again`, from the call for what the chat proposed. So a tab already
// on this item, re-pointed to either, has to build that form again.
const loadKey = $derived(
`${workspaceId}/${kind}/${path}/${version ?? ''}/${pendingRunId ?? ''}/${reloadKey}`
`${workspaceId}/${kind}/${path}/${version ?? ''}/${locationHash}/${pendingRunId ?? ''}/${reloadKey}`
)
// Nothing deployed at this path. Stamped with the load it answers, so a refetch or a
// re-pointed tab is never answered from an earlier load's verdict.
@@ -231,6 +245,7 @@
hash={path}
workspace={workspaceId}
{searchParams}
{locationHash}
{onNavigate}
{active}
embedded
@@ -242,6 +257,7 @@
{path}
workspace={workspaceId}
{searchParams}
{locationHash}
{onNavigate}
{active}
embedded
@@ -3,8 +3,14 @@
import { Loader2 } from 'lucide-svelte'
import { setOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte'
import { setHostedPage, type HostedPage } from '$lib/components/hostedPage'
import { pageItemForListPath, parsePreviewItemRoute } from './previewPaths'
import { previewLocationLabel } from './previewRouter'
import {
RUNS_PATH,
barePath,
isRunsListPath,
pageItemForListPath,
parsePreviewItemRoute
} from './previewPaths'
import { parseRunRoute, previewLocationLabel } from './previewRouter'
import { whereIs } from './sessionPreviewTabs.svelte'
import type { SessionPreviewTab } from './sessionState.svelte'
import type { SessionRuntime } from './sessionRuntime.svelte'
@@ -44,20 +50,48 @@
? { pathname: loc, search: '' }
: { pathname: loc.slice(0, at), search: loc.slice(at) }
}
// The location this page last wrote itself, and the one it was last moved to from outside.
let ownLoc: string | undefined
let movedTo: string | undefined
let moves = 0
// Bumped when the location moves other than by the page's own write. Derived rather than
// effected: it has to flip in the same flush as the location, before the page's own
// effects read the new query and write their defaults back over the stored preferences.
const moved = $derived.by(() => {
const loc = whereIs(tab)
if (loc !== ownLoc && loc !== movedTo) {
movedTo = loc
moves++
}
return moves
})
const host: HostedPage = {
get search() {
return locationParts().search
},
setSearch(search) {
runtime.previewTabs.observeLocation(tab.id, locationParts().pathname + search)
ownLoc = locationParts().pathname + search
movedTo = undefined
runtime.previewTabs.observeLocation(tab.id, ownLoc)
},
openItem(itemPath) {
const ref = pageItemForListPath(path, itemPath)
if (ref) runtime.previewTabs.open({ type: 'pageitem', ref })
},
openLink(href) {
const run = parseRunRoute(href)
if (run) {
runtime.manager.openRunInPreview?.({
jobId: run.jobId,
workspace: run.workspace ?? workspaceId,
label: previewLocationLabel(`/run/${run.jobId}`)
})
return
}
const item = parsePreviewItemRoute(href)
if (item) {
// A link to one version stays a page tab: opening an item focuses a tab already
// on it as it stands, where a page tab's URL is written whole, pin included.
if (item && !new URL(href, 'http://_').searchParams.has('version')) {
runtime.previewTabs.open({
type: 'item',
item: { kind: item.kind, raw_app: item.raw_app, path: item.itemPath, summary: '' },
@@ -83,18 +117,32 @@
if (e.defaultPrevented || e.button !== 0) return
if (e.metaKey || e.ctrlKey || e.shiftKey || e.altKey) return
const a = (e.target as Element | null)?.closest?.('a[href]') as HTMLAnchorElement | null
if (!a || (a.target && a.target !== '_self') || a.hasAttribute('download')) return
if (!a || a.hasAttribute('download')) return
const url = new URL(a.href, window.location.href)
if (url.origin !== window.location.origin) return
const href = url.pathname + url.search + url.hash
// A run's link asks for a new tab so the list keeps its place: here that tab is a
// preview tab, not a browser one that would leave the session.
if (a.target && a.target !== '_self' && !parseRunRoute(href)) return
e.preventDefault()
if (a.getAttribute('href')?.startsWith('#')) return
host.openLink(url.pathname + url.search + url.hash)
host.openLink(href)
}
el.addEventListener('click', onClick)
return () => el.removeEventListener('click', onClick)
})
const listed = $derived(pageItemForListPath(path, ''))
// `/runs/<item path>` lists one item's runs.
const runsOf = $derived.by(() => {
if (!isRunsListPath(path)) return undefined
const itemPath = barePath(path).slice(RUNS_PATH.length + 1)
try {
return decodeURIComponent(itemPath)
} catch {
return itemPath
}
})
</script>
{#snippet loading()}
@@ -130,6 +178,15 @@
{:then Module}
<Module.default triggerKind={listed.triggerKind} />
{/await}
{:else if runsOf !== undefined}
<!-- A move from outside remounts it, so it seeds the filters the user keeps stored. -->
{#key moved}
{#await import('$lib/components/RunsPage.svelte')}
{@render loading()}
{:then Module}
<Module.default initialPath={runsOf} />
{/await}
{/key}
{/if}
{/key}
</div>
@@ -97,15 +97,14 @@
? (new URL(tab.url, 'http://x').searchParams.get('version') ?? undefined)
: undefined
)
// The chat tool call this tab was opened to answer, read the same way and for the same
// reason. Re-pointing the tab rebuilds its url without it, so a tab sent anywhere else —
// another item, another version — stops carrying the call rather than answering it from a
// page it was never confirmed against.
const pendingRunId = $derived(
slot.kind === 'viewer'
? (new URL(tab.url, 'http://x').searchParams.get('pending_run') ?? undefined)
: undefined
)
// What a run's `Run again` hands the deployed page it opens: the run's args in the
// fragment, and for a dbt run the failure it could resume instead.
const viewerUrl = $derived(slot.kind === 'viewer' ? new URL(tab.url, 'http://x') : undefined)
// The chat tool call this tab was opened to answer rides in the same query. Re-pointing
// the tab rebuilds its url without it, so a tab sent anywhere else — another item,
// another version — stops carrying the call rather than answering it from a page it was
// never confirmed against.
const pendingRunId = $derived(viewerUrl?.searchParams.get('pending_run') ?? undefined)
// The path an editor reports is its deployed one, which a staged rename can make
// differ from this tab's — so the view opens on what actually exists.
@@ -156,6 +155,7 @@
// A page item's editor reads its draft only when it loads, so a reload remounts it.
let pageItemReloadNonce = $state(0)
let pageListReloadNonce = $state(0)
let runReloadNonce = $state(0)
export function reload() {
// A live editor shares the runtime store the chat mutates, so generic chat
@@ -177,6 +177,10 @@
pageListReloadNonce++
return
}
if (slot.kind === 'run') {
runReloadNonce++
return
}
try {
const win = frame?.contentWindow
if (!win) return
@@ -403,6 +407,8 @@
path={itemPath}
version={pinnedVersion}
{pendingRunId}
locationHash={viewerUrl?.hash}
dbtRetryFrom={viewerUrl?.searchParams.get('dbt_retry_from') ?? undefined}
{workspaceId}
tabId={tab.id}
container={overlayHostEl}
@@ -454,6 +460,29 @@
{/await}
{/if}
</div>
{:else if slot.kind === 'run' && mounted && runtime}
<div
bind:this={overlayHostEl}
class="absolute inset-0 flex flex-col min-h-0 bg-surface {visibility}"
aria-hidden={!active}
>
{#if overlayHostEl}
{#await import('./RunViewerView.svelte')}
{@render editorLoading()}
{:then Module}
<!-- A run stays in the workspace it ran in, which the tab's URL names: the
session's own can change under an open tab (a staged fork committing). -->
<Module.default
{runtime}
jobId={slot.jobId}
workspaceId={slot.workspace ?? workspaceId}
tabId={tab.id}
container={overlayHostEl}
reloadNonce={runReloadNonce}
/>
{/await}
{/if}
</div>
{:else if slot.kind === 'artifact' && mounted}
<div
bind:this={overlayHostEl}
@@ -0,0 +1,106 @@
<script lang="ts">
/*
* A run in a session preview tab, rendered by the component the workspace `/run/<id>` page
* renders. Everything it reaches is scoped to the run's workspace through `workspace`, and
* every navigation it attempts is caught here and turned into a move inside the preview
* panel, so nothing takes the browser out of the session.
*/
import { setContext, untrack } from 'svelte'
import RunDetail from '$lib/components/runs/RunDetail.svelte'
import type { SessionRuntime } from './sessionRuntime.svelte'
import {
parseHistoricalScriptEdit,
parsePreviewItemRoute,
parseRunRoute,
previewLocationLabel,
barePath
} from './previewRouter'
let {
runtime,
jobId,
workspaceId,
tabId,
container,
reloadNonce = 0
}: {
runtime: SessionRuntime
jobId: string
workspaceId: string
/** The preview tab hosting this run, which a move to another run re-points. */
tabId: string
/** The tab's host element, which also holds the overlays the page portals out. */
container: HTMLElement | undefined
/** Bumped to load the run again from the server. */
reloadNonce?: number
} = $props()
// Mounted in the preview panel, outside the chat's own subtree, as ItemViewerView is: the
// run page's AI entry points would otherwise offer a session from inside one.
setContext(
'aiChatManager',
untrack(() => runtime.manager)
)
// Everything the run page navigates to, resolved against the preview panel. Another run —
// a re-run, a retry, the parent job — is still this tab's subject, so it re-points in
// place; an item or any other page opens beside it. Nothing reaches the top-level router.
function onNavigate(url: string): void {
const parsed = new URL(url, window.location.origin)
const href = `${parsed.pathname}${parsed.search}${parsed.hash}`
if (parseRunRoute(href)) {
runtime.previewTabs.navigate({ type: 'page', href, label: previewLocationLabel(href) }, tabId)
return
}
const route = parsePreviewItemRoute(href)
// Legacy drag-and-drop apps and historical script edits have no in-panel host, so they
// fall through to a page tab. So does `Run again`, whose args ride in the fragment, and
// a link to one version: opening an item focuses a tab already on it as it stands,
// where a page tab's URL is written whole, pin and fragment included.
if (
route &&
(route.kind !== 'app' || route.raw_app) &&
!parseHistoricalScriptEdit(href) &&
!parsed.hash &&
!parsed.searchParams.has('version')
) {
runtime.previewTabs.open({
type: 'item',
item: { path: route.itemPath, summary: '', kind: route.kind, raw_app: route.raw_app },
mode: route.mode
})
return
}
runtime.previewTabs.open({ type: 'page', href, label: previewLocationLabel(barePath(href)) })
}
// Every in-app link under the tab, not only the ones that call `onNavigate`: a plain anchor
// followed by the browser would take it out of the session. Listened for on `body`, as
// ItemViewerView does and for the same reason.
$effect(() => {
const el = container
if (!el) return
function onClick(e: MouseEvent) {
if (e.defaultPrevented || e.button !== 0) return
if (e.metaKey || e.ctrlKey || e.shiftKey || e.altKey) return
const target = e.target as Element | null
if (!target || !el!.contains(target)) return
const a = target.closest?.('a[href]') as HTMLAnchorElement | null
if (!a || (a.target && a.target !== '_self') || a.hasAttribute('download')) return
const url = new URL(a.href, window.location.href)
if (url.origin !== window.location.origin) return
e.preventDefault()
if (a.getAttribute('href')?.startsWith('#')) return
onNavigate(`${url.pathname}${url.search}${url.hash}`)
}
document.body.addEventListener('click', onClick)
return () => document.body.removeEventListener('click', onClick)
})
</script>
<div class="flex h-full min-h-0 flex-col overflow-auto">
<!-- Keyed: the run page binds one job's loaders and panels for its lifetime. -->
{#key `${workspaceId}/${jobId}/${reloadNonce}`}
<RunDetail id={jobId} workspace={workspaceId} {onNavigate} embedded />
{/key}
</div>
@@ -84,6 +84,17 @@ export function isPageItemListPath(path: string): boolean {
return pageItemForListPath(path, '') !== undefined
}
/** Whether a path is the runs list: every run, or `/runs/<item path>` for one item's. */
export function isRunsListPath(path: string): boolean {
const clean = barePath(path)
return clean === RUNS_PATH || clean.startsWith(`${RUNS_PATH}/`)
}
/** Whether a path is a list page a session preview mounts in process rather than frames. */
export function isPreviewListPath(path: string): boolean {
return isPageItemListPath(path) || isRunsListPath(path)
}
/** The page item a list page's row names, or undefined for a page that lists none. */
export function pageItemForListPath(pagePath: string, path: string): PageItemRef | undefined {
const clean = barePath(pagePath)
@@ -145,7 +156,13 @@ export const pageKey = (path: string) => `page:${path}`
/** Strip the query and hash from a preview path so it can be matched against
* `PREVIEW_PAGES` / parsed as an item route. */
export function barePath(path: string): string {
return path.split('?')[0].split('#')[0] || '/'
// The router serves `/runs/` as `/runs`, and links are written both ways.
return (
path
.split('?')[0]
.split('#')[0]
.replace(/(.)\/+$/, '$1') || '/'
)
}
/** Which side of an item a route addresses: `/edit/` builds it, `/get/` shows the
@@ -313,7 +313,30 @@ describe('itemDisplayName', () => {
describe('resolvePreviewTab', () => {
it('routes a static page to the iframe fallback', () => {
expect(resolvePreviewTab('/runs')).toEqual({ kind: 'iframe' })
expect(resolvePreviewTab('/audit_logs')).toEqual({ kind: 'iframe' })
})
it('mounts a run in process, in the workspace its URL names', () => {
expect(resolvePreviewTab('/run/01a0?workspace=fork')).toEqual({
kind: 'run',
jobId: '01a0',
workspace: 'fork'
})
expect(resolvePreviewTab('/run/01a0')).toEqual({ kind: 'run', jobId: '01a0' })
expect(resolvePreviewTab('/run/01a0?workspace=fork&view_token=t')).toEqual({ kind: 'iframe' })
})
it('mounts the runs list in process, for every run or for one item', () => {
expect(resolvePreviewTab('/runs?status=failure')).toEqual({ kind: 'pagelist', path: '/runs' })
expect(resolvePreviewTab('/runs/f/a/b')).toEqual({ kind: 'pagelist', path: '/runs/f/a/b' })
expect(resolvePreviewTab('/runsheet')).toEqual({ kind: 'iframe' })
})
it('tells two runs lists apart by their filters, trailing slash or not', () => {
expect(resolvePreviewTab('/runs/?status=failure')).toEqual({ kind: 'pagelist', path: '/runs' })
expect(showsView('/runs/?schedule_path=a', '/runs/?schedule_path=b')).toBe(false)
expect(showsView('/runs?schedule_path=a', '/runs/?schedule_path=a')).toBe(true)
expect(showsView('/runs/f/a/b?status=failure', '/runs/f/a/b')).toBe(false)
})
it('mounts the list page of a page item kind in process, whatever its filters', () => {
@@ -3,7 +3,8 @@ import {
AUDIT_LOGS_PATH,
FOLDERS_PATH,
GROUPS_PATH,
isPageItemListPath,
isPreviewListPath,
isRunsListPath,
pageItemForListPath,
pageItemListPath,
pageItemPageHref,
@@ -259,9 +260,14 @@ function pageRequestParamTable(): Record<string, readonly string[]> {
})
}
// The page whose filters a path takes: one item's runs filter as every run does.
function filterPageOf(path: string): string {
return isRunsListPath(path) ? RUNS_PATH : path
}
/** The query params a request can set on `path` — empty for a page that takes none. */
export function pageRequestParams(path: string): readonly string[] {
return pageRequestParamTable()[barePath(path)] ?? []
return pageRequestParamTable()[filterPageOf(barePath(path))] ?? []
}
/** What a preview location means, read against the page it points at. */
@@ -298,7 +304,7 @@ export function describeLocation(loc: string): PreviewLocation {
const query = bare.includes('?') ? bare.slice(bare.indexOf('?') + 1) : ''
return {
identity: path,
view: requestedParams(query, pageRequestParamTable()[path]),
view: requestedParams(query, pageRequestParamTable()[filterPageOf(path)]),
anchor: drawerAnchorFor(canonical) ?? ''
}
}
@@ -331,7 +337,7 @@ export function showsView(observed: string, commanded: string): boolean {
const y = describeLocation(commanded)
if (x.identity !== y.identity || x.anchor !== y.anchor) return false
if (x.view === y.view) return true
const seeded = PAGE_SEEDED_PARAMS[x.identity]
const seeded = PAGE_SEEDED_PARAMS[filterPageOf(x.identity)]
if (!seeded?.length) return false
// Both views come out of `requestedParams` already sorted and encoded, so they compare
// as strings once the seeded params are dropped. One direction only: the page may add
@@ -476,9 +482,9 @@ export function previewLocationLabel(url: string): string {
if (page) return page.label
const trigger = triggerLabelForPath(url)
if (trigger) return trigger
const run = barePath(url).match(/^\/run\/([^/?#]+)/)
const run = parseRunRoute(url)
// Job ids are time-ordered, so runs started moments apart share their head.
if (run) return `Run ${decodeURIComponent(run[1]).slice(-8)}`
if (run) return `Run ${run.jobId.slice(-8)}`
// The runs of one item, as a detail page's Runs button links them.
const itemRuns = barePath(url).match(/^\/runs\/(.+)$/)
if (itemRuns) return `Runs · ${decodeURIComponent(itemRuns[1]).split('/').pop()}`
@@ -571,6 +577,25 @@ export function pendingRunIdOf(url: string): string | undefined {
return new URL(url, 'http://x').searchParams.get('pending_run') ?? undefined
}
/** The run a `/run/<id>` location shows, in the workspace its query names (the opener's,
* when it names none). */
export function parseRunRoute(
url: string
): { jobId: string; workspace?: string; viewToken: boolean } | null {
const m = barePath(url).match(/^\/run\/([^/?#]+)$/)
if (!m) return null
try {
const query = new URL(url, 'http://_').searchParams
return {
jobId: decodeURIComponent(m[1]),
workspace: query.get('workspace') || undefined,
viewToken: query.has('view_token')
}
} catch {
return null
}
}
export function runFormUrl(toolCallId: string, label: string): string {
return `runform:${encodeURIComponent(toolCallId)}#${encodeURIComponent(label)}`
}
@@ -596,7 +621,8 @@ export const isArtifactKey = (key: string) => key.startsWith('artifact:')
// flow, raw app) mounts its per-(kind,path) cell editor on `/edit/` and its viewer
// on `/get/`; a `/pipeline/<folder>` route mounts the data-pipeline graph editor of
// that folder (`path` is the folder); the list page of a page item kind mounts that
// list; everything
// list, and so does the runs list; a `/run/<id>` route without a share token mounts that
// run's page; everything
// else (static pages, regular drag-and-drop apps, any other route) stays an iframe.
export type PreviewSlot =
| { kind: 'editor'; editorKind: SessionTargetKind | 'pipeline'; path: string }
@@ -605,6 +631,7 @@ export type PreviewSlot =
| { kind: 'runform'; toolCallId: string }
| { kind: 'pageitem'; ref: PageItemRef }
| { kind: 'pagelist'; path: string }
| { kind: 'run'; jobId: string; workspace?: string }
| { kind: 'iframe' }
export function resolvePreviewTab(url: string): PreviewSlot {
@@ -615,7 +642,13 @@ export function resolvePreviewTab(url: string): PreviewSlot {
const pageItem = parsePageItemRoute(url)
if (pageItem) return { kind: 'pageitem', ref: pageItem }
const route = barePath(url)
if (isPageItemListPath(route)) return { kind: 'pagelist', path: route }
if (isPreviewListPath(route)) return { kind: 'pagelist', path: route }
const run = parseRunRoute(url)
// A share link's token authorizes every request of the window that installs it, so a run
// opened through one keeps a frame of its own rather than lending the token to the session.
if (run && !run.viewToken) {
return { kind: 'run', jobId: run.jobId, ...(run.workspace && { workspace: run.workspace }) }
}
const pipelineFolder = parsePipelineRoute(url)
if (pipelineFolder) {
return { kind: 'editor', editorKind: 'pipeline', path: pipelineFolder }
@@ -649,6 +682,7 @@ export function workspacePageHref(location: string): string | undefined {
case 'editor':
case 'viewer':
case 'pagelist':
case 'run':
case 'iframe':
return location
default: {
@@ -475,6 +475,7 @@ function createRuntime(session: Session): SessionRuntime {
slotKind !== 'iframe' &&
slotKind !== 'pageitem' &&
slotKind !== 'pagelist' &&
slotKind !== 'run' &&
slotKind !== 'viewer'
)
return undefined
@@ -617,8 +618,8 @@ function createRuntime(session: Session): SessionRuntime {
}
}
// Let the jobs tray open a run in this session's preview panel (as an iframe
// tab over the run page). The global side-panel chat leaves this unset and
// Let the jobs tray open a run in this session's preview panel (as a tab on
// the run's page). The global side-panel chat leaves this unset and
// falls back to a new browser tab.
manager.openRunInPreview = ({ jobId, workspace, label }) => {
previewTabs.open({
@@ -23,7 +23,7 @@
import { superadmin } from '$lib/stores'
import { sendUserToast } from '$lib/toast'
import { deepEqual } from 'fast-equals'
import { ADMIN_DATATABLE_ROLE, isDatatableRoleName } from '../dbTypes'
import { ADMIN_DATATABLE_ROLE, NO_DEFAULT_DATATABLE_ROLE, isDatatableRoleName } from '../dbTypes'
import PgAclEditor from '../datatableAcl/PgAclEditor.svelte'
import ConfirmationModal from '../common/confirmationModal/ConfirmationModal.svelte'
import { createAsyncConfirmationModal } from '../common/confirmationModal/asyncConfirmationModal.svelte'
@@ -87,6 +87,8 @@
const pgInstanceName = $derived(pgInstanceNameOf(info?.cluster))
const unusedRoles = $derived(availableRoles.filter((r) => !roles.some((row) => row.id === r.id)))
const confirmationModal = createAsyncConfirmationModal()
// Every member of the governing workspace, as the tenant lists spell it.
const EVERYONE_TENANT = '*'
let aclEditor = $state<PgAclEditor | undefined>(undefined)
const hasUnsavedChanges = $derived(
@@ -111,9 +113,13 @@
(a, b) => Number(b.id === ADMIN_DATATABLE_ROLE) - Number(a.id === ADMIN_DATATABLE_ROLE)
)
// A data table never put under roles comes back with none; admin is what turning the toggle
// on starts from.
// on starts from, open to everyone so that turning it on changes nothing by itself.
if (!loaded.some((r) => r.id === ADMIN_DATATABLE_ROLE)) {
loaded.unshift({ id: ADMIN_DATATABLE_ROLE, name: ADMIN_DATATABLE_ROLE, tenants: [] })
loaded.unshift({
id: ADMIN_DATATABLE_ROLE,
name: ADMIN_DATATABLE_ROLE,
tenants: [EVERYONE_TENANT]
})
}
info = res
catalog = undefined
@@ -137,7 +143,7 @@
FolderService.listFolderNames({ workspace: ws })
])
tenantItems = [
{ value: '*', label: 'Everyone', group: 'Anyone in the workspace' },
{ value: EVERYONE_TENANT, label: 'Everyone', group: 'Anyone in the workspace' },
...users.map((u) => ({ value: `u/${u}`, label: u, group: 'Users' })),
...groups.map((g) => ({ value: `g/${g}`, label: g, group: 'Groups' })),
...folders.map((f) => ({ value: `f/${f}`, label: f, group: 'Folders' }))
@@ -218,11 +224,28 @@
}
}
function removeRole(role: EditedRole) {
async function removeRole(role: EditedRole) {
if (defaultRoleId === role.id) {
const confirmed = await confirmationModal.ask({
title: 'Remove the default role?',
children: `Jobs that name no role connect as ${role.name ?? role.id}. Without it the data table has no default role: they connect with the roles of the person running them, and are refused when that person has none.`,
confirmationText: 'Remove',
type: 'danger'
})
if (!confirmed) return
defaultRoleId = NO_DEFAULT_DATATABLE_ROLE
}
roles = roles.filter((r) => r.id !== role.id)
if (defaultRoleId === role.id) defaultRoleId = ADMIN_DATATABLE_ROLE
}
const defaultRoleName = $derived(roles.find((r) => r.id === defaultRoleId)?.name ?? defaultRoleId)
const defaultsToAdmin = $derived(defaultRoleId === ADMIN_DATATABLE_ROLE)
const hasNoDefault = $derived(defaultRoleId === NO_DEFAULT_DATATABLE_ROLE)
const hasOtherRoles = $derived(roles.some((r) => r.id !== ADMIN_DATATABLE_ROLE))
const adminOpenToEveryone = $derived(
!!roles.find((r) => r.id === ADMIN_DATATABLE_ROLE)?.tenants.includes(EVERYONE_TENANT)
)
async function save() {
saving = true
// Turning roles off clears the whole block, so every role it named is removed.
@@ -320,7 +343,7 @@
options={{
right: 'Put this data table under roles',
rightTooltip:
'Off, every job connects as admin — the connection that owns every table. On, every job resolves to a role, and a caller no tenant covers is refused.'
'Off, every job connects as admin — the connection that owns every table. On, every job resolves to a role, and a caller no tenant covers is refused. It starts with admin open to everyone, so turning it on changes nothing until the roles are edited.'
}}
/>
@@ -391,8 +414,10 @@
<Cell head>
Default
<Tooltip>
The role a job gets when it names none — no `-- role` annotation, no `?role=` in
the reference. Callers still have to be one of its tenants.
The role a job gets when it names none: no `-- role` annotation, no `?role=` in
the reference. There is always one choice: admin until changed, another role, or
none. Unless it is admin, the roles the person running the job is a tenant of
are added to it.
</Tooltip>
</Cell>
<Cell head last />
@@ -440,6 +465,29 @@
</Cell>
</Row>
{/each}
<Row>
<Cell first class="w-56 align-top">
<span class="text-xs text-secondary italic">No default role</span>
</Cell>
<Cell class="align-top">
<span class="text-xs text-secondary">
Jobs that name no role get only the roles of the person running them.
</span>
</Cell>
<Cell class="w-20 align-top">
<div class="flex justify-center">
<Checkbox
checked={hasNoDefault}
disabled={!editable}
title="Use no role when a job names none"
onChange={() => {
defaultRoleId = NO_DEFAULT_DATATABLE_ROLE
}}
/>
</div>
</Cell>
<Cell last class="w-10 align-top" />
</Row>
{#if editable}
<Row>
<Cell colspan={4} class="pt-2 pb-2">
@@ -466,6 +514,47 @@
{/if}
</tbody>
</DataTable>
{#if adminOpenToEveryone}
<Alert type="warning" title="Everyone can connect as admin" size="xs">
Every member of the workspace may use admin{defaultsToAdmin
? ', and a job that names no role connects as it, exactly as with roles off'
: ' by naming it'}. admin owns every table: it reads every row and ignores grants
and row-level security. Once the other roles are set up, remove Everyone from admin's
tenants{defaultsToAdmin ? ' and choose another default, or none' : ''}.
</Alert>
{/if}
{#if defaultsToAdmin && hasOtherRoles && !adminOpenToEveryone}
<Alert type="warning" title="The default role is admin" size="xs">
A job that names no role connects as admin when it runs as a workspace admin or as one
of admin's tenants, which covers every viewer of an app an admin published. admin owns
every table: it reads every row and ignores grants and row-level security. Everyone
else's jobs that name no role are refused. Choose another default, or none, to have
the roles above apply to them.
</Alert>
{:else if !defaultsToAdmin}
<div class="flex flex-col gap-1 text-xs text-secondary">
{#if hasNoDefault}
<span>
A job that names no role connects with the roles here that the person running it
is a tenant of. It is refused when they have none, and so is a job with nobody
behind it: a schedule, a trigger, an anonymous app viewer.
</span>
{:else}
<span>
A job that names no role connects as
<span class="font-mono">{defaultRoleName}</span> when it may use it, plus every other
role here that the person running it is a tenant of. With neither, it is refused.
</span>
{/if}
<span>A job that names a role connects as that role alone.</span>
<span>
A table created over a connection that holds several roles belongs to that
combination of roles alone. Run <span class="font-mono">SET ROLE</span> to one role before
creating it, or name a role.
</span>
</div>
{/if}
{/if}
</div>
+7 -1
View File
@@ -538,8 +538,14 @@ export async function deployDraft(
// no draft. `draft_path` and `summary` are draft-only fields mirrored onto
// the App value (the editor drops them on deploy), so strip them from the
// value and apply them as the deploy path / summary column.
// `parent_version` is the draft's stale-check base, never deployed.
const draft = r.draft as Record<string, any> | undefined
const { draft_path: draftPath, summary: draftSummary, ...appValue } = draft ?? r.value ?? {}
const {
draft_path: draftPath,
summary: draftSummary,
parent_version: _pv,
...appValue
} = draft ?? r.value ?? {}
// Policy isn't carried in the app draft, so it comes from the deployed app
// (or a default). custom_path requires admin on update; non-admins send
// undefined so the backend preserves the existing route. The draft has no
@@ -110,7 +110,7 @@
import { isGlobalAiEnabled, sessionsAllowedFor } from '$lib/components/copilot/chat/global/gate'
import { copilotInfo } from '$lib/aiStore'
import {
isPageItemListPath,
isPreviewListPath,
parsePreviewItemRoute,
barePath
} from '$lib/components/sessions/previewPaths'
@@ -467,7 +467,7 @@
}
// A list page mounts in process too, where its rows open as tabs of their own.
const listUrl = navigation.to?.url
if (listUrl && isPageItemListPath(barePath(listUrl.pathname))) {
if (listUrl && isPreviewListPath(barePath(listUrl.pathname))) {
navigation.cancel()
const u = new URL(listUrl.href)
u.searchParams.delete('nomenubar')
File diff suppressed because it is too large Load Diff
@@ -80,7 +80,7 @@
tabsToReload,
viewerTabsToReload
} from '$lib/components/sessions/previewReload'
import { isPageItemListPath, barePath } from '$lib/components/sessions/previewPaths'
import { isPreviewListPath, barePath } from '$lib/components/sessions/previewPaths'
import {
leafKeyFor,
loadKind,
@@ -928,7 +928,7 @@
// A preview frame navigating to a list page: re-point the active tab to the list
// mounted in process, whose rows open as tabs of their own.
if (d.type === 'wm.session.openList') {
if (typeof d.href !== 'string' || !isPageItemListPath(barePath(d.href))) return
if (typeof d.href !== 'string' || !isPreviewListPath(barePath(d.href))) return
owner?.navigate({ type: 'page', href: d.href, label: previewLocationLabel(d.href) })
return
}
+1
View File
@@ -46,6 +46,7 @@ a CLI-only workflow file (`flow-cli.md`, `script-cli.md`, `raw-app-cli.md`).
| resources | `getResourcePrompt()` | `resources` | `resources.md` |
| workflow-as-code | `getWorkflowAsCodePrompt(lang)` | `write-workflow-as-code` | `workflow-as-code.md`, the WAC SDK |
| pipeline | `getPipelinePrompt()` | `write-pipeline` | `pipeline-base.md` |
| data tables | `getDatatablePrompt()` | `datatables` | `datatable.md` |
| triggers, schedules, preview, CLI commands | — | same names | CLI only |
The app chat embeds `RAW_APP_BASE` directly (`chat/app/core.ts`).
+1
View File
@@ -4,5 +4,6 @@ export declare function getFlowPrompt(): string;
export declare function getResourcePrompt(): string;
export declare function getRawAppPrompt(language?: string): string;
export declare function getPipelinePrompt(): string;
export declare function getDatatablePrompt(): string;
export declare function getDatatableSdkReference(language?: string): string;
export declare function getWorkflowAsCodePrompt(language?: string): string;
+7
View File
@@ -73,6 +73,13 @@ export function getPipelinePrompt(): string {
].filter(Boolean).join('\n\n');
}
// Helper for data table schema changes, migrations and permissions (chat consumers)
export function getDatatablePrompt(): string {
return [
prompts.DATATABLE_BASE
].filter(Boolean).join('\n\n');
}
// Helper to get the datatable SQL SDK reference (wmill.datatable()).
// Pass a language to get only that SDK; omit it to get both.
export function getDatatableSdkReference(language?: string): string {
+1
View File
@@ -5,6 +5,7 @@ export declare const FLOW_BASE: string;
export declare const RESOURCES_BASE: string;
export declare const RAW_APP_BASE: string;
export declare const PIPELINE_BASE: string;
export declare const DATATABLE_BASE: string;
export declare const WORKFLOW_AS_CODE_BASE: string;
export declare const FLOW_CHAT_SPECIAL_MODULES: string;
export declare const SDK_TYPESCRIPT: string;
+42
View File
@@ -1131,6 +1131,7 @@ Data tables are PostgreSQL databases managed by Windmill. Backend runnables quer
3. **No DDL inside runnables**: runnables only read and write rows (SELECT, INSERT, UPDATE, DELETE) on existing tables. Never CREATE, ALTER or DROP a table from a runnable.
4. **Qualify table names**: an unqualified name means the \`public\` schema, so write every other table as \`schema.table\`, in table creation and queries alike. The app's \`data\` config sets the default datatable and schema its tables go in; use them consistently across runnables.
5. **Pass the role**: when the app's \`data.roles\` gives a data table a role, every \`wmill.datatable\` call on it passes that role (\`wmill.datatable('main', { role: 'analyst' })\` in TypeScript, \`wmill.datatable('main', role='analyst')\` in Python). The role only reaches what it was granted, so a query outside it fails with \`permission denied\`.
6. **Per-viewer rows**: a data table with no entry in \`data.roles\` is called with no role. Unless its default role is \`admin\`, the connection then holds the default role, if it has one, plus every role the person using the app is a tenant of, and is refused when that is none. Row-level security policies written as \`pg_has_role(current_user, '<role>', 'member')\` return that viewer's rows with nothing added to the runnable. A role pins the connection to that one role for every viewer, so where rows should differ per viewer give the data table no \`data.roles\` entry.
### Querying in TypeScript (Bun/Deno)
@@ -1337,6 +1338,47 @@ FROM ducklake.main.orders GROUP BY 1
\`\`\`
`;
export const DATATABLE_BASE = `# Data tables: schema changes, migrations and permissions
A data table is a PostgreSQL database managed by Windmill and shared by the whole workspace: scripts, flows and apps reach it by name, \`main\` being the default. It is configured in the workspace settings (Data Tables), never created by SQL.
## Changing the schema
A data table with migrations enabled keeps its schema as an ordered list of versioned SQL migrations, each an \`up\` and an optional \`down\`. That list is what git sync and a workspace merge carry, and what rebuilds the schema in another workspace. So first find out whether migrations are enabled on the data table:
- **Enabled**: make every schema change (CREATE, ALTER, DROP, anything but row data) as a new migration. A change made with plain SQL exists in that one database only, and the history no longer describes it.
- **Not enabled**: schema changes are plain SQL. Enabling migrations is a workspace admin's decision, made in the Data Tables settings, not a step to take on the way to something else.
Writing a migration:
- Wrap the statements in \`BEGIN; ... END;\` so it applies atomically. Leave the frame out only for a statement that cannot run in a transaction, such as \`CREATE INDEX CONCURRENTLY\`: such a migration can fail half applied.
- Give it a \`down\` that undoes the \`up\` whenever the change is reversible. Without one it cannot be rolled back.
- Migrations apply oldest first. One that is applied is not edited: correct it with a new migration.
- Row data is not schema. Insert, update and delete rows with plain SQL, migrations or not.
## A migration is not a draft
A migration is saved to the workspace and changes the database as soon as it is applied, for everything that reads those tables. To try a schema out without touching tables already in use:
- put the new tables in a schema of their own, created by the same migration (\`CREATE SCHEMA IF NOT EXISTS\`), and point what you are building at it. A raw app names its schema in its \`data\` config, which is why a new app gets a new schema by default;
- or work in a workspace fork whose data table was cloned when the fork was created ("Clone schema only" or "Clone schema and data"): migrations applied there change that copy alone, and reach the parent workspace when the fork is merged. A fork that kept the original data table points at the parent's database, so a migration applied in it changes the parent's tables at once.
Altering or dropping a table other scripts or apps use is neither of those: say what depends on it before doing it.
## Roles and permissions
Data table roles are a self-hosted Enterprise Edition feature, for data tables on managed Postgres (the instance database or the external managed instance). Without them, and always on a data table backed by a Postgres resource, everyone who can use the data table reaches all of it: there is no per-user, per-schema or per-row access to promise.
A data table under roles is reached as a Postgres role, and Postgres enforces what that role was granted:
- roles are assigned, per data table, to users, groups or folders. \`admin\` is the connection the data table had before roles: it owns the existing objects and ignores grants;
- a connection either names a role (\`wmill.datatable('main', { role: 'analyst' })\`, \`role='analyst'\` in Python, \`datatable://main?role=analyst\`) and holds that role alone, or names none and is resolved from the data table's default role;
- \`permission denied\` means the role in use lacks a grant. Say which role and which object: the grant is a data table admin's to give, in the roles drawer of the Database Manager. Do not switch to \`admin\` to get past it;
- a migration runs as the role its SQL declares on its first line, \`-- role <name>\`, the same one in its \`down\`. One that declares none runs as \`admin\`, which only those who may use \`admin\` can do.
Read the Permissions section of https://www.windmill.dev/docs/core_concepts/persistent_storage/data_tables before designing who reaches what: it covers tenants, grants, the default role and the roles a connection that names none ends up holding, rows that differ per viewer through row-level security, and how forks and raw apps behave under roles.
`;
export const WORKFLOW_AS_CODE_BASE = `# Windmill Workflow-as-Code Writing Guide
## Scope
@@ -0,0 +1,46 @@
---
name: datatables
description: MUST use when changing a data table's schema, writing or running its migrations, or working with its roles and permissions.
---
# Data tables: schema changes, migrations and permissions
A data table is a PostgreSQL database managed by Windmill and shared by the whole workspace: scripts, flows and apps reach it by name, `main` being the default. It is configured in the workspace settings (Data Tables), never created by SQL.
## Changing the schema
A data table with migrations enabled keeps its schema as an ordered list of versioned SQL migrations, each an `up` and an optional `down`. That list is what git sync and a workspace merge carry, and what rebuilds the schema in another workspace. So first find out whether migrations are enabled on the data table:
- **Enabled**: make every schema change (CREATE, ALTER, DROP, anything but row data) as a new migration. A change made with plain SQL exists in that one database only, and the history no longer describes it.
- **Not enabled**: schema changes are plain SQL. Enabling migrations is a workspace admin's decision, made in the Data Tables settings, not a step to take on the way to something else.
Writing a migration:
- Wrap the statements in `BEGIN; ... END;` so it applies atomically. Leave the frame out only for a statement that cannot run in a transaction, such as `CREATE INDEX CONCURRENTLY`: such a migration can fail half applied.
- Give it a `down` that undoes the `up` whenever the change is reversible. Without one it cannot be rolled back.
- Migrations apply oldest first. One that is applied is not edited: correct it with a new migration.
- Row data is not schema. Insert, update and delete rows with plain SQL, migrations or not.
`wmill datatable migrate new <name>` scaffolds `migrations/datatable/<datatable>/<timestamp>_<name>.up.sql` and `.down.sql`, `wmill datatable migrate up` pushes the local files and applies what is pending, `wmill datatable migrate down` rolls back the most recent one and `wmill datatable migrate status` lists them. All take `-d <datatable>` and default to `main`.
## A migration is not a draft
A migration is saved to the workspace and changes the database as soon as it is applied, for everything that reads those tables. To try a schema out without touching tables already in use:
- put the new tables in a schema of their own, created by the same migration (`CREATE SCHEMA IF NOT EXISTS`), and point what you are building at it. A raw app names its schema in its `data` config, which is why a new app gets a new schema by default;
- or work in a workspace fork whose data table was cloned when the fork was created ("Clone schema only" or "Clone schema and data"): migrations applied there change that copy alone, and reach the parent workspace when the fork is merged. A fork that kept the original data table points at the parent's database, so a migration applied in it changes the parent's tables at once.
Altering or dropping a table other scripts or apps use is neither of those: say what depends on it before doing it.
## Roles and permissions
Data table roles are a self-hosted Enterprise Edition feature, for data tables on managed Postgres (the instance database or the external managed instance). Without them, and always on a data table backed by a Postgres resource, everyone who can use the data table reaches all of it: there is no per-user, per-schema or per-row access to promise.
A data table under roles is reached as a Postgres role, and Postgres enforces what that role was granted:
- roles are assigned, per data table, to users, groups or folders. `admin` is the connection the data table had before roles: it owns the existing objects and ignores grants;
- a connection either names a role (`wmill.datatable('main', { role: 'analyst' })`, `role='analyst'` in Python, `datatable://main?role=analyst`) and holds that role alone, or names none and is resolved from the data table's default role;
- `permission denied` means the role in use lacks a grant. Say which role and which object: the grant is a data table admin's to give, in the roles drawer of the Database Manager. Do not switch to `admin` to get past it;
- a migration runs as the role its SQL declares on its first line, `-- role <name>`, the same one in its `down`. One that declares none runs as `admin`, which only those who may use `admin` can do.
Read the Permissions section of https://www.windmill.dev/docs/core_concepts/persistent_storage/data_tables before designing who reaches what: it covers tenants, grants, the default role and the roles a connection that names none ends up holding, rows that differ per viewer through row-level security, and how forks and raw apps behave under roles.
@@ -188,7 +188,7 @@ data:
- `<datatable>/<table>` — Specific table in public schema
- `<datatable>/<schema>:<table>` — Table in specific schema
**Roles:** when a datatable is under roles, its queries run as a role, which only reaches what it was granted. `roles` records the role the app uses each datatable through; the app's code must pass the same role: `wmill.datatable('main', { role: 'analyst' })` in TypeScript, `wmill.datatable('main', role='analyst')` in Python. A datatable without an entry is used as its default role.
**Roles:** when a datatable is under roles, its queries run as a role, which only reaches what it was granted. `roles` records the role the app uses each datatable through; the app's code must pass the same role: `wmill.datatable('main', { role: 'analyst' })` in TypeScript, `wmill.datatable('main', role='analyst')` in Python. A datatable without an entry is used as its default role, if it has one, plus the roles of whoever is using the app unless that default is `admin`.
## SQL Migrations (sql_to_apply/)
@@ -494,6 +494,7 @@ Data tables are PostgreSQL databases managed by Windmill. Backend runnables quer
3. **No DDL inside runnables**: runnables only read and write rows (SELECT, INSERT, UPDATE, DELETE) on existing tables. Never CREATE, ALTER or DROP a table from a runnable.
4. **Qualify table names**: an unqualified name means the `public` schema, so write every other table as `schema.table`, in table creation and queries alike. The app's `data` config sets the default datatable and schema its tables go in; use them consistently across runnables.
5. **Pass the role**: when the app's `data.roles` gives a data table a role, every `wmill.datatable` call on it passes that role (`wmill.datatable('main', { role: 'analyst' })` in TypeScript, `wmill.datatable('main', role='analyst')` in Python). The role only reaches what it was granted, so a query outside it fails with `permission denied`.
6. **Per-viewer rows**: a data table with no entry in `data.roles` is called with no role. Unless its default role is `admin`, the connection then holds the default role, if it has one, plus every role the person using the app is a tenant of, and is refused when that is none. Row-level security policies written as `pg_has_role(current_user, '<role>', 'member')` return that viewer's rows with nothing added to the runnable. A role pins the connection to that one role for every viewer, so where rows should differ per viewer give the data table no `data.roles` entry.
`wmill datatable list` lists the workspace's data tables. Create or change tables with a migration in `sql_to_apply/` (see "SQL Migrations" above), then add them to `data.tables` in `raw_app.yaml`.
+43
View File
@@ -0,0 +1,43 @@
# Data tables: schema changes, migrations and permissions
A data table is a PostgreSQL database managed by Windmill and shared by the whole workspace: scripts, flows and apps reach it by name, `main` being the default. It is configured in the workspace settings (Data Tables), never created by SQL.
## Changing the schema
A data table with migrations enabled keeps its schema as an ordered list of versioned SQL migrations, each an `up` and an optional `down`. That list is what git sync and a workspace merge carry, and what rebuilds the schema in another workspace. So first find out whether migrations are enabled on the data table:
- **Enabled**: make every schema change (CREATE, ALTER, DROP, anything but row data) as a new migration. A change made with plain SQL exists in that one database only, and the history no longer describes it.
- **Not enabled**: schema changes are plain SQL. Enabling migrations is a workspace admin's decision, made in the Data Tables settings, not a step to take on the way to something else.
Writing a migration:
- Wrap the statements in `BEGIN; ... END;` so it applies atomically. Leave the frame out only for a statement that cannot run in a transaction, such as `CREATE INDEX CONCURRENTLY`: such a migration can fail half applied.
- Give it a `down` that undoes the `up` whenever the change is reversible. Without one it cannot be rolled back.
- Migrations apply oldest first. One that is applied is not edited: correct it with a new migration.
- Row data is not schema. Insert, update and delete rows with plain SQL, migrations or not.
<!-- cli-only -->
`wmill datatable migrate new <name>` scaffolds `migrations/datatable/<datatable>/<timestamp>_<name>.up.sql` and `.down.sql`, `wmill datatable migrate up` pushes the local files and applies what is pending, `wmill datatable migrate down` rolls back the most recent one and `wmill datatable migrate status` lists them. All take `-d <datatable>` and default to `main`.
<!-- /cli-only -->
## A migration is not a draft
A migration is saved to the workspace and changes the database as soon as it is applied, for everything that reads those tables. To try a schema out without touching tables already in use:
- put the new tables in a schema of their own, created by the same migration (`CREATE SCHEMA IF NOT EXISTS`), and point what you are building at it. A raw app names its schema in its `data` config, which is why a new app gets a new schema by default;
- or work in a workspace fork whose data table was cloned when the fork was created ("Clone schema only" or "Clone schema and data"): migrations applied there change that copy alone, and reach the parent workspace when the fork is merged. A fork that kept the original data table points at the parent's database, so a migration applied in it changes the parent's tables at once.
Altering or dropping a table other scripts or apps use is neither of those: say what depends on it before doing it.
## Roles and permissions
Data table roles are a self-hosted Enterprise Edition feature, for data tables on managed Postgres (the instance database or the external managed instance). Without them, and always on a data table backed by a Postgres resource, everyone who can use the data table reaches all of it: there is no per-user, per-schema or per-row access to promise.
A data table under roles is reached as a Postgres role, and Postgres enforces what that role was granted:
- roles are assigned, per data table, to users, groups or folders. `admin` is the connection the data table had before roles: it owns the existing objects and ignores grants;
- a connection either names a role (`wmill.datatable('main', { role: 'analyst' })`, `role='analyst'` in Python, `datatable://main?role=analyst`) and holds that role alone, or names none and is resolved from the data table's default role;
- `permission denied` means the role in use lacks a grant. Say which role and which object: the grant is a data table admin's to give, in the roles drawer of the Database Manager. Do not switch to `admin` to get past it;
- a migration runs as the role its SQL declares on its first line, `-- role <name>`, the same one in its `down`. One that declares none runs as `admin`, which only those who may use `admin` can do.
Read the Permissions section of https://www.windmill.dev/docs/core_concepts/persistent_storage/data_tables before designing who reaches what: it covers tenants, grants, the default role and the roles a connection that names none ends up holding, rows that differ per viewer through row-level security, and how forks and raw apps behave under roles.
+1 -1
View File
@@ -183,7 +183,7 @@ data:
- `<datatable>/<table>` — Specific table in public schema
- `<datatable>/<schema>:<table>` — Table in specific schema
**Roles:** when a datatable is under roles, its queries run as a role, which only reaches what it was granted. `roles` records the role the app uses each datatable through; the app's code must pass the same role: `wmill.datatable('main', { role: 'analyst' })` in TypeScript, `wmill.datatable('main', role='analyst')` in Python. A datatable without an entry is used as its default role.
**Roles:** when a datatable is under roles, its queries run as a role, which only reaches what it was granted. `roles` records the role the app uses each datatable through; the app's code must pass the same role: `wmill.datatable('main', { role: 'analyst' })` in TypeScript, `wmill.datatable('main', role='analyst')` in Python. A datatable without an entry is used as its default role, if it has one, plus the roles of whoever is using the app unless that default is `admin`.
## SQL Migrations (sql_to_apply/)
+1
View File
@@ -264,6 +264,7 @@ Data tables are PostgreSQL databases managed by Windmill. Backend runnables quer
3. **No DDL inside runnables**: runnables only read and write rows (SELECT, INSERT, UPDATE, DELETE) on existing tables. Never CREATE, ALTER or DROP a table from a runnable.
4. **Qualify table names**: an unqualified name means the `public` schema, so write every other table as `schema.table`, in table creation and queries alike. The app's `data` config sets the default datatable and schema its tables go in; use them consistently across runnables.
5. **Pass the role**: when the app's `data.roles` gives a data table a role, every `wmill.datatable` call on it passes that role (`wmill.datatable('main', { role: 'analyst' })` in TypeScript, `wmill.datatable('main', role='analyst')` in Python). The role only reaches what it was granted, so a query outside it fails with `permission denied`.
6. **Per-viewer rows**: a data table with no entry in `data.roles` is called with no role. Unless its default role is `admin`, the connection then holds the default role, if it has one, plus every role the person using the app is a tenant of, and is refused when that is none. Row-level security policies written as `pg_has_role(current_user, '<role>', 'member')` return that viewer's rows with nothing added to the runnable. A role pins the connection to that one role for every viewer, so where rows should differ per viewer give the data table no `data.roles` entry.
<!-- cli-only -->
`wmill datatable list` lists the workspace's data tables. Create or change tables with a migration in `sql_to_apply/` (see "SQL Migrations" above), then add them to `data.tables` in `raw_app.yaml`.
+13
View File
@@ -1696,6 +1696,12 @@ TOPICS: dict[str, dict] = {
'chat_helper': 'getPipelinePrompt',
'parts': ['pipeline-base.md'],
},
'datatable': {
'skill': 'datatables',
'description': "MUST use when changing a data table's schema, writing or running its migrations, or working with its roles and permissions.",
'chat_helper': 'getDatatablePrompt',
'parts': ['datatable.md'],
},
'cli_commands': {
'skill': 'cli-commands',
'description': 'MUST use when using the CLI, including debugging job failures and inspecting run history via `wmill job`.',
@@ -1715,6 +1721,7 @@ CHAT_CONSTANTS = {
'resources.md': 'RESOURCES_BASE',
'raw-app.md': 'RAW_APP_BASE',
'pipeline-base.md': 'PIPELINE_BASE',
'datatable.md': 'DATATABLE_BASE',
'workflow-as-code.md': 'WORKFLOW_AS_CODE_BASE',
'flow-chat-special-modules.md': 'FLOW_CHAT_SPECIAL_MODULES',
}
@@ -2650,6 +2657,11 @@ export function getPipelinePrompt(): string {
__RETURN_pipeline__
}
// Helper for data table schema changes, migrations and permissions (chat consumers)
export function getDatatablePrompt(): string {
__RETURN_datatable__
}
// Helper to get the datatable SQL SDK reference (wmill.datatable()).
// Pass a language to get only that SDK; omit it to get both.
export function getDatatableSdkReference(language?: string): string {
@@ -2711,6 +2723,7 @@ export declare function getFlowPrompt(): string;
export declare function getResourcePrompt(): string;
export declare function getRawAppPrompt(language?: string): string;
export declare function getPipelinePrompt(): string;
export declare function getDatatablePrompt(): string;
export declare function getDatatableSdkReference(language?: string): string;
export declare function getWorkflowAsCodePrompt(language?: string): string;
"""