diff --git a/ai_evals/README.md b/ai_evals/README.md index 44870ba2a2..9b8850c435 100644 --- a/ai_evals/README.md +++ b/ai_evals/README.md @@ -196,7 +196,10 @@ not have read is itself the failure — e.g. `write_variable.value` on a secret Global (and flow) initial fixtures can seed `workspace.datatables` so the `list_datatables`, `get_datatable_table_schema`, and `exec_datatable_sql` tools return seeded data during evals. Each entry is -`{ datatable_name, schemas: { : { : { columns, rows? } } } }`. +`{ datatable_name, schemas: { : {
: { columns, rows? } } } }`, +plus `migrations_enabled: true` and `migrations: [{ timestamp, name, code_up, code_down?, applied? }]` +for a datatable that keeps its schema as migrations, which the migration tools then +read and write in memory. SQL runs through a small in-memory engine (`datatableSqlEngine.ts`), not a real database. Writes are **stateful within a case**: `CREATE`/`DROP`/`INSERT`/`UPDATE`/ `DELETE` mutate the seeded datatable in place, so a later `list_datatables`, diff --git a/ai_evals/adapters/frontend/datatableSqlEngine.ts b/ai_evals/adapters/frontend/datatableSqlEngine.ts index 453eca286e..96a62f6804 100644 --- a/ai_evals/adapters/frontend/datatableSqlEngine.ts +++ b/ai_evals/adapters/frontend/datatableSqlEngine.ts @@ -30,6 +30,17 @@ export interface BenchmarkDatatableSeed { [table: string]: BenchmarkDatatableTableSeed } } + /** Whether the datatable keeps its schema as migrations. */ + migrations_enabled?: boolean + migrations?: BenchmarkDatatableMigrationSeed[] +} + +export interface BenchmarkDatatableMigrationSeed { + timestamp: number + name: string + code_up: string + code_down?: string + applied?: boolean } export interface DatatableSqlResult { diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index 2f47e85be6..d8284039b9 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -692,10 +692,138 @@ export function listBenchmarkDatatables(workspace: string): DataTableTables[] | datatable_name: datatable.datatable_name, schemas: Object.fromEntries( Object.entries(datatable.schemas).map(([schema, tables]) => [schema, Object.keys(tables)]) - ) + ), + ...(datatable.migrations_enabled && { migrations_enabled: true }) })) } +// ============= Datatable migrations ============= + +function benchmarkDatatable(workspace: string, datatableName: string): BenchmarkDatatableSeed { + const datatable = (benchmarkWorkspaceRunnables.get(workspace)?.datatables ?? []).find( + (entry) => entry.datatable_name === datatableName + ) + if (!datatable) { + throw new Error(`datatable "${datatableName}" not found`) + } + return datatable +} + +/** Run a migration body through the in-memory engine, one statement at a time and without the + * transaction frame, so the tables it creates show up in later reads. */ +function applyBenchmarkMigrationSql(datatable: BenchmarkDatatableSeed, sql: string): void { + // Split on the semicolons outside single-quoted literals. + for (const statement of sql.match(/(?:'(?:[^']|'')*'|[^;'])+/g) ?? []) { + const trimmed = statement.replace(/^\s*--.*$/gm, '').trim() + if (trimmed !== '' && !/^(begin|end|commit)$/i.test(trimmed)) { + applyDatatableSql(datatable, trimmed) + } + } +} + +export function getBenchmarkDatatableMigrationsStatus(input: { + workspace: string + datatableName: string +}) { + const datatable = benchmarkDatatable(input.workspace, input.datatableName) + if (!datatable.migrations_enabled) { + return { enabled: false, migrations: [] } + } + return { + enabled: true, + migrations: (datatable.migrations ?? []).map(({ applied, ...migration }) => ({ + ...migration, + status: applied ? ('ran' as const) : ('not_run' as const) + })) + } +} + +export function createBenchmarkDatatableMigration(input: { + workspace: string + datatableName: string + requestBody: { name: string; code_up: string; code_down?: string } +}) { + const datatable = benchmarkDatatable(input.workspace, input.datatableName) + if (!datatable.migrations_enabled) { + throw new Error( + `Migrations are not enabled for data table '${input.datatableName}'. Enable them first.` + ) + } + const migrations = (datatable.migrations ??= []) + const timestamp = Math.max(20260101000000, ...migrations.map((m) => m.timestamp)) + 1 + migrations.push({ timestamp, ...input.requestBody }) + return { datatable: input.datatableName, timestamp, ...input.requestBody } +} + +export function upsertBenchmarkDatatableMigration(input: { + workspace: string + datatableName: string + requestBody: { timestamp: number; name: string; code_up: string; code_down?: string } +}): string { + const datatable = benchmarkDatatable(input.workspace, input.datatableName) + const migrations = (datatable.migrations ??= []) + const existing = migrations.find((m) => m.timestamp === input.requestBody.timestamp) + if (existing?.applied) { + throw new Error( + `Migration ${existing.timestamp} on data table '${input.datatableName}' has already been applied and cannot be modified` + ) + } + if (existing) { + // Replaced whole, as the server does: a down left out is stored as none. + delete existing.code_down + Object.assign(existing, input.requestBody) + } else { + migrations.push({ ...input.requestBody }) + migrations.sort((a, b) => a.timestamp - b.timestamp) + } + return `Upserted migration ${input.requestBody.timestamp} in ${input.datatableName}` +} + +export function runBenchmarkDatatableMigrations(input: { + workspace: string + datatableName: string + upTo?: number + only?: number +}) { + const datatable = benchmarkDatatable(input.workspace, input.datatableName) + const applied: { version: number; name: string }[] = [] + for (const migration of datatable.migrations ?? []) { + if ( + migration.applied || + (input.only !== undefined && migration.timestamp !== input.only) || + (input.only === undefined && input.upTo !== undefined && migration.timestamp > input.upTo) + ) { + continue + } + applyBenchmarkMigrationSql(datatable, migration.code_up) + migration.applied = true + applied.push({ version: migration.timestamp, name: migration.name }) + } + return { applied } +} + +export function rollbackBenchmarkDatatableMigration(input: { + workspace: string + datatableName: string + only?: number +}) { + const datatable = benchmarkDatatable(input.workspace, input.datatableName) + const last = (datatable.migrations ?? []) + .filter((m) => m.applied && (input.only === undefined || m.timestamp === input.only)) + .at(-1) + if (!last) { + return { rolled_back: [] } + } + if (last.code_down === undefined) { + throw new Error( + `Cannot roll back migration ${last.timestamp} (${last.name}): it has no down migration` + ) + } + applyBenchmarkMigrationSql(datatable, last.code_down) + last.applied = false + return { rolled_back: [{ version: last.timestamp, name: last.name }] } +} + // ============= DuckLake catalogs and declared metrics ============= /** Seeded DuckLake names, or `null` for a non-benchmark workspace. */ diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index 288c67e3eb..cb7469b45e 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -92,6 +92,11 @@ vi.mock('$lib/gen', async () => { getBenchmarkCompletedJob, getBenchmarkCompletedJobResultMaybe, getBenchmarkDatatableSchema, + getBenchmarkDatatableMigrationsStatus, + createBenchmarkDatatableMigration, + upsertBenchmarkDatatableMigration, + runBenchmarkDatatableMigrations, + rollbackBenchmarkDatatableMigration, getBenchmarkDraftForUser, getBenchmarkFlowByPath, getBenchmarkFlowAllResults, @@ -400,7 +405,27 @@ vi.mock('$lib/gen', async () => { schemaName: data.schemaName, tableName: data.tableName }) - : actual.WorkspaceService.getDataTableTableSchema(data) + : actual.WorkspaceService.getDataTableTableSchema(data), + getDatatableMigrationsStatus: async (data: any) => + hasBenchmarkWorkspace(data.workspace) + ? getBenchmarkDatatableMigrationsStatus(data) + : actual.WorkspaceService.getDatatableMigrationsStatus(data), + createDatatableMigration: async (data: any) => + hasBenchmarkWorkspace(data.workspace) + ? createBenchmarkDatatableMigration(data) + : actual.WorkspaceService.createDatatableMigration(data), + upsertDatatableMigration: async (data: any) => + hasBenchmarkWorkspace(data.workspace) + ? upsertBenchmarkDatatableMigration(data) + : actual.WorkspaceService.upsertDatatableMigration(data), + runDatatableMigrations: async (data: any) => + hasBenchmarkWorkspace(data.workspace) + ? runBenchmarkDatatableMigrations(data) + : actual.WorkspaceService.runDatatableMigrations(data), + rollbackDatatableMigrations: async (data: any) => + hasBenchmarkWorkspace(data.workspace) + ? rollbackBenchmarkDatatableMigration(data) + : actual.WorkspaceService.rollbackDatatableMigrations(data) }), DataMetricService: wrapService(actual.DataMetricService, { listDataMetrics: async (data: { workspace: string }) => diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index b66af3e8dd..4b6704b229 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -901,6 +901,57 @@ - treats creating the table as a normal SQL statement and does not claim a separate registration step is needed - does not write a script to create the table +# The same request on a data table that keeps its schema as migrations: the table is +# created by a migration, since plain SQL would leave it out of the history. +- id: global-datatable-schema-change-uses-migration + prompt: |- + Add a new table called refunds to our workspace data table, with an order id and a refund amount. + initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_migrations_seed.json + runtime: + maxTurns: 8 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - create_datatable_migration + forbiddenToolsUsed: + - write_script + - deploy_workspace_item + - delete_workspace_item + toolCallArgs: + - tool: create_datatable_migration + field: code_up + stringIncludesAnyOf: + - create table + skipJudge: true + judgeChecklist: + - notices that the data table keeps its schema as migrations + - creates the refunds table through a migration rather than a plain CREATE TABLE statement + - includes an order id and a refund amount column + - gives the migration a down that drops the table + +# Row data is not schema: it is written with plain SQL even where migrations are enabled. +- id: global-datatable-row-change-skips-migration + prompt: |- + Mark order number 2 as shipped in our workspace data table. + initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_migrations_seed.json + runtime: + maxTurns: 12 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - exec_datatable_sql + forbiddenToolsUsed: + - create_datatable_migration + - write_script + - deploy_workspace_item + - delete_workspace_item + skipJudge: true + judgeChecklist: + - runs an UPDATE on the orders table setting the status of order id 2 to shipped + - does not create a migration for a change to row data + - id: global-test25-datatable-mutate-rows prompt: |- Mark order number 2 as shipped in our workspace data table. diff --git a/ai_evals/fixtures/frontend/global/initial/datatable_orders_migrations_seed.json b/ai_evals/fixtures/frontend/global/initial/datatable_orders_migrations_seed.json new file mode 100644 index 0000000000..d5227d9af8 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/datatable_orders_migrations_seed.json @@ -0,0 +1,36 @@ +{ + "workspace": { + "datatables": [ + { + "datatable_name": "main", + "migrations_enabled": true, + "migrations": [ + { + "timestamp": 20260501090000, + "name": "create_orders", + "code_up": "BEGIN;\n\nCREATE TABLE orders (id serial PRIMARY KEY, customer_id int, total numeric, status text, created_at timestamptz DEFAULT now());\n\nEND;", + "code_down": "BEGIN;\n\nDROP TABLE orders;\n\nEND;", + "applied": true + } + ], + "schemas": { + "public": { + "orders": { + "columns": { + "id": "int4", + "customer_id": "int4", + "total": "numeric", + "status": "text", + "created_at": "timestamptz" + }, + "rows": [ + { "id": 1, "customer_id": 1, "total": 42.5, "status": "shipped", "created_at": "2026-05-01T10:00:00Z" }, + { "id": 2, "customer_id": 2, "total": 19.99, "status": "pending", "created_at": "2026-05-02T11:30:00Z" } + ] + } + } + } + } + ] + } +} diff --git a/backend/windmill-api-workspaces/src/datatable_migrations.rs b/backend/windmill-api-workspaces/src/datatable_migrations.rs index e63e69d4d5..21e460359e 100644 --- a/backend/windmill-api-workspaces/src/datatable_migrations.rs +++ b/backend/windmill-api-workspaces/src/datatable_migrations.rs @@ -880,7 +880,11 @@ struct DatatableMigrationsStatusResult { /// Whether the SQL-migrations feature is enabled for a data table. Honors the /// explicit `migrations_enabled` flag; when unset (data tables predating the /// feature) it is considered enabled only if migrations already exist. -async fn datatable_migrations_enabled(db: &DB, w_id: &str, datatable_name: &str) -> Result { +pub(crate) async fn datatable_migrations_enabled( + db: &DB, + w_id: &str, + datatable_name: &str, +) -> Result { let flag: Option = sqlx::query_scalar!( "SELECT (ws.datatable->'datatables'->$2->>'migrations_enabled')::boolean \ FROM workspace_settings ws WHERE ws.workspace_id = $1", diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 3bfcef5c96..bdb3b59e6e 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -2325,6 +2325,9 @@ struct DataTableTables { /// What the role the listing connected as may create. can_create_schema: bool, creatable_schemas: Vec, + /// Whether its schema is kept as versioned migrations, which a change made with plain SQL + /// would bypass. + migrations_enabled: bool, } #[derive(Deserialize)] @@ -2570,8 +2573,17 @@ async fn list_one_datatable_tables( default_role: windmill_common::datatable_roles::ADMIN_DATATABLE_ROLE.to_string(), can_create_schema: false, creatable_schemas: vec![], + migrations_enabled: false, }; let result: Result<()> = async { + // Read from the workspace settings alone, so it is known even when the database is not + // reachable. + entry.migrations_enabled = crate::datatable_migrations::datatable_migrations_enabled( + db, + w_id, + &entry.datatable_name, + ) + .await?; let governing = resolve_governing_datatable(db, w_id, &entry.datatable_name).await?; entry.instance = governing.role_cluster().is_some(); let usable = diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 433d7c00be..53996db1dc 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -37336,6 +37336,7 @@ components: default_role, can_create_schema, creatable_schemas, + migrations_enabled, ] properties: datatable_name: @@ -37369,6 +37370,9 @@ components: description: the schemas the role the listing connected as may create in items: type: string + migrations_enabled: + type: boolean + description: whether its schema is kept as versioned migrations, which a change made with plain SQL would bypass DataTableTableSchema: type: object diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index d90a401260..0a880b05cf 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -34,6 +34,7 @@ export const SKILLS: SkillMetadata[] = [ { name: "resources", description: "MUST use when managing resources." }, { name: "write-workflow-as-code", description: "MUST use when writing or modifying Windmill Workflow-as-Code scripts using workflow, task, step, sleep, approvals, taskScript, taskFlow, task_script, or task_flow." }, { name: "write-pipeline", description: "MUST use when creating or modifying a data pipeline, a set of scripts marked `pipeline` and wired together by `on` / `materialize` annotations." }, + { name: "datatables", description: "MUST use when changing a data table's schema, writing or running its migrations, or working with its roles and permissions." }, { name: "cli-commands", description: "MUST use when using the CLI, including debugging job failures and inspecting run history via `wmill job`." }, { name: "preview", description: "MUST use when opening the Windmill dev page / visual preview of a flow, script, or app. Triggers on words like preview, open, navigate to, visualize, see the flow/app/script, and after writing a flow/script/app for visual verification." }, ]; @@ -8123,6 +8124,53 @@ Node \`f/sales/orders_daily\` (runs when \`orders\` is produced, writes a rollup SELECT date_trunc('day', ts) AS day, count(*) AS n FROM ducklake.main.orders GROUP BY 1 \`\`\` +`, + "datatables": `--- +name: datatables +description: MUST use when changing a data table's schema, writing or running its migrations, or working with its roles and permissions. +--- + +# Data tables: schema changes, migrations and permissions + +A data table is a PostgreSQL database managed by Windmill and shared by the whole workspace: scripts, flows and apps reach it by name, \`main\` being the default. It is configured in the workspace settings (Data Tables), never created by SQL. + +## Changing the schema + +A data table with migrations enabled keeps its schema as an ordered list of versioned SQL migrations, each an \`up\` and an optional \`down\`. That list is what git sync and a workspace merge carry, and what rebuilds the schema in another workspace. So first find out whether migrations are enabled on the data table: + +- **Enabled**: make every schema change (CREATE, ALTER, DROP, anything but row data) as a new migration. A change made with plain SQL exists in that one database only, and the history no longer describes it. +- **Not enabled**: schema changes are plain SQL. Enabling migrations is a workspace admin's decision, made in the Data Tables settings, not a step to take on the way to something else. + +Writing a migration: + +- Wrap the statements in \`BEGIN; ... END;\` so it applies atomically. Leave the frame out only for a statement that cannot run in a transaction, such as \`CREATE INDEX CONCURRENTLY\`: such a migration can fail half applied. +- Give it a \`down\` that undoes the \`up\` whenever the change is reversible. Without one it cannot be rolled back. +- Migrations apply oldest first. One that is applied is not edited: correct it with a new migration. +- Row data is not schema. Insert, update and delete rows with plain SQL, migrations or not. + +\`wmill datatable migrate new \` scaffolds \`migrations/datatable//_.up.sql\` and \`.down.sql\`, \`wmill datatable migrate up\` pushes the local files and applies what is pending, \`wmill datatable migrate down\` rolls back the most recent one and \`wmill datatable migrate status\` lists them. All take \`-d \` and default to \`main\`. + +## A migration is not a draft + +A migration is saved to the workspace and changes the database as soon as it is applied, for everything that reads those tables. To try a schema out without touching tables already in use: + +- put the new tables in a schema of their own, created by the same migration (\`CREATE SCHEMA IF NOT EXISTS\`), and point what you are building at it. A raw app names its schema in its \`data\` config, which is why a new app gets a new schema by default; +- or work in a workspace fork whose data table was cloned when the fork was created ("Clone schema only" or "Clone schema and data"): migrations applied there change that copy alone, and reach the parent workspace when the fork is merged. A fork that kept the original data table points at the parent's database, so a migration applied in it changes the parent's tables at once. + +Altering or dropping a table other scripts or apps use is neither of those: say what depends on it before doing it. + +## Roles and permissions + +Data table roles are a self-hosted Enterprise Edition feature, for data tables on managed Postgres (the instance database or the external managed instance). Without them, and always on a data table backed by a Postgres resource, everyone who can use the data table reaches all of it: there is no per-user, per-schema or per-row access to promise. + +A data table under roles is reached as a Postgres role, and Postgres enforces what that role was granted: + +- roles are assigned, per data table, to users, groups or folders. \`admin\` is the connection the data table had before roles: it owns the existing objects and ignores grants; +- a connection either names a role (\`wmill.datatable('main', { role: 'analyst' })\`, \`role='analyst'\` in Python, \`datatable://main?role=analyst\`) and holds that role alone, or names none and is resolved from the data table's default role; +- \`permission denied\` means the role in use lacks a grant. Say which role and which object: the grant is a data table admin's to give, in the roles drawer of the Database Manager. Do not switch to \`admin\` to get past it; +- a migration runs as the role its SQL declares on its first line, \`-- role \`, the same one in its \`down\`. One that declares none runs as \`admin\`, which only those who may use \`admin\` can do. + +Read the Permissions section of https://www.windmill.dev/docs/core_concepts/persistent_storage/data_tables before designing who reaches what: it covers tenants, grants, the default role and the roles a connection that names none ends up holding, rows that differ per viewer through row-level security, and how forks and raw apps behave under roles. `, "cli-commands": `--- name: cli-commands diff --git a/frontend/src/lib/components/copilot/chat/datatableMigrationTools.test.ts b/frontend/src/lib/components/copilot/chat/datatableMigrationTools.test.ts new file mode 100644 index 0000000000..086adce9e9 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/datatableMigrationTools.test.ts @@ -0,0 +1,141 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { statusMock, createMock, upsertMock, runMock } = vi.hoisted(() => ({ + upsertMock: vi.fn(), + statusMock: vi.fn(), + createMock: vi.fn(), + runMock: vi.fn() +})) + +vi.mock('./shared', () => ({ + createToolDef: (_schema: unknown, name: string, description: string) => ({ + type: 'function', + function: { name, description, parameters: {} } + }) +})) + +vi.mock('$lib/gen', () => ({ + WorkspaceService: { + getDatatableMigrationsStatus: statusMock, + createDatatableMigration: createMock, + upsertDatatableMigration: upsertMock, + runDatatableMigrations: runMock + } +})) + +import { getDatatableMigrationTools } from './datatableMigrationTools' + +function create(args: Record) { + const tool = getDatatableMigrationTools().find( + (entry) => entry.def.function.name === 'create_datatable_migration' + )! + return tool.fn({ + args: { + datatable_name: 'main', + name: 'add_refunds', + code_up: 'BEGIN;\nCREATE TABLE refunds ();\nEND;', + ...args + }, + workspace: 'test-workspace', + helpers: {}, + toolCallbacks: { setToolStatus: vi.fn(), removeToolStatus: vi.fn() } as any, + toolId: 'tool-1' + }) +} + +const applied = { timestamp: 10, name: 'init', code_up: '', status: 'ran' } +const created = { timestamp: 30, name: 'add_refunds', code_up: '', status: 'not_run' } + +beforeEach(() => { + for (const mock of [statusMock, createMock, upsertMock, runMock]) mock.mockReset() + createMock.mockResolvedValue({ timestamp: 30 }) + statusMock.mockResolvedValue({ enabled: true, migrations: [applied, created] }) + runMock.mockResolvedValue({ applied: [{ version: 30, name: 'add_refunds' }] }) +}) + +describe('create_datatable_migration', () => { + it('applies only the migration it created', async () => { + const result = await create({}) + expect(runMock).toHaveBeenCalledWith({ + workspace: 'test-workspace', + datatableName: 'main', + only: 30 + }) + expect(result).toContain('Applied migration 30 (add_refunds)') + }) + + it('leaves it pending when an earlier migration is not applied', async () => { + statusMock.mockResolvedValue({ + enabled: true, + migrations: [ + applied, + { timestamp: 20, name: 'earlier', code_up: '', status: 'not_run' }, + created + ] + }) + const result = await create({}) + expect(runMock).not.toHaveBeenCalled() + expect(result).toContain('20 (earlier)') + }) + + it('keeps a migration that fails to apply and rewrites it on request', async () => { + runMock.mockRejectedValueOnce({ body: 'relation "refunds" already exists' }) + const failed = await create({}) + expect(failed).toContain('relation "refunds" already exists') + expect(failed).toContain('with version 30') + + statusMock.mockResolvedValue({ + enabled: true, + migrations: [applied, { ...created, code_down: 'DROP TABLE refunds;' }] + }) + await create({ version: 30, code_up: 'BEGIN;\nCREATE TABLE refunds2 ();\nEND;' }) + expect(createMock).toHaveBeenCalledTimes(1) + expect(upsertMock).toHaveBeenCalledWith({ + workspace: 'test-workspace', + datatableName: 'main', + requestBody: { + timestamp: 30, + code_down: 'DROP TABLE refunds;', + name: 'add_refunds', + code_up: 'BEGIN;\nCREATE TABLE refunds2 ();\nEND;' + } + }) + expect(runMock).toHaveBeenLastCalledWith({ + workspace: 'test-workspace', + datatableName: 'main', + only: 30 + }) + }) + + it('refuses to rewrite a version that is not pending', async () => { + const result = await create({ version: 10 }) + expect(upsertMock).not.toHaveBeenCalled() + expect(runMock).not.toHaveBeenCalled() + expect(result).toContain('cannot be rewritten') + }) + + it('does not call unreadable earlier migrations pending', async () => { + statusMock.mockResolvedValue({ + enabled: true, + migrations: [ + { ...applied, status: 'unknown' }, + { ...created, status: 'unknown' } + ], + error: 'connection refused' + }) + const result = await create({}) + expect(runMock).not.toHaveBeenCalled() + expect(result).toContain('connection refused') + expect(result).not.toContain('10 (init)') + }) + + it('declares the role above the SQL of both directions', async () => { + await create({ role: 'app_rw', code_down: 'DROP TABLE refunds;', apply: false }) + expect(createMock.mock.calls[0][0].requestBody).toEqual({ + name: 'add_refunds', + code_up: '-- role app_rw\nBEGIN;\nCREATE TABLE refunds ();\nEND;', + code_down: '-- role app_rw\nDROP TABLE refunds;' + }) + expect(runMock).not.toHaveBeenCalled() + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/datatableMigrationTools.ts b/frontend/src/lib/components/copilot/chat/datatableMigrationTools.ts new file mode 100644 index 0000000000..3a2476b9d0 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/datatableMigrationTools.ts @@ -0,0 +1,391 @@ +import { z } from 'zod' +import { WorkspaceService } from '$lib/gen' +import { withMigrationRole } from '$lib/components/datatableMigrationRole' +import { createToolDef, type ToolCallbacks } from './shared' +import { NONE, RUN_PREVIEW, type SessionTool } from './sessionCapabilities' + +/** + * Tools over a data table's versioned SQL migrations (`datatable_migrations.rs`). + * + * A data table with migrations enabled keeps its schema as an ordered list of definitions, each + * applied once per database and recorded there. That list is what git sync and a workspace merge + * carry, so a schema change made with plain SQL exists in one database only. + */ + +const memo = (factory: () => T): (() => T) => { + let cached: T | undefined + return () => (cached ??= factory()) +} + +function apiErrorMessage(e: unknown): string { + const err = e as { body?: unknown; message?: unknown } | undefined + const body = err?.body + if (typeof body === 'string' && body !== '') return body + if (typeof err?.message === 'string') return err.message + return String(e) +} + +function fail(toolCallbacks: ToolCallbacks, toolId: string, message: string): string { + toolCallbacks.setToolStatus(toolId, { content: message, error: message }) + return message +} + +const migrationLabel = (m: { version: number; name: string }) => `${m.version} (${m.name})` + +const datatableNameSchema = () => + z.string().describe('The datatable the migrations belong to, e.g. "main".') + +// ============= Tool definitions ============= + +const getListMigrationsSchema = memo(() => + z.object({ + datatable_name: datatableNameSchema(), + version: z + .number() + .optional() + .describe('Return the SQL of this one migration instead of the list.') + }) +) +const getListMigrationsToolDef = memo(() => + createToolDef( + getListMigrationsSchema(), + 'list_datatable_migrations', + "List a datatable's migrations in order, each with its version, name and whether it is applied or pending. Pass `version` to read one migration's SQL." + ) +) + +const getCreateMigrationSchema = memo(() => + z.object({ + datatable_name: datatableNameSchema(), + name: z + .string() + .regex(/^[A-Za-z0-9_-]+$/) + .describe( + 'Short name of the change, e.g. "add_refunds_table". Letters, digits, "_" and "-" only.' + ), + code_up: z + .string() + .describe( + 'The SQL that makes the change. Wrap the statements in `BEGIN; ... END;` so they apply atomically, unless one of them cannot run in a transaction.' + ), + code_down: z + .string() + .optional() + .describe( + 'The SQL that undoes `code_up`. Without it the migration cannot be rolled back, so provide it whenever the change is reversible.' + ), + role: z + .string() + .regex(/^[A-Za-z0-9_-]{1,63}$/) + .optional() + .describe( + 'The datatable role the migration runs as, on a datatable that has roles. Omit to run as admin, which only the datatable admins may do.' + ), + version: z + .number() + .optional() + .describe( + 'Rewrite this migration instead of adding one: only for a pending migration, typically the one that just failed to apply.' + ), + apply: z + .boolean() + .optional() + .describe('Set false to save the migration as pending without running it. Defaults to true.') + }) +) +const getCreateMigrationToolDef = memo(() => + createToolDef( + getCreateMigrationSchema(), + 'create_datatable_migration', + 'Change the schema of a datatable that has migrations enabled: saves the change as a new migration and applies it. One that fails to apply stays saved as pending: call this again with its `version` to fix it.' + ) +) + +const getRunMigrationsSchema = memo(() => z.object({ datatable_name: datatableNameSchema() })) +const getRunMigrationsToolDef = memo(() => + createToolDef( + getRunMigrationsSchema(), + 'run_datatable_migrations', + "Apply a datatable's pending migrations, oldest first. It stops at the first one that fails." + ) +) + +const getRollbackMigrationSchema = memo(() => + z.object({ + datatable_name: datatableNameSchema(), + version: z + .number() + .describe('The applied migration to undo, normally the most recent one, as listed.') + }) +) +const getRollbackMigrationToolDef = memo(() => + createToolDef( + getRollbackMigrationSchema(), + 'rollback_datatable_migration', + 'Undo one applied migration of a datatable by running its down SQL. The migration stays in the list as pending.' + ) +) + +// ============= Tools ============= + +export function getDatatableMigrationTools(): SessionTool<{}>[] { + return [ + { + requires: NONE, + def: getListMigrationsToolDef(), + planModeSafe: true, + fn: async ({ args, workspace, toolId, toolCallbacks }) => { + const { datatable_name, version } = getListMigrationsSchema().parse(args) + toolCallbacks.setToolStatus(toolId, { + content: `Listing migrations of "${datatable_name}"...` + }) + try { + const status = await WorkspaceService.getDatatableMigrationsStatus({ + workspace, + datatableName: datatable_name + }) + if (!status.enabled) { + toolCallbacks.setToolStatus(toolId, { + content: `Migrations are not enabled on "${datatable_name}"` + }) + return `Migrations are not enabled on datatable "${datatable_name}": its schema is changed with plain SQL and there is no history to list. Enabling them is a workspace admin's decision, made in Workspace settings → Data Tables.` + } + if (version !== undefined) { + const migration = status.migrations.find((m) => m.timestamp === version) + if (!migration) { + return fail( + toolCallbacks, + toolId, + `No migration ${version} on datatable "${datatable_name}"` + ) + } + toolCallbacks.setToolStatus(toolId, { + content: `Read migration ${migrationLabel({ version, name: migration.name })}` + }) + return JSON.stringify( + { + version, + name: migration.name, + status: migrationStatus(migration.status), + code_up: migration.code_up, + ...(migration.code_down != null && { code_down: migration.code_down }) + }, + null, + 2 + ) + } + toolCallbacks.setToolStatus(toolId, { + content: `Listed ${status.migrations.length} migration(s) of "${datatable_name}"` + }) + return JSON.stringify( + { + migrations: status.migrations.map((m) => ({ + version: m.timestamp, + name: m.name, + status: migrationStatus(m.status) + })), + // Set when the database could not be read, which is why every status is unknown. + ...(status.error && { error: status.error }) + }, + null, + 2 + ) + } catch (e) { + return fail(toolCallbacks, toolId, `Error listing migrations: ${apiErrorMessage(e)}`) + } + } + }, + { + // `ensure_not_operator` in datatable_migrations.rs: the condition `run_preview` stands for. + requires: RUN_PREVIEW, + def: getCreateMigrationToolDef(), + requiresConfirmation: true, + confirmationMessage: (args) => + `${args?.version === undefined ? 'Create' : 'Rewrite'}${args?.apply === false ? '' : ' and apply'} data table migration`, + queuedLabel: (args) => `Write migration "${args?.name ?? ''}"`, + showDetails: true, + fn: async ({ args, workspace, toolId, toolCallbacks }) => { + const parsed = getCreateMigrationSchema().parse(args) + const datatableName = parsed.datatable_name + toolCallbacks.setToolStatus(toolId, { + content: `Saving migration "${parsed.name}" on "${datatableName}"...` + }) + // The role is an annotation in the SQL, read from above everything else, and a rollback + // has to run as the role that created what it drops. + const definition = { + name: parsed.name, + code_up: withRole(parsed.code_up, parsed.role), + ...(parsed.code_down !== undefined && { + code_down: withRole(parsed.code_down, parsed.role) + }) + } + let version: number + try { + if (parsed.version === undefined) { + const created = await WorkspaceService.createDatatableMigration({ + workspace, + datatableName, + requestBody: definition + }) + version = created.timestamp + } else { + version = parsed.version + // The server inserts at a version it does not know, which would wedge a new + // migration into the middle of the history, and stores a down left out as none. + const { migrations } = await WorkspaceService.getDatatableMigrationsStatus({ + workspace, + datatableName + }) + const existing = migrations.find((m) => m.timestamp === version) + if (existing?.status !== 'not_run') { + return fail( + toolCallbacks, + toolId, + `Migration ${version} is not a pending migration of datatable "${datatableName}", so it cannot be rewritten. Omit \`version\` to add a new migration.` + ) + } + await WorkspaceService.upsertDatatableMigration({ + workspace, + datatableName, + requestBody: { + timestamp: version, + ...(existing.code_down != null && { code_down: existing.code_down }), + ...definition + } + }) + } + } catch (e) { + return fail(toolCallbacks, toolId, `Error saving migration: ${apiErrorMessage(e)}`) + } + const label = migrationLabel({ version, name: parsed.name }) + if (parsed.apply === false) { + toolCallbacks.setToolStatus(toolId, { content: `Saved migration ${label} as pending` }) + return `Saved migration ${label} as pending. It is not applied yet.` + } + + // Applying it alone would run it ahead of migrations it may depend on, and applying + // those with it would run SQL the user was never shown. + let earlierPending: string[] + try { + const status = await WorkspaceService.getDatatableMigrationsStatus({ + workspace, + datatableName + }) + const earlier = status.migrations.filter((m) => m.timestamp < version) + // Unreadable is not pending: reporting it so would send the user to apply + // migrations that already ran. + if (status.error || earlier.some((m) => m.status === 'unknown')) { + throw new Error(status.error ?? 'the applied migrations could not be read') + } + earlierPending = earlier + .filter((m) => m.status === 'not_run') + .map((m) => migrationLabel({ version: m.timestamp, name: m.name })) + } catch (e) { + return fail( + toolCallbacks, + toolId, + `Saved migration ${label} as pending, but could not check what else is pending, so it was not applied: ${apiErrorMessage(e)}` + ) + } + if (earlierPending.length > 0) { + toolCallbacks.setToolStatus(toolId, { + content: `Saved migration ${label}; ${earlierPending.length} earlier migration(s) still pending` + }) + return `Saved migration ${label} as pending without applying it: earlier migrations are not applied yet (${earlierPending.join(', ')}) and it may depend on them. Tell the user, then apply them all in order with run_datatable_migrations.` + } + + toolCallbacks.setToolStatus(toolId, { content: `Applying migration ${label}...` }) + try { + await WorkspaceService.runDatatableMigrations({ + workspace, + datatableName, + only: version + }) + } catch (e) { + // Kept whatever failed: part of it may be applied, or all of it with only the record + // missing, and the definition with its down is then the one trace of that. + return fail( + toolCallbacks, + toolId, + `Migration ${label} is saved but failed to apply: ${apiErrorMessage(e)}. It stays pending and blocks later migrations. If nothing of it was applied, fix it by calling this tool again with version ${version}; otherwise, or to drop it, tell the user, who can edit or delete it in Workspace settings → Data Tables.` + ) + } + toolCallbacks.setToolStatus(toolId, { content: `Applied migration ${label}` }) + return `Applied migration ${label} on datatable "${datatableName}".` + } + }, + { + requires: RUN_PREVIEW, + def: getRunMigrationsToolDef(), + requiresConfirmation: true, + confirmationMessage: 'Apply pending data table migrations', + queuedLabel: (args) => `Apply migrations on "${args?.datatable_name ?? 'datatable'}"`, + showDetails: true, + fn: async ({ args, workspace, toolId, toolCallbacks }) => { + const { datatable_name } = getRunMigrationsSchema().parse(args) + toolCallbacks.setToolStatus(toolId, { + content: `Applying migrations on "${datatable_name}"...` + }) + try { + const { applied } = await WorkspaceService.runDatatableMigrations({ + workspace, + datatableName: datatable_name + }) + if (applied.length === 0) { + toolCallbacks.setToolStatus(toolId, { content: 'No pending migration to apply' }) + return `No pending migration to apply on datatable "${datatable_name}".` + } + toolCallbacks.setToolStatus(toolId, { + content: `Applied ${applied.length} migration(s) on "${datatable_name}"` + }) + return `Applied ${applied.length} migration(s) on datatable "${datatable_name}": ${applied.map(migrationLabel).join(', ')}.` + } catch (e) { + // Those before the failing one are applied: the list is what says where it stopped. + return fail( + toolCallbacks, + toolId, + `Error applying migrations: ${apiErrorMessage(e)}. Migrations before the failing one may have been applied: list them to see.` + ) + } + } + }, + { + requires: RUN_PREVIEW, + def: getRollbackMigrationToolDef(), + requiresConfirmation: true, + confirmationMessage: (args) => `Roll back data table migration ${args?.version ?? ''}`, + queuedLabel: (args) => `Roll back migration ${args?.version ?? ''}`, + showDetails: true, + fn: async ({ args, workspace, toolId, toolCallbacks }) => { + const { datatable_name, version } = getRollbackMigrationSchema().parse(args) + toolCallbacks.setToolStatus(toolId, { + content: `Rolling back migration ${version} on "${datatable_name}"...` + }) + try { + const { rolled_back } = await WorkspaceService.rollbackDatatableMigrations({ + workspace, + datatableName: datatable_name, + only: version + }) + if (rolled_back.length === 0) { + toolCallbacks.setToolStatus(toolId, { content: `Migration ${version} is not applied` }) + return `Migration ${version} is not applied on datatable "${datatable_name}", so there is nothing to roll back.` + } + const label = rolled_back.map(migrationLabel).join(', ') + toolCallbacks.setToolStatus(toolId, { content: `Rolled back migration ${label}` }) + return `Rolled back migration ${label} on datatable "${datatable_name}". It is pending again.` + } catch (e) { + return fail(toolCallbacks, toolId, `Error rolling back: ${apiErrorMessage(e)}`) + } + } + } + ] +} + +function migrationStatus(status: 'ran' | 'not_run' | 'unknown'): 'applied' | 'pending' | 'unknown' { + return status === 'ran' ? 'applied' : status === 'not_run' ? 'pending' : 'unknown' +} + +function withRole(sql: string, role: string | undefined): string { + return role === undefined ? sql : withMigrationRole(sql, role) +} diff --git a/frontend/src/lib/components/copilot/chat/datatableTools.ts b/frontend/src/lib/components/copilot/chat/datatableTools.ts index 7a743ab783..e05b2e4325 100644 --- a/frontend/src/lib/components/copilot/chat/datatableTools.ts +++ b/frontend/src/lib/components/copilot/chat/datatableTools.ts @@ -286,6 +286,7 @@ export function getDatatableTools(): SessionTool<{}>[] { datatable_name: d.datatable_name, schemas: d.schemas, ...(d.error && { error: d.error }), + ...(d.migrations_enabled && { migrations_enabled: true }), ...(d.permissioned && { usable_roles: d.usable_roles, default_role: d.default_role }) })), null, diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index 5295712626..c935e1bc02 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -121,6 +121,7 @@ import { } from '../skills/skillResources' import { MAX_SKILL_DESCRIPTION_LENGTH, MAX_SKILL_INSTRUCTIONS_LENGTH } from '../skills/skillMd' import { + getDatatablePrompt, getDatatableSdkReference, getFlowPrompt, getPipelinePrompt, @@ -161,6 +162,7 @@ import { searchDocsTool, readDocsPageTool } from '../docs/core' import { createDbSchemaTool } from '../script/core' import type { ContextElement } from '../context' import { getDatatableTools } from '../datatableTools' +import { getDatatableMigrationTools } from '../datatableMigrationTools' import { getDucklakeTools } from '../ducklakeTools' import { fileTools } from '../files/fileTools' import type { AttachedFilesStore } from '../files/attachedFiles.svelte' @@ -363,7 +365,7 @@ const scriptLangSchema = z.enum($ScriptLang.enum) const getInstructionsSchema = z.object({ subject: instructionSubjectSchema.describe( - 'What to get authoring instructions for: a workspace item type (script, flow, resource, app), "pipeline" for building a data pipeline (a DAG of annotated scripts wired by storage assets — NOT a flow), or "datatable" for the wmill.datatable() SQL SDK used inside runnables. Schedules, triggers, and variables don\'t need instructions — their tool schemas describe everything.' + 'What to get authoring instructions for: a workspace item type (script, flow, resource, app), "pipeline" for building a data pipeline (a DAG of annotated scripts wired by storage assets — NOT a flow), or "datatable" for how data tables work (schema changes and migrations, roles and permissions) plus the wmill.datatable() SQL SDK used inside runnables. Schedules, triggers, and variables don\'t need instructions — their tool schemas describe everything.' ), language: scriptLangSchema .optional() @@ -1453,6 +1455,16 @@ const buildGlobalSystemPrompt = ( ? '\n- If the user message includes an ACTIVE PREVIEW section, that is the page the side panel is showing — resolve "this page", "here" and "it" against it, and against `open` (the item the user has open in its editor) when there is one. It already tells you what get_preview_status would, so do not call that tool to learn what is on screen; call it only to check the panel\'s *other* tabs.' : '' const pipelineBullet = `- A "data pipeline" is NOT a flow: it is a DAG of independent scripts in one folder, wired by storage assets (DuckLake/data tables/S3) and triggers via top-of-file \`pipeline\` / \`on \` annotation comments written in each script's comment syntax (\`--\` for SQL, \`#\` for Python/Bash, \`//\` for TS — a \`//\` line in a SQL node is a syntax error). When the user asks for a data pipeline (or to ingest/transform/materialize data across steps), call get_instructions with subject "pipeline" and build annotated script drafts — do not build a flow. list_pipelines lists the existing ones by folder.${pipelineAlphaNote}` + // Roles need a self-hosted Enterprise Edition. The edition is knowable only in the browser, + // so elsewhere the guidance stays and nothing is claimed about the instance. + const datatableRolesAvailable = !isCloudHosted() && (!BROWSER || !!get(enterpriseLicense)) + const datatableRolesBullet = datatableRolesAvailable + ? ` +- A datatable can be under roles; list_datatables then reports its \`usable_roles\` and \`default_role\`. When working on a raw app that pins a role (\`data.roles\` in its raw_app.yaml), pass that role to the datatable tools, and to wmill.datatable in its runnables, so you see and change only what the app itself can. For anything else about roles, grants or a \`permission denied\`, call get_instructions with subject "datatable".` + : BROWSER + ? ` +- Datatable roles and permissions are not available on this instance (they need a self-hosted Enterprise Edition): everyone who can use a datatable reaches all of it, so do not promise per-user, per-schema or per-row access on one.` + : '' // Hosting and edition come from the hostname and a store the app populates at init, so // they are knowable only in the browser: a non-browser caller reads false for both and // would be told "self-hosted Community Edition" whatever it targets. No base URL here @@ -1623,13 +1635,9 @@ Data Tables: - Use get_datatable_table_schema only when you need a table's column names/types; list_datatables is enough for table-list or availability summaries.${when( canRunPreview, ` -- Use exec_datatable_sql to explore data, run queries, mutate rows, or change schema (CREATE/ALTER/DROP). Creating a table is a normal CREATE TABLE statement — it appears in list_datatables afterward, with no registration step.` - )}${ - isCloudHosted() - ? '' - : ` -- A raw app may use a datatable through a role (\`data.roles\` in its raw_app.yaml). When working on such an app, pass that role to the datatable tools, and to wmill.datatable in its runnables, so you see and change only what the app itself can.` - }${when( +- Use exec_datatable_sql to explore data, run queries, mutate rows, or change schema (CREATE/ALTER/DROP). Creating a table is a normal CREATE TABLE statement — it appears in list_datatables afterward, with no registration step. +- A datatable that list_datatables reports with \`migrations_enabled\` keeps its schema as versioned migrations: change its schema with create_datatable_migration, never exec_datatable_sql. list_datatable_migrations reads them, run_datatable_migrations applies what is pending, rollback_datatable_migration undoes one. A migration is applied for real, not drafted, so call get_instructions with subject "datatable" before your first one.` + )}${datatableRolesBullet}${when( canWriteDraft, ` - When writing runnable code (inline app runnables, scripts, flow modules) that reads or writes datatable data at runtime, it accesses a datatable via wmill.datatable(). Default to TypeScript (bun) unless the user asked for another language. Call get_instructions with subject "datatable" and language "bun" for the TypeScript SQL SDK reference (or language "python3" for Python) — it returns only that language so you get just what you need.` @@ -2525,7 +2533,9 @@ ${getResourcePrompt()}` function getDatatableInstructions(language?: ScriptLang): string { // Default to the TypeScript SDK so we return only what's needed, not both. const lang = language ?? 'bun' - return `# Datatable SQL SDK reference + return `${getDatatablePrompt()} + +# Datatable SQL SDK reference Datatables are workspace-scoped managed PostgreSQL databases. In chat, explore and shape them with the datatable tools you were given. The reference below is for code you author inside runnables (inline app runnables, scripts, or flow rawscript modules) that reads or writes datatable data at runtime. @@ -3483,7 +3493,7 @@ export const globalTools: SessionTool<{}>[] = [ def: createToolDef( getInstructionsSchema, 'get_instructions', - 'Get authoring guidance for scripts, flows, data pipelines, resources, apps, or the datatable SQL SDK (wmill.datatable()) used inside runnables.' + 'Get authoring guidance for scripts, flows, data pipelines, resources, apps, or data tables: their migrations and permissions, and the SQL SDK (wmill.datatable()) used inside runnables.' ), planModeSafe: true, fn: async (ctx) => { @@ -4773,6 +4783,7 @@ export const globalTools: SessionTool<{}>[] = [ }, // Workspace-scoped datatable tools (unrestricted: no whitelist, no creation policy) ...getDatatableTools(), + ...getDatatableMigrationTools(), // Workspace DuckLake: pipeline storage prerequisite, and declared measures ...getDucklakeTools(), // Read-only tools over files the user attached to the conversation diff --git a/system_prompts/README.md b/system_prompts/README.md index cc16473c6c..a651c48692 100644 --- a/system_prompts/README.md +++ b/system_prompts/README.md @@ -46,6 +46,7 @@ a CLI-only workflow file (`flow-cli.md`, `script-cli.md`, `raw-app-cli.md`). | resources | `getResourcePrompt()` | `resources` | `resources.md` | | workflow-as-code | `getWorkflowAsCodePrompt(lang)` | `write-workflow-as-code` | `workflow-as-code.md`, the WAC SDK | | pipeline | `getPipelinePrompt()` | `write-pipeline` | `pipeline-base.md` | +| data tables | `getDatatablePrompt()` | `datatables` | `datatable.md` | | triggers, schedules, preview, CLI commands | — | same names | CLI only | The app chat embeds `RAW_APP_BASE` directly (`chat/app/core.ts`). diff --git a/system_prompts/auto-generated/index.d.ts b/system_prompts/auto-generated/index.d.ts index d77f699a33..8ba83f7f74 100644 --- a/system_prompts/auto-generated/index.d.ts +++ b/system_prompts/auto-generated/index.d.ts @@ -4,5 +4,6 @@ export declare function getFlowPrompt(): string; export declare function getResourcePrompt(): string; export declare function getRawAppPrompt(language?: string): string; export declare function getPipelinePrompt(): string; +export declare function getDatatablePrompt(): string; export declare function getDatatableSdkReference(language?: string): string; export declare function getWorkflowAsCodePrompt(language?: string): string; diff --git a/system_prompts/auto-generated/index.ts b/system_prompts/auto-generated/index.ts index ade95efe9e..60bb79b031 100644 --- a/system_prompts/auto-generated/index.ts +++ b/system_prompts/auto-generated/index.ts @@ -73,6 +73,13 @@ export function getPipelinePrompt(): string { ].filter(Boolean).join('\n\n'); } +// Helper for data table schema changes, migrations and permissions (chat consumers) +export function getDatatablePrompt(): string { + return [ + prompts.DATATABLE_BASE + ].filter(Boolean).join('\n\n'); +} + // Helper to get the datatable SQL SDK reference (wmill.datatable()). // Pass a language to get only that SDK; omit it to get both. export function getDatatableSdkReference(language?: string): string { diff --git a/system_prompts/auto-generated/prompts.d.ts b/system_prompts/auto-generated/prompts.d.ts index e74555504e..cfdde52495 100644 --- a/system_prompts/auto-generated/prompts.d.ts +++ b/system_prompts/auto-generated/prompts.d.ts @@ -5,6 +5,7 @@ export declare const FLOW_BASE: string; export declare const RESOURCES_BASE: string; export declare const RAW_APP_BASE: string; export declare const PIPELINE_BASE: string; +export declare const DATATABLE_BASE: string; export declare const WORKFLOW_AS_CODE_BASE: string; export declare const FLOW_CHAT_SPECIAL_MODULES: string; export declare const SDK_TYPESCRIPT: string; diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index c3310351b7..54d869f727 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -1338,6 +1338,47 @@ FROM ducklake.main.orders GROUP BY 1 \`\`\` `; +export const DATATABLE_BASE = `# Data tables: schema changes, migrations and permissions + +A data table is a PostgreSQL database managed by Windmill and shared by the whole workspace: scripts, flows and apps reach it by name, \`main\` being the default. It is configured in the workspace settings (Data Tables), never created by SQL. + +## Changing the schema + +A data table with migrations enabled keeps its schema as an ordered list of versioned SQL migrations, each an \`up\` and an optional \`down\`. That list is what git sync and a workspace merge carry, and what rebuilds the schema in another workspace. So first find out whether migrations are enabled on the data table: + +- **Enabled**: make every schema change (CREATE, ALTER, DROP, anything but row data) as a new migration. A change made with plain SQL exists in that one database only, and the history no longer describes it. +- **Not enabled**: schema changes are plain SQL. Enabling migrations is a workspace admin's decision, made in the Data Tables settings, not a step to take on the way to something else. + +Writing a migration: + +- Wrap the statements in \`BEGIN; ... END;\` so it applies atomically. Leave the frame out only for a statement that cannot run in a transaction, such as \`CREATE INDEX CONCURRENTLY\`: such a migration can fail half applied. +- Give it a \`down\` that undoes the \`up\` whenever the change is reversible. Without one it cannot be rolled back. +- Migrations apply oldest first. One that is applied is not edited: correct it with a new migration. +- Row data is not schema. Insert, update and delete rows with plain SQL, migrations or not. + +## A migration is not a draft + +A migration is saved to the workspace and changes the database as soon as it is applied, for everything that reads those tables. To try a schema out without touching tables already in use: + +- put the new tables in a schema of their own, created by the same migration (\`CREATE SCHEMA IF NOT EXISTS\`), and point what you are building at it. A raw app names its schema in its \`data\` config, which is why a new app gets a new schema by default; +- or work in a workspace fork whose data table was cloned when the fork was created ("Clone schema only" or "Clone schema and data"): migrations applied there change that copy alone, and reach the parent workspace when the fork is merged. A fork that kept the original data table points at the parent's database, so a migration applied in it changes the parent's tables at once. + +Altering or dropping a table other scripts or apps use is neither of those: say what depends on it before doing it. + +## Roles and permissions + +Data table roles are a self-hosted Enterprise Edition feature, for data tables on managed Postgres (the instance database or the external managed instance). Without them, and always on a data table backed by a Postgres resource, everyone who can use the data table reaches all of it: there is no per-user, per-schema or per-row access to promise. + +A data table under roles is reached as a Postgres role, and Postgres enforces what that role was granted: + +- roles are assigned, per data table, to users, groups or folders. \`admin\` is the connection the data table had before roles: it owns the existing objects and ignores grants; +- a connection either names a role (\`wmill.datatable('main', { role: 'analyst' })\`, \`role='analyst'\` in Python, \`datatable://main?role=analyst\`) and holds that role alone, or names none and is resolved from the data table's default role; +- \`permission denied\` means the role in use lacks a grant. Say which role and which object: the grant is a data table admin's to give, in the roles drawer of the Database Manager. Do not switch to \`admin\` to get past it; +- a migration runs as the role its SQL declares on its first line, \`-- role \`, the same one in its \`down\`. One that declares none runs as \`admin\`, which only those who may use \`admin\` can do. + +Read the Permissions section of https://www.windmill.dev/docs/core_concepts/persistent_storage/data_tables before designing who reaches what: it covers tenants, grants, the default role and the roles a connection that names none ends up holding, rows that differ per viewer through row-level security, and how forks and raw apps behave under roles. +`; + export const WORKFLOW_AS_CODE_BASE = `# Windmill Workflow-as-Code Writing Guide ## Scope diff --git a/system_prompts/auto-generated/skills/datatables/SKILL.md b/system_prompts/auto-generated/skills/datatables/SKILL.md new file mode 100644 index 0000000000..1c1cb9eede --- /dev/null +++ b/system_prompts/auto-generated/skills/datatables/SKILL.md @@ -0,0 +1,46 @@ +--- +name: datatables +description: MUST use when changing a data table's schema, writing or running its migrations, or working with its roles and permissions. +--- + +# Data tables: schema changes, migrations and permissions + +A data table is a PostgreSQL database managed by Windmill and shared by the whole workspace: scripts, flows and apps reach it by name, `main` being the default. It is configured in the workspace settings (Data Tables), never created by SQL. + +## Changing the schema + +A data table with migrations enabled keeps its schema as an ordered list of versioned SQL migrations, each an `up` and an optional `down`. That list is what git sync and a workspace merge carry, and what rebuilds the schema in another workspace. So first find out whether migrations are enabled on the data table: + +- **Enabled**: make every schema change (CREATE, ALTER, DROP, anything but row data) as a new migration. A change made with plain SQL exists in that one database only, and the history no longer describes it. +- **Not enabled**: schema changes are plain SQL. Enabling migrations is a workspace admin's decision, made in the Data Tables settings, not a step to take on the way to something else. + +Writing a migration: + +- Wrap the statements in `BEGIN; ... END;` so it applies atomically. Leave the frame out only for a statement that cannot run in a transaction, such as `CREATE INDEX CONCURRENTLY`: such a migration can fail half applied. +- Give it a `down` that undoes the `up` whenever the change is reversible. Without one it cannot be rolled back. +- Migrations apply oldest first. One that is applied is not edited: correct it with a new migration. +- Row data is not schema. Insert, update and delete rows with plain SQL, migrations or not. + +`wmill datatable migrate new ` scaffolds `migrations/datatable//_.up.sql` and `.down.sql`, `wmill datatable migrate up` pushes the local files and applies what is pending, `wmill datatable migrate down` rolls back the most recent one and `wmill datatable migrate status` lists them. All take `-d ` and default to `main`. + +## A migration is not a draft + +A migration is saved to the workspace and changes the database as soon as it is applied, for everything that reads those tables. To try a schema out without touching tables already in use: + +- put the new tables in a schema of their own, created by the same migration (`CREATE SCHEMA IF NOT EXISTS`), and point what you are building at it. A raw app names its schema in its `data` config, which is why a new app gets a new schema by default; +- or work in a workspace fork whose data table was cloned when the fork was created ("Clone schema only" or "Clone schema and data"): migrations applied there change that copy alone, and reach the parent workspace when the fork is merged. A fork that kept the original data table points at the parent's database, so a migration applied in it changes the parent's tables at once. + +Altering or dropping a table other scripts or apps use is neither of those: say what depends on it before doing it. + +## Roles and permissions + +Data table roles are a self-hosted Enterprise Edition feature, for data tables on managed Postgres (the instance database or the external managed instance). Without them, and always on a data table backed by a Postgres resource, everyone who can use the data table reaches all of it: there is no per-user, per-schema or per-row access to promise. + +A data table under roles is reached as a Postgres role, and Postgres enforces what that role was granted: + +- roles are assigned, per data table, to users, groups or folders. `admin` is the connection the data table had before roles: it owns the existing objects and ignores grants; +- a connection either names a role (`wmill.datatable('main', { role: 'analyst' })`, `role='analyst'` in Python, `datatable://main?role=analyst`) and holds that role alone, or names none and is resolved from the data table's default role; +- `permission denied` means the role in use lacks a grant. Say which role and which object: the grant is a data table admin's to give, in the roles drawer of the Database Manager. Do not switch to `admin` to get past it; +- a migration runs as the role its SQL declares on its first line, `-- role `, the same one in its `down`. One that declares none runs as `admin`, which only those who may use `admin` can do. + +Read the Permissions section of https://www.windmill.dev/docs/core_concepts/persistent_storage/data_tables before designing who reaches what: it covers tenants, grants, the default role and the roles a connection that names none ends up holding, rows that differ per viewer through row-level security, and how forks and raw apps behave under roles. diff --git a/system_prompts/base/datatable.md b/system_prompts/base/datatable.md new file mode 100644 index 0000000000..c90fb5e349 --- /dev/null +++ b/system_prompts/base/datatable.md @@ -0,0 +1,43 @@ +# Data tables: schema changes, migrations and permissions + +A data table is a PostgreSQL database managed by Windmill and shared by the whole workspace: scripts, flows and apps reach it by name, `main` being the default. It is configured in the workspace settings (Data Tables), never created by SQL. + +## Changing the schema + +A data table with migrations enabled keeps its schema as an ordered list of versioned SQL migrations, each an `up` and an optional `down`. That list is what git sync and a workspace merge carry, and what rebuilds the schema in another workspace. So first find out whether migrations are enabled on the data table: + +- **Enabled**: make every schema change (CREATE, ALTER, DROP, anything but row data) as a new migration. A change made with plain SQL exists in that one database only, and the history no longer describes it. +- **Not enabled**: schema changes are plain SQL. Enabling migrations is a workspace admin's decision, made in the Data Tables settings, not a step to take on the way to something else. + +Writing a migration: + +- Wrap the statements in `BEGIN; ... END;` so it applies atomically. Leave the frame out only for a statement that cannot run in a transaction, such as `CREATE INDEX CONCURRENTLY`: such a migration can fail half applied. +- Give it a `down` that undoes the `up` whenever the change is reversible. Without one it cannot be rolled back. +- Migrations apply oldest first. One that is applied is not edited: correct it with a new migration. +- Row data is not schema. Insert, update and delete rows with plain SQL, migrations or not. + + +`wmill datatable migrate new ` scaffolds `migrations/datatable//_.up.sql` and `.down.sql`, `wmill datatable migrate up` pushes the local files and applies what is pending, `wmill datatable migrate down` rolls back the most recent one and `wmill datatable migrate status` lists them. All take `-d ` and default to `main`. + + +## A migration is not a draft + +A migration is saved to the workspace and changes the database as soon as it is applied, for everything that reads those tables. To try a schema out without touching tables already in use: + +- put the new tables in a schema of their own, created by the same migration (`CREATE SCHEMA IF NOT EXISTS`), and point what you are building at it. A raw app names its schema in its `data` config, which is why a new app gets a new schema by default; +- or work in a workspace fork whose data table was cloned when the fork was created ("Clone schema only" or "Clone schema and data"): migrations applied there change that copy alone, and reach the parent workspace when the fork is merged. A fork that kept the original data table points at the parent's database, so a migration applied in it changes the parent's tables at once. + +Altering or dropping a table other scripts or apps use is neither of those: say what depends on it before doing it. + +## Roles and permissions + +Data table roles are a self-hosted Enterprise Edition feature, for data tables on managed Postgres (the instance database or the external managed instance). Without them, and always on a data table backed by a Postgres resource, everyone who can use the data table reaches all of it: there is no per-user, per-schema or per-row access to promise. + +A data table under roles is reached as a Postgres role, and Postgres enforces what that role was granted: + +- roles are assigned, per data table, to users, groups or folders. `admin` is the connection the data table had before roles: it owns the existing objects and ignores grants; +- a connection either names a role (`wmill.datatable('main', { role: 'analyst' })`, `role='analyst'` in Python, `datatable://main?role=analyst`) and holds that role alone, or names none and is resolved from the data table's default role; +- `permission denied` means the role in use lacks a grant. Say which role and which object: the grant is a data table admin's to give, in the roles drawer of the Database Manager. Do not switch to `admin` to get past it; +- a migration runs as the role its SQL declares on its first line, `-- role `, the same one in its `down`. One that declares none runs as `admin`, which only those who may use `admin` can do. + +Read the Permissions section of https://www.windmill.dev/docs/core_concepts/persistent_storage/data_tables before designing who reaches what: it covers tenants, grants, the default role and the roles a connection that names none ends up holding, rows that differ per viewer through row-level security, and how forks and raw apps behave under roles. diff --git a/system_prompts/generate.py b/system_prompts/generate.py index 918deef0fb..055f03d543 100644 --- a/system_prompts/generate.py +++ b/system_prompts/generate.py @@ -1696,6 +1696,12 @@ TOPICS: dict[str, dict] = { 'chat_helper': 'getPipelinePrompt', 'parts': ['pipeline-base.md'], }, + 'datatable': { + 'skill': 'datatables', + 'description': "MUST use when changing a data table's schema, writing or running its migrations, or working with its roles and permissions.", + 'chat_helper': 'getDatatablePrompt', + 'parts': ['datatable.md'], + }, 'cli_commands': { 'skill': 'cli-commands', 'description': 'MUST use when using the CLI, including debugging job failures and inspecting run history via `wmill job`.', @@ -1715,6 +1721,7 @@ CHAT_CONSTANTS = { 'resources.md': 'RESOURCES_BASE', 'raw-app.md': 'RAW_APP_BASE', 'pipeline-base.md': 'PIPELINE_BASE', + 'datatable.md': 'DATATABLE_BASE', 'workflow-as-code.md': 'WORKFLOW_AS_CODE_BASE', 'flow-chat-special-modules.md': 'FLOW_CHAT_SPECIAL_MODULES', } @@ -2650,6 +2657,11 @@ export function getPipelinePrompt(): string { __RETURN_pipeline__ } +// Helper for data table schema changes, migrations and permissions (chat consumers) +export function getDatatablePrompt(): string { + __RETURN_datatable__ +} + // Helper to get the datatable SQL SDK reference (wmill.datatable()). // Pass a language to get only that SDK; omit it to get both. export function getDatatableSdkReference(language?: string): string { @@ -2711,6 +2723,7 @@ export declare function getFlowPrompt(): string; export declare function getResourcePrompt(): string; export declare function getRawAppPrompt(language?: string): string; export declare function getPipelinePrompt(): string; +export declare function getDatatablePrompt(): string; export declare function getDatatableSdkReference(language?: string): string; export declare function getWorkflowAsCodePrompt(language?: string): string; """