From dc6b99775b550e7433fee8a159c30eaf296500c5 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Mon, 6 Jul 2026 16:56:32 +0200 Subject: [PATCH 01/16] fix(cli): quote non-identifier property names in resource-type namespace (#9964) Co-authored-by: Claude Opus 4.8 (1M context) --- cli/src/utils/resource_types.ts | 10 ++-- cli/test/resource_types_unit.test.ts | 68 ++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 cli/test/resource_types_unit.test.ts diff --git a/cli/src/utils/resource_types.ts b/cli/src/utils/resource_types.ts index 741cb9fb71..5bd56c9a8e 100644 --- a/cli/src/utils/resource_types.ts +++ b/cli/src/utils/resource_types.ts @@ -1,5 +1,9 @@ import { Schema, SchemaProperty } from "../../bootstrap/common.ts"; +function quotePropName(name: string): string { + return /^[a-zA-Z_$][a-zA-Z0-9_$]*$/.test(name) ? name : JSON.stringify(name); +} + export function compileResourceTypeToTsType(schema: Schema) { function rec(x: { [name: string]: SchemaProperty }, root = false) { let res = "{\n"; @@ -10,15 +14,15 @@ export function compileResourceTypeToTsType(schema: Schema) { let i = 0; for (let [name, prop] of entries) { if (prop.type == "object") { - res += ` ${name}: ${rec(prop.properties ?? {})}`; + res += ` ${quotePropName(name)}: ${rec(prop.properties ?? {})}`; } else if (prop.type == "array") { - res += ` ${name}: ${prop?.items?.type ?? "any"}[]`; + res += ` ${quotePropName(name)}: ${prop?.items?.type ?? "any"}[]`; } else { let typ = prop?.type ?? "any"; if (typ == "integer") { typ = "number"; } - res += ` ${name}: ${typ}`; + res += ` ${quotePropName(name)}: ${typ}`; } i++; if (i < entries.length) { diff --git a/cli/test/resource_types_unit.test.ts b/cli/test/resource_types_unit.test.ts new file mode 100644 index 0000000000..0966ae25bf --- /dev/null +++ b/cli/test/resource_types_unit.test.ts @@ -0,0 +1,68 @@ +import { expect, test } from "bun:test"; + +import { compileResourceTypeToTsType } from "../src/utils/resource_types.ts"; +import type { Schema } from "../bootstrap/common.ts"; + +// ============================================================================= +// Resource-type namespace generation (WIN-2132) +// +// `compileResourceTypeToTsType` renders a JSON Schema into the body of a +// TypeScript type used in the generated `rt.d.ts` (RT namespace). JSON Schema +// property names are unconstrained, so a name with a colon, hyphen, or space +// is legal in the schema but not a valid bare TS identifier. Emitting it raw +// produced syntactically invalid output that broke `tsc`. These tests pin that +// such names are quoted while plain identifiers stay bare. +// ============================================================================= + +function schema(properties: Schema["properties"]): Schema { + return { + $schema: undefined, + type: "object", + properties, + required: [], + }; +} + +test("plain identifiers are emitted without quotes", () => { + const out = compileResourceTypeToTsType( + schema({ + host: { type: "string" }, + _port: { type: "integer" }, + $ref: { type: "boolean" }, + }) + ); + expect(out).toContain(" host: string"); + expect(out).toContain(" _port: number"); + expect(out).toContain(" $ref: boolean"); + expect(out).not.toContain('"host"'); +}); + +test("non-identifier property names are double-quoted", () => { + const out = compileResourceTypeToTsType( + schema({ + "content-type": { type: "string" }, + "x:api:key": { type: "string" }, + "with space": { type: "integer" }, + "3leading": { type: "boolean" }, + }) + ); + expect(out).toContain(' "content-type": string'); + expect(out).toContain(' "x:api:key": string'); + expect(out).toContain(' "with space": number'); + expect(out).toContain(' "3leading": boolean'); +}); + +test("nested object and array property names are quoted too", () => { + const out = compileResourceTypeToTsType( + schema({ + "nested-obj": { + type: "object", + properties: { "inner-key": { type: "string" } }, + }, + "arr-field": { type: "array", items: { type: "string" } }, + }) + ); + expect(out).toContain('"nested-obj": {'); + expect(out).toContain('"inner-key": string'); + expect(out).toContain('"arr-field": string[]'); +}); From cc2f638de6cebeffb9fee1d4835a0cfd565af86c Mon Sep 17 00:00:00 2001 From: hugocasa Date: Mon, 6 Jul 2026 18:36:21 +0200 Subject: [PATCH 02/16] fix(ai): centralize Anthropic Messages API routing across completion paths (#9960) Co-authored-by: Claude Opus 4.8 (1M context) --- .../copilot/lib.anthropicRouting.test.ts | 214 ++++++++++++++++++ frontend/src/lib/components/copilot/lib.ts | 165 ++++++++++---- 2 files changed, 330 insertions(+), 49 deletions(-) create mode 100644 frontend/src/lib/components/copilot/lib.anthropicRouting.test.ts diff --git a/frontend/src/lib/components/copilot/lib.anthropicRouting.test.ts b/frontend/src/lib/components/copilot/lib.anthropicRouting.test.ts new file mode 100644 index 0000000000..2f2b0add26 --- /dev/null +++ b/frontend/src/lib/components/copilot/lib.anthropicRouting.test.ts @@ -0,0 +1,214 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AIProviderModel } from '$lib/gen' +import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs' + +// getCurrentModel/getMetadataModel are read per call, so a hoisted holder lets +// each test point the routing at a different provider/model. +const h = vi.hoisted(() => ({ currentModel: undefined as AIProviderModel | undefined })) + +vi.mock('monaco-editor', () => ({ editor: {} })) + +vi.mock('$lib/stores', () => ({ + workspaceStore: { subscribe: () => () => undefined } +})) + +vi.mock('$lib/components/flows/flowTree', () => ({ + findModuleInModules: () => undefined +})) + +vi.mock('$lib/gen', () => ({ + OpenAPI: { BASE: '/api', TOKEN: undefined }, + ResourceService: {}, + ScriptService: {}, + FlowService: {}, + JobService: {}, + ScheduleService: {}, + HttpTriggerService: {}, + WebsocketTriggerService: {}, + KafkaTriggerService: {}, + NatsTriggerService: {}, + PostgresTriggerService: {}, + MqttTriggerService: {}, + SqsTriggerService: {}, + GcpTriggerService: {}, + AzureTriggerService: {} +})) + +vi.mock('$lib/utils', () => ({ + emptyString: (value: string | undefined | null) => !value, + generateRandomString: () => 'generated_id' +})) + +vi.mock('$lib/scripts', () => ({ + scriptLangToEditorLang: (language: string) => language +})) + +vi.mock('$lib/aiStore', () => ({ + getCurrentModel: () => h.currentModel, + getMetadataModel: () => h.currentModel, + copilotInfo: { + subscribe: (run: (value: unknown) => void) => { + run({}) + return () => undefined + } + } +})) + +vi.mock('@leeoniya/ufuzzy', () => ({ + default: class { + search() { + return [[], [], []] + } + } +})) + +function streamOf(chunks: unknown[]): any { + return (async function* () { + for (const chunk of chunks) { + yield chunk + } + })() +} + +function textDelta(text: string) { + return { type: 'content_block_delta', delta: { type: 'text_delta', text } } +} + +const messages: ChatCompletionMessageParam[] = [{ role: 'user', content: 'hi' }] + +let anthropicCreate: ReturnType +let anthropicStream: ReturnType +let openaiCreate: ReturnType + +async function setupClients() { + const { workspaceAIClients } = await import('./lib') + + anthropicCreate = vi.fn().mockResolvedValue({ + content: [ + { type: 'text', text: 'Hel' }, + { type: 'thinking', thinking: 'ignored' }, + { type: 'text', text: 'lo' } + ] + }) + anthropicStream = vi + .fn() + .mockReturnValue( + streamOf([ + { type: 'message_start' }, + textDelta('Hel'), + { type: 'content_block_delta', delta: { type: 'input_json_delta', partial_json: '{' } }, + textDelta('lo'), + { type: 'message_stop' } + ]) + ) + openaiCreate = vi.fn().mockResolvedValue({ choices: [{ message: { content: 'openai text' } }] }) + + vi.spyOn(workspaceAIClients, 'getAnthropicClient').mockReturnValue({ + messages: { create: anthropicCreate, stream: anthropicStream } + } as any) + vi.spyOn(workspaceAIClients, 'getOpenaiClient').mockReturnValue({ + chat: { completions: { create: openaiCreate } } + } as any) +} + +beforeEach(async () => { + await setupClients() +}) + +afterEach(() => { + vi.restoreAllMocks() + h.currentModel = undefined +}) + +describe('Anthropic Messages API routing', () => { + it('getNonStreamingCompletion routes Foundry Claude through the Anthropic client', async () => { + const { getNonStreamingCompletion } = await import('./lib') + h.currentModel = { provider: 'azure_foundry', model: 'claude-sonnet-5' } + + const response = await getNonStreamingCompletion(messages, new AbortController()) + + expect(anthropicCreate).toHaveBeenCalledTimes(1) + expect(openaiCreate).not.toHaveBeenCalled() + // text blocks concatenated, non-text blocks dropped + expect(response).toBe('Hello') + + const headers = anthropicCreate.mock.calls[0][1].headers + // X-Provider must carry the real provider so the backend resolves Foundry + // credentials/URL; the SDK header selects the Messages API path. + expect(headers['X-Provider']).toBe('azure_foundry') + expect(headers['X-Anthropic-SDK']).toBe('true') + }) + + it('getNonStreamingCompletion routes native Anthropic through the Anthropic client', async () => { + const { getNonStreamingCompletion } = await import('./lib') + h.currentModel = { provider: 'anthropic', model: 'claude-opus-4-8' } + + await getNonStreamingCompletion(messages, new AbortController()) + + expect(anthropicCreate).toHaveBeenCalledTimes(1) + expect(anthropicCreate.mock.calls[0][1].headers['X-Provider']).toBe('anthropic') + }) + + it('getNonStreamingCompletion keeps non-Claude Foundry models on the OpenAI path', async () => { + const { getNonStreamingCompletion } = await import('./lib') + h.currentModel = { provider: 'azure_foundry', model: 'gpt-4o' } + + await getNonStreamingCompletion(messages, new AbortController()) + + expect(anthropicCreate).not.toHaveBeenCalled() + expect(openaiCreate).toHaveBeenCalledTimes(1) + }) + + it('getCompletion adapts the Anthropic stream into OpenAI text chunks', async () => { + const { getCompletion, getResponseFromEvent } = await import('./lib') + h.currentModel = { provider: 'azure_foundry', model: 'claude-sonnet-5' } + + const completion = await getCompletion(messages, new AbortController()) + + let text = '' + let chunks = 0 + for await (const part of completion) { + chunks++ + text += getResponseFromEvent(part) + } + + expect(anthropicStream).toHaveBeenCalledTimes(1) + // only the two text deltas surface; message_start/stop and input_json are dropped + expect(chunks).toBe(2) + expect(text).toBe('Hello') + }) + + it('testKey routes Foundry Claude through the Anthropic client', async () => { + const { testKey } = await import('./lib') + + await testKey({ + resourcePath: 'u/admin/foundry', + model: 'claude-sonnet-5', + abortController: new AbortController(), + messages, + aiProvider: 'azure_foundry' + }) + + expect(anthropicCreate).toHaveBeenCalledTimes(1) + const headers = anthropicCreate.mock.calls[0][1].headers + expect(headers['X-Provider']).toBe('azure_foundry') + expect(headers['X-Resource-Path']).toBe('u/admin/foundry') + }) + + it('getFimCompletion no-ops for Anthropic Messages API models', async () => { + const { getFimCompletion } = await import('./lib') + const fetchSpy = vi.spyOn(globalThis, 'fetch') + + for (const provider of ['anthropic', 'azure_foundry'] as const) { + const result = await getFimCompletion( + 'prefix', + 'suffix', + { provider, model: 'claude-sonnet-5' }, + new AbortController() + ) + expect(result).toBeUndefined() + } + // no autocomplete request should be issued for these models + expect(fetchSpy).not.toHaveBeenCalled() + }) +}) diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index 323b552cca..808f9e0988 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -302,10 +302,10 @@ export function getModelMaxTokens(provider: AIProvider, model: string) { return 8192 } -function getModelSpecificConfig( - modelProvider: AIProviderModel, - tools?: OpenAI.Chat.Completions.ChatCompletionTool[] -) { +// Resolves the completion token cap for a model: the workspace's per-model +// override when set, otherwise the built-in default. Shared by the OpenAI and +// Anthropic request paths so both honor the same limit. +function resolveMaxTokens(modelProvider: AIProviderModel): number { const defaultMaxTokens = getModelMaxTokens(modelProvider.provider, modelProvider.model) const modelKey = `${modelProvider.provider}:${modelProvider.model}` let customMaxTokensStore: Record | undefined @@ -314,7 +314,14 @@ function getModelSpecificConfig( } catch { // copilotInfo store may not be initialized in vitest } - const maxTokens = customMaxTokensStore?.[modelKey] ?? defaultMaxTokens + return customMaxTokensStore?.[modelKey] ?? defaultMaxTokens +} + +function getModelSpecificConfig( + modelProvider: AIProviderModel, + tools?: OpenAI.Chat.Completions.ChatCompletionTool[] +) { + const maxTokens = resolveMaxTokens(modelProvider) if ( (modelProvider.provider === 'openai' || modelProvider.provider === 'azure_openai' || @@ -466,23 +473,10 @@ export async function testKey({ throw new Error('Missing a model to test') } - // Providers served through the Anthropic Messages API (native Anthropic, and - // Claude deployments on Azure Foundry) must use the Anthropic SDK path rather - // than OpenAI chat completions. Mirrors the chat loop's routing so the test - // key exercises the same request shape the chat actually sends. - if (usesAnthropicMessagesApi(aiProvider, modelToTest)) { - await testAnthropicKey({ - apiKey, - workspace, - resourcePath, - model: modelToTest, - abortController, - messages, - aiProvider - }) - return - } - + // getNonStreamingCompletion routes Anthropic-Messages-API models (native + // Anthropic and Claude on Azure Foundry) through the Anthropic SDK and + // everything else through OpenAI chat completions, so the test exercises the + // same request shape the feature actually sends. await getNonStreamingCompletion(messages, abortController, { apiKey, workspace, @@ -494,30 +488,35 @@ export async function testKey({ }) } -async function testAnthropicKey({ - apiKey, - workspace, - resourcePath, - model, - abortController, - messages, - aiProvider -}: { +// Providers served through the Anthropic Messages API (native Anthropic, and +// Claude deployments on Azure Foundry) require the Anthropic SDK request shape: +// OpenAI chat-completions requests fail against them because the proxy forwards +// the body verbatim and, for Foundry, rewrites the URL to the /anthropic/v1 +// surface that only serves /messages. This centralizes the client/header/message +// setup so every completion entry point routes them the same way the chat does. +interface AnthropicCompletionParams { + messages: ChatCompletionMessageParam[] + modelProvider: AIProviderModel + abortController: AbortController apiKey?: string workspace?: string resourcePath?: string - model: string - abortController: AbortController - messages: ChatCompletionMessageParam[] - aiProvider: AIProvider -}) { +} + +function buildAnthropicProxyRequest({ + messages, + modelProvider, + apiKey, + workspace, + resourcePath +}: Omit) { const { system, messages: anthropicMessages } = convertOpenAIToAnthropicMessages(messages) // X-Provider must be the real provider (e.g. azure_foundry) so the backend // resolves the right credentials and Anthropic URL; the SDK headers tell it to // route through the Anthropic Messages API. const headers: Record = { - 'X-Provider': aiProvider, + 'X-Provider': modelProvider.provider, 'anthropic-version': '2023-06-01', 'X-Anthropic-SDK': 'true' } @@ -528,24 +527,65 @@ async function testAnthropicKey({ headers['X-API-Key'] = apiKey } - const anthropicClient = apiKey + const client = apiKey ? createAnthropicProxyClient(getAiProxyBaseURL()) : workspace ? workspaceAIClients.createAnthropicClient(workspace) : workspaceAIClients.getAnthropicClient() - await anthropicClient.messages.create( - { - model, - max_tokens: 100, - messages: anthropicMessages, - ...(system && { system }) - }, - { - signal: abortController.signal, - headers + const body = { + model: modelProvider.model, + max_tokens: resolveMaxTokens(modelProvider), + messages: anthropicMessages, + ...(system && { system }) + } + + return { client, headers, body } +} + +async function getAnthropicNonStreamingCompletion({ + abortController, + ...params +}: AnthropicCompletionParams): Promise { + const { client, headers, body } = buildAnthropicProxyRequest(params) + + const message = await client.messages.create(body, { + signal: abortController.signal, + headers + }) + + return message.content.map((block) => (block.type === 'text' ? block.text : '')).join('') +} + +// Adapts an Anthropic Messages stream into the OpenAI ChatCompletionChunk shape +// the completion consumers already iterate, so they need no Anthropic-specific +// handling. Only text deltas are surfaced (these paths don't use tool calls). +function getAnthropicStreamingCompletion({ + abortController, + ...params +}: AnthropicCompletionParams): Stream { + const { client, headers, body } = buildAnthropicProxyRequest(params) + + const stream = client.messages.stream(body, { + signal: abortController.signal, + headers + }) + + async function* toOpenAIChunks(): AsyncGenerator { + for await (const event of stream) { + if (event.type === 'content_block_delta' && event.delta.type === 'text_delta') { + yield { + id: '', + object: 'chat.completion.chunk', + created: 0, + model: params.modelProvider.model, + choices: [{ index: 0, delta: { content: event.delta.text }, finish_reason: null }] + } + } } - ) + } + + return toOpenAIChunks() as unknown as Stream } interface BaseOptions { @@ -773,6 +813,19 @@ export async function getNonStreamingCompletion( forceModelProvider?: AIProviderModel } ) { + const modelProvider = options?.forceModelProvider ?? getCurrentModel() + + if (usesAnthropicMessagesApi(modelProvider.provider, modelProvider.model)) { + return getAnthropicNonStreamingCompletion({ + messages, + modelProvider, + abortController, + apiKey: options?.apiKey, + workspace: options?.workspace, + resourcePath: options?.resourcePath + }) + } + let response: string | undefined = '' const { provider, config } = getProviderAndCompletionConfig({ messages, @@ -846,6 +899,14 @@ export async function getFimCompletion( providerModel: AIProviderModel, abortController: AbortController ): Promise { + // The Anthropic Messages API has no fill-in-the-middle endpoint, and Foundry + // Claude deployments don't expose the OpenAI-compatible completions surface the + // FIM proxy targets. Skip autocomplete for these models rather than issuing a + // request that can't succeed. + if (usesAnthropicMessagesApi(providerModel.provider, providerModel.model)) { + return undefined + } + const fetchOptions: { signal: AbortSignal headers: Record @@ -908,6 +969,12 @@ export async function getCompletion( reasoningEffort?: string } ): Promise> { + const modelProvider = options?.forceModelProvider ?? getCurrentModel() + + if (usesAnthropicMessagesApi(modelProvider.provider, modelProvider.model)) { + return getAnthropicStreamingCompletion({ messages, modelProvider, abortController }) + } + const { provider, config } = getProviderAndCompletionConfig({ messages, stream: true, From 91e1b087a206efb7189824b4184e1f3f4cda7211 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 6 Jul 2026 19:02:51 +0200 Subject: [PATCH 03/16] feat(auth): add runtime NO_AUTH mode for authentication bypass (#9962) * feat(auth): add runtime NO_AUTH mode for authentication bypass Adds a runtime `NO_AUTH` env flag that makes every request resolve as the `admin@windmill.dev` superadmin with no login required, so self-hosted deployments can front Windmill with their own authenticating gateway without building a dedicated `oss` (compile-time `no_auth`) binary. - `NO_AUTH` is honored in any build but is force-disabled when `CLOUD_HOSTED` is set, so the managed cloud always enforces real auth. - The existing compile-time `no_auth` feature keeps its always-on behavior (`cfg!(feature = "no_auth") || *NO_AUTH`), so `oss` builds are unchanged. - `Tokened` now yields a synthetic token in no-auth mode so handlers that require it (e.g. global_whoami, called by the frontend on load) resolve. - A loud startup banner warns when the mode is on; `HIDE_NO_AUTH_BANNER` silences it once the operator has deliberately deployed behind a gateway. Fixes WIN-2131 Co-Authored-By: Claude Opus 4.8 (1M context) * feat(auth): dismissable NO_AUTH warning banner via global setting Replaces the HIDE_NO_AUTH_BANNER env flag with a UI warning banner that can be permanently dismissed for all users from within the running instance (not exposed in instance settings). - New `no_auth_banner_dismissed` global setting, only ever written by dismissing the banner itself. - `GET /api/settings/no_auth_banner` returns whether to show the banner (true only when NO_AUTH is active and it hasn't been dismissed). - NoAuthBanner.svelte renders a top-of-app warning in NO_AUTH mode; its dismiss button opens a confirmation modal, then writes the global setting via the existing setGlobal endpoint so it stays hidden for everyone. - The server still logs the startup NO_AUTH warning unconditionally. Fixes WIN-2131 Co-Authored-By: Claude Opus 4.8 (1M context) * fix(auth): resolve NO_AUTH in AuthCache so all_runnables works Codex/Pi review flagged that `/api/users/all_runnables` still failed in NO_AUTH mode: `get_all_runnables` extracts `Tokened` and re-validates the request token per workspace via `AuthCache::get_authed`, which rejected the fabricated `"no_auth"` token (no matching DB row) with a 400. Short-circuit `AuthCache::get_opt_job_authed` (the resolver behind `get_authed`) to the admin superadmin in no-auth mode, so any direct cache caller resolves without a real token. Single-source the mode check and the synthetic identity via `is_no_auth()` / `no_auth_admin_authed()` and reuse them across the extractor, resolver, and login paths. Co-Authored-By: Claude Opus 4.8 (1M context) * revert(auth): drop the NO_AUTH dismissable UI banner The in-app banner added a GET /api/settings/no_auth_banner request to every instance load for little benefit. The startup log warning already surfaces that auth is bypassed to operators, so drop the banner, its endpoint, and the no_auth_banner_dismissed global setting entirely. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- backend/src/main.rs | 10 +++++ backend/windmill-api-auth/src/auth.rs | 57 ++++++++++++++++++------- backend/windmill-api-auth/src/lib.rs | 4 +- backend/windmill-api-users/src/users.rs | 6 ++- backend/windmill-common/src/worker.rs | 8 ++++ 5 files changed, 66 insertions(+), 19 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index 0186f271e6..4115e2a12b 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -578,6 +578,7 @@ fn print_help() { println!(" JSON_FMT = false Output logs in JSON instead of logfmt"); println!(" METRICS_ADDR = None (EE only) Prometheus metrics addr at /metrics; set \"true\" to use :8001"); println!(" SUPERADMIN_SECRET = None Virtual superadmin token (server)"); + println!(" NO_AUTH = false Bypass all auth; every request acts as the admin@windmill.dev superadmin (only behind a trusted gateway; ignored when CLOUD_HOSTED)"); println!(" LICENSE_KEY = None (EE only) Enterprise license key (workers require valid key)"); println!(" RUN_UPDATE_CA_CERTIFICATE_AT_START = false Run system CA update at startup"); println!(" RUN_UPDATE_CA_CERTIFICATE_PATH = /usr/sbin/update-ca-certificates Path to CA update tool"); @@ -641,6 +642,15 @@ async fn windmill_main() -> anyhow::Result<()> { println!("Running in MCP mode"); } + if *windmill_common::worker::NO_AUTH { + println!("############################################################"); + println!("# NO_AUTH mode is ENABLED: authentication is fully #"); + println!("# bypassed and every request is treated as the #"); + println!("# admin@windmill.dev superadmin. Only run this behind a #"); + println!("# trusted authenticating gateway on a private network. #"); + println!("############################################################"); + } + #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] println!("jemalloc enabled"); diff --git a/backend/windmill-api-auth/src/auth.rs b/backend/windmill-api-auth/src/auth.rs index dea200e417..59d9cd80b3 100644 --- a/backend/windmill-api-auth/src/auth.rs +++ b/backend/windmill-api-auth/src/auth.rs @@ -131,6 +131,13 @@ impl AuthCache { w_id: Option, token: &str, ) -> Option { + // In no-auth mode there are no real tokens: resolve directly as the + // admin superadmin so direct cache callers (e.g. get_all_runnables, + // which re-validates the request token per workspace) don't reject the + // fabricated token. + if is_no_auth() { + return Some(OptJobAuthed { authed: no_auth_admin_authed(), job_id: None }); + } let key = ( w_id.as_ref().unwrap_or(&"".to_string()).to_string(), token.to_string(), @@ -598,6 +605,12 @@ where let tokened = Self { token }; parts.extensions.insert(tokened.clone()); Ok(tokened) + } else if is_no_auth() { + // In `--no-auth` mode requests carry no token, but handlers that + // also require Tokened (e.g. global_whoami) must still resolve. + let tokened = Self { token: "no_auth".to_string() }; + parts.extensions.insert(tokened.clone()); + Ok(tokened) } else { BRUTE_FORCE_COUNTER.increment().await; Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned())) @@ -677,6 +690,30 @@ fn maybe_get_workspace_id_from_path(path_vec: &[&str]) -> Option { workspace_id } +/// `--no-auth` mode: compiled-in `oss` builds, or the `NO_AUTH` runtime flag on +/// any build (the runtime flag is force-disabled on CLOUD_HOSTED). When on, +/// every request resolves as the admin superadmin so a fronting gateway can +/// handle authentication instead. +pub fn is_no_auth() -> bool { + cfg!(feature = "no_auth") || *windmill_common::worker::NO_AUTH +} + +/// The synthetic superadmin identity returned for every request in no-auth mode. +fn no_auth_admin_authed() -> ApiAuthed { + ApiAuthed { + email: "admin@windmill.dev".to_string(), + username: "admin".to_string(), + is_admin: true, + is_operator: false, + groups: Vec::new(), + folders: Vec::new(), + scopes: None, + username_override: None, + token_prefix: None, + read_only: false, + } +} + /// Resolves OptJobAuthed from request parts. /// Takes ownership of Parts and returns them back. #[allow(unreachable_code, unused_mut)] @@ -687,21 +724,11 @@ pub async fn resolve_opt_job_authed( return Ok((OptJobAuthed::default(), parts)); }; - #[cfg(feature = "no_auth")] - { - let authed = ApiAuthed { - email: "admin@windmill.dev".to_string(), - username: "admin".to_string(), - is_admin: true, - is_operator: false, - groups: Vec::new(), - folders: Vec::new(), - scopes: None, - username_override: None, - token_prefix: None, - read_only: false, - }; - return Ok((OptJobAuthed { authed, job_id: None }, parts)); + if is_no_auth() { + return Ok(( + OptJobAuthed { authed: no_auth_admin_authed(), job_id: None }, + parts, + )); } let already_authed = parts.extensions.get::().cloned(); diff --git a/backend/windmill-api-auth/src/lib.rs b/backend/windmill-api-auth/src/lib.rs index d2347aa22e..b1bf2d4273 100644 --- a/backend/windmill-api-auth/src/lib.rs +++ b/backend/windmill-api-auth/src/lib.rs @@ -31,8 +31,8 @@ use scopes::ScopeDefinition; // Re-export key auth types and functions pub use auth::{ - get_end_user_email, invalidate_token_from_cache, AuthCache, ExpiringAuthCache, OptTokened, - Tokened, TruncatedTokenWithEmail, AUTH_CACHE, + get_end_user_email, invalidate_token_from_cache, is_no_auth, AuthCache, ExpiringAuthCache, + OptTokened, Tokened, TruncatedTokenWithEmail, AUTH_CACHE, }; // ------------ ApiAuthed & OptJobAuthed types ------------ diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index 3fcebdc083..ff0e8e754e 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -1942,8 +1942,10 @@ async fn login( Extension(argon2): Extension>>, Json(Login { email, password }): Json, ) -> Result { - #[cfg(feature = "no_auth")] - { + // In `--no-auth` mode there is no real login; the frontend never needs a + // session cookie because every request already resolves as the admin + // superadmin (see resolve_opt_job_authed). + if windmill_api_auth::is_no_auth() { return Ok("no_auth".to_string()); } diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 3a1469d5e3..ea2b5455f3 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -273,6 +273,14 @@ lazy_static::lazy_static! { /// production `app.windmill.dev` cluster, not on staging or self-hosted. pub static ref CLOUD_PRODUCTION_HOST: &'static str = "app.windmill.dev"; + /// `--no-auth` mode: when set, every API request is treated as + /// authenticated as the `admin@windmill.dev` superadmin and no login is + /// ever required. Meant for self-hosted deployments that front Windmill + /// with their own authenticating gateway. Never honored on the managed + /// cloud (`CLOUD_HOSTED`), which must always enforce real authentication. + pub static ref NO_AUTH: bool = !*CLOUD_HOSTED + && std::env::var("NO_AUTH").ok().is_some_and(|x| x == "1" || x == "true"); + pub static ref CUSTOM_TAGS: Vec = std::env::var("CUSTOM_TAGS") .ok() .map(|x| x.split(',').map(|x| x.to_string()).collect::>()).unwrap_or_default(); From 3dcd3949a14199b106506994ea31ca3de7e636b3 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 6 Jul 2026 20:12:12 +0200 Subject: [PATCH 04/16] feat(pipelines): auto-derive cascade edges from ducklake/s3 reads (+ muted-read badge) (#9963) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(pipelines): auto-derive cascade trigger edges from ducklake/s3 reads Within a `// pipeline`, a read of a ducklake table or s3 object now auto-wires its cascade trigger edge straight from the FROM clause, so `// on ` is only needed for edges inference can't see (dynamic SQL) or to carry per-edge opts. Two opt-outs: `// mute ` suppresses a single derived edge (a lookup / SCD input read every run but not cascaded on), and `// mute all` opts the script out of derivation entirely (back to explicit-`// on`-only). Explicit `// on` still wins the dedup. Scoped to ducklake + s3 reads; resource/datatable/volume stay explicit. Read-write (RW) and write inputs are excluded so a self-referential merge can't loop-trigger itself; ambiguous (None) access is skipped. - parser: `mute` / `mute_all` in PipelineAnnotations (Rust + TS mirror) - deploy: derive_pipeline_asset_trigger_refs → script_trigger rows - frontend: resolveGraph mirrors derivation for the live edit-mode canvas - tests: shared parity corpus + derive-helper units + resolveGraph overlays Co-Authored-By: Claude Opus 4.8 (1M context) * feat(pipelines): mark auto-derived cascade edges with a persisted derived flag + "auto" badge Persist script_trigger.derived (deploy: true for ducklake/s3-read derivation, false for explicit // on) and return it from the asset-graph endpoint so the canvas renders a Sparkles "auto" badge on auto-wired edges — the inference is now visible on both the deployed graph and the live edit canvas, not just implied. Dispatch (fetch_subscribers) ignores the flag, so a derived edge fires identically to an explicit // on. Also copy derived in the workspace-clone trigger copy, and backfill muteAssets/muteAll into two empty PipelineAnnotations literals the base commit left stale (check:fast). Co-Authored-By: Claude Opus 4.8 (1M context) * fix(pipelines): derive cascade edge from effective (alt-fallback) asset access derive_pipeline_asset_trigger_refs gated on the raw parser access_type, but the persisted asset.usage_access_type and the frontend canvas both use access_type.or(alt_access_type). An ambiguous parse with a manual read override was persisted/drawn as a read yet derived no edge, so the auto edge silently vanished on deploy. Gate on the effective access type for parity. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(pipelines): badge muted reads instead of auto-derived edges Auto-derivation is the default now, so badging every derived cascade edge is noise. Drop the "auto" badge and the persisted `script_trigger.derived` flag (migration + insert param + graph field + clone copy) that only powered it, and instead badge the exception: a ducklake/s3 asset a script reads but does NOT cascade — `// mute ` / `// mute all`. `computeMutedReadKeys` marks a read-only ('r') supported read with no cascade trigger and no self-write; the canvas renders a bell-off "muted" badge on that read edge. Also fixes two review parity nits: - TS `// on` parser now strips trailing `key=value` opts (e.g. `debounce=60s`) like the Rust `split_trailing_kv_opts`, so the ref dedups against inference. - A `// materialize` producer reading its own target is upgraded to `rw` (deploy) / excluded via the materialize write refs (canvas), so it neither self-cascades nor shows as a muted read. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(pipelines): drop redundant // on for auto-derived reads; gate muted badge to pipeline scripts - Templates no longer scaffold `// on ` for a ducklake/s3 input the body reads — the read auto-wires the cascade now that derivation is the default. Kept for datatable/resource (not auto-derived) and native triggers. The discoverability hint now mentions `// mute` (the newly relevant annotation). - computeMutedReadKeys only badges reads by `// pipeline` scripts. A plain script or flow reading a ducklake/s3 asset never had an auto trigger to suppress, so it must render as ordinary lineage, not "muted" (Codex review). Co-Authored-By: Claude Opus 4.8 (1M context) * fix(pipelines): only drop template // on when the body actually reads the input The redundant-`// on` removal assumed the generated body reads the ducklake/s3 input, but postgres/bash/generic bodies (and `data_upload`, which reads the picker file) ignore `input` — dropping `// on` there left the asset-created script with no cascade at all. Gate the drop on READS_INPUT_LANGS (bun/deno/python/duckdb) so non-reading templates keep the explicit trigger. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- .../windmill-parser/src/asset_parser.rs | 27 +++ .../tests/fixtures/pipeline_annotations.json | 104 ++++++++++ .../tests/pipeline_annotations_parity.rs | 20 ++ backend/windmill-api-scripts/src/scripts.rs | 77 ++++++- backend/windmill-common/src/assets.rs | 190 ++++++++++++++++++ .../assets/AssetGraph/AssetGraphCanvas.svelte | 22 +- .../AssetGraph/AssetGraphDetailsPane.svelte | 4 +- .../assets/AssetGraph/AssetGraphEdge.svelte | 31 ++- .../parsePipelineAnnotations.parity.test.ts | 16 +- .../parsePipelineAnnotations.test.ts | 7 + .../AssetGraph/parsePipelineAnnotations.ts | 49 ++++- .../AssetGraph/pipelineTemplates.test.ts | 67 ++++++ .../assets/AssetGraph/pipelineTemplates.ts | 30 ++- .../assets/AssetGraph/resolveGraph.test.ts | 168 +++++++++++++++- .../assets/AssetGraph/resolveGraph.ts | 153 +++++++++++++- .../(logged)/pipeline/[folder]/+page.svelte | 4 +- 16 files changed, 950 insertions(+), 19 deletions(-) diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 045cac87e6..bdce9ec764 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -528,6 +528,16 @@ pub struct PipelineAnnotations { pub column_lineage: Vec, pub macros: bool, pub use_libs: Vec, + // `// mute ` — suppress the auto-derived cascade edge for a read + // that would otherwise trigger this script (a lookup / slowly-changing + // dimension you read every run but don't want to re-run on). Only Asset + // specs are stored; native trigger kinds are never auto-derived, so + // muting them is meaningless. + pub mute: Vec, + // `// mute all` — opt out of auto-derivation entirely for this script. + // Falls back to explicit-`// on`-only semantics. Explicit `// on` edges + // are unaffected. + pub mute_all: bool, } impl ParseAssetsOutput { @@ -956,6 +966,23 @@ pub fn parse_pipeline_annotations(code: &str) -> PipelineAnnotations { continue; } + // `// mute all` opts out of auto-derived cascade edges entirely; + // `// mute ` suppresses the one edge. Only asset refs are + // muteable — native trigger kinds are never auto-derived. Checked + // before the generic `on`/asset shorthand (a complete word, so + // prose like `// muted for now` never matches). + if let Some(after_kw) = consume_keyword(rest, "mute") { + let arg = after_kw.trim(); + if arg == "all" { + out.mute_all = true; + } else if let Some(spec @ TriggerSpec::Asset { .. }) = parse_trigger_spec(arg) { + if !out.mute.contains(&spec) { + out.mute.push(spec); + } + } + continue; + } + // `data_test` is checked before `on`/asset shorthands and is a complete // word (so it never collides with the `// test:` CI annotation, which // has no whitespace after `test`). Accumulates — every well-formed line diff --git a/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json b/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json index cabfe20ddf..226f32914c 100644 --- a/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json +++ b/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json @@ -769,5 +769,109 @@ "tag": null, "retry": null } + }, + { + "name": "mute suppresses a single ducklake read edge", + "code": "// pipeline\n// mute ducklake://main.orders\nselect 1", + "expected": { + "in_pipeline": true, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "mute": [ + "ducklake:main.orders" + ] + } + }, + { + "name": "mute all opts out of all auto-derivation", + "code": "// pipeline\n// mute all\nselect 1", + "expected": { + "in_pipeline": true, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "mute_all": true + } + }, + { + "name": "mute accumulates in order and dedups", + "code": "// pipeline\n// mute ducklake://main.a\n// mute s3://raw/b\n// mute ducklake://main.a\nselect 1", + "expected": { + "in_pipeline": true, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "mute": [ + "ducklake:main.a", + "s3object:raw/b" + ] + } + }, + { + "name": "mute of a native trigger kind is dropped (only assets are muteable)", + "code": "// pipeline\n// mute kafka\nselect 1", + "expected": { + "in_pipeline": true, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "mute prose without an asset ref never false-positives", + "code": "// pipeline\n// muted for now\nselect 1", + "expected": { + "in_pipeline": true, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "mute all coexists with explicit on edges", + "code": "// pipeline\n// mute all\n// on ducklake://main.orders\nselect 1", + "expected": { + "in_pipeline": true, + "asset_triggers": [ + "ducklake:main.orders" + ], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "mute_all": true + } + }, + { + "name": "on asset ref strips trailing key=value opts", + "code": "// pipeline\n// on ducklake://main.orders debounce=60s\nselect 1", + "expected": { + "in_pipeline": true, + "asset_triggers": [ + "ducklake:main.orders" + ], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } } ] diff --git a/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs b/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs index 7adb07b55c..fff818edb2 100644 --- a/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs +++ b/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs @@ -53,6 +53,13 @@ struct Expected { // `// use ` accumulation, declaration order, deduped. Absent === []. #[serde(default)] use_libs: Vec, + // `// mute ` accumulation as `kind:path`, declaration order, deduped. + // Absent === []. + #[serde(default)] + mute: Vec, + // `// mute all` marker. Absent === false. + #[serde(default)] + mute_all: bool, } #[derive(Deserialize)] @@ -251,5 +258,18 @@ fn pipeline_annotation_fixtures_match() { assert_eq!(got.macros, f.expected.macros, "{ctx}: macros"); assert_eq!(got.use_libs, f.expected.use_libs, "{ctx}: use_libs"); + + let mute: Vec = got + .mute + .iter() + .filter_map(|t| match t { + TriggerSpec::Asset { asset_kind, path, .. } => { + Some(format!("{}:{}", kind_str(*asset_kind), path)) + } + _ => None, + }) + .collect(); + assert_eq!(mute, f.expected.mute, "{ctx}: mute"); + assert_eq!(got.mute_all, f.expected.mute_all, "{ctx}: mute_all"); } } diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index d7a461dafa..bfc7dd1ac7 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -45,8 +45,9 @@ use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap; use windmill_common::{ assets::{ clear_script_triggers, clear_static_asset_usage, clear_static_asset_usage_by_script_hash, - insert_script_trigger, parse_duration_secs, parse_pipeline_annotations, - replace_static_asset_usage, trigger_spec_to_row, AssetUsageKind, TriggerSpec, + derive_pipeline_asset_trigger_refs, insert_script_trigger, parse_duration_secs, + parse_pipeline_annotations, replace_static_asset_usage, trigger_spec_to_row, + AssetUsageKind, ScriptTriggerKind, TriggerSpec, }, error::{self, to_anyhow}, min_version::{MIN_VERSION_SUPPORTS_DEBOUNCING, MIN_VERSION_SUPPORTS_DEBOUNCING_V2}, @@ -1457,11 +1458,22 @@ async fn create_script_internal<'c>( // fire and the view would be an orphan node in the lineage graph. for (target_kind, path) in m.write_targets() { let kind = windmill_common::assets::asset_kind_from_parser(target_kind); - if !a.iter().any(|x| x.kind == kind && x.path == path) { + use windmill_common::assets::AssetUsageAccessType; + if let Some(existing) = a.iter_mut().find(|x| x.kind == kind && x.path == path) { + // The body reads its own managed target (an incremental/merge + // model `SELECT`ing from the table it materializes). The runtime + // still generates the write, so the effective access is RW — mark + // it so, otherwise it stays a plain read and (a) auto-derives a + // self-cascade edge back to this producer and (b) shows as a muted + // read on the canvas. Both are wrong: it's the script's own output. + if existing.access_type != Some(AssetUsageAccessType::W) { + existing.access_type = Some(AssetUsageAccessType::RW); + } + } else { a.push(windmill_common::assets::AssetWithAltAccessType { path, kind, - access_type: Some(windmill_common::assets::AssetUsageAccessType::W), + access_type: Some(AssetUsageAccessType::W), alt_access_type: None, columns: None, }); @@ -2035,6 +2047,63 @@ async fn create_script_internal<'c>( .await?; } + // Auto-derived cascade edges: within a `// pipeline`, a read of a ducklake + // table or s3 object wires the cascade edge straight from the FROM clause, + // so `// on ` is only needed for edges inference can't see (dynamic + // SQL) or to carry per-edge opts. `// mute ` / `// mute all` opt out. + // Explicit `// on` asset edges (inserted just above) win the dedup — they + // carry the per-edge debounce, so a derived row must not shadow them. + if in_pipeline && !pipeline_annotations.mute_all { + let asset_ref = |spec: &TriggerSpec| { + trigger_spec_to_row(spec) + .filter(|(k, _)| *k == ScriptTriggerKind::Asset) + .map(|(_, r)| r) + }; + let explicit_refs: std::collections::HashSet = + pipeline_triggers.iter().filter_map(asset_ref).collect(); + let muted_refs: std::collections::HashSet = pipeline_annotations + .mute + .iter() + .filter_map(asset_ref) + .collect(); + let derived = derive_pipeline_asset_trigger_refs( + effective_assets.as_deref().unwrap_or(&[]), + &explicit_refs, + &muted_refs, + pipeline_annotations.mute_all, + ); + // Derived edges have no per-`// on` opts, so they take the script-level + // `// debounce` default and `// retry` policy — same as writing a bare + // `// on ` would. + let derived_debounce_s = pipeline_debounce_default + .as_deref() + .and_then(parse_duration_secs); + let derived_retry_count = pipeline_annotations + .retry + .as_ref() + .map(|r| r.count.min(i16::MAX as u32) as i16); + let derived_retry_delay_s = pipeline_annotations + .retry + .as_ref() + .and_then(|r| r.delay.as_deref()) + .and_then(parse_duration_secs); + for trigger_ref in derived { + insert_script_trigger( + &mut *tx, + &w_id, + AssetUsageKind::Script, + &ns.path, + ScriptTriggerKind::Asset, + &trigger_ref, + pipeline_join_all, + derived_debounce_s, + derived_retry_count, + derived_retry_delay_s, + ) + .await?; + } + } + // Schedule annotations (`// on schedule`) are marker-only — the binding // lives on the schedule row's own `script_path` field, which the user // creates separately via the schedule editor. No script-create-time diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index 862bd645c3..62f6d32070 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -169,6 +169,68 @@ fn is_write_access(access: Option) -> bool { ) } +/// Kinds whose *read* usage auto-derives a cascade trigger edge inside a +/// `// pipeline`. Scoped to the two intra-pipeline data kinds — a ducklake +/// table read (the core case) and an s3 object read (file-ingestion +/// producers). Resource / datatable / volume reads stay explicit-`// on`: +/// a config/lookup read cascading is more often surprising than wanted. +fn is_auto_trigger_kind(kind: AssetKind) -> bool { + matches!(kind, AssetKind::Ducklake | AssetKind::S3Object) +} + +/// Trigger refs auto-derived from a pipeline script's inferred reads, so the +/// FROM clause alone wires the cascade edge (no redundant `// on `). +/// +/// Included: an input read read-*only* (`R`) of a supported kind +/// ([`is_auto_trigger_kind`]). The effective access type is +/// `access_type.or(alt_access_type)` — same precedence as the persisted +/// `asset.usage_access_type` and the frontend mirror's `access_type ?? +/// alt_access_type`, so a manual read override on an ambiguous parse still +/// derives an edge (and the live canvas and the deployed graph agree). +/// Excluded, each for a reason: +/// - `RW` / `W` — the script also writes the asset; an edge would be a +/// self-triggering loop. +/// - `None` access — usage is ambiguous (poisoned merge) with no override; +/// can't confirm a read, so fail safe and don't cascade. +/// - already in `explicit_refs` — the author wrote `// on `, which +/// wins (it carries the per-edge debounce/opts). +/// - in `muted_refs` — a `// mute ` opt-out (lookup / SCD input). +/// +/// `mute_all` (from `// mute all`) short-circuits to no derivation, leaving +/// only the explicit `// on` edges. Returns canonical refs (e.g. +/// `ducklake://main.orders`), deduped, in input order. +pub fn derive_pipeline_asset_trigger_refs( + assets: &[AssetWithAltAccessType], + explicit_refs: &HashSet, + muted_refs: &HashSet, + mute_all: bool, +) -> Vec { + if mute_all { + return vec![]; + } + let mut out = vec![]; + let mut seen = HashSet::new(); + for a in assets { + // Effective access mirrors the persisted `usage_access_type` and the + // frontend derivation: an explicit parse wins, else the manual override. + let access = a.access_type.or(a.alt_access_type); + if access != Some(AssetUsageAccessType::R) || !is_auto_trigger_kind(a.kind) { + continue; + } + let Some(prefix) = a.kind.canonical_prefix() else { + continue; + }; + let r = format!("{}{}", prefix, a.path); + if explicit_refs.contains(&r) || muted_refs.contains(&r) { + continue; + } + if seen.insert(r.clone()) { + out.push(r); + } + } + out +} + /// Clear and reinsert the full static-asset usage set of a script in one tx, /// invalidating the producer-writes cache at most once and only on a real /// change. The cache (asset_dispatch::ASSET_PRODUCER_WRITES_CACHE) keys a @@ -369,6 +431,134 @@ mod debounce_duration_tests { } } +#[cfg(test)] +mod derive_trigger_tests { + use super::{derive_pipeline_asset_trigger_refs, AssetKind, AssetUsageAccessType}; + use std::collections::HashSet; + use windmill_types::assets::AssetWithAltAccessType; + + fn asset( + kind: AssetKind, + path: &str, + at: Option, + ) -> AssetWithAltAccessType { + AssetWithAltAccessType { + path: path.to_string(), + kind, + access_type: at, + alt_access_type: None, + columns: None, + } + } + + fn derive(assets: &[AssetWithAltAccessType]) -> Vec { + derive_pipeline_asset_trigger_refs(assets, &HashSet::new(), &HashSet::new(), false) + } + + #[test] + fn read_only_ducklake_and_s3_derive_an_edge() { + use AssetUsageAccessType::R; + let a = [ + asset(AssetKind::Ducklake, "main.orders", Some(R)), + asset(AssetKind::S3Object, "raw/events", Some(R)), + ]; + assert_eq!( + derive(&a), + vec![ + "ducklake://main.orders".to_string(), + "s3://raw/events".to_string() + ] + ); + } + + #[test] + fn writes_and_rw_are_skipped_to_avoid_self_edges() { + use AssetUsageAccessType::*; + // W (pure producer) and RW (reads *and* writes the same table — a + // self-cascade if edged) both derive nothing. + let a = [ + asset(AssetKind::Ducklake, "main.out", Some(W)), + asset(AssetKind::Ducklake, "main.self", Some(RW)), + ]; + assert!(derive(&a).is_empty()); + } + + #[test] + fn ambiguous_access_and_unsupported_kinds_are_skipped() { + use AssetUsageAccessType::R; + let a = [ + asset(AssetKind::Ducklake, "main.ambiguous", None), // poisoned merge + asset(AssetKind::Resource, "f/db", Some(R)), // out of scope + asset(AssetKind::DataTable, "main.dt", Some(R)), // out of scope + ]; + assert!(derive(&a).is_empty()); + } + + #[test] + fn manual_read_override_on_ambiguous_parse_derives_an_edge() { + use AssetUsageAccessType::{R, W}; + // Parser can't confirm access (`access_type: None`) but the user manually + // overrode it. Effective access = `access_type.or(alt_access_type)`, the + // same value persisted to `asset.usage_access_type` and used by the + // frontend canvas — so a read override derives an edge (parity, no + // silently-vanishing edge on deploy) and a write override does not. + let read_override = AssetWithAltAccessType { + path: "main.override_r".to_string(), + kind: AssetKind::Ducklake, + access_type: None, + alt_access_type: Some(R), + columns: None, + }; + let write_override = AssetWithAltAccessType { + path: "main.override_w".to_string(), + kind: AssetKind::Ducklake, + access_type: None, + alt_access_type: Some(W), + columns: None, + }; + assert_eq!( + derive(&[read_override, write_override]), + vec!["ducklake://main.override_r".to_string()] + ); + } + + #[test] + fn explicit_and_muted_refs_are_excluded() { + use AssetUsageAccessType::R; + let a = [ + asset(AssetKind::Ducklake, "main.explicit", Some(R)), + asset(AssetKind::Ducklake, "main.muted", Some(R)), + asset(AssetKind::Ducklake, "main.keep", Some(R)), + ]; + let explicit: HashSet = ["ducklake://main.explicit".to_string()].into(); + let muted: HashSet = ["ducklake://main.muted".to_string()].into(); + assert_eq!( + derive_pipeline_asset_trigger_refs(&a, &explicit, &muted, false), + vec!["ducklake://main.keep".to_string()] + ); + } + + #[test] + fn mute_all_derives_nothing() { + use AssetUsageAccessType::R; + let a = [asset(AssetKind::Ducklake, "main.orders", Some(R))]; + assert!( + derive_pipeline_asset_trigger_refs(&a, &HashSet::new(), &HashSet::new(), true) + .is_empty() + ); + } + + #[test] + fn duplicate_reads_dedup() { + use AssetUsageAccessType::R; + let a = [ + asset(AssetKind::Ducklake, "main.orders", Some(R)), + asset(AssetKind::Ducklake, "main.orders", Some(R)), + ]; + assert_eq!(derive(&a), vec!["ducklake://main.orders".to_string()]); + } +} + #[cfg(test)] mod trigger_ref_roundtrip_tests { use super::{parse_asset_trigger_ref, trigger_spec_to_row, AssetKind, ScriptTriggerKind}; diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte index fbaf3ed6ed..64bd2790cb 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte @@ -18,6 +18,7 @@ import PanToNode from './PanToNode.svelte' import InitialFitView from './InitialFitView.svelte' import { layoutAssetGraph } from './assetGraphLayout' + import { computeMutedReadKeys } from './resolveGraph' import { buildDownstreamMap } from './graphTraversal' import { buildLineageDownstreamMap } from './boundedCascade' import type { AssetGraphResponse, AssetGraphSelection, NativeTriggerKind } from './types' @@ -228,6 +229,11 @@ | 'macro' | 'test-dependency' unsaved?: boolean + // Muted read edge: a ducklake/s3 input read every run whose (default) + // auto cascade trigger is suppressed by `// mute` / `// mute all`. + // Rendered with a bell-off badge — auto-wiring is the norm, so we mark + // the read that deliberately does NOT cascade, not every derived edge. + muted?: boolean // Edge from a missing-trigger placeholder — styled red dashed to // signal "this script declared `// on kafka` but no trigger row // targets it; create one or remove the annotation". @@ -490,6 +496,10 @@ }) } + // Read edges of a ducklake/s3 asset with no cascade trigger = muted + // (`// mute` / `// mute all` opted the default auto trigger out). Gated + // on pipeline scripts inside the helper (non-pipeline reads never derive). + const mutedReadKeys = computeMutedReadKeys(g.edges, g.triggers, g.runnables) for (const e of g.edges) { const runnableId = `${e.runnable_kind}:${e.runnable_path}` const assetId = `asset:${e.asset_kind}:${e.asset_path}` @@ -542,7 +552,13 @@ source: assetId, target: runnableId, kind: 'lineage-read', - unsaved: e.unsaved + unsaved: e.unsaved, + // Only a pure `'r'` read can be muted; `'rw'` is a self-read. + muted: + access === 'r' && + mutedReadKeys.has( + `${e.asset_kind}:${e.asset_path}->${e.runnable_kind}:${e.runnable_path}` + ) }) } } @@ -1079,7 +1095,9 @@ // Macro-edge badge: which of the library's macros the consumer // calls (all of them when pulled in via `// use`). macro_names: e.macro_names, - via_use: e.via_use + via_use: e.via_use, + // Muted read edge — bell-off badge on the read link. + muted: e.muted }, animated, label, diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte index a097f09b6c..4e5acaee08 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte @@ -636,7 +636,9 @@ dataTests: [], columnLineage: [], macros: false, - useLibs: [] + useLibs: [], + muteAssets: [], + muteAll: false } ) // `// macros` library: the defined signatures for the strip above the diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphEdge.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphEdge.svelte index 329fad6902..e16e11566d 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphEdge.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphEdge.svelte @@ -1,7 +1,7 @@ @@ -197,10 +186,14 @@ {#snippet headerRight()} {#if $superadmin || $userStore?.is_admin} + {#snippet trigger()}
@@ -246,8 +239,8 @@ {#if $superadmin} {#snippet trigger()}
@@ -304,6 +297,7 @@ 0 } + // Load the channel state on mount so the "no channels" warning doesn't depend on + // there being unacknowledged alerts to trigger a refresh (muting auto-acks them). + onMount(() => { + if ($superadmin) checkCriticalAlertChannels() + }) + async function acknowledgeAlert(id: number) { await acknowledgeCriticalAlert({ id }) getAlerts(false) @@ -133,7 +143,7 @@ - {#if !hasCriticalAlertChannels && $superadmin} + {#if $superadmin && isMuted && !hasCriticalAlertChannels}
Go to the From fd8e64d11fea3ffdb7858100c67cb2e9ca841ed6 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Mon, 6 Jul 2026 20:43:17 +0200 Subject: [PATCH 08/16] feat: add cosmetic dev/staging label for dev workspaces (#9959) * feat: add cosmetic dev/staging label for dev workspaces Co-Authored-By: Claude Opus 4.8 (1M context) * feat: prefill dev fork name and use a link to switch its label Co-Authored-By: Claude Opus 4.8 (1M context) * style: reword the dev/staging label link copy Co-Authored-By: Claude Opus 4.8 (1M context) * style: preview the dev/staging label as a badge in the switch link Co-Authored-By: Claude Opus 4.8 (1M context) * feat: show the dev/staging badge in the session diff drawer header Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- ...dd530a653081619e6d3132bf07996520b1e25.json | 19 ++++ ...a29a8f88010891fcf3eaf4761a57461f97703.json | 23 +++++ ...0c956bea94dffb46b3b838c096f04b66d6c52.json | 34 +++++++ ...1493fcdbab4ae91cae271ea911f5b14ecf0d4.json | 18 ++++ ...2d0bbe1f5aff27187505778b6abe2c87a5d01.json | 16 ++++ ...da5c60aa6878b2c77f6028a68d64f797c3322.json | 70 ++++++++++++++ ...706094033_add_dev_workspace_label.down.sql | 1 + ...60706094033_add_dev_workspace_label.up.sql | 4 + .../windmill-api-workspaces/src/workspaces.rs | 91 +++++++++++++++++-- .../src/workspaces_extra.rs | 10 +- backend/windmill-api/openapi.yaml | 41 +++++++++ .../lib/components/DevWorkspaceSetting.svelte | 69 ++++++++++++-- .../lib/components/ForkWorkspaceBanner.svelte | 5 +- .../lib/components/NoDirectDeployAlert.svelte | 9 +- .../sessions/SessionDiffDrawer.svelte | 5 + .../sessions/SessionWorkspaceBar.svelte | 3 +- .../sessions/WorkspaceFamilyPicker.svelte | 5 +- .../components/sidebar/WorkspaceMenu.svelte | 6 +- .../components/workspace/WorkspaceCard.svelte | 4 +- .../components/workspace/WorkspaceIcon.svelte | 5 +- .../CreateWorkspaceInner.svelte | 40 +++++++- frontend/src/lib/stores.ts | 1 + frontend/src/lib/utils/devWorkspaceLabel.ts | 25 +++++ 23 files changed, 472 insertions(+), 32 deletions(-) create mode 100644 backend/.sqlx/query-530a797e67ff352471f1b34f260dd530a653081619e6d3132bf07996520b1e25.json create mode 100644 backend/.sqlx/query-63d6d968905cf82fb3bb0577d41a29a8f88010891fcf3eaf4761a57461f97703.json create mode 100644 backend/.sqlx/query-868985685d95197efc534bb2f3e0c956bea94dffb46b3b838c096f04b66d6c52.json create mode 100644 backend/.sqlx/query-8ed229e88dc49b0ba7328d48f991493fcdbab4ae91cae271ea911f5b14ecf0d4.json create mode 100644 backend/.sqlx/query-9f567f04f67ce3b197eaa641eaf2d0bbe1f5aff27187505778b6abe2c87a5d01.json create mode 100644 backend/.sqlx/query-af19b9e3deb4f5c9e6ba77963a5da5c60aa6878b2c77f6028a68d64f797c3322.json create mode 100644 backend/migrations/20260706094033_add_dev_workspace_label.down.sql create mode 100644 backend/migrations/20260706094033_add_dev_workspace_label.up.sql create mode 100644 frontend/src/lib/utils/devWorkspaceLabel.ts diff --git a/backend/.sqlx/query-530a797e67ff352471f1b34f260dd530a653081619e6d3132bf07996520b1e25.json b/backend/.sqlx/query-530a797e67ff352471f1b34f260dd530a653081619e6d3132bf07996520b1e25.json new file mode 100644 index 0000000000..465ee41633 --- /dev/null +++ b/backend/.sqlx/query-530a797e67ff352471f1b34f260dd530a653081619e6d3132bf07996520b1e25.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace\n (id, name, owner, parent_workspace_id, is_dev_workspace, dev_workspace_label)\n VALUES ($1, $2, $3, $4, $5, $6)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "530a797e67ff352471f1b34f260dd530a653081619e6d3132bf07996520b1e25" +} diff --git a/backend/.sqlx/query-63d6d968905cf82fb3bb0577d41a29a8f88010891fcf3eaf4761a57461f97703.json b/backend/.sqlx/query-63d6d968905cf82fb3bb0577d41a29a8f88010891fcf3eaf4761a57461f97703.json new file mode 100644 index 0000000000..3959f4e8cc --- /dev/null +++ b/backend/.sqlx/query-63d6d968905cf82fb3bb0577d41a29a8f88010891fcf3eaf4761a57461f97703.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace SET dev_workspace_label = $1 WHERE id = $2 AND is_dev_workspace RETURNING id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "63d6d968905cf82fb3bb0577d41a29a8f88010891fcf3eaf4761a57461f97703" +} diff --git a/backend/.sqlx/query-868985685d95197efc534bb2f3e0c956bea94dffb46b3b838c096f04b66d6c52.json b/backend/.sqlx/query-868985685d95197efc534bb2f3e0c956bea94dffb46b3b838c096f04b66d6c52.json new file mode 100644 index 0000000000..34d99a7e61 --- /dev/null +++ b/backend/.sqlx/query-868985685d95197efc534bb2f3e0c956bea94dffb46b3b838c096f04b66d6c52.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, name, dev_workspace_label FROM workspace WHERE parent_workspace_id = $1 AND is_dev_workspace AND deleted = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "dev_workspace_label", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + true + ] + }, + "hash": "868985685d95197efc534bb2f3e0c956bea94dffb46b3b838c096f04b66d6c52" +} diff --git a/backend/.sqlx/query-8ed229e88dc49b0ba7328d48f991493fcdbab4ae91cae271ea911f5b14ecf0d4.json b/backend/.sqlx/query-8ed229e88dc49b0ba7328d48f991493fcdbab4ae91cae271ea911f5b14ecf0d4.json new file mode 100644 index 0000000000..3f8deb8835 --- /dev/null +++ b/backend/.sqlx/query-8ed229e88dc49b0ba7328d48f991493fcdbab4ae91cae271ea911f5b14ecf0d4.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace (id, name, owner, deleted, premium, parent_workspace_id, is_dev_workspace, dev_workspace_label)\n SELECT $1, $2, owner, false, premium,\n CASE WHEN $4 THEN parent_workspace_id ELSE NULL END, $5,\n CASE WHEN $5 THEN dev_workspace_label ELSE NULL END\n FROM workspace WHERE id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + "Bool", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "8ed229e88dc49b0ba7328d48f991493fcdbab4ae91cae271ea911f5b14ecf0d4" +} diff --git a/backend/.sqlx/query-9f567f04f67ce3b197eaa641eaf2d0bbe1f5aff27187505778b6abe2c87a5d01.json b/backend/.sqlx/query-9f567f04f67ce3b197eaa641eaf2d0bbe1f5aff27187505778b6abe2c87a5d01.json new file mode 100644 index 0000000000..a29b5f0924 --- /dev/null +++ b/backend/.sqlx/query-9f567f04f67ce3b197eaa641eaf2d0bbe1f5aff27187505778b6abe2c87a5d01.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace SET parent_workspace_id = $1, is_dev_workspace = true, dev_workspace_label = $3 WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "9f567f04f67ce3b197eaa641eaf2d0bbe1f5aff27187505778b6abe2c87a5d01" +} diff --git a/backend/.sqlx/query-af19b9e3deb4f5c9e6ba77963a5da5c60aa6878b2c77f6028a68d64f797c3322.json b/backend/.sqlx/query-af19b9e3deb4f5c9e6ba77963a5da5c60aa6878b2c77f6028a68d64f797c3322.json new file mode 100644 index 0000000000..c21a21163e --- /dev/null +++ b/backend/.sqlx/query-af19b9e3deb4f5c9e6ba77963a5da5c60aa6878b2c77f6028a68d64f797c3322.json @@ -0,0 +1,70 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace.id, workspace.name, usr.username, workspace_settings.color, workspace.parent_workspace_id,\n workspace.is_dev_workspace, workspace.dev_workspace_label,\n CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings,\n usr.disabled\n FROM workspace\n JOIN usr ON usr.workspace_id = workspace.id\n JOIN workspace_settings ON workspace_settings.workspace_id = workspace.id\n WHERE usr.email = $1 AND workspace.deleted = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "username", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "color", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "parent_workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "is_dev_workspace", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "dev_workspace_label", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "operator_settings", + "type_info": "Jsonb" + }, + { + "ordinal": 8, + "name": "disabled", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true, + true, + false, + true, + null, + false + ] + }, + "hash": "af19b9e3deb4f5c9e6ba77963a5da5c60aa6878b2c77f6028a68d64f797c3322" +} diff --git a/backend/migrations/20260706094033_add_dev_workspace_label.down.sql b/backend/migrations/20260706094033_add_dev_workspace_label.down.sql new file mode 100644 index 0000000000..0b9b81636a --- /dev/null +++ b/backend/migrations/20260706094033_add_dev_workspace_label.down.sql @@ -0,0 +1 @@ +ALTER TABLE workspace DROP COLUMN dev_workspace_label; diff --git a/backend/migrations/20260706094033_add_dev_workspace_label.up.sql b/backend/migrations/20260706094033_add_dev_workspace_label.up.sql new file mode 100644 index 0000000000..951f1ec3c9 --- /dev/null +++ b/backend/migrations/20260706094033_add_dev_workspace_label.up.sql @@ -0,0 +1,4 @@ +-- Cosmetic display label for a dev workspace: NULL/'dev' render as "dev", 'staging' renders as "stg". +-- Only meaningful when is_dev_workspace = true; changes nothing about behavior (locking, promote and +-- compare all key off is_dev_workspace / parent_workspace_id). The value is validated in the handler. +ALTER TABLE workspace ADD COLUMN dev_workspace_label VARCHAR; diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 228b8d1831..8d53562635 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -156,6 +156,7 @@ pub fn workspaced_service() -> Router { .route("/create_fork", post(create_workspace_fork)) .route("/attach_dev_workspace", post(attach_dev_workspace)) .route("/detach_dev_workspace", post(detach_dev_workspace)) + .route("/set_dev_workspace_label", post(set_dev_workspace_label)) .route("/get_dev_workspace", get(get_dev_workspace)) .route("/change_workspace_name", post(change_workspace_name)) .route("/change_workspace_color", post(change_workspace_color)) @@ -472,6 +473,10 @@ struct CreateWorkspaceFork { /// the team can work in it. Defaults off; the dev-workspace UI defaults it on. #[serde(default)] copy_members: bool, + /// Cosmetic display label for the dev workspace: 'dev' | 'staging'. Purely visual (badge text + + /// wording); ignored for non-dev forks. None defaults to 'dev'. + #[serde(default)] + dev_workspace_label: Option, } #[derive(Deserialize)] @@ -501,6 +506,7 @@ struct UserWorkspace { pub operator_settings: Option>, pub parent_workspace_id: Option, pub is_dev_workspace: bool, + pub dev_workspace_label: Option, pub disabled: bool, } @@ -678,6 +684,20 @@ async fn exists_workspace( struct DevWorkspaceInfo { id: String, name: String, + dev_workspace_label: Option, +} + +/// Normalize/validate the cosmetic dev-workspace display label. None or 'dev' both render as "dev"; +/// 'staging' renders as "stg". Anything else is rejected. Stored explicitly ('dev'/'staging') so it +/// round-trips, but a NULL column is treated as 'dev' on the read side too. +fn normalize_dev_workspace_label(label: Option) -> Result> { + match label.as_deref() { + None | Some("dev") => Ok(Some("dev".to_string())), + Some("staging") => Ok(Some("staging".to_string())), + Some(other) => Err(Error::BadRequest(format!( + "invalid dev workspace label '{other}' (expected 'dev' or 'staging')" + ))), + } } /// This workspace's active canonical dev workspace, if any. The create-fork UI and the dev-workspace @@ -691,7 +711,7 @@ async fn get_dev_workspace( ) -> JsonResult> { let dev = sqlx::query_as!( DevWorkspaceInfo, - "SELECT id, name FROM workspace WHERE parent_workspace_id = $1 AND is_dev_workspace AND deleted = false", + "SELECT id, name, dev_workspace_label FROM workspace WHERE parent_workspace_id = $1 AND is_dev_workspace AND deleted = false", &w_id ) .fetch_optional(&db) @@ -3697,7 +3717,7 @@ async fn user_workspaces( let workspaces = sqlx::query_as!( UserWorkspace, "SELECT workspace.id, workspace.name, usr.username, workspace_settings.color, workspace.parent_workspace_id, - workspace.is_dev_workspace, + workspace.is_dev_workspace, workspace.dev_workspace_label, CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings, usr.disabled FROM workspace @@ -5187,6 +5207,8 @@ async fn create_workspace_fork_branch( // that second call. Validating early lets a bad request fail before any branch is created. if nw.is_dev_workspace { validate_dev_workspace_id(&nw.id)?; + // Reject a bad cosmetic label before any git branch is created (acted on in create_workspace_fork). + normalize_dev_workspace_label(nw.dev_workspace_label.clone())?; ensure_dev_parent_is_root(&db, &w_id).await?; // Reject before creating any git branch if the parent already has a dev workspace, // otherwise the deferred branch-creation job leaves a dangling branch on the synced repos. @@ -5418,6 +5440,12 @@ async fn create_workspace_fork( validate_fork_workspace_id(&nw.id)?; } validate_workspace_name(&nw.name)?; + // Cosmetic label only applies to dev workspaces; a non-dev fork stores NULL. + let dev_workspace_label = if nw.is_dev_workspace { + normalize_dev_workspace_label(nw.dev_workspace_label.clone())? + } else { + None + }; // Check the id conflict before the CE workspace-count limit so that // re-using a taken (possibly archived) fork id reports the actual // conflict instead of a misleading "maximum number of workspaces" error. @@ -5495,13 +5523,14 @@ async fn create_workspace_fork( sqlx::query!( "INSERT INTO workspace - (id, name, owner, parent_workspace_id, is_dev_workspace) - VALUES ($1, $2, $3, $4, $5)", + (id, name, owner, parent_workspace_id, is_dev_workspace, dev_workspace_label) + VALUES ($1, $2, $3, $4, $5, $6)", forked_id, nw.name, authed.email, parent_workspace_id, nw.is_dev_workspace, + dev_workspace_label, ) .execute(&mut *tx) .await?; @@ -5634,6 +5663,9 @@ struct AttachDevWorkspace { lock_prod_deploy: bool, #[serde(default)] lock_prod_forking: bool, + /// Cosmetic display label for the attached dev workspace: 'dev' | 'staging'. None defaults to 'dev'. + #[serde(default)] + dev_workspace_label: Option, } #[derive(Deserialize)] @@ -5687,6 +5719,7 @@ async fn attach_dev_workspace( // The id is interpolated into a `wm-fork//` branch name like any fork. validate_dev_workspace_id(&dev_w_id)?; + let dev_workspace_label = normalize_dev_workspace_label(req.dev_workspace_label.clone())?; let dev = sqlx::query!( r#"SELECT parent_workspace_id, deleted FROM workspace WHERE id = $1"#, @@ -5754,9 +5787,10 @@ async fn attach_dev_workspace( let mut tx = db.begin().await?; sqlx::query!( - "UPDATE workspace SET parent_workspace_id = $1, is_dev_workspace = true WHERE id = $2", + "UPDATE workspace SET parent_workspace_id = $1, is_dev_workspace = true, dev_workspace_label = $3 WHERE id = $2", &prod_w_id, - &dev_w_id + &dev_w_id, + dev_workspace_label, ) .execute(&mut *tx) .await?; @@ -5822,6 +5856,51 @@ async fn attach_dev_workspace( )) } +#[derive(Deserialize)] +struct SetDevWorkspaceLabel { + #[serde(default)] + dev_workspace_label: Option, +} + +/// Change the cosmetic display label ('dev' | 'staging') of the current workspace, which must itself +/// be a dev workspace. Purely visual (badge text + wording); requires admin of the dev workspace. +async fn set_dev_workspace_label( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + let label = normalize_dev_workspace_label(req.dev_workspace_label)?; + + let mut tx = db.begin().await?; + let updated = sqlx::query_scalar!( + "UPDATE workspace SET dev_workspace_label = $1 WHERE id = $2 AND is_dev_workspace RETURNING id", + label, + &w_id, + ) + .fetch_optional(&mut *tx) + .await?; + if updated.is_none() { + return Err(Error::BadRequest(format!( + "Workspace '{w_id}' is not a dev workspace" + ))); + } + + audit_log( + &mut *tx, + &authed, + "workspaces.set_dev_workspace_label", + ActionKind::Update, + &w_id, + label.as_deref(), + None, + ) + .await?; + tx.commit().await?; + Ok(format!("Updated dev workspace label for {w_id}")) +} + /// Reverse [`attach_dev_workspace`] / clear the dev designation: unset the dev flag and remove the /// prod lock. The workspace keeps its `parent_workspace_id` (it remains an ordinary fork). async fn detach_dev_workspace( diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index 9fbaad2bf3..6b4ace5f0a 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -91,9 +91,10 @@ pub(crate) async fn change_workspace_id( .await?; } sqlx::query!( - "INSERT INTO workspace (id, name, owner, deleted, premium, parent_workspace_id, is_dev_workspace) + "INSERT INTO workspace (id, name, owner, deleted, premium, parent_workspace_id, is_dev_workspace, dev_workspace_label) SELECT $1, $2, owner, false, premium, - CASE WHEN $4 THEN parent_workspace_id ELSE NULL END, $5 + CASE WHEN $4 THEN parent_workspace_id ELSE NULL END, $5, + CASE WHEN $5 THEN dev_workspace_label ELSE NULL END FROM workspace WHERE id = $3", &rw.new_id, &rw.new_name, @@ -1095,7 +1096,10 @@ pub(crate) async fn delete_workspace( // effort: failures are logged — the workspace row is already gone, and broken storage // credentials must not have made it undeletable. for e in cleanup_fork_ducklake_namespaces(&db, &w_id, fork_ducklake_cleanups).await { - tracing::warn!("deleted workspace {w_id}: ducklake namespace cleanup: {}", e.msg); + tracing::warn!( + "deleted workspace {w_id}: ducklake namespace cleanup: {}", + e.msg + ); } if let Some(parent) = dev_lock_parent { diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 16a1f88ffc..ebeeb999f9 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1205,6 +1205,9 @@ paths: type: boolean lock_prod_forking: type: boolean + dev_workspace_label: + type: string + enum: [dev, staging] required: - dev_workspace_id responses: @@ -1263,10 +1266,40 @@ paths: type: string name: type: string + dev_workspace_label: + type: string + nullable: true + description: "Cosmetic display label ('dev' | 'staging'); null defaults to 'dev'" required: - id - name + /w/{workspace}/workspaces/set_dev_workspace_label: + post: + summary: set the cosmetic display label (dev/staging) of this dev workspace + operationId: setDevWorkspaceLabel + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + dev_workspace_label: + type: string + enum: [dev, staging] + responses: + "200": + description: dev workspace label updated + content: + text/plain: + schema: + type: string + /workspaces/exists: post: summary: exists workspace @@ -28180,6 +28213,10 @@ components: nullable: true is_dev_workspace: type: boolean + dev_workspace_label: + type: string + nullable: true + description: "Cosmetic display label of the dev workspace ('dev' | 'staging'); null defaults to 'dev'" created_by: type: string nullable: true @@ -28249,6 +28286,10 @@ components: copy_members: type: boolean description: "Copy the parent's members (users + group memberships) into the fork so the team can work in it" + dev_workspace_label: + type: string + enum: [dev, staging] + description: "Cosmetic display label for the dev workspace (badge text + wording only); ignored for non-dev forks" required: - id - name diff --git a/frontend/src/lib/components/DevWorkspaceSetting.svelte b/frontend/src/lib/components/DevWorkspaceSetting.svelte index bdba23c57f..e6295d24cd 100644 --- a/frontend/src/lib/components/DevWorkspaceSetting.svelte +++ b/frontend/src/lib/components/DevWorkspaceSetting.svelte @@ -1,7 +1,7 @@ @@ -54,7 +56,9 @@ {tooltip} {/if}
- {#if actionButton} + {#if headerAction} + {@render headerAction()} + {:else if actionButton} +
+ {/if} +{/snippet} + -
-
+ {:else if runtimeError} + {/if} {#if logs}
Date: Tue, 7 Jul 2026 10:13:55 +0200 Subject: [PATCH 15/16] feat: update base image to debian 13 (trixie) (#9973) Co-authored-by: Claude Fable 5 --- Dockerfile | 35 ++++++++++++++++---------------- docker/DockerfileFullEe | 5 ++++- docker/DockerfileSlim | 16 +++++++-------- docker/DockerfileSlimEe | 16 +++++++-------- sandbox-image/Dockerfile.sandbox | 2 +- 5 files changed, 39 insertions(+), 35 deletions(-) diff --git a/Dockerfile b/Dockerfile index df03ffbdc0..cb5cbb57e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ -ARG DEBIAN_IMAGE=debian:bookworm-slim -ARG RUST_IMAGE=rust:1.93-slim-bookworm +ARG DEBIAN_IMAGE=debian:trixie-slim +ARG RUST_IMAGE=rust:1.93-slim-trixie -FROM debian:bookworm-slim AS nsjail +FROM debian:trixie-slim AS nsjail WORKDIR /nsjail @@ -9,12 +9,12 @@ RUN apt-get -y update \ && apt-get install -y \ bison=2:3.8.* \ flex=2.6.* \ - g++=4:12.2.* \ - gcc=4:12.2.* \ - git=1:2.39.* \ + g++=4:14.2.* \ + gcc=4:14.2.* \ + git=1:2.47.* \ libprotobuf-dev=3.21.* \ libnl-route-3-dev=3.7.* \ - make=4.3-4.1 \ + make=4.4.* \ pkg-config=1.8.* \ protobuf-compiler=3.21.* @@ -44,7 +44,7 @@ FROM rust_base AS windmill_duckdb_ffi_internal_builder WORKDIR /windmill-duckdb-ffi-internal -RUN apt-get update && apt-get install -y clang=1:14.0-55.* libclang-dev=1:14.0-55.* cmake=3.25.* && \ +RUN apt-get update && apt-get install -y clang=1:19.0* libclang-dev=1:19.0* cmake=3.31.* && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* @@ -98,7 +98,7 @@ ARG features="" COPY --from=planner /windmill/recipe.json recipe.json -RUN apt-get update && apt-get install -y libxml2-dev=2.9.* libxmlsec1-dev=1.2.* libkrb5-dev libsasl2-dev libcurl4-openssl-dev clang=1:14.0-55.* libclang-dev=1:14.0-55.* cmake=3.25.* && \ +RUN apt-get update && apt-get install -y libxml2-dev=2.12.* libxmlsec1-dev=1.2.* libkrb5-dev libsasl2-dev libcurl4-openssl-dev clang=1:19.0* libclang-dev=1:19.0* cmake=3.31.* && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* @@ -135,7 +135,6 @@ FROM ${DEBIAN_IMAGE} ARG TARGETPLATFORM ARG POWERSHELL_VERSION=7.5.0 -ARG POWERSHELL_DEB_VERSION=7.5.0-1 ARG KUBECTL_VERSION=1.28.7 ARG HELM_VERSION=3.14.3 # NOTE: If changing, also change go version in workspace dependencies template at WorkspaceDependenciesEditor.svelte @@ -183,12 +182,14 @@ RUN if [ "$WITH_GIT" = "true" ]; then \ && rm -rf /var/lib/apt/lists/*; \ else echo 'Building the image without git'; fi; +# PowerShell ships as a tarball: the upstream .deb depends on libicu<=74 which no longer exists in trixie RUN if [ "$WITH_POWERSHELL" = "true" ]; then \ - if [ "$TARGETPLATFORM" = "linux/amd64" ]; then apt-get update -y && apt install libicu72 -y && wget -O 'pwsh.deb' "https://github.com/PowerShell/PowerShell/releases/download/v${POWERSHELL_VERSION}/powershell_${POWERSHELL_DEB_VERSION}.deb_amd64.deb" && apt-get clean \ - && rm -rf /var/lib/apt/lists/* && \ - dpkg --install 'pwsh.deb' && \ - rm 'pwsh.deb'; \ - elif [ "$TARGETPLATFORM" = "linux/arm64" ]; then apt-get update -y && apt install libicu72 -y && wget -O powershell.tar.gz "https://github.com/PowerShell/PowerShell/releases/download/v${POWERSHELL_VERSION}/powershell-${POWERSHELL_VERSION}-linux-arm64.tar.gz" && apt-get clean \ + case "$TARGETPLATFORM" in \ + "linux/amd64") pwsh_arch=x64 ;; \ + "linux/arm64") pwsh_arch=arm64 ;; \ + *) pwsh_arch="" ;; \ + esac; \ + if [ -n "$pwsh_arch" ]; then apt-get update -y && apt install libicu76 -y && wget -O powershell.tar.gz "https://github.com/PowerShell/PowerShell/releases/download/v${POWERSHELL_VERSION}/powershell-${POWERSHELL_VERSION}-linux-${pwsh_arch}.tar.gz" && apt-get clean \ && rm -rf /var/lib/apt/lists/* && \ mkdir -p /opt/microsoft/powershell/7 && \ tar zxf powershell.tar.gz -C /opt/microsoft/powershell/7 && \ @@ -292,7 +293,7 @@ RUN bun install -g windmill-cli \ RUN curl -fsSL https://claude.ai/install.sh | bash \ && cp /root/.local/share/claude/versions/* /usr/bin/claude -COPY --from=php:8.3.30-cli-bookworm /usr/local/bin/php /usr/bin/php +COPY --from=php:8.3.30-cli-trixie /usr/local/bin/php /usr/bin/php COPY --from=composer:2.9.5 /usr/bin/composer /usr/bin/composer # add the docker client to call docker from a worker if enabled @@ -303,7 +304,7 @@ ENV CARGO_HOME="/tmp/windmill/cache/cargo" ENV LD_LIBRARY_PATH="." # nsjail runtime deps and binary -RUN apt-get update && apt-get install -y --no-install-recommends libprotobuf32 libnl-route-3-200 libnl-3-200 \ +RUN apt-get update && apt-get install -y --no-install-recommends libprotobuf32t64 libnl-route-3-200 libnl-3-200 \ && apt-get clean && rm -rf /var/lib/apt/lists/* COPY --from=nsjail /nsjail/nsjail /bin/nsjail diff --git a/docker/DockerfileFullEe b/docker/DockerfileFullEe index 3fbb06da71..b82a45b89a 100644 --- a/docker/DockerfileFullEe +++ b/docker/DockerfileFullEe @@ -35,7 +35,10 @@ RUN wget https://dot.net/v1/dotnet-install.sh -O dotnet-install.sh \ # Oracle DB Client COPY --from=oracledb-client /opt/oracle/23/lib /opt/oracle/23/lib -RUN apt-get -y update && apt-get install -y libaio1 +# libaio1t64 only ships libaio.so.1t64; Oracle instantclient loads libaio.so.1, so add a compat symlink +RUN apt-get -y update && apt-get install -y libaio1t64 \ + && libaio="$(find /usr/lib -name 'libaio.so.1t64' -print -quit)" \ + && ln -s "$(basename "$libaio")" "$(dirname "$libaio")/libaio.so.1" RUN echo /opt/oracle/23/lib > /etc/ld.so.conf.d/oracle-instantclient.conf && ldconfig # Nushell diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index ef3fb1261f..31e42cb2c9 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -1,6 +1,6 @@ -ARG DEBIAN_IMAGE=debian:bookworm-slim +ARG DEBIAN_IMAGE=debian:trixie-slim -FROM debian:bookworm-slim AS nsjail +FROM debian:trixie-slim AS nsjail WORKDIR /nsjail @@ -9,12 +9,12 @@ RUN apt-get -y update \ bison=2:3.8.* \ ca-certificates \ flex=2.6.* \ - g++=4:12.2.* \ - gcc=4:12.2.* \ - git=1:2.39.* \ + g++=4:14.2.* \ + gcc=4:14.2.* \ + git=1:2.47.* \ libprotobuf-dev=3.21.* \ libnl-route-3-dev=3.7.* \ - make=4.3-4.1 \ + make=4.4.* \ pkg-config=1.8.* \ protobuf-compiler=3.21.* \ && apt-get clean \ @@ -39,7 +39,7 @@ ENV PATH=/usr/local/bin:/root/.local/bin:/tmp/.local/bin:$PATH # Install system dependencies RUN apt-get update \ - && apt-get install -y --no-install-recommends ca-certificates wget curl git jq unzip unixodbc xmlsec1 gnupg lsb-release libgnutls30 libgcrypt20 \ + && apt-get install -y --no-install-recommends ca-certificates wget curl git jq unzip unixodbc xmlsec1 gnupg lsb-release libgnutls30t64 libgcrypt20 \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -86,7 +86,7 @@ COPY --from=docker:29-dind /usr/local/bin/docker /usr/local/bin/ # nsjail runtime deps and binary RUN apt-get update \ - && apt-get install -y --no-install-recommends libprotobuf32 libnl-route-3-200 libnl-3-200 \ + && apt-get install -y --no-install-recommends libprotobuf32t64 libnl-route-3-200 libnl-3-200 \ && apt-get clean && rm -rf /var/lib/apt/lists/* COPY --from=nsjail /nsjail/nsjail /bin/nsjail diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index 3e36d67a12..e850718469 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -1,6 +1,6 @@ -ARG DEBIAN_IMAGE=debian:bookworm-slim +ARG DEBIAN_IMAGE=debian:trixie-slim -FROM debian:bookworm-slim AS nsjail +FROM debian:trixie-slim AS nsjail WORKDIR /nsjail @@ -9,12 +9,12 @@ RUN apt-get -y update \ bison=2:3.8.* \ ca-certificates \ flex=2.6.* \ - g++=4:12.2.* \ - gcc=4:12.2.* \ - git=1:2.39.* \ + g++=4:14.2.* \ + gcc=4:14.2.* \ + git=1:2.47.* \ libprotobuf-dev=3.21.* \ libnl-route-3-dev=3.7.* \ - make=4.3-4.1 \ + make=4.4.* \ pkg-config=1.8.* \ protobuf-compiler=3.21.* \ && apt-get clean \ @@ -39,7 +39,7 @@ ENV PATH=/usr/local/bin:/root/.local/bin:/tmp/.local/bin:$PATH # Install system dependencies RUN apt-get update \ - && apt-get install -y --no-install-recommends ca-certificates wget curl git jq unzip unixodbc xmlsec1 gnupg lsb-release libgnutls30 libgcrypt20 \ + && apt-get install -y --no-install-recommends ca-certificates wget curl git jq unzip unixodbc xmlsec1 gnupg lsb-release libgnutls30t64 libgcrypt20 \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -86,7 +86,7 @@ COPY --from=docker:29-dind /usr/local/bin/docker /usr/local/bin/ # nsjail runtime deps and binary RUN apt-get update \ - && apt-get install -y --no-install-recommends libprotobuf32 libnl-route-3-200 libnl-3-200 \ + && apt-get install -y --no-install-recommends libprotobuf32t64 libnl-route-3-200 libnl-3-200 \ && apt-get clean && rm -rf /var/lib/apt/lists/* COPY --from=nsjail /nsjail/nsjail /bin/nsjail diff --git a/sandbox-image/Dockerfile.sandbox b/sandbox-image/Dockerfile.sandbox index 7c9ca181a4..7666366720 100644 --- a/sandbox-image/Dockerfile.sandbox +++ b/sandbox-image/Dockerfile.sandbox @@ -1,4 +1,4 @@ -FROM debian:bookworm-slim +FROM debian:trixie-slim # ── Minimal system deps ────────────────────────────────────────────────────── RUN apt-get update && apt-get install -y --no-install-recommends \ From e47aedac0a4af40dd697d5fc4d54dd3c8efe9ab8 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Tue, 7 Jul 2026 10:25:16 +0200 Subject: [PATCH 16/16] feat: add SQL migrations for data tables (#9693) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add datatable_migrations table * feat: add route to run datatable migrations * feat: sync datatable migrations as .up.sql/.down.sql files * feat: add datatable migrate up/down commands and post-push run prompt * feat: add datatable migrate new command to scaffold migrations * feat: add datatable migrations management UI * feat: prompt to create migration on DDL in datatable SQL editors * feat: support running a single specific datatable migration * feat: view migration content, run single migration, fix stacked modal * feat: per-row revert button with out-of-order warning * fix: avoid migrations list flicker on refresh after an action * feat: generate initial datatable migration via pg_dump * fix: surface datatable migration API error details in toasts * fix: revert created migration if create-and-run fails to run * fix: include postgres error detail in migration run/rollback failures * feat: sync datatable migrations as files via the workspace export * refactor: move datatable migrations to migrations/datatable/ path * fix: drop redundant datatable_migration label in sync output * fix: exclude datatable migration sql files from script metadata generation * feat: run datatable migrations as user-permissioned labeled jobs * feat: reject invalid datatable migrations on sync push * feat: datatable migrate up/down default to all datatables, --datatable to target one * fix: surface postgres error detail when datatable migrations fail to run * chore: regenerate CLI docs for datatable migrate commands * feat: default new datatable migration to a BEGIN/END transaction template * fix: validate datatable migration name and datatable at the API boundary * fix: ensure detected DDL ends with semicolon when wrapped in transaction * fix: re-prompt instead of stripping DDL when new-migration modal is cancelled * feat: refresh datatable schema after running a migration from the SQL REPL * feat: record db manager DDL on data tables as migrations * feat: make datatable migrations opt-in per data table * fix: make migration view editor read-only so its code can scroll * fix: don't re-prompt DDL guard when creating a migration without running * feat: generate down migrations for db manager DDL (postgres) * fix: correct down migration for db manager alters (no double-wrap, serial) * feat: explain migrations purpose with a tooltip in the migrations modal * compare paeg * feat: add datatable_migration kind to workspace diff pipeline * chore: point ee-repo-ref at datatable_migration git-sync companion * fix: harden datatable migration version allocation and initial-migration bookkeeping, add tests * feat: deploy and run datatable migrations on workspace merge Co-Authored-By: Claude Opus 4.8 (1M context) * Refactor + handle datatable setting delete/rename * refactor: move datatable migration rename/delete cascade into module Co-Authored-By: Claude Opus 4.8 (1M context) * chore(windmill-utils-internal): bump to 1.7.1 for datatable migration deploy provider methods Co-Authored-By: Claude Opus 4.8 (1M context) * feat(db-manager): add Migrations button to top bar, make Refresh icon-only Co-Authored-By: Claude Opus 4.8 (1M context) * BEGIN/END placeholder in down migration * feat: autofocus migration name input and flag it red when empty Co-Authored-By: Claude Opus 4.8 (1M context) * feat(datatable-migrations): allow non-admins to create/run/revert migrations, gate only opt in/out Co-Authored-By: Claude Opus 4.8 (1M context) * border nits * refresh db manager schema on migrations * BEGIN/END scaffold in CLI * feat(cli): push local datatable migrations before running on migrate up Co-Authored-By: Claude Opus 4.8 (1M context) * feat: flag invalid migration name with red border, not just empty Co-Authored-By: Claude Opus 4.8 (1M context) * refactor: drop random slug from auto-generated migration names Co-Authored-By: Claude Opus 4.8 (1M context) * feat: offer revert-and-delete when deleting an installed migration Co-Authored-By: Claude Opus 4.8 (1M context) * feat: record fork merge as a migration when target datatable opts in * nit * clone migrations on fork * windmill-utils-internal * fix(datatable-migrations): serialize run/rollback with a per-db advisory lock Co-Authored-By: Claude Opus 4.8 (1M context) * fix(db-manager): fail closed when migrations-status check errors on DDL apply Co-Authored-By: Claude Opus 4.8 (1M context) * docs: fix generate_initial migration ordering comment to match code * chore(datatable-migrations): remove unused update_datatable_migrations endpoint Co-Authored-By: Claude Opus 4.8 (1M context) * fix: run DDL migration guard on the script editor Test button Co-Authored-By: Claude Opus 4.8 (1M context) * split * ee-repo-ref * chore(frontend): sync package-lock with package.json (@emnapi deps) Co-Authored-By: Claude Opus 4.8 (1M context) * fix(datatable-migrations): never resolve instance credentials into migration job args datatable_database_arg eagerly resolved instance data-table credentials (including the shared instance-wide Postgres password) and passed them as the migration job's plaintext `database` arg, landing in v2_job.args. Since the run route has no admin gate, a non-admin could run a migration and read args.database to recover the password, granting cross-workspace psql access to all instance data-table DBs. Pass a `datatable://` reference for both resource-backed and instance data tables instead; the pg executor already resolves it to real credentials server-side at run time, so nothing sensitive is ever stored in the job args. Co-Authored-By: Claude Opus 4.8 (1M context) * nit * fix: handle dollar-quoting and comments when splitting SQL statements * feat: deploy datatable migrations on merge with explicit opt-in error * fix(frontend): sync package-lock with npm 11 peer-dep resolution npm ci failed with 'Missing: @emnapi/core@1.11.2 / @emnapi/runtime@1.11.2 from lock file'. @napi-rs/wasm-runtime declares @emnapi/core|runtime ^1.7.1 as peerDependencies while @rolldown/binding-wasm32-wasi pins them to exactly 1.10.0. Newer npm (bundled with node 24 in CI) installs the peer deps at the highest match (1.11.2) alongside rolldown's nested 1.10.0, so the ideal tree needs both versions; the committed lock only had 1.10.0. Regenerate the lock with npm 11.18 so it carries both 1.11.2 (top-level, for the peer deps) and 1.10.0 (nested, for rolldown's pin). Verified npm ci passes. Co-Authored-By: Claude Opus 4.8 (1M context) * nit npm publish * fix: fail closed on migrations-status error in fork schema merge * nit CI emnapi/core version * prevent initial_datatable_migration if migrations already exist * fix(datatable-migrations): validate persisted data table names as path segments edit_datatable_config only validated rename segments, not the actual settings.datatables keys, so a data table could be saved directly under a name like '..' or one containing '/'. Since new tables default to migrations_enabled = true, generate_initial_datatable_migration would then insert a migration row and the sync export would build migrations/datatable//... paths from that name, producing malformed or directory-escaping export paths. Validate every persisted data table name in edit_datatable_config (alongside the existing rename checks) and add validate_datatable_path_segment to generate_initial_datatable_migration for defense in depth. Co-Authored-By: Claude Opus 4.8 (1M context) * fix: scope datatable _wm_migrations by data table and cascade renames/deletes Co-Authored-By: Claude Opus 4.8 (1M context) * fix(system_prompts): resolve nested local command groups in CLI docs generator The CLI docs generator anchored on the first `new Command()` in a file and never resolved locally-defined command groups passed as `.command("name", localCmd)`. For datatable this flattened the nested `migrate` group: it emitted `datatable new/up/down` plus a bare `datatable migrate`, and mislabeled the datatable command with the migrate group's description. jobs was broken the same way (its description was pull's, and pull/push rendered empty). Anchor block extraction on the `export default`ed command, recurse into locally-defined `const x = new Command()` groups mounted as subcommands, and render nested sub-subcommands. Regenerated docs now show `datatable migrate new/up/down` and `jobs pull/push` with their real options. Co-Authored-By: Claude Opus 4.8 (1M context) * refactor: drop unreleased _wm_migrations legacy-upgrade handling Co-Authored-By: Claude Opus 4.8 (1M context) * fix: return datatable migration SQL from getItemValue for the diff drawer Co-Authored-By: Claude Opus 4.8 (1M context) * chore(frontend): use windmill-utils-internal 1.8.2 for migration diff drawer Co-Authored-By: Claude Opus 4.8 (1M context) * nit * nit * fix: handle datatable migration renames on push and dedupe timestamps * fix: reject rewriting an already-applied datatable migration on upsert Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): add missing @emnapi/core and @emnapi/runtime lockfile entries Resolves npm ci EUSAGE failure: the optional cpu:wasm32 @rolldown/binding-wasm32-wasi declares deps on @emnapi/core@1.11.2 and @emnapi/runtime@1.11.2 that had no resolved lockfile entries. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(cli): datatable migrate up/down default to main datatable, not all Co-Authored-By: Claude Opus 4.8 (1M context) * fix: fail closed when applied status unreadable on datatable migration rewrite Co-Authored-By: Claude Opus 4.8 (1M context) * fix: surface full error detail in Database Manager DDL/query errors * "See migration" button in the toast * feat: add Enter shortcut to Create-a-migration in the DDL guard * fix(frontend): warn before running a newly-created datatable migration out of order The row-level Run action warns when earlier migrations are still pending, but the create-and-run paths ran a just-created migration with `only` directly, applying it ahead of older pending migrations without that confirmation. Reuse the same "Run migration out of order" confirmation across all create-and-run paths via a shared helper (datatableMigrationUtils): - NewDataTableMigrationModal "Create and run" (and the DDL guard path) - DatatableSchemaDiff fork→parent merge - dbOps schema ops (DB manager create/alter/drop) — the pure factory throws a MigrationRunCancelled sentinel on decline, which DBTableEditor treats as a silent cancel Co-Authored-By: Claude Opus 4.8 (1M context) * fix: keep renamed datatable migrations visible in compare view * fix: record per-migration deployment on datatable migrations disable * fix(cli): run deployed datatable migrations after workspace merge The merge command upserted datatable_migration definitions into the target workspace and reported the item as successfully deployed, but never ran the migrations. For forked datatables backed by separate databases, this left the target schema unchanged until someone manually ran `wmill datatable migrate up`, while the CLI reported a successful merge. Collect the datatable migrations deployed (not deleted) into the target and, after the deploy loop, offer to run them via the existing offerToRunNewMigrations helper — the same post-deploy run prompt the push/sync path uses (interactive only; `--yes`/non-TTY skip the mutating run, matching push behavior). Export parseDatatableMigrationDeployPath so the merge path can parse the deployed items. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(backend): serialize datatable migration edits/deletes with the run lock A migration run snapshots a migration's code_up from datatable_migrations and only records its version in the data table's _wm_migrations after the job succeeds. upsert_datatable_migration checked _wm_migrations before allowing an edit but took no lock, so a concurrent edit could read "not applied yet", rewrite code_up/code_down, and then the in-flight run would record the version for the old SQL — leaving _wm_migrations pointing at SQL that was never applied (migrate up then skips it; rollback runs a down that doesn't match). Serialize definition rewrites and deletes with the same per-database advisory lock the run/rollback paths use: - Factor the connect+advisory-lock into lock_datatable_migration_runs and the applied-versions read into read_applied_versions_on_client. - run_datatable_migrations now snapshots the definitions AFTER taking the lock, so code_up can't change between snapshot and version-record. - upsert (when changing an existing def) and delete take the lock across the applied-check and the write; delete now rejects deleting an already-applied migration (would orphan its _wm_migrations record), symmetric with upsert. Both fail closed if the data table database is unreachable. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): stack the out-of-order migration confirm above the DB editor preview Creating a table on a migrations-enabled data table opened the DB table editor's "Confirm running the following" preview modal, whose confirm triggers applyDdl, which then asks for out-of-order confirmation. Both are ConfirmationModals with a hardcoded z-[9999]; the out-of-order one lives in DBManagerContent (mounted before the editor), so it rendered behind the still-open preview modal. Add an optional zIndexClass prop to ConfirmationModal (default z-[9999], backward-compatible) and give the DB-manager out-of-order confirm z-[10000] so it stacks on top. Co-Authored-By: Claude Opus 4.8 (1M context) * chore: update ee-repo-ref to 27672e37df5d9dfde94f19963d5ffcdf8dd5448c This commit updates the EE repository reference after PR #623 was merged in windmill-ee-private. Previous ee-repo-ref: 6c287041cd7edd4a77a4bc07ad0e156cec32cce4 New ee-repo-ref: 27672e37df5d9dfde94f19963d5ffcdf8dd5448c Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: windmill-internal-app[bot] --- ...04ab3569a0611b33c261602d62e9c847c583e.json | 15 + ...1f7f387f5055c47f493271d26731336257384.json | 10 +- ...4324c47e010e3554c14e755a7e045453745bb.json | 23 + ...974f2ce577b78decd6b821096c9f2f252ae8b.json | 22 - ...741a2e193f18ffa11d08e8ca49cef5c3b850c.json | 16 + ...3c1293c4b72d0b52abeefd1e954617984a2d8.json | 24 + ...d40dfe1ffe51839884f3e9e9360d9e17b5afd.json | 36 + ...95bde7f479877018980cffe1c1e9d34aad59e.json | 29 + ...153c43903f929ae5d62fbba12610f89c36d55.json | 2 +- ...7535d4d962c328a060f33c07191ac3e033e82.json | 16 + ...82734e4ca6cad2d849f75a8f8a249f83df83f.json | 23 + ...332c696e12b52066a5ae2ab10fe3b5bd0f3d0.json | 30 + ...030390d9510e73e9b7df347b697d6dc7aefe6.json | 19 + ...9f909bf4babb29910514753cb822dde4e7ca9.json | 23 + ...7a20a5773113e290d69a016363543067baf14.json | 23 + ...c1d18df0ab30f528ef4ce7910c43dd44f4cfb.json | 23 + ...07ba6333ad5feb4557a9341427e9e68608025.json | 46 + ...89730dd215f4e90d105f0e86025f0ac42f020.json | 15 + ...1cf520939fa4fa4b65371d11db7416ba0b14c.json | 23 + ...8d887920beaec97696a17da020a2adfb052f0.json | 17 + ...ae590d0ed2f4b7831956bb429a47c478f7c1f.json | 19 + ...0d503a92e44e291a76a53ef09ded619edfacb.json | 22 + ...0099ee5f46dafba8f323cf002e329ac69d1ac.json | 30 + ...0e74e14afde3c02000a88e696d7b6adcad0d6.json | 41 + ...7a3e43ef12b97d55353f86a9a368617efccca.json | 23 + ...4b94fd84c1bb64e9659f798df2888848894bd.json | 46 + ...358581f5716b3b58dc2e6b9b9282c50b1b66b.json | 15 + ...6f85238cbf189ccf18191493d6357e269d12d.json | 35 + backend/Cargo.lock | 1 + backend/ee-repo-ref.txt | 2 +- ...260617081932_datatable_migrations.down.sql | 1 + ...20260617081932_datatable_migrations.up.sql | 21 + backend/windmill-api-workspaces/Cargo.toml | 1 + .../src/datatable_migrations.rs | 1712 +++++++++++++++++ backend/windmill-api-workspaces/src/lib.rs | 1 + .../windmill-api-workspaces/src/workspaces.rs | 162 +- backend/windmill-api/openapi.yaml | 358 ++++ backend/windmill-api/src/workspaces_export.rs | 28 + backend/windmill-common/src/workspaces.rs | 6 + backend/windmill-git-sync/src/lib.rs | 123 +- cli/src/commands/datatable/datatable.ts | 71 + cli/src/commands/datatable_migrations.ts | 340 ++++ .../generate-metadata/generate-metadata.ts | 6 +- cli/src/commands/sync/sync.ts | 106 +- cli/src/commands/workspace/merge.ts | 32 + cli/src/guidance/skills.gen.ts | 26 +- cli/src/types.ts | 40 +- cli/test/datatable_migrations_unit.test.ts | 122 ++ cli/windmill-utils-internal/package.json | 2 +- cli/windmill-utils-internal/src/deploy.ts | 124 ++ frontend/package-lock.json | 171 +- frontend/package.json | 2 +- .../lib/components/CompareWorkspaces.svelte | 93 +- frontend/src/lib/components/DBManager.svelte | 10 +- .../lib/components/DBManagerContent.svelte | 21 +- .../src/lib/components/DBManagerDrawer.svelte | 28 +- .../src/lib/components/DBTableEditor.svelte | 13 +- .../lib/components/DatatableSchemaDiff.svelte | 273 ++- .../lib/components/DdlMigrationGuard.svelte | 159 ++ frontend/src/lib/components/Editor.svelte | 40 +- .../src/lib/components/ScriptEditor.svelte | 15 +- .../src/lib/components/SimpleEditor.svelte | 8 + frontend/src/lib/components/SqlRepl.svelte | 83 +- .../ConfirmationModal.svelte | 6 +- .../lib/components/common/table/Row.svelte | 1 + .../components/common/table/RowIcon.svelte | 3 + frontend/src/lib/components/dbOps.ts | 168 +- frontend/src/lib/components/sqlDdl.ts | 142 ++ .../DataTableMigrationsButton.svelte | 596 ++++++ .../DataTableSettings.svelte | 32 +- .../NewDataTableMigrationModal.svelte | 230 +++ .../datatableMigrationUtils.ts | 30 + frontend/src/lib/utils_deployable.ts | 2 + frontend/src/lib/utils_workspace_deploy.ts | 9 +- .../auto-generated/cli/cli-commands.md | 26 +- system_prompts/auto-generated/prompts.ts | 26 +- .../skills/cli-commands/SKILL.md | 26 +- system_prompts/generate.py | 86 +- 78 files changed, 5791 insertions(+), 459 deletions(-) create mode 100644 backend/.sqlx/query-00e9fc3fed9379264881c58df9404ab3569a0611b33c261602d62e9c847c583e.json create mode 100644 backend/.sqlx/query-15c0584e9eb078442f6928adb894324c47e010e3554c14e755a7e045453745bb.json delete mode 100644 backend/.sqlx/query-16a67b92dbd32024838983184e6974f2ce577b78decd6b821096c9f2f252ae8b.json create mode 100644 backend/.sqlx/query-2c543583bcf7bcaf2f422638ce0741a2e193f18ffa11d08e8ca49cef5c3b850c.json create mode 100644 backend/.sqlx/query-3c057e0284f1219865b8f7dc6eb3c1293c4b72d0b52abeefd1e954617984a2d8.json create mode 100644 backend/.sqlx/query-44c4e9848bc97b66a49d5454d30d40dfe1ffe51839884f3e9e9360d9e17b5afd.json create mode 100644 backend/.sqlx/query-564f4bdf43135c603518ef35a3e95bde7f479877018980cffe1c1e9d34aad59e.json create mode 100644 backend/.sqlx/query-5b67c3af6477d7029d118ac259a7535d4d962c328a060f33c07191ac3e033e82.json create mode 100644 backend/.sqlx/query-5cca3509374b1ddd8707c930bc382734e4ca6cad2d849f75a8f8a249f83df83f.json create mode 100644 backend/.sqlx/query-6ac00d65b3b7707cba9456171d4332c696e12b52066a5ae2ab10fe3b5bd0f3d0.json create mode 100644 backend/.sqlx/query-760908f44cafb500e4ba2515d1b030390d9510e73e9b7df347b697d6dc7aefe6.json create mode 100644 backend/.sqlx/query-798dc8ce1c80b5ebd8120f55b6c9f909bf4babb29910514753cb822dde4e7ca9.json create mode 100644 backend/.sqlx/query-95e370a82ef46d9310f77a99b437a20a5773113e290d69a016363543067baf14.json create mode 100644 backend/.sqlx/query-9eea74f68a0c99d5ce6601fd854c1d18df0ab30f528ef4ce7910c43dd44f4cfb.json create mode 100644 backend/.sqlx/query-a8cec061756c7be61af829f4a5207ba6333ad5feb4557a9341427e9e68608025.json create mode 100644 backend/.sqlx/query-aef927110ef4cff51d3faabb0c389730dd215f4e90d105f0e86025f0ac42f020.json create mode 100644 backend/.sqlx/query-b3200181380df2f645bfce270dc1cf520939fa4fa4b65371d11db7416ba0b14c.json create mode 100644 backend/.sqlx/query-b96c1a720654ae8ae52e8098cc18d887920beaec97696a17da020a2adfb052f0.json create mode 100644 backend/.sqlx/query-c5639d48c92d6863f16f97de91eae590d0ed2f4b7831956bb429a47c478f7c1f.json create mode 100644 backend/.sqlx/query-c9d9c08505e69790154876f50f90d503a92e44e291a76a53ef09ded619edfacb.json create mode 100644 backend/.sqlx/query-cf2e74dcd0992f22eb3b62995fd0099ee5f46dafba8f323cf002e329ac69d1ac.json create mode 100644 backend/.sqlx/query-e5d8d46b9431033fa7572a880ee0e74e14afde3c02000a88e696d7b6adcad0d6.json create mode 100644 backend/.sqlx/query-ec8e502f9c2926e578e02cd164a7a3e43ef12b97d55353f86a9a368617efccca.json create mode 100644 backend/.sqlx/query-f89b024949f29eb5142744635914b94fd84c1bb64e9659f798df2888848894bd.json create mode 100644 backend/.sqlx/query-fc59d9b911529de75c466593252358581f5716b3b58dc2e6b9b9282c50b1b66b.json create mode 100644 backend/.sqlx/query-fcb0cf55f5c9047066a6fdd62bf6f85238cbf189ccf18191493d6357e269d12d.json create mode 100644 backend/migrations/20260617081932_datatable_migrations.down.sql create mode 100644 backend/migrations/20260617081932_datatable_migrations.up.sql create mode 100644 backend/windmill-api-workspaces/src/datatable_migrations.rs create mode 100644 cli/src/commands/datatable_migrations.ts create mode 100644 cli/test/datatable_migrations_unit.test.ts create mode 100644 frontend/src/lib/components/DdlMigrationGuard.svelte create mode 100644 frontend/src/lib/components/sqlDdl.ts create mode 100644 frontend/src/lib/components/workspaceSettings/DataTableMigrationsButton.svelte create mode 100644 frontend/src/lib/components/workspaceSettings/NewDataTableMigrationModal.svelte create mode 100644 frontend/src/lib/components/workspaceSettings/datatableMigrationUtils.ts diff --git a/backend/.sqlx/query-00e9fc3fed9379264881c58df9404ab3569a0611b33c261602d62e9c847c583e.json b/backend/.sqlx/query-00e9fc3fed9379264881c58df9404ab3569a0611b33c261602d62e9c847c583e.json new file mode 100644 index 0000000000..13536a244e --- /dev/null +++ b/backend/.sqlx/query-00e9fc3fed9379264881c58df9404ab3569a0611b33c261602d62e9c847c583e.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = ANY($2::text[])", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "00e9fc3fed9379264881c58df9404ab3569a0611b33c261602d62e9c847c583e" +} diff --git a/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json b/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json index e7ed0aee65..d29a18c691 100644 --- a/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json +++ b/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json @@ -46,11 +46,11 @@ ] }, "nullable": [ - false, - false, - false, - false, - false, + true, + true, + true, + true, + true, true, true ] diff --git a/backend/.sqlx/query-15c0584e9eb078442f6928adb894324c47e010e3554c14e755a7e045453745bb.json b/backend/.sqlx/query-15c0584e9eb078442f6928adb894324c47e010e3554c14e755a7e045453745bb.json new file mode 100644 index 0000000000..47bfec9e5c --- /dev/null +++ b/backend/.sqlx/query-15c0584e9eb078442f6928adb894324c47e010e3554c14e755a7e045453745bb.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "15c0584e9eb078442f6928adb894324c47e010e3554c14e755a7e045453745bb" +} diff --git a/backend/.sqlx/query-16a67b92dbd32024838983184e6974f2ce577b78decd6b821096c9f2f252ae8b.json b/backend/.sqlx/query-16a67b92dbd32024838983184e6974f2ce577b78decd6b821096c9f2f252ae8b.json deleted file mode 100644 index d099d97bd3..0000000000 --- a/backend/.sqlx/query-16a67b92dbd32024838983184e6974f2ce577b78decd6b821096c9f2f252ae8b.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT ws.datatable->'datatables' AS datatable_name\n FROM workspace_settings ws\n WHERE ws.workspace_id = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "datatable_name", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "16a67b92dbd32024838983184e6974f2ce577b78decd6b821096c9f2f252ae8b" -} diff --git a/backend/.sqlx/query-2c543583bcf7bcaf2f422638ce0741a2e193f18ffa11d08e8ca49cef5c3b850c.json b/backend/.sqlx/query-2c543583bcf7bcaf2f422638ce0741a2e193f18ffa11d08e8ca49cef5c3b850c.json new file mode 100644 index 0000000000..2f1ad395b6 --- /dev/null +++ b/backend/.sqlx/query-2c543583bcf7bcaf2f422638ce0741a2e193f18ffa11d08e8ca49cef5c3b850c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE datatable_migrations SET datatable = $3 WHERE workspace_id = $1 AND datatable = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "2c543583bcf7bcaf2f422638ce0741a2e193f18ffa11d08e8ca49cef5c3b850c" +} diff --git a/backend/.sqlx/query-3c057e0284f1219865b8f7dc6eb3c1293c4b72d0b52abeefd1e954617984a2d8.json b/backend/.sqlx/query-3c057e0284f1219865b8f7dc6eb3c1293c4b72d0b52abeefd1e954617984a2d8.json new file mode 100644 index 0000000000..3206ba4433 --- /dev/null +++ b/backend/.sqlx/query-3c057e0284f1219865b8f7dc6eb3c1293c4b72d0b52abeefd1e954617984a2d8.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3 RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "3c057e0284f1219865b8f7dc6eb3c1293c4b72d0b52abeefd1e954617984a2d8" +} diff --git a/backend/.sqlx/query-44c4e9848bc97b66a49d5454d30d40dfe1ffe51839884f3e9e9360d9e17b5afd.json b/backend/.sqlx/query-44c4e9848bc97b66a49d5454d30d40dfe1ffe51839884f3e9e9360d9e17b5afd.json new file mode 100644 index 0000000000..1e1c5f20c3 --- /dev/null +++ b/backend/.sqlx/query-44c4e9848bc97b66a49d5454d30d40dfe1ffe51839884f3e9e9360d9e17b5afd.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT name, code_up, code_down FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "code_up", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "code_down", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false, + false, + true + ] + }, + "hash": "44c4e9848bc97b66a49d5454d30d40dfe1ffe51839884f3e9e9360d9e17b5afd" +} diff --git a/backend/.sqlx/query-564f4bdf43135c603518ef35a3e95bde7f479877018980cffe1c1e9d34aad59e.json b/backend/.sqlx/query-564f4bdf43135c603518ef35a3e95bde7f479877018980cffe1c1e9d34aad59e.json new file mode 100644 index 0000000000..20ac675432 --- /dev/null +++ b/backend/.sqlx/query-564f4bdf43135c603518ef35a3e95bde7f479877018980cffe1c1e9d34aad59e.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 RETURNING timestamp, name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "timestamp", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "564f4bdf43135c603518ef35a3e95bde7f479877018980cffe1c1e9d34aad59e" +} diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 36ddb8ab9f..713ccb9dd3 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - true + null ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-5b67c3af6477d7029d118ac259a7535d4d962c328a060f33c07191ac3e033e82.json b/backend/.sqlx/query-5b67c3af6477d7029d118ac259a7535d4d962c328a060f33c07191ac3e033e82.json new file mode 100644 index 0000000000..12628305cc --- /dev/null +++ b/backend/.sqlx/query-5b67c3af6477d7029d118ac259a7535d4d962c328a060f33c07191ac3e033e82.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "5b67c3af6477d7029d118ac259a7535d4d962c328a060f33c07191ac3e033e82" +} diff --git a/backend/.sqlx/query-5cca3509374b1ddd8707c930bc382734e4ca6cad2d849f75a8f8a249f83df83f.json b/backend/.sqlx/query-5cca3509374b1ddd8707c930bc382734e4ca6cad2d849f75a8f8a249f83df83f.json new file mode 100644 index 0000000000..14b085db2e --- /dev/null +++ b/backend/.sqlx/query-5cca3509374b1ddd8707c930bc382734e4ca6cad2d849f75a8f8a249f83df83f.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ws.datatable->'datatables'->$2 FROM workspace_settings ws WHERE ws.workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "5cca3509374b1ddd8707c930bc382734e4ca6cad2d849f75a8f8a249f83df83f" +} diff --git a/backend/.sqlx/query-6ac00d65b3b7707cba9456171d4332c696e12b52066a5ae2ab10fe3b5bd0f3d0.json b/backend/.sqlx/query-6ac00d65b3b7707cba9456171d4332c696e12b52066a5ae2ab10fe3b5bd0f3d0.json new file mode 100644 index 0000000000..3a1dc9da28 --- /dev/null +++ b/backend/.sqlx/query-6ac00d65b3b7707cba9456171d4332c696e12b52066a5ae2ab10fe3b5bd0f3d0.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT name, code_down FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "code_down", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "6ac00d65b3b7707cba9456171d4332c696e12b52066a5ae2ab10fe3b5bd0f3d0" +} diff --git a/backend/.sqlx/query-760908f44cafb500e4ba2515d1b030390d9510e73e9b7df347b697d6dc7aefe6.json b/backend/.sqlx/query-760908f44cafb500e4ba2515d1b030390d9510e73e9b7df347b697d6dc7aefe6.json new file mode 100644 index 0000000000..384686eb35 --- /dev/null +++ b/backend/.sqlx/query-760908f44cafb500e4ba2515d1b030390d9510e73e9b7df347b697d6dc7aefe6.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) VALUES ($1, $2, $3, $4, $5, $6)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "760908f44cafb500e4ba2515d1b030390d9510e73e9b7df347b697d6dc7aefe6" +} diff --git a/backend/.sqlx/query-798dc8ce1c80b5ebd8120f55b6c9f909bf4babb29910514753cb822dde4e7ca9.json b/backend/.sqlx/query-798dc8ce1c80b5ebd8120f55b6c9f909bf4babb29910514753cb822dde4e7ca9.json new file mode 100644 index 0000000000..f7ff5d527a --- /dev/null +++ b/backend/.sqlx/query-798dc8ce1c80b5ebd8120f55b6c9f909bf4babb29910514753cb822dde4e7ca9.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT (ws.datatable->'datatables'->$2->>'migrations_enabled')::boolean FROM workspace_settings ws WHERE ws.workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "bool", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "798dc8ce1c80b5ebd8120f55b6c9f909bf4babb29910514753cb822dde4e7ca9" +} diff --git a/backend/.sqlx/query-95e370a82ef46d9310f77a99b437a20a5773113e290d69a016363543067baf14.json b/backend/.sqlx/query-95e370a82ef46d9310f77a99b437a20a5773113e290d69a016363543067baf14.json new file mode 100644 index 0000000000..eb94fe6ab5 --- /dev/null +++ b/backend/.sqlx/query-95e370a82ef46d9310f77a99b437a20a5773113e290d69a016363543067baf14.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT MAX(timestamp) FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "max", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "95e370a82ef46d9310f77a99b437a20a5773113e290d69a016363543067baf14" +} diff --git a/backend/.sqlx/query-9eea74f68a0c99d5ce6601fd854c1d18df0ab30f528ef4ce7910c43dd44f4cfb.json b/backend/.sqlx/query-9eea74f68a0c99d5ce6601fd854c1d18df0ab30f528ef4ce7910c43dd44f4cfb.json new file mode 100644 index 0000000000..5ac3174af6 --- /dev/null +++ b/backend/.sqlx/query-9eea74f68a0c99d5ce6601fd854c1d18df0ab30f528ef4ce7910c43dd44f4cfb.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET datatable = jsonb_set(datatable, ARRAY['datatables', $2::text, 'migrations_enabled'], 'false'::jsonb) WHERE workspace_id = $1 AND jsonb_exists(datatable->'datatables', $2) RETURNING 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "9eea74f68a0c99d5ce6601fd854c1d18df0ab30f528ef4ce7910c43dd44f4cfb" +} diff --git a/backend/.sqlx/query-a8cec061756c7be61af829f4a5207ba6333ad5feb4557a9341427e9e68608025.json b/backend/.sqlx/query-a8cec061756c7be61af829f4a5207ba6333ad5feb4557a9341427e9e68608025.json new file mode 100644 index 0000000000..c262f3d1f6 --- /dev/null +++ b/backend/.sqlx/query-a8cec061756c7be61af829f4a5207ba6333ad5feb4557a9341427e9e68608025.json @@ -0,0 +1,46 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT datatable, timestamp, name, code_up, code_down FROM datatable_migrations WHERE workspace_id = $1 ORDER BY datatable, timestamp ASC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "datatable", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "timestamp", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "code_up", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "code_down", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + true + ] + }, + "hash": "a8cec061756c7be61af829f4a5207ba6333ad5feb4557a9341427e9e68608025" +} diff --git a/backend/.sqlx/query-aef927110ef4cff51d3faabb0c389730dd215f4e90d105f0e86025f0ac42f020.json b/backend/.sqlx/query-aef927110ef4cff51d3faabb0c389730dd215f4e90d105f0e86025f0ac42f020.json new file mode 100644 index 0000000000..2a90d12ad0 --- /dev/null +++ b/backend/.sqlx/query-aef927110ef4cff51d3faabb0c389730dd215f4e90d105f0e86025f0ac42f020.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down)\n SELECT $2, datatable, timestamp, name, code_up, code_down\n FROM datatable_migrations WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "aef927110ef4cff51d3faabb0c389730dd215f4e90d105f0e86025f0ac42f020" +} diff --git a/backend/.sqlx/query-b3200181380df2f645bfce270dc1cf520939fa4fa4b65371d11db7416ba0b14c.json b/backend/.sqlx/query-b3200181380df2f645bfce270dc1cf520939fa4fa4b65371d11db7416ba0b14c.json new file mode 100644 index 0000000000..f3cc23c5b8 --- /dev/null +++ b/backend/.sqlx/query-b3200181380df2f645bfce270dc1cf520939fa4fa4b65371d11db7416ba0b14c.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_xact_lock", + "type_info": "Void" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "b3200181380df2f645bfce270dc1cf520939fa4fa4b65371d11db7416ba0b14c" +} diff --git a/backend/.sqlx/query-b96c1a720654ae8ae52e8098cc18d887920beaec97696a17da020a2adfb052f0.json b/backend/.sqlx/query-b96c1a720654ae8ae52e8098cc18d887920beaec97696a17da020a2adfb052f0.json new file mode 100644 index 0000000000..b3861c0111 --- /dev/null +++ b/backend/.sqlx/query-b96c1a720654ae8ae52e8098cc18d887920beaec97696a17da020a2adfb052f0.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) VALUES ($1, $2, $3, 'initial', $4, NULL)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b96c1a720654ae8ae52e8098cc18d887920beaec97696a17da020a2adfb052f0" +} diff --git a/backend/.sqlx/query-c5639d48c92d6863f16f97de91eae590d0ed2f4b7831956bb429a47c478f7c1f.json b/backend/.sqlx/query-c5639d48c92d6863f16f97de91eae590d0ed2f4b7831956bb429a47c478f7c1f.json new file mode 100644 index 0000000000..fa1396235f --- /dev/null +++ b/backend/.sqlx/query-c5639d48c92d6863f16f97de91eae590d0ed2f4b7831956bb429a47c478f7c1f.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (workspace_id, datatable, timestamp) DO UPDATE SET name = EXCLUDED.name, code_up = EXCLUDED.code_up, code_down = EXCLUDED.code_down", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "c5639d48c92d6863f16f97de91eae590d0ed2f4b7831956bb429a47c478f7c1f" +} diff --git a/backend/.sqlx/query-c9d9c08505e69790154876f50f90d503a92e44e291a76a53ef09ded619edfacb.json b/backend/.sqlx/query-c9d9c08505e69790154876f50f90d503a92e44e291a76a53ef09ded619edfacb.json new file mode 100644 index 0000000000..b089e466cf --- /dev/null +++ b/backend/.sqlx/query-c9d9c08505e69790154876f50f90d503a92e44e291a76a53ef09ded619edfacb.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ws.datatable->'datatables' FROM workspace_settings ws WHERE ws.workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "c9d9c08505e69790154876f50f90d503a92e44e291a76a53ef09ded619edfacb" +} diff --git a/backend/.sqlx/query-cf2e74dcd0992f22eb3b62995fd0099ee5f46dafba8f323cf002e329ac69d1ac.json b/backend/.sqlx/query-cf2e74dcd0992f22eb3b62995fd0099ee5f46dafba8f323cf002e329ac69d1ac.json new file mode 100644 index 0000000000..83c306f4d8 --- /dev/null +++ b/backend/.sqlx/query-cf2e74dcd0992f22eb3b62995fd0099ee5f46dafba8f323cf002e329ac69d1ac.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT datatable, timestamp FROM datatable_migrations WHERE workspace_id = $1 AND datatable = ANY($2) AND timestamp = ANY($3)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "datatable", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "timestamp", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Int8Array" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "cf2e74dcd0992f22eb3b62995fd0099ee5f46dafba8f323cf002e329ac69d1ac" +} diff --git a/backend/.sqlx/query-e5d8d46b9431033fa7572a880ee0e74e14afde3c02000a88e696d7b6adcad0d6.json b/backend/.sqlx/query-e5d8d46b9431033fa7572a880ee0e74e14afde3c02000a88e696d7b6adcad0d6.json new file mode 100644 index 0000000000..d6c6a5c02f --- /dev/null +++ b/backend/.sqlx/query-e5d8d46b9431033fa7572a880ee0e74e14afde3c02000a88e696d7b6adcad0d6.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT timestamp, name, code_up, code_down FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 ORDER BY timestamp ASC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "timestamp", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "code_up", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "code_down", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true + ] + }, + "hash": "e5d8d46b9431033fa7572a880ee0e74e14afde3c02000a88e696d7b6adcad0d6" +} diff --git a/backend/.sqlx/query-ec8e502f9c2926e578e02cd164a7a3e43ef12b97d55353f86a9a368617efccca.json b/backend/.sqlx/query-ec8e502f9c2926e578e02cd164a7a3e43ef12b97d55353f86a9a368617efccca.json new file mode 100644 index 0000000000..fbcf0743be --- /dev/null +++ b/backend/.sqlx/query-ec8e502f9c2926e578e02cd164a7a3e43ef12b97d55353f86a9a368617efccca.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET datatable = jsonb_set(datatable, ARRAY['datatables', $2::text, 'migrations_enabled'], 'true'::jsonb) WHERE workspace_id = $1 AND jsonb_exists(datatable->'datatables', $2) RETURNING 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ec8e502f9c2926e578e02cd164a7a3e43ef12b97d55353f86a9a368617efccca" +} diff --git a/backend/.sqlx/query-f89b024949f29eb5142744635914b94fd84c1bb64e9659f798df2888848894bd.json b/backend/.sqlx/query-f89b024949f29eb5142744635914b94fd84c1bb64e9659f798df2888848894bd.json new file mode 100644 index 0000000000..8a6d989157 --- /dev/null +++ b/backend/.sqlx/query-f89b024949f29eb5142744635914b94fd84c1bb64e9659f798df2888848894bd.json @@ -0,0 +1,46 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT datatable, timestamp, name, code_up, code_down FROM datatable_migrations WHERE workspace_id = $1 ORDER BY datatable, timestamp", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "datatable", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "timestamp", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "code_up", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "code_down", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + true + ] + }, + "hash": "f89b024949f29eb5142744635914b94fd84c1bb64e9659f798df2888848894bd" +} diff --git a/backend/.sqlx/query-fc59d9b911529de75c466593252358581f5716b3b58dc2e6b9b9282c50b1b66b.json b/backend/.sqlx/query-fc59d9b911529de75c466593252358581f5716b3b58dc2e6b9b9282c50b1b66b.json new file mode 100644 index 0000000000..1efaaa32e8 --- /dev/null +++ b/backend/.sqlx/query-fc59d9b911529de75c466593252358581f5716b3b58dc2e6b9b9282c50b1b66b.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job SET labels = (\n SELECT array_agg(DISTINCT l)\n FROM unnest(coalesce(labels, ARRAY[]::TEXT[]) || $2) l\n ) WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "fc59d9b911529de75c466593252358581f5716b3b58dc2e6b9b9282c50b1b66b" +} diff --git a/backend/.sqlx/query-fcb0cf55f5c9047066a6fdd62bf6f85238cbf189ccf18191493d6357e269d12d.json b/backend/.sqlx/query-fcb0cf55f5c9047066a6fdd62bf6f85238cbf189ccf18191493d6357e269d12d.json new file mode 100644 index 0000000000..c630f5dd05 --- /dev/null +++ b/backend/.sqlx/query-fcb0cf55f5c9047066a6fdd62bf6f85238cbf189ccf18191493d6357e269d12d.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT timestamp, name, code_up FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 ORDER BY timestamp ASC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "timestamp", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "code_up", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "fcb0cf55f5c9047066a6fdd62bf6f85238cbf189ccf18191493d6357e269d12d" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index b2acfc2e66..177509c02e 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -14473,6 +14473,7 @@ dependencies = [ "sqlx", "strum", "tokio", + "tokio-postgres", "tracing", "uuid", "windmill-api-auth", diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 087459205e..1b702a3a54 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -0de2412ff0734b11e12ba378c9bcc373ff9ae800 +27672e37df5d9dfde94f19963d5ffcdf8dd5448c diff --git a/backend/migrations/20260617081932_datatable_migrations.down.sql b/backend/migrations/20260617081932_datatable_migrations.down.sql new file mode 100644 index 0000000000..26efc2bdcc --- /dev/null +++ b/backend/migrations/20260617081932_datatable_migrations.down.sql @@ -0,0 +1 @@ +DROP TABLE datatable_migrations; diff --git a/backend/migrations/20260617081932_datatable_migrations.up.sql b/backend/migrations/20260617081932_datatable_migrations.up.sql new file mode 100644 index 0000000000..72c7dc5ec6 --- /dev/null +++ b/backend/migrations/20260617081932_datatable_migrations.up.sql @@ -0,0 +1,21 @@ +-- SQL migrations defined per data table within a workspace. +-- `datatable` is the target data table name, `name` is the migration name +-- (e.g. add_index_to_customers), and `timestamp` is the migration version +-- (YYYYMMDDHHMMSS), recorded as `version` in the data table's `_wm_migrations` +-- table once applied. +CREATE TABLE datatable_migrations ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + datatable VARCHAR(255) NOT NULL, + timestamp BIGINT NOT NULL, + name VARCHAR(255) NOT NULL, + code_up TEXT NOT NULL, + code_down TEXT, + PRIMARY KEY (workspace_id, datatable, timestamp) +); + +-- No standalone index: the (workspace_id, datatable, timestamp) primary-key btree +-- already serves both `WHERE workspace_id = $1` and `WHERE workspace_id = $1 AND +-- datatable = $2` lookups via its leading columns. + +GRANT ALL ON datatable_migrations TO windmill_user; +GRANT ALL ON datatable_migrations TO windmill_admin; diff --git a/backend/windmill-api-workspaces/Cargo.toml b/backend/windmill-api-workspaces/Cargo.toml index c25bb8f42b..52a58aa1a3 100644 --- a/backend/windmill-api-workspaces/Cargo.toml +++ b/backend/windmill-api-workspaces/Cargo.toml @@ -47,6 +47,7 @@ serde_json.workspace = true sha2.workspace = true sqlx.workspace = true tokio.workspace = true +tokio-postgres.workspace = true tracing.workspace = true uuid.workspace = true strum.workspace = true diff --git a/backend/windmill-api-workspaces/src/datatable_migrations.rs b/backend/windmill-api-workspaces/src/datatable_migrations.rs new file mode 100644 index 0000000000..fef9963933 --- /dev/null +++ b/backend/windmill-api-workspaces/src/datatable_migrations.rs @@ -0,0 +1,1712 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Data table SQL migrations: CRUD endpoints, run/rollback execution, opt-in +//! management, and the workspace-merge diff helper. Split out of `workspaces.rs` +//! to keep that file focused on core workspace configuration. + +use crate::workspaces::{pg_dump_database, ItemComparison}; + +use axum::{ + extract::{Extension, Path, Query}, + routing::{delete, get, post}, + Json, Router, +}; +use chrono::Utc; +use serde::{Deserialize, Serialize}; +use sqlx::{Postgres, Transaction}; +use std::collections::{HashMap, HashSet}; + +use windmill_api_auth::{require_super_admin, ApiAuthed}; +use windmill_api_jobs::run_wait_result_internal; +use windmill_audit::audit_oss::audit_log; +use windmill_audit::ActionKind; +use windmill_common::db::UserDB; +use windmill_common::error::{Error, JsonResult, Result}; +use windmill_common::jobs::{JobPayload, RawCode}; +use windmill_common::runnable_settings::{ConcurrencySettingsWithCustom, DebouncingSettings}; +use windmill_common::scripts::ScriptLang; +use windmill_common::users::username_to_permissioned_as; +use windmill_common::worker::to_raw_value; +use windmill_common::workspaces::get_datatable_resource_from_db_unchecked; +use windmill_common::{PgDatabase, DB}; +use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; +use windmill_queue::{push, PushArgs, PushIsolationLevel}; + +pub(crate) fn routes() -> Router { + Router::new() + .route( + "/run_datatable_migrations/{datatable_name}", + post(run_datatable_migrations), + ) + .route( + "/rollback_datatable_migrations/{datatable_name}", + post(rollback_datatable_migrations), + ) + .route("/list_datatable_migrations", get(list_datatable_migrations)) + .route( + "/datatable_migrations_status/{datatable_name}", + get(datatable_migrations_status), + ) + .route( + "/enable_datatable_migrations/{datatable_name}", + post(enable_datatable_migrations), + ) + .route( + "/disable_datatable_migrations/{datatable_name}", + post(disable_datatable_migrations), + ) + .route( + "/create_datatable_migration/{datatable_name}", + post(create_datatable_migration), + ) + .route( + "/delete_datatable_migration/{datatable_name}/{timestamp}", + delete(delete_datatable_migration), + ) + .route( + "/upsert_datatable_migration/{datatable_name}", + post(upsert_datatable_migration), + ) + .route( + "/generate_initial_datatable_migration/{datatable_name}", + post(generate_initial_datatable_migration), + ) +} + +#[derive(Serialize)] +struct AppliedMigration { + version: i64, + name: String, +} + +#[derive(Serialize)] +struct RunDatatableMigrationsResult { + applied: Vec, +} + +#[derive(Deserialize)] +struct RunDatatableMigrationsQuery { + /// When set, only apply pending migrations up to and including this version. + up_to: Option, + /// When set, apply only this specific migration version (if not already + /// applied), ignoring any other pending migrations. Takes precedence over + /// `up_to`. + only: Option, +} + +/// Build the `database` argument for a migration job. Both resource-backed and +/// instance data tables pass a `datatable://` reference; the pg executor +/// resolves it to real credentials server-side at run time. It must never be +/// resolved here: the resolved instance credentials include a single +/// instance-wide Postgres password, and the job's `args` are readable by the — +/// possibly non-admin — user who ran the migration. +async fn datatable_database_arg( + db: &DB, + w_id: &str, + datatable_name: &str, +) -> Result> { + // Fail fast with a clear error if the data table doesn't exist. + sqlx::query_scalar!( + "SELECT ws.datatable->'datatables'->$2 FROM workspace_settings ws WHERE ws.workspace_id = $1", + w_id, + datatable_name, + ) + .fetch_one(db) + .await? + .ok_or_else(|| Error::internal_err(format!("datatable {datatable_name} not found")))?; + + Ok(to_raw_value(&format!("datatable://{datatable_name}"))) +} + +/// Run a migration's SQL as a normal Windmill `postgresql` job, permissioned as +/// the requesting user and labelled `datatable_migration` for traceability, then +/// wait for it. Errors if the job fails. +async fn run_datatable_migration_job( + db: &DB, + user_db: &UserDB, + authed: &ApiAuthed, + w_id: &str, + database_arg: &Box, + sql: &str, +) -> Result<()> { + let mut args = HashMap::new(); + args.insert("database".to_string(), database_arg.clone()); + let push_args = PushArgs { extra: None, args: &args }; + + let (uuid, mut tx) = push( + db, + PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()), + w_id, + JobPayload::Code(RawCode { + content: sql.to_string(), + path: Some("datatable_migration".to_string()), + hash: None, + language: ScriptLang::Postgresql, + lock: None, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + tag: None, + concurrency_settings: ConcurrencySettingsWithCustom::default(), + debouncing_settings: DebouncingSettings::default(), + modules: None, + }), + push_args, + authed.display_username(), + &authed.email, + username_to_permissioned_as(&authed.username), + authed.token_prefix.as_deref(), + None, + None, + None, + None, + None, + None, + false, + false, + None, + true, + None, + None, + None, + None, + Some(&authed.clone().into()), + false, + None, + None, + None, + ) + .await?; + + // Tag the job so migration runs are easy to find in the run history. + sqlx::query!( + "UPDATE v2_job SET labels = ( + SELECT array_agg(DISTINCT l) + FROM unnest(coalesce(labels, ARRAY[]::TEXT[]) || $2) l + ) WHERE id = $1", + uuid, + &vec!["datatable_migration".to_string()], + ) + .execute(&mut *tx) + .await?; + tx.commit().await?; + + let (result, success) = + run_wait_result_internal(db, uuid, w_id, None, false, &authed.username).await?; + if !success { + // On failure the job result is `{"error": {"name", "message", ...}}`; + // surface the executor's message (the Postgres error, e.g. `relation + // "foo" does not exist`) instead of the raw JSON envelope. + let detail = serde_json::from_str::(result.get()) + .ok() + .as_ref() + .and_then(|v| v.get("error")) + .and_then(|e| e.get("message")) + .and_then(|m| m.as_str()) + .map(str::to_string) + .unwrap_or_else(|| result.get().to_string()); + return Err(Error::internal_err(detail)); + } + Ok(()) +} + +/// Ensure the `_wm_migrations` bookkeeping table exists. Migration versions are +/// only unique per data table, but several data-table configs can point at one +/// physical database, so it is keyed by `(datatable, version)` — a version-only +/// key would let one data table's migration mark another's same-version +/// migration as already applied (and rollback could touch the wrong row). +async fn ensure_wm_migrations_schema(client: &tokio_postgres::Client) -> Result<()> { + client + .batch_execute( + "CREATE TABLE IF NOT EXISTS _wm_migrations (\ + datatable TEXT NOT NULL, \ + version BIGINT NOT NULL, \ + installed_at TIMESTAMPTZ NOT NULL DEFAULT now(), \ + PRIMARY KEY (datatable, version))", + ) + .await + .map_err(|e| { + Error::internal_err(format!("Failed to ensure _wm_migrations table: {}", e)) + })?; + Ok(()) +} + +/// Open a connection to a data table's own database and hold the session-level +/// advisory lock that serializes migration runs/rollbacks. The lock is released +/// when the returned client is dropped, so callers must keep it in scope for the +/// whole critical section. +/// +/// Runs, rollbacks *and* definition rewrites/deletes all take this lock: a run +/// snapshots a migration's `code_up` from `datatable_migrations` and only records +/// its version in `_wm_migrations` after the job succeeds, so an unserialized edit +/// could rewrite the definition in that window and leave `_wm_migrations` pointing +/// at SQL that was never applied. `_wm_migrations` is per-database, so a single +/// key is sufficient. +async fn lock_datatable_migration_runs( + db: &DB, + w_id: &str, + datatable_name: &str, +) -> Result { + let db_resource = get_datatable_resource_from_db_unchecked(db, w_id, datatable_name).await?; + let pg_db: PgDatabase = serde_json::from_value(db_resource) + .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?; + let (client, connection) = pg_db.connect(Some(db)).await?; + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::error!("Datatable connection error: {}", e); + } + }); + client + .batch_execute("SELECT pg_advisory_lock(hashtext('windmill_datatable_migrations')::int8)") + .await + .map_err(|e| Error::internal_err(format!("Failed to acquire migration lock: {}", e)))?; + Ok(client) +} + +/// Read the versions recorded as applied in a data table's `_wm_migrations`, +/// scoped to that data table, using an existing connection. An absent table +/// (`42P01`) means nothing has been migrated yet. +async fn read_applied_versions_on_client( + client: &tokio_postgres::Client, + datatable_name: &str, +) -> Result> { + match client + .query( + "SELECT version FROM _wm_migrations WHERE datatable = $1", + &[&datatable_name], + ) + .await + { + Ok(rows) => Ok(rows.iter().map(|row| row.get::<_, i64>(0)).collect()), + Err(e) if e.as_db_error().map(|d| d.code().code()) == Some("42P01") => Ok(HashSet::new()), + Err(e) => Err(Error::internal_err(format!( + "Failed to read _wm_migrations: {}", + e + ))), + } +} + +/// Apply the workspace's pending data table migrations to a given data table. +/// Each migration runs as a normal Windmill `postgresql` job (permissioned as +/// the requester, labelled `datatable_migration`); applied versions are then +/// recorded in the data table's own `_wm_migrations` table, so only migrations +/// not recorded there are run, in ascending `timestamp` order. +async fn run_datatable_migrations( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, + Query(query): Query, +) -> JsonResult { + audit_log( + &db, + &authed, + "workspaces.run_datatable_migrations", + ActionKind::Update, + &w_id, + Some(datatable_name.as_str()), + None, + ) + .await?; + + let database_arg = datatable_database_arg(&db, &w_id, &datatable_name).await?; + + // Take the run-serialization lock before snapshotting the migration + // definitions: a concurrent definition rewrite/delete takes the same lock, so + // the `code_up` we read here can't change between now and when we record its + // version below. The lock is held until `client` drops at return. + let client = lock_datatable_migration_runs(&db, &w_id, &datatable_name).await?; + + let migrations = sqlx::query!( + "SELECT timestamp, name, code_up FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 ORDER BY timestamp ASC", + &w_id, + &datatable_name, + ) + .fetch_all(&db) + .await?; + + ensure_wm_migrations_schema(&client).await?; + + let applied_versions = read_applied_versions_on_client(&client, &datatable_name).await?; + + let mut applied = Vec::new(); + for m in migrations { + if let Some(only) = query.only { + // Run a single specific migration, skipping every other one. + if m.timestamp != only { + continue; + } + } else if query.up_to.is_some_and(|up_to| m.timestamp > up_to) { + // Migrations are ordered ascending, so once we pass `up_to` we're done. + break; + } + if applied_versions.contains(&m.timestamp) { + continue; + } + run_datatable_migration_job(&db, &user_db, &authed, &w_id, &database_arg, &m.code_up) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to apply migration {} ({}): {}", + m.timestamp, m.name, e + )) + })?; + // Record the migration as installed once its job has succeeded. + client + .execute( + "INSERT INTO _wm_migrations (datatable, version) VALUES ($1, $2) \ + ON CONFLICT (datatable, version) DO NOTHING", + &[&datatable_name, &m.timestamp], + ) + .await + .map_err(|e| Error::internal_err(format!("Failed to record migration: {}", e)))?; + applied.push(AppliedMigration { version: m.timestamp, name: m.name }); + } + + Ok(Json(RunDatatableMigrationsResult { applied })) +} + +#[derive(Serialize)] +struct RolledBackMigration { + version: i64, + name: String, +} + +#[derive(Serialize)] +struct RollbackDatatableMigrationsResult { + rolled_back: Vec, +} + +#[derive(Deserialize)] +struct RollbackDatatableMigrationsQuery { + /// When set, roll back this specific applied migration version instead of + /// the most recently applied one. + only: Option, +} + +/// Roll back a migration on a given data table: run its `code_down` as a normal +/// Windmill `postgresql` job (permissioned as the requester, labelled +/// `datatable_migration`) then drop its `_wm_migrations` row. Without `only` this +/// targets the most recently applied migration (one step); with `only` it +/// targets that specific applied version. +async fn rollback_datatable_migrations( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, + Query(query): Query, +) -> JsonResult { + audit_log( + &db, + &authed, + "workspaces.rollback_datatable_migrations", + ActionKind::Update, + &w_id, + Some(datatable_name.as_str()), + None, + ) + .await?; + + // The data table's `_wm_migrations` bookkeeping is read here and the version + // dropped after the job succeeds; the down SQL itself runs in the job. The + // lock is held (until `client` drops at return) so a concurrent run or + // definition rewrite can't interleave with this rollback. + let client = lock_datatable_migration_runs(&db, &w_id, &datatable_name).await?; + + ensure_wm_migrations_schema(&client).await?; + + // Resolve which applied version to roll back: a specific one when `only` is + // given (and actually applied), otherwise the most recently applied. Scoped + // to this data table so a shared physical database can't surface another + // data table's version. + let target = match query.only { + Some(only) => client + .query_opt( + "SELECT version FROM _wm_migrations WHERE datatable = $1 AND version = $2", + &[&datatable_name, &only], + ) + .await + .map_err(|e| Error::internal_err(format!("Failed to read _wm_migrations: {}", e)))?, + None => client + .query_opt( + "SELECT version FROM _wm_migrations WHERE datatable = $1 \ + ORDER BY version DESC LIMIT 1", + &[&datatable_name], + ) + .await + .map_err(|e| Error::internal_err(format!("Failed to read _wm_migrations: {}", e)))?, + }; + + let version: i64 = match target { + Some(row) => row.get::<_, i64>(0), + None => { + return Ok(Json(RollbackDatatableMigrationsResult { + rolled_back: vec![], + })) + } + }; + + let definition = sqlx::query!( + "SELECT name, code_down FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + &w_id, + &datatable_name, + version + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| { + Error::BadRequest(format!( + "Cannot roll back migration {version}: its definition no longer exists" + )) + })?; + + let code_down = definition.code_down.ok_or_else(|| { + Error::BadRequest(format!( + "Cannot roll back migration {} ({}): it has no down migration", + version, definition.name + )) + })?; + + let database_arg = datatable_database_arg(&db, &w_id, &datatable_name).await?; + run_datatable_migration_job(&db, &user_db, &authed, &w_id, &database_arg, &code_down) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to roll back migration {} ({}): {}", + version, definition.name, e + )) + })?; + + // Forget the version once its down job has succeeded. + client + .execute( + "DELETE FROM _wm_migrations WHERE datatable = $1 AND version = $2", + &[&datatable_name, &version], + ) + .await + .map_err(|e| Error::internal_err(format!("Failed to drop migration record: {}", e)))?; + + Ok(Json(RollbackDatatableMigrationsResult { + rolled_back: vec![RolledBackMigration { version, name: definition.name }], + })) +} + +#[derive(Serialize, Deserialize)] +pub struct DatatableMigration { + pub datatable: String, + pub timestamp: i64, + pub name: String, + pub code_up: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub code_down: Option, +} + +async fn list_datatable_migrations( + _authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult> { + let migrations = sqlx::query_as!( + DatatableMigration, + "SELECT datatable, timestamp, name, code_up, code_down FROM datatable_migrations \ + WHERE workspace_id = $1 ORDER BY datatable, timestamp ASC", + &w_id + ) + .fetch_all(&db) + .await?; + + Ok(Json(migrations)) +} + +#[derive(Serialize)] +#[serde(rename_all = "snake_case")] +enum DatatableMigrationRunStatus { + /// Recorded in the data table's `_wm_migrations` table. + Ran, + /// Defined but not yet applied. + NotRun, + /// Applied status could not be determined (connection failure). + Unknown, +} + +#[derive(Serialize)] +struct DatatableMigrationWithStatus { + timestamp: i64, + name: String, + code_up: String, + #[serde(skip_serializing_if = "Option::is_none")] + code_down: Option, + status: DatatableMigrationRunStatus, +} + +#[derive(Serialize)] +struct DatatableMigrationsStatusResult { + /// Whether the migrations feature is opted in for this data table. + enabled: bool, + migrations: Vec, + /// Set when the applied status couldn't be read from the data table. + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, +} + +/// Whether the SQL-migrations feature is enabled for a data table. Honors the +/// explicit `migrations_enabled` flag; when unset (data tables predating the +/// feature) it is considered enabled only if migrations already exist. +async fn datatable_migrations_enabled(db: &DB, w_id: &str, datatable_name: &str) -> Result { + let flag: Option = sqlx::query_scalar!( + "SELECT (ws.datatable->'datatables'->$2->>'migrations_enabled')::boolean \ + FROM workspace_settings ws WHERE ws.workspace_id = $1", + w_id, + datatable_name, + ) + .fetch_optional(db) + .await? + .flatten(); + + match flag { + Some(v) => Ok(v), + None => Ok(sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2)", + w_id, + datatable_name, + ) + .fetch_one(db) + .await? + .unwrap_or(false)), + } +} + +/// Reject the request when migrations are not enabled for the data table. +async fn ensure_datatable_migrations_enabled( + db: &DB, + w_id: &str, + datatable_name: &str, +) -> Result<()> { + if !datatable_migrations_enabled(db, w_id, datatable_name).await? { + return Err(Error::BadRequest(format!( + "Migrations are not enabled for data table '{}'. Enable them first.", + datatable_name + ))); + } + Ok(()) +} + +/// Read the versions recorded in a data table's `_wm_migrations` table. A +/// missing table means nothing has been applied yet (empty set, not an error). +async fn read_applied_datatable_versions( + db: &DB, + w_id: &str, + datatable_name: &str, +) -> Result> { + let db_resource = get_datatable_resource_from_db_unchecked(db, w_id, datatable_name).await?; + let pg_db: PgDatabase = serde_json::from_value(db_resource) + .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?; + let (client, connection) = pg_db.connect(Some(db)).await?; + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::error!("Datatable connection error: {}", e); + } + }); + + // Read-only status path: don't create the table here, and don't take the run + // lock — a stale-by-a-moment applied set is fine for display. + read_applied_versions_on_client(&client, datatable_name).await +} + +/// List a data table's migrations annotated with whether each has been applied. +async fn datatable_migrations_status( + _authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, +) -> JsonResult { + let enabled = datatable_migrations_enabled(&db, &w_id, &datatable_name).await?; + if !enabled { + return Ok(Json(DatatableMigrationsStatusResult { + enabled: false, + migrations: vec![], + error: None, + })); + } + + let defs = sqlx::query!( + "SELECT timestamp, name, code_up, code_down FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 ORDER BY timestamp ASC", + &w_id, + &datatable_name, + ) + .fetch_all(&db) + .await?; + + let (applied, error) = match read_applied_datatable_versions(&db, &w_id, &datatable_name).await + { + Ok(set) => (Some(set), None), + Err(e) => (None, Some(e.to_string())), + }; + + let migrations = defs + .into_iter() + .map(|m| { + let status = match &applied { + Some(set) if set.contains(&m.timestamp) => DatatableMigrationRunStatus::Ran, + Some(_) => DatatableMigrationRunStatus::NotRun, + None => DatatableMigrationRunStatus::Unknown, + }; + DatatableMigrationWithStatus { + timestamp: m.timestamp, + name: m.name, + code_up: m.code_up, + code_down: m.code_down, + status, + } + }) + .collect(); + + Ok(Json(DatatableMigrationsStatusResult { + enabled: true, + migrations, + error, + })) +} + +/// Only workspace admins and super admins may opt a data table in or out of +/// migrations. +async fn require_datatable_migrations_manager(db: &DB, authed: &ApiAuthed) -> Result<()> { + if authed.is_admin || require_super_admin(db, &authed.email).await.is_ok() { + Ok(()) + } else { + Err(Error::BadRequest( + "Only workspace admins and super admins can enable or disable data table migrations" + .to_string(), + )) + } +} + +async fn enable_datatable_migrations( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, +) -> Result { + require_datatable_migrations_manager(&db, &authed).await?; + + let updated = sqlx::query_scalar!( + "UPDATE workspace_settings \ + SET datatable = jsonb_set(datatable, ARRAY['datatables', $2::text, 'migrations_enabled'], 'true'::jsonb) \ + WHERE workspace_id = $1 AND jsonb_exists(datatable->'datatables', $2) \ + RETURNING 1", + &w_id, + &datatable_name, + ) + .fetch_optional(&db) + .await?; + if updated.is_none() { + return Err(Error::NotFound(format!( + "data table {datatable_name} not found" + ))); + } + + audit_log( + &db, + &authed, + "workspaces.enable_datatable_migrations", + ActionKind::Update, + &w_id, + Some(datatable_name.as_str()), + None, + ) + .await?; + + Ok(format!( + "Enabled migrations for data table {datatable_name}" + )) +} + +/// Opt a data table out of migrations. Deletes ALL of its migration definitions. +async fn disable_datatable_migrations( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, +) -> Result { + require_datatable_migrations_manager(&db, &authed).await?; + + let mut tx = db.begin().await?; + + let updated = sqlx::query_scalar!( + "UPDATE workspace_settings \ + SET datatable = jsonb_set(datatable, ARRAY['datatables', $2::text, 'migrations_enabled'], 'false'::jsonb) \ + WHERE workspace_id = $1 AND jsonb_exists(datatable->'datatables', $2) \ + RETURNING 1", + &w_id, + &datatable_name, + ) + .fetch_optional(&mut *tx) + .await?; + if updated.is_none() { + return Err(Error::NotFound(format!( + "data table {datatable_name} not found" + ))); + } + + // Capture the deleted definitions so each removal is tallied as a deployed + // object (like single-migration deletion), keeping workspace comparison and + // git-sync callbacks in sync when a fork opts back out of migrations. + let deleted = sqlx::query!( + "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 \ + RETURNING timestamp, name", + &w_id, + &datatable_name, + ) + .fetch_all(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "workspaces.disable_datatable_migrations", + ActionKind::Delete, + &w_id, + Some(datatable_name.as_str()), + None, + ) + .await?; + + tx.commit().await?; + + for m in deleted { + record_datatable_migration_deployment( + &authed, + &db, + &w_id, + &datatable_name, + m.timestamp, + &m.name, + ) + .await?; + } + + Ok(format!( + "Disabled migrations for data table {datatable_name} and deleted its migrations" + )) +} + +#[derive(Deserialize)] +pub struct CreateDatatableMigration { + pub name: String, + pub code_up: String, + #[serde(default)] + pub code_down: Option, +} + +/// Migration names map onto on-disk file names and the `_wm_migrations` record, +/// so keep them to a safe path-segment charset (matches the CLI scaffold). +fn validate_migration_name(name: &str) -> Result<()> { + if name.is_empty() + || !name + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') + { + return Err(Error::BadRequest(format!( + "Invalid migration name '{name}': use only letters, digits, '_' and '-'" + ))); + } + Ok(()) +} + +/// The data table name becomes a directory segment in the sync export +/// (`migrations/datatable//...`); reject anything that could escape it. +pub(crate) fn validate_datatable_path_segment(datatable: &str) -> Result<()> { + if datatable.is_empty() + || datatable.contains('/') + || datatable.contains('\\') + || datatable.contains("..") + { + return Err(Error::BadRequest(format!( + "Invalid data table name '{datatable}': must not contain '/', '\\' or '..'" + ))); + } + Ok(()) +} + +/// Record a data table migration change as a deployed object so it is tallied +/// into `workspace_diff` and shows up as a `datatable_migration` item in the +/// workspace-merge diff. The diff path is `/_`, +/// matching `parse_datatable_migration_diff_path`. +async fn record_datatable_migration_deployment( + authed: &ApiAuthed, + db: &DB, + w_id: &str, + datatable: &str, + timestamp: i64, + name: &str, +) -> Result<()> { + handle_deployment_metadata( + &authed.email, + &authed.username, + db, + w_id, + DeployedObject::DatatableMigration { path: format!("{datatable}/{timestamp}_{name}") }, + Some(format!( + "Data table migration {name} ({timestamp}) on {datatable}" + )), + false, + None, + ) + .await +} + +/// Allocate the next version for a data table and insert the migration +/// definition, in one transaction. A per-(workspace, data table) advisory lock +/// serializes concurrent version allocation so two creates can't read the same +/// `MAX(timestamp)` and collide on the `(workspace_id, datatable, timestamp)` +/// primary key. The version is the current UTC `YYYYMMDDHHMMSS`, bumped past any +/// existing version to stay unique and monotonically increasing. +async fn insert_datatable_migration_def( + tx: &mut Transaction<'_, Postgres>, + w_id: &str, + datatable: &str, + name: &str, + code_up: &str, + code_down: Option<&str>, +) -> Result { + sqlx::query!( + "SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))", + w_id, + datatable, + ) + .execute(&mut **tx) + .await?; + + let now_ts: i64 = Utc::now() + .format("%Y%m%d%H%M%S") + .to_string() + .parse() + .map_err(|e| Error::internal_err(format!("Failed to build migration version: {}", e)))?; + let max_existing: Option = sqlx::query_scalar!( + "SELECT MAX(timestamp) FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2", + w_id, + datatable, + ) + .fetch_one(&mut **tx) + .await?; + let timestamp = match max_existing { + Some(m) if m >= now_ts => m + 1, + _ => now_ts, + }; + + sqlx::query!( + "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) \ + VALUES ($1, $2, $3, $4, $5, $6)", + w_id, + datatable, + timestamp, + name, + code_up, + code_down, + ) + .execute(&mut **tx) + .await?; + + Ok(timestamp) +} + +/// Mark a version as already installed in a data table's `_wm_migrations` table +/// (ensuring the table exists first). +async fn mark_datatable_version_installed( + db: &DB, + pg_db: &PgDatabase, + datatable: &str, + version: i64, +) -> Result<()> { + let (client, connection) = pg_db.connect(Some(db)).await?; + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::error!("Datatable connection error: {}", e); + } + }); + ensure_wm_migrations_schema(&client).await?; + client + .execute( + "INSERT INTO _wm_migrations (datatable, version) VALUES ($1, $2) \ + ON CONFLICT (datatable, version) DO NOTHING", + &[&datatable, &version], + ) + .await + .map_err(|e| { + Error::internal_err(format!("Failed to mark initial migration installed: {}", e)) + })?; + Ok(()) +} + +/// Create a single migration for a data table. The version is generated +/// server-side (current UTC `YYYYMMDDHHMMSS`), bumped past any existing version +/// so it stays unique and monotonically increasing. +async fn create_datatable_migration( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, + Json(payload): Json, +) -> JsonResult { + validate_datatable_path_segment(&datatable_name)?; + validate_migration_name(&payload.name)?; + ensure_datatable_migrations_enabled(&db, &w_id, &datatable_name).await?; + + let mut tx = db.begin().await?; + let timestamp = insert_datatable_migration_def( + &mut tx, + &w_id, + &datatable_name, + &payload.name, + &payload.code_up, + payload.code_down.as_deref(), + ) + .await?; + tx.commit().await?; + + audit_log( + &db, + &authed, + "workspaces.create_datatable_migration", + ActionKind::Create, + &w_id, + Some(datatable_name.as_str()), + None, + ) + .await?; + + record_datatable_migration_deployment( + &authed, + &db, + &w_id, + &datatable_name, + timestamp, + &payload.name, + ) + .await?; + + Ok(Json(DatatableMigration { + datatable: datatable_name, + timestamp, + name: payload.name, + code_up: payload.code_up, + code_down: payload.code_down, + })) +} + +/// Delete a single migration definition from a data table. +async fn delete_datatable_migration( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name, timestamp)): Path<(String, String, i64)>, +) -> Result { + // Hold the run-serialization lock across the applied-check and the delete: a + // run snapshots a migration's SQL before recording its version, so an + // unserialized delete could race it and leave `_wm_migrations` pointing at a + // definition that no longer exists (breaking rollback and hiding the applied + // version). Held until the handler returns. Fail closed if we can't verify. + let unreachable = |e| { + Error::internal_err(format!( + "Cannot verify whether migration {} on data table '{}' has already been applied \ + (its database is unreachable: {}). Refusing to delete it; retry once the database \ + is reachable.", + timestamp, datatable_name, e + )) + }; + let lock_client = lock_datatable_migration_runs(&db, &w_id, &datatable_name) + .await + .map_err(unreachable)?; + let applied = read_applied_versions_on_client(&lock_client, &datatable_name) + .await + .map_err(unreachable)?; + if applied.contains(×tamp) { + return Err(Error::BadRequest(format!( + "Migration {} on data table '{}' has already been applied and cannot be deleted. \ + Revert it first.", + timestamp, datatable_name + ))); + } + + let deleted_name = sqlx::query_scalar!( + "DELETE FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3 \ + RETURNING name", + &w_id, + &datatable_name, + timestamp, + ) + .fetch_optional(&db) + .await?; + // The definition is removed; runs may resume (audit/deploy metadata below + // don't need the lock). + drop(lock_client); + + audit_log( + &db, + &authed, + "workspaces.delete_datatable_migration", + ActionKind::Delete, + &w_id, + Some(datatable_name.as_str()), + None, + ) + .await?; + + // Only tally a change if a migration was actually deleted. + if let Some(name) = deleted_name { + record_datatable_migration_deployment( + &authed, + &db, + &w_id, + &datatable_name, + timestamp, + &name, + ) + .await?; + } + + Ok(format!( + "Deleted migration {} from {}", + timestamp, datatable_name + )) +} + +#[derive(Deserialize)] +pub struct UpsertDatatableMigration { + pub timestamp: i64, + pub name: String, + pub code_up: String, + #[serde(default)] + pub code_down: Option, +} + +/// Insert or update a single migration at an explicit version. Used by +/// `wmill sync` to push a `migrations/datatable/
/_.up.sql` +/// (and `.down.sql`) file as the source of truth. +async fn upsert_datatable_migration( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, + Json(payload): Json, +) -> Result { + validate_datatable_path_segment(&datatable_name)?; + validate_migration_name(&payload.name)?; + ensure_datatable_migrations_enabled(&db, &w_id, &datatable_name).await?; + + // Guard against silently rewriting a migration that has already run in the + // data table's database: its `_wm_migrations` record would no longer match + // its SQL, so a later `migrate up` would skip it and a rollback would run a + // `down` that doesn't correspond to what was applied. Only an actual change + // to an existing migration is guarded; unchanged re-pushes (e.g. + // `wmill sync push`) always proceed. + let existing = sqlx::query!( + "SELECT name, code_up, code_down FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + &w_id, + &datatable_name, + payload.timestamp, + ) + .fetch_optional(&db) + .await?; + // When modifying an existing definition, hold the run-serialization lock + // across the applied-check and the write below so an in-flight run can't + // record a version for the SQL we're about to overwrite. Held until the end + // of the handler (well past the write); a new/unchanged upsert needs no lock. + let _run_lock = match existing { + Some(existing) + if !(existing.name == payload.name + && existing.code_up == payload.code_up + && existing.code_down == payload.code_down) => + { + // Fail closed: if we can't lock/read the applied set (e.g. the + // data-table database is temporarily unreachable), refuse the change + // rather than risk overwriting a migration that has already run. + let unreachable = |e| { + Error::internal_err(format!( + "Cannot verify whether migration {} on data table '{}' has already been \ + applied (its database is unreachable: {}). Refusing to modify it; retry \ + once the database is reachable.", + payload.timestamp, datatable_name, e + )) + }; + let client = lock_datatable_migration_runs(&db, &w_id, &datatable_name) + .await + .map_err(unreachable)?; + let applied = read_applied_versions_on_client(&client, &datatable_name) + .await + .map_err(unreachable)?; + if applied.contains(&payload.timestamp) { + return Err(Error::BadRequest(format!( + "Migration {} on data table '{}' has already been applied and cannot be modified. \ + Revert it first, or add a new migration instead.", + payload.timestamp, datatable_name + ))); + } + Some(client) + } + _ => None, + }; + + sqlx::query!( + "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) \ + VALUES ($1, $2, $3, $4, $5, $6) \ + ON CONFLICT (workspace_id, datatable, timestamp) DO UPDATE \ + SET name = EXCLUDED.name, code_up = EXCLUDED.code_up, code_down = EXCLUDED.code_down", + &w_id, + &datatable_name, + payload.timestamp, + &payload.name, + &payload.code_up, + payload.code_down.as_deref(), + ) + .execute(&db) + .await?; + // The definition is written; runs may resume (audit/deploy metadata below + // don't need the lock). + drop(_run_lock); + + audit_log( + &db, + &authed, + "workspaces.upsert_datatable_migration", + ActionKind::Update, + &w_id, + Some(datatable_name.as_str()), + None, + ) + .await?; + + record_datatable_migration_deployment( + &authed, + &db, + &w_id, + &datatable_name, + payload.timestamp, + &payload.name, + ) + .await?; + + Ok(format!( + "Upserted migration {} in {}", + payload.timestamp, datatable_name + )) +} + +/// Generate the first migration for a data table by snapshotting its current +/// schema with `pg_dump`. The migration is recorded as already installed (the +/// definition is written first, then its version is marked in the data table's +/// `_wm_migrations`, so it ends up considered applied and is never re-run) and +/// has no down migration. +async fn generate_initial_datatable_migration( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, +) -> JsonResult { + validate_datatable_path_segment(&datatable_name)?; + ensure_datatable_migrations_enabled(&db, &w_id, &datatable_name).await?; + + // The initial snapshot only makes sense on a data table with no migrations + // yet; reject otherwise so repeated calls don't pile up duplicate "initial" + // definitions (each at a distinct timestamp, each marked installed). + let has_migrations: bool = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2)", + &w_id, + &datatable_name, + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + if has_migrations { + return Err(Error::BadRequest(format!( + "Data table '{datatable_name}' already has migrations; the initial migration can only be generated when there are none." + ))); + } + + let db_resource = get_datatable_resource_from_db_unchecked(&db, &w_id, &datatable_name).await?; + let pg_db: PgDatabase = serde_json::from_value(db_resource) + .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?; + + // Snapshot the schema, excluding Windmill's own migration bookkeeping table. + let dump_file = pg_dump_database(&pg_db, true, &["_wm_migrations"]).await?; + let raw_dump = tokio::fs::read_to_string(&dump_file.path) + .await + .map_err(|e| Error::internal_err(format!("Failed to read schema dump: {}", e)))?; + // pg_dump emits psql meta-commands (\restrict / \unrestrict) that aren't + // valid SQL; drop them so the migration body can run via a plain query. + let code_up: String = raw_dump + .lines() + .filter(|line| !line.trim_start().starts_with('\\')) + .collect::>() + .join("\n"); + + // Record the definition first, then mark it installed. If marking fails we + // delete the definition, so a failure leaves no phantom "initial" (rather + // than a `_wm_migrations` version with no definition that the UI can't + // clear). The narrow window where it briefly shows "not run" is benign: + // running it would just no-op/fail harmlessly against the existing schema. + let mut tx = db.begin().await?; + let timestamp = + insert_datatable_migration_def(&mut tx, &w_id, &datatable_name, "initial", &code_up, None) + .await?; + tx.commit().await?; + + if let Err(e) = mark_datatable_version_installed(&db, &pg_db, &datatable_name, timestamp).await + { + let _ = sqlx::query!( + "DELETE FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + &w_id, + &datatable_name, + timestamp, + ) + .execute(&db) + .await; + return Err(e); + } + + audit_log( + &db, + &authed, + "workspaces.generate_initial_datatable_migration", + ActionKind::Create, + &w_id, + Some(datatable_name.as_str()), + None, + ) + .await?; + + record_datatable_migration_deployment( + &authed, + &db, + &w_id, + &datatable_name, + timestamp, + "initial", + ) + .await?; + + Ok(Json(DatatableMigration { + datatable: datatable_name, + timestamp, + name: "initial".to_string(), + code_up, + code_down: None, + })) +} + +/// A datatable migration diff item has path `/_`. +/// Parse out the (datatable, timestamp) needed to look it up. +fn parse_datatable_migration_diff_path(path: &str) -> Option<(String, i64)> { + let (datatable, file) = path.split_once('/')?; + let ts_str: String = file.chars().take_while(|c| c.is_ascii_digit()).collect(); + let timestamp = ts_str.parse::().ok()?; + Some((datatable.to_string(), timestamp)) +} + +pub(crate) async fn compare_two_datatable_migration( + db: &DB, + source_workspace_id: &str, + fork_workspace_id: &str, + path: &str, +) -> Result { + let (datatable, timestamp) = match parse_datatable_migration_diff_path(path) { + Some(v) => v, + None => { + return Ok(ItemComparison { + has_changes: false, + exists_in_source: false, + exists_in_fork: false, + }) + } + }; + + let source = sqlx::query!( + "SELECT name, code_up, code_down FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + source_workspace_id, + datatable, + timestamp, + ) + .fetch_optional(db) + .await?; + let target = sqlx::query!( + "SELECT name, code_up, code_down FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + fork_workspace_id, + datatable, + timestamp, + ) + .fetch_optional(db) + .await?; + + let has_changes = match (&source, &target) { + (Some(s), Some(t)) => { + s.name != t.name || s.code_up != t.code_up || s.code_down != t.code_down + } + _ => source.is_some() || target.is_some(), + }; + + Ok(ItemComparison { + has_changes, + exists_in_source: source.is_some(), + exists_in_fork: target.is_some(), + }) +} + +#[derive(Deserialize, Debug)] +pub(crate) struct DatatableRename { + pub(crate) from: String, + pub(crate) to: String, +} + +async fn resolve_datatable_pg(db: &DB, w_id: &str, datatable: &str) -> Result { + let db_resource = get_datatable_resource_from_db_unchecked(db, w_id, datatable).await?; + serde_json::from_value(db_resource) + .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e))) +} + +/// Tolerate a data table whose database has no `_wm_migrations` yet (42P01 = +/// undefined_table): it has never run a migration, so nothing to rename or forget. +fn ignore_missing_wm_migrations(e: tokio_postgres::Error) -> Result<()> { + match e.as_db_error().map(|d| d.code().code()) { + Some("42P01") => Ok(()), + _ => Err(Error::internal_err(format!( + "Failed to update _wm_migrations: {}", + e + ))), + } +} + +/// Drop a data table's rows from its own database's `_wm_migrations`. +async fn remote_forget_datatable_migrations(db: &DB, w_id: &str, datatable: &str) -> Result<()> { + let pg_db = resolve_datatable_pg(db, w_id, datatable).await?; + let (client, connection) = pg_db.connect(Some(db)).await?; + tokio::spawn(async move { + let _ = connection.await; + }); + client + .execute( + "DELETE FROM _wm_migrations WHERE datatable = $1", + &[&datatable], + ) + .await + .map(|_| ()) + .or_else(ignore_missing_wm_migrations) +} + +/// Relabel a data table's rows in its own database's `_wm_migrations`. `resolve_by` +/// names the config entry used to find the database (the old name, still present +/// pre-commit); `from`/`to` are the `datatable` column values to move between. +async fn remote_rename_datatable_migrations( + db: &DB, + w_id: &str, + resolve_by: &str, + from: &str, + to: &str, +) -> Result<()> { + let pg_db = resolve_datatable_pg(db, w_id, resolve_by).await?; + let (client, connection) = pg_db.connect(Some(db)).await?; + tokio::spawn(async move { + let _ = connection.await; + }); + client + .execute( + "UPDATE _wm_migrations SET datatable = $2 WHERE datatable = $1", + &[&from, &to], + ) + .await + .map(|_| ()) + .or_else(ignore_missing_wm_migrations) +} + +/// Keep migration bookkeeping in sync when data tables are renamed or deleted in +/// the workspace config: the control table `datatable_migrations` (this database, +/// in `tx`) and each data table's own `_wm_migrations` (its own database, keyed +/// by data table name — see [`ensure_wm_migrations_schema`]). +/// +/// Renames are applied in two phases through a temporary key so that a rename +/// chain or a swap (A->B, B->A) can't transiently collide on the +/// (datatable, ...) uniqueness mid-update. +/// +/// The `_wm_migrations` updates are best-effort: they run just before `tx` +/// commits (resolved via the pool, which still exposes the old names), and a +/// temporarily unreachable data-table database is logged rather than failing the +/// whole config edit. If one is missed, the next run re-applies its migrations +/// against the existing schema. +pub(crate) async fn cascade_datatable_migration_renames_and_deletes( + db: &DB, + tx: &mut Transaction<'_, Postgres>, + w_id: &str, + renames: &[DatatableRename], + deleted_datatables: &[String], +) -> Result<()> { + if !deleted_datatables.is_empty() { + sqlx::query!( + "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = ANY($2::text[])", + w_id, + deleted_datatables + ) + .execute(&mut **tx) + .await?; + } + + for (i, r) in renames.iter().enumerate() { + let tmp = format!("__wm_rename_tmp/{i}"); + sqlx::query!( + "UPDATE datatable_migrations SET datatable = $3 WHERE workspace_id = $1 AND datatable = $2", + w_id, + &r.from, + &tmp + ) + .execute(&mut **tx) + .await?; + } + for (i, r) in renames.iter().enumerate() { + let tmp = format!("__wm_rename_tmp/{i}"); + sqlx::query!( + "UPDATE datatable_migrations SET datatable = $3 WHERE workspace_id = $1 AND datatable = $2", + w_id, + &tmp, + &r.to + ) + .execute(&mut **tx) + .await?; + } + + for name in deleted_datatables { + if let Err(e) = remote_forget_datatable_migrations(db, w_id, name).await { + tracing::warn!("Failed to clear _wm_migrations for deleted data table {name}: {e}"); + } + } + for (i, r) in renames.iter().enumerate() { + let tmp = format!("__wm_rename_tmp/{i}"); + if let Err(e) = remote_rename_datatable_migrations(db, w_id, &r.from, &r.from, &tmp).await { + tracing::warn!( + "Failed to stage _wm_migrations rename {} -> {}: {e}", + r.from, + r.to + ); + } + } + for (i, r) in renames.iter().enumerate() { + let tmp = format!("__wm_rename_tmp/{i}"); + if let Err(e) = remote_rename_datatable_migrations(db, w_id, &r.from, &tmp, &r.to).await { + tracing::warn!( + "Failed to finish _wm_migrations rename {} -> {}: {e}", + r.from, + r.to + ); + } + } + + Ok(()) +} + +/// Copy a workspace's migration definitions to another workspace, so a fork +/// inherits the same per-data-table migration history as its parent. +pub(crate) async fn clone_datatable_migrations( + tx: &mut Transaction<'_, Postgres>, + source_workspace_id: &str, + target_workspace_id: &str, +) -> Result<()> { + sqlx::query!( + "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) + SELECT $2, datatable, timestamp, name, code_up, code_down + FROM datatable_migrations WHERE workspace_id = $1", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn validate_migration_name_accepts_safe_names() { + for name in ["initial", "add_index_to_customers", "fix-bug_2", "ABC123"] { + assert!( + validate_migration_name(name).is_ok(), + "{name} should be valid" + ); + } + } + + #[test] + fn validate_migration_name_rejects_unsafe_names() { + for name in [ + "", + "add index", + "a/b", + "a\\b", + "a..b", + "a.b", + "naïve", + "a/../b", + ] { + assert!( + validate_migration_name(name).is_err(), + "{name} should be rejected" + ); + } + } + + #[test] + fn validate_datatable_path_segment_accepts_and_rejects() { + for ok in ["mydt", "my-dt", "main", "a b"] { + assert!( + validate_datatable_path_segment(ok).is_ok(), + "{ok} should be ok" + ); + } + for bad in ["", "a/b", "a\\b", "..", "a..b", "../etc", "x/.."] { + assert!( + validate_datatable_path_segment(bad).is_err(), + "{bad} should be rejected" + ); + } + } + + #[test] + fn parse_datatable_migration_diff_path_roundtrips() { + assert_eq!( + parse_datatable_migration_diff_path("mydt/20260101000001_create_users.up.sql"), + Some(("mydt".to_string(), 20260101000001)) + ); + // up and down map to the same (datatable, timestamp) record. + assert_eq!( + parse_datatable_migration_diff_path("mydt/20260101000001_create_users.down.sql"), + Some(("mydt".to_string(), 20260101000001)) + ); + // datatable names may themselves be hyphenated. + assert_eq!( + parse_datatable_migration_diff_path("my-dt/42_x.up.sql"), + Some(("my-dt".to_string(), 42)) + ); + } + + #[test] + fn parse_datatable_migration_diff_path_rejects_malformed() { + // no slash → not a migration path + assert_eq!(parse_datatable_migration_diff_path("nofile"), None); + // filename not starting with digits → no timestamp + assert_eq!( + parse_datatable_migration_diff_path("mydt/create_users.up.sql"), + None + ); + // empty filename + assert_eq!(parse_datatable_migration_diff_path("mydt/"), None); + } + + async fn seed_migration(pool: &DB, w_id: &str, datatable: &str, timestamp: i64, name: &str) { + sqlx::query( + "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up) \ + VALUES ($1, $2, $3, $4, 'select 1;')", + ) + .bind(w_id) + .bind(datatable) + .bind(timestamp) + .bind(name) + .execute(pool) + .await + .unwrap(); + } + + async fn migration_keys(pool: &DB, w_id: &str) -> Vec<(String, String)> { + sqlx::query_as::<_, (String, String)>( + "SELECT datatable, name FROM datatable_migrations WHERE workspace_id = $1 \ + ORDER BY datatable, timestamp", + ) + .bind(w_id) + .fetch_all(pool) + .await + .unwrap() + } + + // The cascade keeps each data table's migrations attached to its name when a + // data table is renamed, drops them when it is deleted, and survives a swap + // (A->B, B->A) at a shared timestamp without a primary-key collision. + #[sqlx::test(migrations = "../migrations")] + async fn cascade_renames_and_deletes_datatable_migrations(pool: DB) { + let w_id = format!("dtmig{}", uuid::Uuid::new_v4().simple()); + sqlx::query("INSERT INTO workspace (id, name, owner) VALUES ($1, $1, 'test@windmill.dev')") + .bind(&w_id) + .execute(&pool) + .await + .unwrap(); + + // rename a -> a2, delete d, and swap sa <-> sb (both at timestamp 5000) + seed_migration(&pool, &w_id, "a", 1, "a_mig").await; + seed_migration(&pool, &w_id, "d", 1, "d_mig").await; + seed_migration(&pool, &w_id, "sa", 5000, "sa_mig").await; + seed_migration(&pool, &w_id, "sb", 5000, "sb_mig").await; + + let mut tx = pool.begin().await.unwrap(); + cascade_datatable_migration_renames_and_deletes( + &pool, + &mut tx, + &w_id, + &[ + DatatableRename { from: "a".to_string(), to: "a2".to_string() }, + DatatableRename { from: "sa".to_string(), to: "sb".to_string() }, + DatatableRename { from: "sb".to_string(), to: "sa".to_string() }, + ], + &["d".to_string()], + ) + .await + .unwrap(); + tx.commit().await.unwrap(); + + assert_eq!( + migration_keys(&pool, &w_id).await, + vec![ + ("a2".to_string(), "a_mig".to_string()), + ("sa".to_string(), "sb_mig".to_string()), + ("sb".to_string(), "sa_mig".to_string()), + ] + ); + } + + // A fork inherits its parent's migration definitions unchanged. + #[sqlx::test(migrations = "../migrations")] + async fn clone_copies_datatable_migrations_to_target(pool: DB) { + let src = format!("src{}", uuid::Uuid::new_v4().simple()); + let dst = format!("dst{}", uuid::Uuid::new_v4().simple()); + for w in [&src, &dst] { + sqlx::query( + "INSERT INTO workspace (id, name, owner) VALUES ($1, $1, 'test@windmill.dev')", + ) + .bind(w) + .execute(&pool) + .await + .unwrap(); + } + seed_migration(&pool, &src, "customers", 1, "create_customers").await; + seed_migration(&pool, &src, "customers", 2, "add_index").await; + seed_migration(&pool, &src, "orders", 3, "create_orders").await; + + let mut tx = pool.begin().await.unwrap(); + clone_datatable_migrations(&mut tx, &src, &dst) + .await + .unwrap(); + tx.commit().await.unwrap(); + + // the target ends up with an identical set, and the source is untouched. + let expected = vec![ + ("customers".to_string(), "create_customers".to_string()), + ("customers".to_string(), "add_index".to_string()), + ("orders".to_string(), "create_orders".to_string()), + ]; + assert_eq!(migration_keys(&pool, &dst).await, expected); + assert_eq!(migration_keys(&pool, &src).await, expected); + } +} diff --git a/backend/windmill-api-workspaces/src/lib.rs b/backend/windmill-api-workspaces/src/lib.rs index b3e9853de9..c2b62d450b 100644 --- a/backend/windmill-api-workspaces/src/lib.rs +++ b/backend/windmill-api-workspaces/src/lib.rs @@ -1,3 +1,4 @@ +pub mod datatable_migrations; pub mod deployment_requests; pub mod workspaces; pub mod workspaces_extra; diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 8d53562635..0b5b600bf1 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -132,6 +132,7 @@ pub fn workspaced_service() -> Router { get(get_datatable_table_schema), ) .route("/edit_datatable_config", post(edit_datatable_config)) + .merge(crate::datatable_migrations::routes()) .route("/git_sync_enabled", get(get_git_sync_enabled)) .route("/edit_git_sync_config", post(edit_git_sync_config)) .route("/edit_git_sync_repository", post(edit_git_sync_repository)) @@ -430,6 +431,12 @@ pub struct DucklakeSettings { #[derive(Deserialize, Debug)] struct EditDataTableConfig { settings: DataTableSettings, + // Data table renames (old -> new) and deletions, tracked client-side by a + // stable id, so we can cascade or drop each data table's migrations. + #[serde(default)] + renames: Vec, + #[serde(default)] + deleted_datatables: Vec, } #[derive(Deserialize, Serialize, Debug)] @@ -1973,8 +1980,8 @@ pub(crate) async fn resolve_pg_source_checked( } /// A temporary file for pg_dump output that is automatically deleted when dropped. -struct DumpFile { - path: std::path::PathBuf, +pub(crate) struct DumpFile { + pub(crate) path: std::path::PathBuf, } impl DumpFile { @@ -2021,7 +2028,11 @@ impl Drop for DumpFile { /// Run pg_dump against a PgDatabase, writing output to a temp file on disk. /// Returns a DumpFile handle; the file is deleted when the handle is dropped. -async fn pg_dump_database(pg_db: &PgDatabase, schema_only: bool) -> Result { +pub(crate) async fn pg_dump_database( + pg_db: &PgDatabase, + schema_only: bool, + exclude_tables: &[&str], +) -> Result { let dump_file = DumpFile::new()?; let host = &pg_db.host; @@ -2034,6 +2045,9 @@ async fn pg_dump_database(pg_db: &PgDatabase, schema_only: bool) -> Result, ) -> Result { let pg = resolve_pg_source_checked(&db, &user_db, &authed, &w_id, &req.source).await?; - let dump_file = pg_dump_database(&pg, true).await?; + let dump_file = pg_dump_database(&pg, true, &[]).await?; tokio::fs::read_to_string(&dump_file.path) .await .map_err(|e| Error::internal_err(format!("Failed to read dump file: {}", e))) @@ -2416,13 +2430,57 @@ async fn edit_datatable_config( Extension(db): Extension, Path(w_id): Path, ApiAuthed { is_admin, username, email, .. }: ApiAuthed, - Json(new_config): Json, + Json(mut new_config): Json, ) -> Result { require_admin(is_admin, &username)?; let is_superadmin = require_super_admin(&db, &email).await.is_ok(); let mut tx = db.begin().await?; + let old_datatables: HashMap = serde_json::from_value( + sqlx::query_scalar!( + "SELECT ws.datatable->'datatables' FROM workspace_settings ws WHERE ws.workspace_id = $1", + &w_id + ) + .fetch_one(&db) + .await? + .unwrap_or(serde_json::Value::Null), + ) + .unwrap_or_default(); + + // Validate every persisted data table name and rename segment before + // touching anything, since they become directory segments in migration + // storage/export keys (`migrations/datatable//...`). + for name in new_config.settings.datatables.keys() { + crate::datatable_migrations::validate_datatable_path_segment(name)?; + } + for r in &new_config.renames { + crate::datatable_migrations::validate_datatable_path_segment(&r.from)?; + crate::datatable_migrations::validate_datatable_path_segment(&r.to)?; + } + + // Map new name -> old name so a renamed data table inherits the previous + // flag instead of being treated as brand new. + let rename_src: HashMap<&str, &str> = new_config + .renames + .iter() + .map(|r| (r.to.as_str(), r.from.as_str())) + .collect(); + + // Migrations opt-in is owned by the enable/disable endpoints, not this config + // form: preserve each existing data table's flag, and default brand-new data + // tables to enabled. + for (name, dt) in new_config.settings.datatables.iter_mut() { + let lookup = rename_src + .get(name.as_str()) + .copied() + .unwrap_or(name.as_str()); + dt.migrations_enabled = match old_datatables.get(lookup) { + Some(old) => old.migrations_enabled, + None => Some(true), + }; + } + let args_for_audit = format!("{:?}", new_config.settings); audit_log( &mut *tx, @@ -2437,19 +2495,6 @@ async fn edit_datatable_config( // Check that non-superadmins are not abusing Instance databases if !is_superadmin { - let old_datatables = sqlx::query_scalar!( - r#" - SELECT ws.datatable->'datatables' AS datatable_name - FROM workspace_settings ws - WHERE ws.workspace_id = $1 - "#, - &w_id - ) - .fetch_one(&mut *tx) - .await? - .unwrap_or(serde_json::Value::Null); - let old_datatables: HashMap = - serde_json::from_value(old_datatables).unwrap_or_default(); for (name, dt) in new_config.settings.datatables.iter() { if dt.database.resource_type == DataTableCatalogResourceType::Instance { let old_dt = old_datatables.get(name); @@ -2478,6 +2523,15 @@ async fn edit_datatable_config( .execute(&mut *tx) .await?; + crate::datatable_migrations::cascade_datatable_migration_renames_and_deletes( + &db, + &mut tx, + &w_id, + &new_config.renames, + &new_config.deleted_datatables, + ) + .await?; + tx.commit().await?; Ok(format!("Edit datatable config for workspace {}", &w_id)) @@ -4068,6 +4122,15 @@ async fn clone_workspace_data( // Clone workspace settings (merge with existing basic settings) update_workspace_settings(tx, source_workspace_id, target_workspace_id).await?; + // Clone data table migration definitions (the settings above carry the data + // table config; this carries their migration history). + crate::datatable_migrations::clone_datatable_migrations( + tx, + source_workspace_id, + target_workspace_id, + ) + .await?; + // Clone workspace environment variables clone_workspace_env(tx, source_workspace_id, target_workspace_id).await?; @@ -7418,6 +7481,7 @@ pub struct CompareSummary { pub folders_changed: usize, pub schedules_changed: usize, pub triggers_changed: usize, + pub datatable_migrations_changed: usize, pub conflicts: usize, // Items that are both ahead and behind } @@ -7609,6 +7673,15 @@ async fn compare_workspaces( compare_two_folders(&db, &source_workspace_id, &fork_workspace_id, &item.path) .await?, ), + "datatable_migration" => Some( + crate::datatable_migrations::compare_two_datatable_migration( + &db, + &source_workspace_id, + &fork_workspace_id, + &item.path, + ) + .await?, + ), // Triggers and schedules are diffed against a hardcoded ignore list // (mode/enabled/server_id/last_server_ping/edited_at/by/error/extra_perms/permissioned_as/email) // so that fork-clones — which differ from the parent only in the runtime @@ -7731,6 +7804,10 @@ async fn compare_workspaces( .iter() .filter(|s| s.kind.ends_with("_trigger")) .count(), + datatable_migrations_changed: visible_diffs + .iter() + .filter(|s| s.kind == "datatable_migration") + .count(), conflicts: visible_diffs .iter() .filter(|s| s.ahead > 0 && s.behind > 0) @@ -8036,6 +8113,45 @@ async fn query_visible_items<'c>( .fetch_all(&mut **tx) .await? } + "datatable_migration" => { + // Match by (datatable, timestamp), not the full path: a migration + // keeps its identity across a rename, so the candidate path's + // `name` segment can differ from the stored one. Parse each + // `/_` candidate, probe existence by + // (datatable, timestamp), and return the *original* candidate path + // so the visibility set stays keyed by the diff's path. + let parsed: Vec<(String, i64, String)> = paths_vec + .iter() + .filter_map(|p| { + let (dt, rest) = p.split_once('/')?; + let ts = rest.split_once('_')?.0.parse::().ok()?; + Some((dt.to_string(), ts, p.clone())) + }) + .collect(); + if parsed.is_empty() { + vec![] + } else { + let dts: Vec = parsed.iter().map(|(d, _, _)| d.clone()).collect(); + let tss: Vec = parsed.iter().map(|(_, t, _)| *t).collect(); + let existing: HashSet<(String, i64)> = sqlx::query!( + "SELECT datatable, timestamp FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = ANY($2) AND timestamp = ANY($3)", + workspace_id, + &dts, + &tss, + ) + .fetch_all(&mut **tx) + .await? + .into_iter() + .map(|r| (r.datatable, r.timestamp)) + .collect(); + parsed + .into_iter() + .filter(|(d, t, _)| existing.contains(&(d.clone(), *t))) + .map(|(_, _, p)| p) + .collect() + } + } k if TRIGGER_OR_SCHEDULE_TABLES.contains(&k) => { // SAFETY: `kind` comes from a hardcoded allowlist // TRIGGER_OR_SCHEDULE_TABLES, not user input. @@ -8103,10 +8219,10 @@ async fn existing_runnables( } #[derive(Debug)] -struct ItemComparison { - has_changes: bool, - exists_in_source: bool, - exists_in_fork: bool, +pub(crate) struct ItemComparison { + pub(crate) has_changes: bool, + pub(crate) exists_in_source: bool, + pub(crate) exists_in_fork: bool, } async fn compare_two_scripts( diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 323de509bd..177b22662d 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -4631,6 +4631,22 @@ paths: properties: settings: $ref: "#/components/schemas/DataTableSettings" + renames: + description: data tables renamed in this save, so their migrations cascade + type: array + items: + type: object + required: [from, to] + properties: + from: + type: string + to: + type: string + deleted_datatables: + description: data tables removed in this save, so their migrations are deleted + type: array + items: + type: string responses: "200": description: status @@ -4638,6 +4654,307 @@ paths: application/json: schema: {} + /w/{workspace}/workspaces/run_datatable_migrations/{datatable_name}: + post: + summary: run pending datatable migrations against a datatable + operationId: runDatatableMigrations + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + - name: up_to + in: query + required: false + description: only apply pending migrations up to and including this version + schema: + type: integer + format: int64 + - name: only + in: query + required: false + description: apply only this specific migration version, ignoring others + schema: + type: integer + format: int64 + responses: + "200": + description: applied migrations + content: + application/json: + schema: + type: object + required: [applied] + properties: + applied: + type: array + items: + type: object + required: [version, name] + properties: + version: + type: integer + format: int64 + name: + type: string + + /w/{workspace}/workspaces/rollback_datatable_migrations/{datatable_name}: + post: + summary: roll back the most recently applied migration on a datatable + operationId: rollbackDatatableMigrations + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + - name: only + in: query + required: false + description: roll back this specific applied migration version instead of the latest + schema: + type: integer + format: int64 + responses: + "200": + description: rolled back migrations + content: + application/json: + schema: + type: object + required: [rolled_back] + properties: + rolled_back: + type: array + items: + type: object + required: [version, name] + properties: + version: + type: integer + format: int64 + name: + type: string + + /w/{workspace}/workspaces/list_datatable_migrations: + get: + summary: list datatable migrations for a workspace + operationId: listDatatableMigrations + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: datatable migrations + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/DatatableMigration" + + /w/{workspace}/workspaces/datatable_migrations_status/{datatable_name}: + get: + summary: list a datatable's migrations with their applied status + operationId: getDatatableMigrationsStatus + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + responses: + "200": + description: migrations with status + content: + application/json: + schema: + type: object + required: [enabled, migrations] + properties: + enabled: + type: boolean + migrations: + type: array + items: + $ref: "#/components/schemas/DatatableMigrationWithStatus" + error: + type: string + + /w/{workspace}/workspaces/enable_datatable_migrations/{datatable_name}: + post: + summary: opt a datatable in to migrations (admins / super admins only) + operationId: enableDatatableMigrations + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/disable_datatable_migrations/{datatable_name}: + post: + summary: opt a datatable out of migrations, deleting all of them (admins / super admins only) + operationId: disableDatatableMigrations + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/create_datatable_migration/{datatable_name}: + post: + summary: create a single datatable migration (version generated server-side) + operationId: createDatatableMigration + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [name, code_up] + properties: + name: + type: string + code_up: + type: string + code_down: + type: string + responses: + "200": + description: created migration + content: + application/json: + schema: + $ref: "#/components/schemas/DatatableMigration" + + /w/{workspace}/workspaces/delete_datatable_migration/{datatable_name}/{timestamp}: + delete: + summary: delete a single datatable migration definition + operationId: deleteDatatableMigration + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + - name: timestamp + in: path + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/upsert_datatable_migration/{datatable_name}: + post: + summary: insert or update a single datatable migration at an explicit version + operationId: upsertDatatableMigration + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [timestamp, name, code_up] + properties: + timestamp: + type: integer + format: int64 + name: + type: string + code_up: + type: string + code_down: + type: string + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/generate_initial_datatable_migration/{datatable_name}: + post: + summary: snapshot the current schema as an already-installed initial migration + operationId: generateInitialDatatableMigration + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + responses: + "200": + description: created migration + content: + application/json: + schema: + $ref: "#/components/schemas/DatatableMigration" + /w/{workspace}/workspaces/create_pg_database: post: summary: create a new PostgreSQL database for a datatable @@ -29067,6 +29384,9 @@ components: type: string required: - resource_type + migrations_enabled: + type: boolean + description: Whether the SQL migrations feature is opted in for this data table forked_from: type: object description: Fork origin info with schema snapshot @@ -29075,6 +29395,40 @@ components: type: object description: Schema snapshot at fork time additionalProperties: true + DatatableMigration: + type: object + required: [datatable, timestamp, name, code_up] + properties: + datatable: + type: string + timestamp: + type: integer + format: int64 + name: + type: string + code_up: + type: string + code_down: + type: string + DatatableMigrationWithStatus: + type: object + required: [timestamp, name, code_up, status] + properties: + timestamp: + type: integer + format: int64 + name: + type: string + code_up: + type: string + code_down: + type: string + status: + type: string + enum: + - ran + - not_run + - unknown DataTableSchema: type: object required: [datatable_name, schemas] @@ -29786,6 +30140,7 @@ components: - folders_changed - schedules_changed - triggers_changed + - datatable_migrations_changed - conflicts properties: total_diffs: @@ -29824,6 +30179,9 @@ components: triggers_changed: type: integer description: Number of triggers with differences (sum across all trigger kinds) + datatable_migrations_changed: + type: integer + description: Number of data table migrations with differences conflicts: type: integer description: Number of items that are both ahead and behind (conflicts) diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index fa112ef4e1..839057b4b9 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -1546,6 +1546,34 @@ pub(crate) async fn tarball_workspace( .await?; } + { + // Data table migrations live in the `datatable_migrations` table; surface + // them in the export as `migrations/datatable//_` + // .up.sql (and .down.sql when present) so `wmill sync` treats them like any + // other workspace item. + let migrations = sqlx::query!( + "SELECT datatable, timestamp, name, code_up, code_down FROM datatable_migrations \ + WHERE workspace_id = $1 ORDER BY datatable, timestamp", + &w_id + ) + .fetch_all(&mut *tx) + .await?; + for m in migrations { + let base = format!( + "migrations/datatable/{}/{}_{}", + m.datatable, m.timestamp, m.name + ); + archive + .write_to_archive(&m.code_up, &format!("{base}.up.sql")) + .await?; + if let Some(code_down) = m.code_down { + archive + .write_to_archive(&code_down, &format!("{base}.down.sql")) + .await?; + } + } + } + archive.finish().await?; let file = tokio::fs::File::open(&file_path).await?; diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 3b5220d46f..a8c6e26f0e 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -164,6 +164,7 @@ pub enum ObjectType { Settings, Key, WorkspaceDependencies, + DatatableMigration, } pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28719/sync-script-to-git-repo-windmill"; @@ -771,6 +772,11 @@ pub struct DataTable { pub database: DataTableDatabase, #[serde(default, skip_serializing_if = "Option::is_none")] pub forked_from: Option, + /// Whether the SQL-migrations feature is opted in for this data table. + /// Absent on data tables created before the feature: treated as enabled only + /// when migrations already exist (see `datatable_migrations_enabled`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub migrations_enabled: Option, } #[derive(Deserialize, Serialize, Debug)] diff --git a/backend/windmill-git-sync/src/lib.rs b/backend/windmill-git-sync/src/lib.rs index b10f62e7a5..6c1b9bf89b 100644 --- a/backend/windmill-git-sync/src/lib.rs +++ b/backend/windmill-git-sync/src/lib.rs @@ -24,30 +24,102 @@ pub use git_sync_oss::{ #[derive(Clone, Debug)] pub enum DeployedObject { - Script { hash: ScriptHash, path: String, parent_path: Option }, - Flow { path: String, parent_path: Option, version: i64 }, - App { path: String, version: i64, parent_path: Option }, - RawApp { path: String, version: i64, parent_path: Option }, - Folder { path: String }, - Resource { path: String, parent_path: Option }, - Variable { path: String, parent_path: Option }, - Schedule { path: String }, - ResourceType { path: String }, - User { email: String }, - Group { name: String }, - HttpTrigger { path: String, parent_path: Option }, - WebsocketTrigger { path: String, parent_path: Option }, - KafkaTrigger { path: String, parent_path: Option }, - NatsTrigger { path: String, parent_path: Option }, - PostgresTrigger { path: String, parent_path: Option }, - MqttTrigger { path: String, parent_path: Option }, - SqsTrigger { path: String, parent_path: Option }, - GcpTrigger { path: String, parent_path: Option }, - AzureTrigger { path: String, parent_path: Option }, - EmailTrigger { path: String, parent_path: Option }, - Settings { setting_type: String }, - Key { key_type: String }, - WorkspaceDependencies { path: String }, + Script { + hash: ScriptHash, + path: String, + parent_path: Option, + }, + Flow { + path: String, + parent_path: Option, + version: i64, + }, + App { + path: String, + version: i64, + parent_path: Option, + }, + RawApp { + path: String, + version: i64, + parent_path: Option, + }, + Folder { + path: String, + }, + Resource { + path: String, + parent_path: Option, + }, + Variable { + path: String, + parent_path: Option, + }, + Schedule { + path: String, + }, + ResourceType { + path: String, + }, + User { + email: String, + }, + Group { + name: String, + }, + HttpTrigger { + path: String, + parent_path: Option, + }, + WebsocketTrigger { + path: String, + parent_path: Option, + }, + KafkaTrigger { + path: String, + parent_path: Option, + }, + NatsTrigger { + path: String, + parent_path: Option, + }, + PostgresTrigger { + path: String, + parent_path: Option, + }, + MqttTrigger { + path: String, + parent_path: Option, + }, + SqsTrigger { + path: String, + parent_path: Option, + }, + GcpTrigger { + path: String, + parent_path: Option, + }, + AzureTrigger { + path: String, + parent_path: Option, + }, + EmailTrigger { + path: String, + parent_path: Option, + }, + Settings { + setting_type: String, + }, + Key { + key_type: String, + }, + WorkspaceDependencies { + path: String, + }, + /// A single data table migration, identified by `/_`. + DatatableMigration { + path: String, + }, } impl DeployedObject { @@ -77,6 +149,7 @@ impl DeployedObject { DeployedObject::Settings { .. } => "settings.yaml".to_string(), DeployedObject::Key { .. } => "encryption_key.yaml".to_string(), DeployedObject::WorkspaceDependencies { path, .. } => path.to_owned(), + DeployedObject::DatatableMigration { path } => path.to_owned(), } } @@ -118,6 +191,7 @@ impl DeployedObject { DeployedObject::Settings { .. } => None, DeployedObject::Key { .. } => None, DeployedObject::WorkspaceDependencies { .. } => None, + DeployedObject::DatatableMigration { .. } => None, } } @@ -147,6 +221,7 @@ impl DeployedObject { DeployedObject::Settings { .. } => "settings", DeployedObject::Key { .. } => "key", DeployedObject::WorkspaceDependencies { .. } => "workspace_dependencies", + DeployedObject::DatatableMigration { .. } => "datatable_migration", } .to_string() } diff --git a/cli/src/commands/datatable/datatable.ts b/cli/src/commands/datatable/datatable.ts index 1291d4cc13..6609d08c50 100644 --- a/cli/src/commands/datatable/datatable.ts +++ b/cli/src/commands/datatable/datatable.ts @@ -9,6 +9,13 @@ import { GlobalOptions } from "../../types.ts"; import { runCatalogQuery } from "../../utils/catalog.ts"; import { psql as psqlDatatable } from "./psql.ts"; import { serve as serveDatatable } from "./serve.ts"; +import { + createMigration, + pushLocalMigrations, + rollbackMigrations, + runMigrations, + validateLocalMigrations, +} from "../datatable_migrations.ts"; const DEFAULT_DATATABLE_NAME = "main"; @@ -41,6 +48,69 @@ async function run( await runCatalogQuery(opts, "datatable", name, sql); } +function migrateNew( + opts: GlobalOptions & { datatable?: string }, + name: string, +) { + createMigration(opts.datatable ?? DEFAULT_DATATABLE_NAME, name); +} + +async function migrateUp(opts: GlobalOptions & { datatable?: string }) { + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + const dt = opts.datatable ?? DEFAULT_DATATABLE_NAME; + // Reject malformed local migrations (duplicate timestamps, orphan downs) before + // pushing — the same check `wmill sync push` runs — so a duplicate timestamp + // can't silently overwrite one migration on upsert. + const errors = validateLocalMigrations(new Set([dt])); + if (errors.length > 0) { + log.error( + "Invalid datatable migrations, aborting:\n" + + errors.map((e) => ` - ${e}`).join("\n"), + ); + process.exit(1); + } + // Push any locally-created/edited migration files first (without running + // them), so `migrate up` works even before a `wmill sync push`. + await pushLocalMigrations(workspace.workspaceId, dt); + await runMigrations(workspace.workspaceId, dt); +} + +async function migrateDown(opts: GlobalOptions & { datatable?: string }) { + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + const dt = opts.datatable ?? DEFAULT_DATATABLE_NAME; + await rollbackMigrations(workspace.workspaceId, dt); +} + +const migrateCommand = new Command() + .description("manage datatable migrations") + .command("new", "scaffold a new migration (.up.sql / .down.sql files)") + .arguments("") + .option( + "-d --datatable ", + "Target datatable (default: main)", + ) + .action(migrateNew as any) + .command( + "up", + "apply all pending migrations to the main datatable (or one via --datatable)", + ) + .option( + "-d --datatable ", + "Target datatable (default: main)", + ) + .action(migrateUp as any) + .command( + "down", + "roll back the most recent migration on the main datatable (or one via --datatable)", + ) + .option( + "-d --datatable ", + "Target datatable (default: main)", + ) + .action(migrateDown as any); + async function create( opts: GlobalOptions & { resource?: string; force?: boolean }, name?: string, @@ -124,6 +194,7 @@ const command = new Command() "Output only the final result as JSON. Useful for scripting.", ) .action(run as any) + .command("migrate", migrateCommand) .command( "create", "register a datatable database in the workspace (default: instance-backed 'main') so scripts can use datatable://", diff --git a/cli/src/commands/datatable_migrations.ts b/cli/src/commands/datatable_migrations.ts new file mode 100644 index 0000000000..4fcaffd4fa --- /dev/null +++ b/cli/src/commands/datatable_migrations.ts @@ -0,0 +1,340 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; +import * as log from "../core/log.ts"; +import { colors } from "@cliffy/ansi/colors"; +import * as wmill from "../../gen/services.gen.ts"; +import { readTextFile } from "../utils/utils.ts"; +import { Confirm } from "@cliffy/prompt/confirm"; + +// Migrations live under /migrations/datatable//, one folder per +// target data table, as `_.up.sql` (and optional `.down.sql`). +// They are synced as ordinary workspace files (see the workspace tarball export +// and the `datatable_migration` handling in sync.ts); this module only holds the +// `wmill datatable migrate` command helpers and the per-file push primitive. +const MIGRATIONS_DIR = path.join("migrations", "datatable"); + +// Migration names map directly onto file names and the DB `name` column. +const MIGRATION_NAME_RE = /^[a-zA-Z0-9_-]+$/; + +/** Current UTC time as a YYYYMMDDHHMMSS migration version. */ +function migrationTimestamp(): string { + const d = new Date(); + const p = (n: number) => String(n).padStart(2, "0"); + return ( + `${d.getUTCFullYear()}${p(d.getUTCMonth() + 1)}${p(d.getUTCDate())}` + + `${p(d.getUTCHours())}${p(d.getUTCMinutes())}${p(d.getUTCSeconds())}` + ); +} + +/** + * A migration version unique within a data table folder: the current UTC + * timestamp bumped past any existing version, so two migrations scaffolded in + * the same second don't collide on the `(datatable, timestamp)` identity used to + * upsert them. + */ +function nextMigrationTimestamp(dir: string): string { + const now = Number(migrationTimestamp()); + let max = 0; + if (fs.existsSync(dir)) { + for (const file of fs.readdirSync(dir)) { + const m = file.match(/^(\d+)_.*\.(up|down)\.sql$/); + if (m) max = Math.max(max, Number(m[1])); + } + } + return String(max >= now ? max + 1 : now); +} + +/** + * Scaffold a new migration under migrations/datatable// as empty + * `_.up.sql` and `.down.sql` files. Purely local — no network. + */ +export function createMigration(datatable: string, name: string): void { + if (!MIGRATION_NAME_RE.test(name)) { + throw new Error( + `Invalid migration name '${name}': use only letters, digits, '_' and '-'`, + ); + } + const dir = path.join(process.cwd(), MIGRATIONS_DIR, datatable); + fs.mkdirSync(dir, { recursive: true }); + + const timestamp = nextMigrationTimestamp(dir); + const base = `${timestamp}_${name}`; + const up = path.join(dir, `${base}.up.sql`); + const down = path.join(dir, `${base}.down.sql`); + // Frame the body in an explicit transaction so it applies atomically, matching + // the template the UI's "New migration" modal seeds. + const template = (direction: string) => + `-- ${direction} migration: ${name}\nBEGIN;\n\n-- Add your migration here\n\nEND;\n`; + fs.writeFileSync(up, template("up"), "utf-8"); + fs.writeFileSync(down, template("down"), "utf-8"); + + log.info( + colors.green(`Created migration ${base} in ${MIGRATIONS_DIR}/${datatable}/`), + ); + for (const f of [up, down]) { + log.info(colors.gray(` ${path.relative(process.cwd(), f)}`)); + } +} + +/** + * Apply the workspace's pending migrations to a data table (forwards migrations + * recorded in `_wm_migrations`). Mirrors `wmill datatable migrate up`. + */ +export async function runMigrations( + workspace: string, + datatableName: string, +): Promise { + const result = await wmill.runDatatableMigrations({ + workspace, + datatableName, + }); + const applied = result.applied ?? []; + if (applied.length === 0) { + log.info(colors.gray(`No pending migrations to run on '${datatableName}'`)); + return; + } + log.info( + colors.green(`Applied ${applied.length} migration(s) to '${datatableName}':`), + ); + for (const m of applied) { + log.info(colors.gray(` ${m.version} ${m.name}`)); + } +} + +/** + * Roll back the most recently applied migration on a data table (one step). + * Mirrors `wmill datatable migrate down`. + */ +export async function rollbackMigrations( + workspace: string, + datatableName: string, +): Promise { + const result = await wmill.rollbackDatatableMigrations({ + workspace, + datatableName, + }); + const rolledBack = result.rolled_back ?? []; + if (rolledBack.length === 0) { + log.info( + colors.gray(`No applied migrations to roll back on '${datatableName}'`), + ); + return; + } + for (const m of rolledBack) { + log.info( + colors.green(`Rolled back migration ${m.version} ${m.name} on '${datatableName}'`), + ); + } +} + +/** + * Validate the on-disk migration files for the given data tables (or all of + * them when `datatables` is omitted). Returns a list of human-readable problems; + * an empty list means the migrations are well-formed. Two states are invalid: + * - two up (or two down) files sharing the same timestamp, which collide on the + * `(datatable, timestamp)` identity used to upsert; and + * - a `.down.sql` with no matching `.up.sql` (an up file is mandatory). + */ +export function validateLocalMigrations(datatables?: Set): string[] { + const errors: string[] = []; + const root = path.join(process.cwd(), MIGRATIONS_DIR); + if (!fs.existsSync(root)) return errors; + + for (const datatable of fs.readdirSync(root)) { + if (datatables && !datatables.has(datatable)) continue; + const dtDir = path.join(root, datatable); + if (!fs.statSync(dtDir).isDirectory()) continue; + + const upNamesByTs = new Map(); + const downNamesByTs = new Map(); + const upBases = new Set(); + const downBases: { ts: number; name: string }[] = []; + + for (const file of fs.readdirSync(dtDir)) { + const m = file.match(/^(\d+)_(.*)\.(up|down)\.sql$/); + if (!m) continue; + const ts = Number(m[1]); + const name = m[2]; + if (m[3] === "up") { + (upNamesByTs.get(ts) ?? upNamesByTs.set(ts, []).get(ts)!).push(name); + upBases.add(`${ts}_${name}`); + } else { + (downNamesByTs.get(ts) ?? downNamesByTs.set(ts, []).get(ts)!).push(name); + downBases.push({ ts, name }); + } + } + + for (const [ts, names] of upNamesByTs) { + if (names.length > 1) { + errors.push( + `${datatable}: ${names.length} up migrations share timestamp ${ts} (${names.join(", ")})`, + ); + } + } + for (const [ts, names] of downNamesByTs) { + if (names.length > 1) { + errors.push( + `${datatable}: ${names.length} down migrations share timestamp ${ts} (${names.join(", ")})`, + ); + } + } + for (const d of downBases) { + if (!upBases.has(`${d.ts}_${d.name}`)) { + errors.push( + `${datatable}: ${d.ts}_${d.name}.down.sql has no matching ${d.ts}_${d.name}.up.sql`, + ); + } + } + } + + return errors; +} + +/** + * Sync a single migration to the workspace based on the current on-disk state of + * its `/_.up.sql` file: upsert it when the up file + * exists, otherwise delete it. Called by `wmill sync push` for each changed + * `datatable_migration` file. + */ +export async function pushMigrationFromDisk( + workspace: string, + m: { datatable: string; timestamp: number }, +): Promise { + const dir = path.join(process.cwd(), MIGRATIONS_DIR, m.datatable); + // Find the up file for this timestamp regardless of its name segment. A rename + // (`123_old.up.sql` -> `123_new.up.sql`) keeps the (datatable, timestamp) + // identity but changes the name; the diff sorter may process the deleted old + // path before the added new one, so keying off the passed name would delete + // the record. Scanning by timestamp upserts the surviving file instead. + const upFile = fs.existsSync(dir) + ? fs.readdirSync(dir).find((f) => { + const parsed = f.match(/^(\d+)_(.*)\.up\.sql$/); + return parsed !== null && Number(parsed[1]) === m.timestamp; + }) + : undefined; + + if (upFile === undefined) { + log.info(colors.red(`Deleting datatable_migration ${m.datatable}/${m.timestamp}`)); + await wmill.deleteDatatableMigration({ + workspace, + datatableName: m.datatable, + timestamp: m.timestamp, + }); + return; + } + + const name = upFile.match(/^(\d+)_(.*)\.up\.sql$/)![2]; + const base = `${m.timestamp}_${name}`; + const code_up = await readTextFile(path.join(dir, upFile)); + const downPath = path.join(dir, `${base}.down.sql`); + const code_down = fs.existsSync(downPath) ? await readTextFile(downPath) : undefined; + + log.info(colors.green(`Pushing datatable_migration ${m.datatable}/${base}`)); + await wmill.upsertDatatableMigration({ + workspace, + datatableName: m.datatable, + requestBody: { + timestamp: m.timestamp, + name, + code_up, + ...(code_down !== undefined ? { code_down } : {}), + }, + }); +} + +/** + * Upsert the on-disk migrations of a data table to the workspace, so a freshly + * created migration file works with `wmill datatable migrate up` even without a + * prior `wmill sync push`. Pushes only migrations that are new or edited + * (compared against the workspace's current definitions); it never deletes + * remote migrations absent on disk and never touches other item kinds. + */ +export async function pushLocalMigrations( + workspace: string, + datatableName: string, +): Promise { + const dir = path.join(process.cwd(), MIGRATIONS_DIR, datatableName); + if (!fs.existsSync(dir)) return; + + // Local migrations are identified by their `.up.sql` file (the up file is + // mandatory); this deliberately ignores files that were only deleted locally. + const local: { timestamp: number; name: string }[] = []; + for (const file of fs.readdirSync(dir)) { + const m = file.match(/^(\d+)_(.*)\.up\.sql$/); + if (m) local.push({ timestamp: Number(m[1]), name: m[2] }); + } + if (local.length === 0) return; + + const remote = await wmill.listDatatableMigrations({ workspace }); + const remoteByTs = new Map( + remote + .filter((r) => r.datatable === datatableName) + .map((r) => [r.timestamp, r] as const), + ); + + for (const { timestamp, name } of local) { + const base = `${timestamp}_${name}`; + const code_up = await readTextFile(path.join(dir, `${base}.up.sql`)); + const downPath = path.join(dir, `${base}.down.sql`); + const code_down = fs.existsSync(downPath) + ? await readTextFile(downPath) + : undefined; + + const r = remoteByTs.get(timestamp); + const unchanged = + r !== undefined && + r.name === name && + r.code_up === code_up && + (r.code_down ?? undefined) === code_down; + if (unchanged) continue; + + log.info(colors.green(`Pushing datatable_migration ${datatableName}/${base}`)); + await wmill.upsertDatatableMigration({ + workspace, + datatableName, + requestBody: { + timestamp, + name, + code_up, + ...(code_down !== undefined ? { code_down } : {}), + }, + }); + } +} + +/** + * After a push that introduced new migrations, list them and (interactively) + * offer to run them, equivalent to `wmill datatable migrate up` on each affected + * data table. + */ +export async function offerToRunNewMigrations( + workspace: string, + newMigrations: { datatable: string; timestamp: number; name: string }[], + opts?: { yes?: boolean; jsonOutput?: boolean }, +): Promise { + if (newMigrations.length === 0) return; + + log.info(colors.green("New migrations were pushed:")); + for (const m of newMigrations) { + log.info(colors.gray(` ${m.datatable}: ${m.timestamp} ${m.name}`)); + } + + // Running migrations mutates the data tables, so skip the prompt in + // non-interactive contexts (--yes, --json, no TTY). + const interactive = !opts?.jsonOutput && !opts?.yes && !!process.stdin.isTTY; + if (!interactive) { + return; + } + + const shouldRun = await Confirm.prompt({ + message: "New migrations were pushed, run them?", + default: false, + }); + if (!shouldRun) { + return; + } + + for (const datatable of new Set(newMigrations.map((m) => m.datatable))) { + await runMigrations(workspace, datatable); + } +} diff --git a/cli/src/commands/generate-metadata/generate-metadata.ts b/cli/src/commands/generate-metadata/generate-metadata.ts index 8517ac0912..7eab79b52d 100644 --- a/cli/src/commands/generate-metadata/generate-metadata.ts +++ b/cli/src/commands/generate-metadata/generate-metadata.ts @@ -2,7 +2,7 @@ import { Command } from "@cliffy/command"; import { Confirm } from "@cliffy/prompt/confirm"; import { colors } from "@cliffy/ansi/colors"; import { sep as SEP } from "node:path"; -import { GlobalOptions } from "../../types.ts"; +import { GlobalOptions, isDatatableMigrationPath } from "../../types.ts"; import { SyncOptions, mergeConfigWithConfigFile } from "../../core/conf.ts"; import { resolveWorkspace } from "../../core/context.ts"; import { requireLogin } from "../../core/auth.ts"; @@ -54,6 +54,8 @@ async function walkLocalScripts( (!isD && !exts.some((ext) => p.endsWith(ext))) || ignore(p, isD) || isFolderResourcePathAnyFormat(p) || + // Datatable migration `.sql` files aren't Windmill scripts. + isDatatableMigrationPath(p) || (isScriptModulePath(p) && !isModuleEntryPoint(p)), false, {}, @@ -221,6 +223,8 @@ function categorizeLocalFiles( } else if ( exts.some((ext) => p.endsWith(ext)) && !isFolderResourcePathAnyFormat(p) && + // Datatable migration `.sql` files aren't Windmill scripts. + !isDatatableMigrationPath(p) && !(isScriptModulePath(p) && !isModuleEntryPoint(p)) ) { scripts.push(p); diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 8e1ecdfef6..65ce10eaad 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -23,10 +23,17 @@ import { showDiff, extractNativeTriggerInfo, redactEncryptionKey, + isDatatableMigrationPath, + parseDatatableMigrationPath, } from "../../types.ts"; import { downloadZip } from "./pull.ts"; import { runLint, printReport, checkMissingLocks } from "../lint/lint.ts"; import { pullSharedUi, pushSharedUi } from "../shared_ui.ts"; +import { + pushMigrationFromDisk, + offerToRunNewMigrations, + validateLocalMigrations, +} from "../datatable_migrations.ts"; import { exts, @@ -2477,7 +2484,8 @@ const isNotWmillFile = (p: string, isDirectory: boolean) => { !p.startsWith("g" + SEP) && !p.startsWith("users" + SEP) && !p.startsWith("groups" + SEP) && - !p.startsWith("dependencies" + SEP) + !p.startsWith("dependencies" + SEP) && + !p.startsWith("migrations" + SEP) ); } @@ -2488,6 +2496,11 @@ const isNotWmillFile = (p: string, isDirectory: boolean) => { try { const typ = getTypeStrFromPath(p); + // Datatable migrations live under migrations/datatable//, outside + // the u/f/g namespaces, but are valid wmill files. + if (typ == "datatable_migration") { + return false; + } if ( typ == "resource-type" || typ == "settings" || @@ -2519,7 +2532,8 @@ export const isWhitelisted = (p: string) => { p == "ui" || p == "users" || p == "groups" || - p == "dependencies" + p == "dependencies" || + p == "migrations" ); }; @@ -2614,6 +2628,11 @@ interface ChangeTracker { } async function addToChangedIfNotExists(p: string, tracker: ChangeTracker) { + // Datatable migration .sql files are not scripts; they're synced via the + // dedicated datatable_migration handler in the push loop. + if (isDatatableMigrationPath(p)) { + return; + } const isScript = exts.some((e) => p.endsWith(e)) && !isFileResource(p) && !isFilesetResource(p); if (isScript) { if (isFlowPath(p)) { @@ -3119,7 +3138,7 @@ export async function pull( change.path.endsWith(".json") ) { log.info( - `Editing ${getTypeStrFromPath(change.path)} ${targetPath}${ + `Editing ${changeTypeLabel(change.path)}${targetPath}${ targetPath !== change.path ? colors.gray(` (workspace-specific override for ${change.path})`) : "" @@ -3137,7 +3156,7 @@ export async function pull( if (opts.stateful) { await mkdir(path.dirname(stateTarget), { recursive: true }); log.info( - `Adding ${getTypeStrFromPath(change.path)} ${targetPath}${ + `Adding ${changeTypeLabel(change.path)}${targetPath}${ targetPath !== change.path ? colors.gray(` (workspace-specific override for ${change.path})`) : "" @@ -3146,7 +3165,7 @@ export async function pull( } await writeFile(target, change.content, "utf-8"); log.info( - `Writing ${getTypeStrFromPath(change.path)} ${targetPath}${ + `Writing ${changeTypeLabel(change.path)}${targetPath}${ targetPath !== change.path ? colors.gray(` (workspace-specific override for ${change.path})`) : "" @@ -3158,7 +3177,7 @@ export async function pull( } else if (change.name === "deleted") { try { log.info( - `Deleting ${getTypeStrFromPath(change.path)} ${change.path}`, + `Deleting ${changeTypeLabel(change.path)}${change.path}`, ); await rm(target); if (opts.stateful) { @@ -3350,6 +3369,9 @@ export async function pull( log.warn(`Failed to pull shared UI folder: ${e}`); } + // Datatable migrations are part of the workspace export now, so they flow + // through the normal diff/apply above as `datatable_migration` items. + // Git-sync deployment-callback mode stops here: branch checkout + pull have // happened, but commit + push are the caller's job. The hub script does // them in-process with `set_gpg_signing_secret` so the agent's pre-warmed @@ -3425,6 +3447,14 @@ export async function gitDeploy( } as any); } +// Display label for a change's type, with a trailing space. Datatable migrations +// are self-describing via their `migrations/datatable/...` path, so they get no +// label prefix. +function changeTypeLabel(p: string): string { + const t = getTypeStrFromPath(p); + return t === "datatable_migration" ? "" : `${t} `; +} + function prettyChanges( changes: Change[], specificItems?: SpecificItemsConfig, @@ -3456,7 +3486,7 @@ function prettyChanges( if (change.name === "added") { log.info( colors.green( - `+ ${getTypeStrFromPath(change.path)} ` + + `+ ${changeTypeLabel(change.path)}` + displayPath + colors.gray(wsNote), ) + extraNote, @@ -3464,7 +3494,7 @@ function prettyChanges( } else if (change.name === "deleted") { log.info( colors.red( - `- ${getTypeStrFromPath(change.path)} ` + + `- ${changeTypeLabel(change.path)}` + displayPath + colors.gray(wsNote), ), @@ -3473,7 +3503,7 @@ function prettyChanges( const changeType = getTypeStrFromPath(change.path); log.info( colors.yellow( - `~ ${changeType} ` + + `~ ${changeTypeLabel(change.path)}` + displayPath + colors.gray(wsNote) + (change.codebase ? ` (codebase changed)` : ""), @@ -4221,6 +4251,24 @@ export async function push( )); } + // Reject malformed datatable migrations (duplicate timestamps, orphan downs) + // before touching the remote, scanning only the data tables in this push. + const migrationDatatables = new Set( + changes + .map((c) => parseDatatableMigrationPath(c.path)?.datatable) + .filter((d): d is string => !!d), + ); + if (migrationDatatables.size > 0) { + const migrationErrors = validateLocalMigrations(migrationDatatables); + if (migrationErrors.length > 0) { + log.error( + "Invalid datatable migrations, aborting push:\n" + + migrationErrors.map((e) => ` - ${e}`).join("\n"), + ); + process.exit(1); + } + } + if ( !opts.yes && !(await Confirm.prompt({ @@ -4293,6 +4341,21 @@ export async function push( // Cache git branch at the start to avoid repeated execSync calls per change const cachedWsNameForPush = wsNameForFiles || (isGitRepository() ? getCurrentGitBranch() : null); + // Datatable migrations are two files (.up.sql/.down.sql) for one record, so + // dedupe upsert/delete by (datatable, version) across the whole push. + const pushedMigrationKeys = new Set(); + // Migrations newly added by this push (an added .up.sql) — offered to run once + // the push has completed. + const newDatatableMigrations = changes + .filter((c) => c.name === "added") + .map((c) => parseDatatableMigrationPath(c.path)) + .filter((p) => !!p && p.kind === "up") + .map((p) => ({ + datatable: p!.datatable, + timestamp: p!.timestamp, + name: p!.name, + })); + while (queue.length > 0 || pool.size > 0) { // Fill the pool until we reach the effective parallelism limit. // During the folder-meta phase this is 1 (sequential) so no item change @@ -4320,6 +4383,20 @@ export async function push( } for await (const change of changes) { + // A datatable migration is one record across two files; upsert/delete + // it from disk once (deduped), regardless of which file changed. + if (isDatatableMigrationPath(change.path)) { + const parsed = parseDatatableMigrationPath(change.path); + if (parsed) { + const key = `${parsed.datatable}\0${parsed.timestamp}`; + if (!pushedMigrationKeys.has(key)) { + pushedMigrationKeys.add(key); + await pushMigrationFromDisk(workspace.workspaceId, parsed); + } + } + continue; + } + let stateTarget = undefined; if (stateful) { try { @@ -4974,6 +5051,16 @@ export async function push( } catch (e) { log.warn(`Failed to push shared UI folder: ${e}`); } + try { + await offerToRunNewMigrations(workspace.workspaceId, newDatatableMigrations, { + yes: opts.yes, + jsonOutput: opts.jsonOutput, + }); + } catch (e: any) { + log.warn( + `Failed to run new datatable migrations: ${e?.body ?? e?.message ?? e}`, + ); + } const lockJobs = await checkServerLockJobs( workspace.workspaceId, pushStartedAt, @@ -5039,6 +5126,7 @@ export async function push( } catch (e) { log.warn(`Failed to push shared UI folder: ${e}`); } + // No changes pushed, so no new datatable migrations to run. if (opts.jsonOutput) { console.log( JSON.stringify( diff --git a/cli/src/commands/workspace/merge.ts b/cli/src/commands/workspace/merge.ts index b8ef24f7d4..585772a9d0 100644 --- a/cli/src/commands/workspace/merge.ts +++ b/cli/src/commands/workspace/merge.ts @@ -11,10 +11,12 @@ import { deleteItemInWorkspace, getOnBehalfOf, isTriggerOrScheduleKind, + parseDatatableMigrationDeployPath, type DeployKind, type DeployProvider, type TriggerDeployKind, } from "../../../windmill-utils-internal/src/deploy.ts"; +import { offerToRunNewMigrations } from "../datatable_migrations.ts"; // --------------------------------------------------------------------------- // Provider adapter — wraps CLI's standalone API functions @@ -85,6 +87,10 @@ const provider: DeployProvider = { createSchedule: wmill.createSchedule, updateSchedule: wmill.updateSchedule, deleteSchedule: wmill.deleteSchedule, + // Datatable migrations + listDatatableMigrations: wmill.listDatatableMigrations, + upsertDatatableMigration: wmill.upsertDatatableMigration, + deleteDatatableMigration: wmill.deleteDatatableMigration, }; /** @@ -530,6 +536,14 @@ async function mergeWorkspaces( // 10. Deploy let successCount = 0; let failCount = 0; + // Datatable migrations deployed (not deleted) into the target. Deploying a + // migration only upserts its definition — the target schema is unchanged until + // the migration is run — so offer to run them afterwards (like the push path). + const deployedMigrations: { + datatable: string; + timestamp: number; + name: string; + }[] = []; for (const diff of sorted) { const label = `${diff.kind}:${diff.path}`; @@ -573,6 +587,12 @@ async function mergeWorkspaces( if (result.success) { log.info(colors.green(` ✓ ${label}`)); successCount++; + if ( + !itemDeletedInSource && + (diff.kind as DeployKind) === "datatable_migration" + ) { + deployedMigrations.push(parseDatatableMigrationDeployPath(diff.path)); + } } else { log.info(colors.red(` ✗ ${label}: ${result.error}`)); failCount++; @@ -606,6 +626,18 @@ async function mergeWorkspaces( ) ); } + + // 13. Deployed migration definitions don't touch the target schema until run; + // offer to run them on the target now (interactive only, like the push path). + if (deployedMigrations.length > 0) { + try { + await offerToRunNewMigrations(workspaceTo, deployedMigrations, { + yes: opts.yes, + }); + } catch (e) { + log.warn(colors.yellow(`Failed to run deployed migrations: ${e}`)); + } + } } export { mergeWorkspaces }; diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index e2cf0be86b..03a5c09eb7 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -6593,6 +6593,13 @@ datatable related commands - \`datatable run \` - run a SQL query on a datatable - \`-n --name \` - Datatable name (default: main) - \`-s --silent\` - Output only the final result as JSON. Useful for scripting. +- \`datatable migrate\` - manage datatable migrations + - \`datatable migrate new \` - scaffold a new migration (.up.sql / .down.sql files) + - \`-d --datatable \` - Target datatable (default: main) + - \`datatable migrate up\` - apply all pending migrations to the main datatable (or one via --datatable) + - \`-d --datatable \` - Target datatable (default: main) + - \`datatable migrate down\` - roll back the most recent migration on the main datatable (or one via --datatable) + - \`-d --datatable \` - Target datatable (default: main) - \`datatable create [name:string]\` - register a datatable database in the workspace (default: instance-backed 'main') so scripts can use datatable:// - \`--resource \` - Back the datatable with an existing postgresql resource path instead of the instance database - \`--force\` - Allow adding to a workspace that already has datatables (fork metadata on existing ones is not preserved) @@ -6861,19 +6868,18 @@ Manage jobs (list, inspect, cancel) ### jobs -Pull completed and queued jobs from workspace - -**Arguments:** \`[workspace:string]\` - -**Options:** -- \`-c, --completed-output \` - Completed jobs output file (default: completed_jobs.json) -- \`-q, --queued-output \` - Queued jobs output file (default: queued_jobs.json) -- \`--skip-worker-check\` - Skip checking for active workers before export +Manage jobs (import/export) **Subcommands:** -- \`jobs pull\` -- \`jobs push\` +- \`jobs pull [workspace:string]\` - Pull completed and queued jobs from workspace + - \`-c, --completed-output \` - Completed jobs output file (default: completed_jobs.json) + - \`-q, --queued-output \` - Queued jobs output file (default: queued_jobs.json) + - \`--skip-worker-check\` - Skip checking for active workers before export +- \`jobs push [workspace:string]\` - Push completed and queued jobs to workspace + - \`-c, --completed-file \` - Completed jobs input file (default: completed_jobs.json) + - \`-q, --queued-file \` - Queued jobs input file (default: queued_jobs.json) + - \`--skip-worker-check\` - Skip checking for active workers before import ### lint diff --git a/cli/src/types.ts b/cli/src/types.ts index 5de6cea48b..c125ae2c2e 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -274,10 +274,10 @@ export function parseFromPath(p: string, content: string): any { return isWorkspaceDependencies(p) ? content : p.endsWith(".yaml") - ? yamlParseContent(p, content) - : p.endsWith(".json") - ? JSON.parse(content) - : content; + ? yamlParseContent(p, content) + : p.endsWith(".json") + ? JSON.parse(content) + : content; } export function parseFromFile(p: string): any { if (p.endsWith(".json")) { @@ -288,9 +288,38 @@ export function parseFromFile(p: string): any { throw new Error("Could not read file " + p); } } +/** + * Parse a `migrations/datatable//_.(up|down).sql` + * path into its parts. Returns undefined for any other path. + */ +export function parseDatatableMigrationPath(p: string): + | { datatable: string; timestamp: number; name: string; kind: "up" | "down" } + | undefined { + const parts = p.split("/"); + if ( + parts[0] !== "migrations" || + parts[1] !== "datatable" || + parts.length !== 4 + ) + return undefined; + const m = parts[3].match(/^(\d+)_(.*)\.(up|down)\.sql$/); + if (!m) return undefined; + return { + datatable: parts[2], + timestamp: Number(m[1]), + name: m[2], + kind: m[3] as "up" | "down", + }; +} + +export function isDatatableMigrationPath(p: string): boolean { + return parseDatatableMigrationPath(p) !== undefined; +} + export function getTypeStrFromPath( p: string ): + | "datatable_migration" | "script" | "variable" | "flow" @@ -316,6 +345,9 @@ export function getTypeStrFromPath( | "settings" | "encryption_key" | "workspace_dependencies" { + if (isDatatableMigrationPath(p)) { + return "datatable_migration"; + } if (isScriptModulePath(p)) { return "script"; } diff --git a/cli/test/datatable_migrations_unit.test.ts b/cli/test/datatable_migrations_unit.test.ts new file mode 100644 index 0000000000..09af49f890 --- /dev/null +++ b/cli/test/datatable_migrations_unit.test.ts @@ -0,0 +1,122 @@ +/** + * Unit tests for datatable-migration path parsing and local validation. + * + * These exercise pure logic with no backend: + * - `parseDatatableMigrationPath` recognizes only the + * `migrations/datatable/
/_.(up|down).sql` shape. + * - `validateLocalMigrations` rejects the two invalid on-disk states a push + * must catch: two up (or two down) files sharing a timestamp, and a + * `.down.sql` with no matching `.up.sql`. + */ + +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import * as os from "node:os"; +import { parseDatatableMigrationPath } from "../src/types.ts"; +import { validateLocalMigrations } from "../src/commands/datatable_migrations.ts"; + +describe("parseDatatableMigrationPath", () => { + test("parses up and down files of the new layout", () => { + expect( + parseDatatableMigrationPath( + "migrations/datatable/mydt/20260101000001_create_users.up.sql", + ), + ).toEqual({ + datatable: "mydt", + timestamp: 20260101000001, + name: "create_users", + kind: "up", + }); + expect( + parseDatatableMigrationPath( + "migrations/datatable/my-dt/42_x.down.sql", + ), + ).toEqual({ datatable: "my-dt", timestamp: 42, name: "x", kind: "down" }); + }); + + test("rejects unrelated, legacy and malformed paths", () => { + for ( + const p of [ + // legacy top-level layout + "datatable_migrations/mydt/20260101000001_x.up.sql", + // wrong sub-namespace / depth + "migrations/ducklake/mydt/1_x.up.sql", + "migrations/datatable/1_x.up.sql", + "migrations/datatable/mydt/sub/1_x.up.sql", + // not a migration file + "migrations/datatable/mydt/notes.txt", + "migrations/datatable/mydt/x.up.sql", // no numeric timestamp prefix + // unrelated workspace files + "f/foo/bar.script.yaml", + "u/admin/script.ts", + ] + ) { + expect(parseDatatableMigrationPath(p)).toBeUndefined(); + } + }); +}); + +describe("validateLocalMigrations", () => { + let prevCwd: string; + let tmp: string; + + beforeEach(() => { + prevCwd = process.cwd(); + tmp = fs.mkdtempSync(path.join(os.tmpdir(), "dtmig-")); + process.chdir(tmp); + }); + afterEach(() => { + process.chdir(prevCwd); + fs.rmSync(tmp, { recursive: true, force: true }); + }); + + function write(datatable: string, file: string) { + const dir = path.join(tmp, "migrations", "datatable", datatable); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, file), "-- sql\n"); + } + + test("accepts up+down pairs and up-only migrations", () => { + write("mydt", "20260101000001_create_users.up.sql"); + write("mydt", "20260101000001_create_users.down.sql"); + write("mydt", "20260101000002_add_email.up.sql"); // down is optional + expect(validateLocalMigrations()).toEqual([]); + }); + + test("flags two up files sharing a timestamp", () => { + write("mydt", "20260101000003_foo.up.sql"); + write("mydt", "20260101000003_bar.up.sql"); + const errors = validateLocalMigrations(); + expect(errors.length).toBe(1); + expect(errors[0]).toContain("20260101000003"); + }); + + test("flags two down files sharing a timestamp", () => { + write("mydt", "20260101000004_a.up.sql"); + write("mydt", "20260101000004_a.down.sql"); + write("mydt", "20260101000004_b.down.sql"); + const errors = validateLocalMigrations(); + // duplicate down + the b.down orphan (no b.up) + expect(errors.some((e) => e.includes("down") && e.includes("20260101000004"))).toBe(true); + }); + + test("flags a down file with no matching up", () => { + write("mydt", "20260101000005_orphan.down.sql"); + const errors = validateLocalMigrations(); + expect(errors.length).toBe(1); + expect(errors[0]).toContain("20260101000005_orphan"); + }); + + test("only validates the requested datatables", () => { + write("bad", "20260101000006_x.up.sql"); + write("bad", "20260101000006_y.up.sql"); // duplicate, but in 'bad' + write("good", "20260101000007_ok.up.sql"); + expect(validateLocalMigrations(new Set(["good"]))).toEqual([]); + expect(validateLocalMigrations(new Set(["bad"])).length).toBe(1); + }); + + test("returns no errors when the migrations folder is absent", () => { + expect(validateLocalMigrations()).toEqual([]); + }); +}); diff --git a/cli/windmill-utils-internal/package.json b/cli/windmill-utils-internal/package.json index 37cfbfaef4..525bb0fc0b 100644 --- a/cli/windmill-utils-internal/package.json +++ b/cli/windmill-utils-internal/package.json @@ -1,6 +1,6 @@ { "name": "windmill-utils-internal", - "version": "1.7.0", + "version": "1.8.2", "description": "Internal utility functions for Windmill", "main": "dist/cjs/index.js", "module": "dist/esm/index.js", diff --git a/cli/windmill-utils-internal/src/deploy.ts b/cli/windmill-utils-internal/src/deploy.ts index 990fd47edd..4c2bd735b3 100644 --- a/cli/windmill-utils-internal/src/deploy.ts +++ b/cli/windmill-utils-internal/src/deploy.ts @@ -21,6 +21,7 @@ export type DeployKind = | "resource_type" | "folder" | "schedule" + | "datatable_migration" | "http_trigger" | "websocket_trigger" | "kafka_trigger" @@ -161,6 +162,24 @@ export interface DeployProvider { requestBody: any; }): Promise; deleteFolder(p: { workspace: string; name: string }): Promise; + // Datatable migrations. In the diff, an item's `path` is + // `/_` (see `parseDatatableMigrationDeployPath`). + listDatatableMigrations(p: { workspace: string }): Promise; + upsertDatatableMigration(p: { + workspace: string; + datatableName: string; + requestBody: { + timestamp: number; + name: string; + code_up: string; + code_down?: string; + }; + }): Promise; + deleteDatatableMigration(p: { + workspace: string; + datatableName: string; + timestamp: number; + }): Promise; // Triggers — per-kind dispatch is delegated to the implementor so the shared // module doesn't need to know about each of the 9 trigger services. existsTriggerByKind( @@ -299,6 +318,40 @@ function toError(e: unknown): string { return err.body || err.message || String(e); } +// A datatable-migration diff item's path is `/_` +// (mirrors the backend, e.g. `mydt/20260101000001_create_users`). +export function parseDatatableMigrationDeployPath(path: string): { + datatable: string; + timestamp: number; + name: string; +} { + const slash = path.indexOf("/"); + const underscore = slash >= 0 ? path.indexOf("_", slash + 1) : -1; + if (slash < 0 || underscore < 0) { + throw new Error(`Invalid datatable migration path: ${path}`); + } + const datatable = path.slice(0, slash); + const timestamp = Number(path.slice(slash + 1, underscore)); + const name = path.slice(underscore + 1); + if (!datatable || !Number.isFinite(timestamp) || !name) { + throw new Error(`Invalid datatable migration path: ${path}`); + } + return { datatable, timestamp, name }; +} + +// The backend rejects `upsertDatatableMigration` when the target data table +// hasn't opted in to migrations. Turn that opaque 400 into an explicit, +// deploy-context message (falls back to the original error otherwise). +function asMigrationsDisabledError(e: unknown, datatable: string): unknown { + const msg = (e as { body?: string; message?: string })?.body ?? '' + if (typeof msg === "string" && /migrations are not enabled/i.test(msg)) { + return new Error( + `Data table '${datatable}' has not opted in to migrations on the target workspace; enable migrations for it there before deploying its migrations.` + ); + } + return e; +} + // --------------------------------------------------------------------------- // checkItemExists // --------------------------------------------------------------------------- @@ -325,6 +378,12 @@ export async function checkItemExists( return provider.existsFolder({ workspace, name: folderName(path) }); } else if (kind === "schedule") { return provider.existsSchedule({ workspace, path }); + } else if (kind === "datatable_migration") { + const { datatable, timestamp } = parseDatatableMigrationDeployPath(path); + const migrations = await provider.listDatatableMigrations({ workspace }); + return (migrations as { datatable: string; timestamp: number }[]).some( + (m) => m.datatable === datatable && m.timestamp === timestamp + ); } else if (isTriggerKind(kind)) { return provider.existsTriggerByKind(kind, { workspace, path }); } @@ -639,6 +698,41 @@ export async function deployItem( requestBody, }); } + } else if (kind === "datatable_migration") { + const { datatable, timestamp } = parseDatatableMigrationDeployPath(path); + const migrations = await provider.listDatatableMigrations({ + workspace: workspaceFrom, + }); + const migration = ( + migrations as { + datatable: string; + timestamp: number; + name: string; + code_up: string; + code_down?: string; + }[] + ).find((m) => m.datatable === datatable && m.timestamp === timestamp); + if (!migration) { + throw new Error( + `Datatable migration ${path} not found in ${workspaceFrom}` + ); + } + try { + await provider.upsertDatatableMigration({ + workspace: workspaceTo, + datatableName: datatable, + requestBody: { + timestamp: migration.timestamp, + name: migration.name, + code_up: migration.code_up, + ...(migration.code_down != null + ? { code_down: migration.code_down } + : {}), + }, + }); + } catch (e) { + throw asMigrationsDisabledError(e, datatable); + } } else { throw new Error(`Unknown kind: ${kind}`); } @@ -684,6 +778,13 @@ export async function deleteItemInWorkspace( await provider.deleteFolder({ workspace, name: folderName(path) }); } else if (kind === "schedule") { await provider.deleteSchedule({ workspace, path }); + } else if (kind === "datatable_migration") { + const { datatable, timestamp } = parseDatatableMigrationDeployPath(path); + await provider.deleteDatatableMigration({ + workspace, + datatableName: datatable, + timestamp, + }); } else if (isTriggerKind(kind)) { await provider.deleteTriggerByKind(kind, { workspace, path }); } else { @@ -767,6 +868,29 @@ export async function getItemValue( } else if (isTriggerKind(kind)) { const trigger = await provider.getTriggerValue(kind, { workspace, path }); return stripTriggerOrScheduleRuntimeFields(trigger); + } else if (kind === "datatable_migration") { + // Surface the migration SQL so the diff drawer shows the up/down bodies a + // reviewer needs to inspect before deploying. + const { datatable, timestamp } = parseDatatableMigrationDeployPath(path); + const migrations = (await provider.listDatatableMigrations({ + workspace, + })) as { + datatable: string; + timestamp: number; + name: string; + code_up: string; + code_down?: string; + }[]; + const migration = migrations.find( + (m) => m.datatable === datatable && m.timestamp === timestamp + ); + if (migration) { + return { + name: migration.name, + code_up: migration.code_up, + code_down: migration.code_down ?? null, + }; + } } } catch { // Item may not exist diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 3023846ce6..0df7eef6f6 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -94,7 +94,7 @@ "windmill-parser-wasm-wac": "1.668.6", "windmill-parser-wasm-yaml": "1.593.0", "windmill-sql-datatype-parser-wasm": "1.512.0", - "windmill-utils-internal": "^1.7.0", + "windmill-utils-internal": "1.8.2", "xterm": "^5.3.0", "xterm-readline": "^1.1.2", "y-monaco": "^0.1.4", @@ -291,7 +291,6 @@ "integrity": "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/helper-validator-identifier": "^7.28.5", "js-tokens": "^4.0.0", @@ -307,7 +306,6 @@ "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=6.9.0" } @@ -865,7 +863,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": "^14 || ^16 || >=18" }, @@ -875,21 +872,21 @@ } }, "node_modules/@emnapi/core": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", - "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", + "version": "1.11.2", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.2.tgz", + "integrity": "sha512-TC8MkTuZUtcTSiFeuC0ksCh9QIJ5+F21MvZ4Wn4ORfYaFJ/0dsiudv5tVkejgwZlwQ39jL9WWDe2lz8x0WglOA==", "dev": true, "license": "MIT", "optional": true, "dependencies": { - "@emnapi/wasi-threads": "1.2.1", + "@emnapi/wasi-threads": "1.2.2", "tslib": "^2.4.0" } }, "node_modules/@emnapi/runtime": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", - "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "version": "1.11.2", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.2.tgz", + "integrity": "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA==", "dev": true, "license": "MIT", "optional": true, @@ -898,9 +895,9 @@ } }, "node_modules/@emnapi/wasi-threads": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", - "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", + "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", "dev": true, "license": "MIT", "optional": true, @@ -1392,6 +1389,7 @@ "integrity": "sha512-Jer+M7DgIwT5IHfTayb4Iw/fkkxWNmC/mqn/nMh9JrbPbkxmyabfLQnhJ+JDn5HK77f84j34lubO3iqFtYAfMg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@floating-ui/core": "^1.3.1", "@floating-ui/dom": "^1.4.5", @@ -1548,6 +1546,7 @@ "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.8.tgz", "integrity": "sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A==", "license": "MIT", + "peer": true, "funding": { "type": "opencollective", "url": "https://opencollective.com/popperjs" @@ -1774,8 +1773,8 @@ "license": "MIT", "optional": true, "dependencies": { - "@emnapi/core": "1.10.0", - "@emnapi/runtime": "1.10.0", + "@emnapi/core": "1.11.2", + "@emnapi/runtime": "1.11.2", "@napi-rs/wasm-runtime": "^1.1.4" }, "engines": { @@ -1912,6 +1911,7 @@ "integrity": "sha512-iAIPEahFgDJJyvz8g0jP08KvqnM6JvdW8YfsygZ+pMeMvyM2zssWMltcsotETvjSZ82G3VlitgDtBIvpQSZrTA==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@standard-schema/spec": "^1.0.0", "@sveltejs/acorn-typescript": "^1.0.5", @@ -2007,6 +2007,7 @@ "integrity": "sha512-ILXmxC7HAsnkK2eslgPetrqqW1BKSL7LktsFgqzNj83MaivMGZzluWq32m25j2mDOjmSKX7GGWahePhuEs7P/g==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "deepmerge": "^4.3.1", "magic-string": "^0.30.21", @@ -2468,8 +2469,7 @@ "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.5.tgz", "integrity": "sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@types/ms": { "version": "2.1.0", @@ -2482,8 +2482,7 @@ "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.4.tgz", "integrity": "sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@types/semver": { "version": "7.7.1", @@ -2552,6 +2551,7 @@ "integrity": "sha512-VlJEV0fOQ7BExOsHYAGrgbEiZoi8D+Bl2+f6V2RrXerRSylnp+ZBHmPvaIa8cz0Ajx7WO7Z5RqfgYg7ED1nRhA==", "dev": true, "license": "BSD-2-Clause", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "5.62.0", "@typescript-eslint/types": "5.62.0", @@ -3079,6 +3079,7 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -3132,6 +3133,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -3277,7 +3279,6 @@ "integrity": "sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -3305,7 +3306,6 @@ "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" } @@ -3386,8 +3386,7 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-2.0.0.tgz", "integrity": "sha512-1ugUSr8BHXRnK23KfuYS+gVMC3LB8QGH9W1iGtDPsNWoQbgtXSExkBu2aDR4epiGWZOjZsj6lDl/N/AqqTC3UA==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/base64-js": { "version": "1.5.1", @@ -3514,6 +3513,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.8.9", "caniuse-lite": "^1.0.30001746", @@ -3711,7 +3711,6 @@ "integrity": "sha512-Rjs1H+A9R+Ig+4E/9oyB66UC5Mj9Xq3N//vcLf2WzgdTi/3gUu3Z9KoqmlrEG4VuuLK8wJHofxzdQXz/knhiYg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "camelcase": "^6.3.0", "map-obj": "^4.1.0", @@ -3731,7 +3730,6 @@ "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10" }, @@ -3745,7 +3743,6 @@ "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10" }, @@ -3759,7 +3756,6 @@ "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", "dev": true, "license": "(MIT OR CC0-1.0)", - "peer": true, "engines": { "node": ">=10" }, @@ -3868,6 +3864,7 @@ "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.5.1.tgz", "integrity": "sha512-GIjfiT9dbmHRiYi6Nl2yFCq7kkwdkp1W/lp2J99rX0yo9tgJGn3lKQATztIjb5tVtevcBtIdICNWqlq5+E8/Pw==", "license": "MIT", + "peer": true, "dependencies": { "@kurkle/color": "^0.3.0" }, @@ -4144,7 +4141,6 @@ "integrity": "sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "import-fresh": "^3.3.0", "js-yaml": "^4.1.0", @@ -4209,7 +4205,6 @@ "integrity": "sha512-8HFEBPKhOpJPEPu70wJJetjKta86Gw9+CCyCnB3sui2qQfOvRyqBy4IKLKKAwdMpWb2lHXWk9Wb4Z6AmaUT1Pg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" } @@ -4397,6 +4392,7 @@ "resolved": "https://registry.npmjs.org/cytoscape/-/cytoscape-3.34.0.tgz", "integrity": "sha512-62rNSrioXw93uliKFBwjukeQyeWwH2PqDrTac31r2P6464u3AUvTk0xS4LVvT251g7IgkFunrI48ZEZGjywSOg==", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10" } @@ -4819,6 +4815,7 @@ "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", "license": "ISC", + "peer": true, "engines": { "node": ">=12" } @@ -4918,6 +4915,7 @@ "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz", "integrity": "sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==", "license": "MIT", + "peer": true, "dependencies": { "@babel/runtime": "^7.21.0" }, @@ -4958,7 +4956,6 @@ "integrity": "sha512-VfxadyCECXgQlkoEAjeghAr5gY3Hf+IKjKb+X8tGVDtveCjN+USwprd2q3QXBR9T1+x2DG0XZF5/w+7HAtSaXA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10" }, @@ -4972,7 +4969,6 @@ "integrity": "sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "decamelize": "^1.1.0", "map-obj": "^1.0.0" @@ -4990,7 +4986,6 @@ "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -5001,7 +4996,6 @@ "integrity": "sha512-7N/q3lyZ+LVCp7PzuxrJr4KMbBE2hW7BT7YNia330OFxIf4d3r5zVpicP2650l7CPN6RM9zOJRl3NGpqSiw3Eg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -5495,7 +5489,6 @@ "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "is-arrayish": "^0.2.1" } @@ -5593,6 +5586,7 @@ "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -6149,7 +6143,6 @@ "integrity": "sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">= 4.9.1" } @@ -6509,7 +6502,6 @@ "integrity": "sha512-NGbfmJBp9x8IxyJSd1P+otYK8vonoJactOogrVfFRIAEY1ukil8RSKDz2Yo7wh1oihl51l/r6W4epkeKJHqL8A==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "global-prefix": "^3.0.0" }, @@ -6523,7 +6515,6 @@ "integrity": "sha512-awConJSVCHVGND6x3tmMaKcQvwXLhjdkmomy2W+Goaui8YPgYgXJZewhg3fWC+DlfqqQuWg8AwqjGTD2nAPVWg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "ini": "^1.3.5", "kind-of": "^6.0.2", @@ -6539,7 +6530,6 @@ "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "isexe": "^2.0.0" }, @@ -6589,8 +6579,7 @@ "resolved": "https://registry.npmjs.org/globjoin/-/globjoin-0.1.4.tgz", "integrity": "sha512-xYfnw62CKG8nLkZBfWbhWwDw02CHty86jfPcc2cr3ZfeuK9ysoVPPEUxf21bAD/rWAgk52SuBrLJlefNy8mvFg==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/gopd": { "version": "1.2.0", @@ -6673,7 +6662,6 @@ "integrity": "sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=6" } @@ -6910,7 +6898,6 @@ "integrity": "sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "lru-cache": "^6.0.0" }, @@ -6924,7 +6911,6 @@ "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "yallist": "^4.0.0" }, @@ -6937,8 +6923,7 @@ "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", "dev": true, - "license": "ISC", - "peer": true + "license": "ISC" }, "node_modules/html-tags": { "version": "3.3.1", @@ -6946,7 +6931,6 @@ "integrity": "sha512-ztqyC3kLto0e9WbNp0aeP+M3kTt+nbaIveGmUxAtZa+8iFgKLUOD4YKM5j+f3QD89bra7UeumolZHKuOXnTmeQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" }, @@ -7042,7 +7026,6 @@ "integrity": "sha512-rKtvo6a868b5Hu3heneU+L4yEQ4jYKLtjpnPeUdK7h0yzXGmyBTypknlkCvHFBqfX9YlorEiMM6Dnq/5atfHkw==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" } @@ -7073,7 +7056,6 @@ "integrity": "sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -7143,8 +7125,7 @@ "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/is-binary-path": { "version": "2.1.0", @@ -7249,7 +7230,6 @@ "integrity": "sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -7260,7 +7240,6 @@ "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -7365,8 +7344,7 @@ "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/js-yaml": { "version": "4.1.0", @@ -7402,8 +7380,7 @@ "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/json-refs": { "version": "3.0.15", @@ -7600,7 +7577,6 @@ "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -8206,8 +8182,7 @@ "resolved": "https://registry.npmjs.org/lodash.truncate/-/lodash.truncate-4.4.2.tgz", "integrity": "sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/lodash.uniq": { "version": "4.5.0", @@ -8275,7 +8250,6 @@ "integrity": "sha512-hdN1wVrZbb29eBGiGjJbeP8JbKjq1urkHJ/LIP/NY48MZ1QVXUsQBV1G1zvYFHn1XE06cwjBsOI2K3Ulnj1YXQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" }, @@ -8326,7 +8300,6 @@ "integrity": "sha512-APMBEanjybaPzUrfqU0IMU5I0AswKMH7k8OTLs0vvV4KZpExkTkY87nR/zpbuTPj+gARop7aGUbl11pnDfW6xg==", "dev": true, "license": "MIT", - "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/wooorm" @@ -8567,7 +8540,6 @@ "integrity": "sha512-/d+PQ4GKmGvM9Bee/DPa8z3mXs/pkvJE2KEThngVNOqtmljC6K7NMPxtc2JeZYTmpWb9k/TmxjeL18ez3h7vCw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/minimist": "^1.2.2", "camelcase-keys": "^7.0.0", @@ -8595,7 +8567,6 @@ "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", "dev": true, "license": "(MIT OR CC0-1.0)", - "peer": true, "engines": { "node": ">=10" }, @@ -9330,7 +9301,6 @@ "integrity": "sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "arrify": "^1.0.1", "is-plain-obj": "^1.1.0", @@ -9396,6 +9366,7 @@ "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-editor-api/-/monaco-vscode-editor-api-25.0.0.tgz", "integrity": "sha512-uiY06RTWFo2WZdh6OybkLlDhuG+8LlkjUDpr9/wW55uucqHo4X8fx4XKEtD98cscC+6FKQkbG2yyUiOJ/npHOw==", "license": "MIT", + "peer": true, "dependencies": { "@codingame/monaco-vscode-api": "25.0.0" } @@ -9632,7 +9603,6 @@ "integrity": "sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA==", "dev": true, "license": "BSD-2-Clause", - "peer": true, "dependencies": { "hosted-git-info": "^4.0.1", "is-core-module": "^2.5.0", @@ -9984,7 +9954,6 @@ "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.0.0", "error-ex": "^1.3.1", @@ -10306,6 +10275,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -10494,6 +10464,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "lilconfig": "^3.0.0", "yaml": "^2.3.4" @@ -10883,8 +10854,7 @@ "resolved": "https://registry.npmjs.org/postcss-resolve-nested-selector/-/postcss-resolve-nested-selector-0.1.6.tgz", "integrity": "sha512-0sglIs9Wmkzbr8lQwEyIzlDOOC9bGmfVKcJTaxv3vMmd3uo4o4DerC3En0bnmgceeql9BfC8hRkp7cg0fjdVqw==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/postcss-safe-parser": { "version": "6.0.0", @@ -11060,6 +11030,7 @@ "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", "dev": true, "license": "MIT", + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -11363,7 +11334,6 @@ "integrity": "sha512-X1Fu3dPuk/8ZLsMhEj5f4wFAF0DWoK7qhGJvgaijocXxBmSToKfbFtqbxMO7bVjNA1dmE5huAzjXj/ey86iw9Q==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/normalize-package-data": "^2.4.0", "normalize-package-data": "^3.0.2", @@ -11383,7 +11353,6 @@ "integrity": "sha512-snVCqPczksT0HS2EC+SxUndvSzn6LRCwpfSvLrIfR5BKDQQZMaI6jPRC9dYvYFDRAuFEAnkwww8kBBNE/3VvzQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "find-up": "^5.0.0", "read-pkg": "^6.0.0", @@ -11402,7 +11371,6 @@ "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", "dev": true, "license": "(MIT OR CC0-1.0)", - "peer": true, "engines": { "node": ">=10" }, @@ -11416,7 +11384,6 @@ "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", "dev": true, "license": "(MIT OR CC0-1.0)", - "peer": true, "engines": { "node": ">=10" }, @@ -11459,7 +11426,6 @@ "integrity": "sha512-tYkDkVVtYkSVhuQ4zBgfvciymHaeuel+zFKXShfDnFP5SyVEP7qo70Rf1jTOTCx3vGNAbnEi/xFkcfQVMIBWag==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "indent-string": "^5.0.0", "strip-indent": "^4.0.0" @@ -12100,7 +12066,6 @@ "integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "ansi-styles": "^4.0.0", "astral-regex": "^2.0.0", @@ -12177,7 +12142,6 @@ "integrity": "sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA==", "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "spdx-expression-parse": "^3.0.0", "spdx-license-ids": "^3.0.0" @@ -12188,8 +12152,7 @@ "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.5.0.tgz", "integrity": "sha512-PiU42r+xO4UbUS1buo3LPJkjlO7430Xn5SVAhdpzzsPHsjbYVflnnFdATgabnLude+Cqu25p6N+g2lw/PFsa4w==", "dev": true, - "license": "CC-BY-3.0", - "peer": true + "license": "CC-BY-3.0" }, "node_modules/spdx-expression-parse": { "version": "3.0.1", @@ -12197,7 +12160,6 @@ "integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "spdx-exceptions": "^2.1.0", "spdx-license-ids": "^3.0.0" @@ -12208,8 +12170,7 @@ "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.23.tgz", "integrity": "sha512-CWLcCCH7VLu13TgOH+r8p1O/Znwhqv/dbb6lqWy67G+pT1kHmeD/+V36AVb/vq8QMIQwVShJ6Ssl5FPh0fuSdw==", "dev": true, - "license": "CC0-1.0", - "peer": true + "license": "CC0-1.0" }, "node_modules/sprintf-js": { "version": "1.0.3", @@ -12303,7 +12264,6 @@ "integrity": "sha512-SlyRoSkdh1dYP0PzclLE7r0M9sgbFKKMFXpFRUMNuKhQSbC6VQIGzq3E0qsfvGJaUFJPGv6Ws1NZ/haTAjfbMA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -12329,8 +12289,7 @@ "resolved": "https://registry.npmjs.org/style-search/-/style-search-0.1.0.tgz", "integrity": "sha512-Dj1Okke1C3uKKwQcetra4jSuk0DqbzbYtXipzFlFMZtowbF1x7BKJwB9AayVMyFARvU8EDrZdcax4At/452cAg==", "dev": true, - "license": "ISC", - "peer": true + "license": "ISC" }, "node_modules/style-to-object": { "version": "0.4.4", @@ -12379,7 +12338,6 @@ "integrity": "sha512-78O4c6IswZ9TzpcIiQJIN49K3qNoXTM8zEJzhaTE/xRTCZswaovSEVIa/uwbOltZrk16X4jAxjaOhzz/hTm1Kw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@csstools/css-parser-algorithms": "^2.3.1", "@csstools/css-tokenizer": "^2.2.0", @@ -12462,7 +12420,6 @@ } ], "license": "MIT-0", - "peer": true, "engines": { "node": "^14 || ^16 || >=18" }, @@ -12476,7 +12433,6 @@ "integrity": "sha512-TfW7/1iI4Cy7Y8L6iqNdZQVvdXn0f8B4QcIXmkIbtTIe/Okm/nSlHb4IwGzRVOd3WfSieCgvf5cMzEfySAIl0g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "flat-cache": "^3.2.0" }, @@ -12489,8 +12445,7 @@ "resolved": "https://registry.npmjs.org/known-css-properties/-/known-css-properties-0.29.0.tgz", "integrity": "sha512-Ne7wqW7/9Cz54PDt4I3tcV+hAyat8ypyOGzYRJQfdxnnjeWsTxt1cy8pjvvKeI5kfXuyvULyeeAvwvvtAX3ayQ==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/stylelint/node_modules/postcss-selector-parser": { "version": "6.1.2", @@ -12513,7 +12468,6 @@ "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" } @@ -12654,7 +12608,6 @@ "integrity": "sha512-zFObLMyZeEwzAoKCyu1B91U79K2t7ApXuQfo8OuxwXLDgcKxuwM+YvcbIhm6QWqz7mHUH1TVytR1PwVVjEuMig==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "has-flag": "^4.0.0", "supports-color": "^7.0.0" @@ -12684,6 +12637,7 @@ "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.53.5.tgz", "integrity": "sha512-YkqERnF05g8KLdDZwZrF8/i1eSbj6Eoat8Jjr2IfruZz9StLuBqo8sfCSzjosNKd+ZrQ8DkKZDjpO5y3ht1Pow==", "license": "MIT", + "peer": true, "dependencies": { "@jridgewell/remapping": "^2.3.4", "@jridgewell/sourcemap-codec": "^1.5.0", @@ -12781,21 +12735,6 @@ } } }, - "node_modules/svelte-check/node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/svelte-eslint-parser": { "version": "0.43.0", "resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-0.43.0.tgz", @@ -13025,8 +12964,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/svg-tags/-/svg-tags-1.0.0.tgz", "integrity": "sha512-ovssysQTa+luh7A5Weu3Rta6FJlFBBbInjOh722LIt6klpU2/HtdUbszju/G4devcvk8PGt7FCLv5wftu3THUA==", - "dev": true, - "peer": true + "dev": true }, "node_modules/svgo": { "version": "3.3.2", @@ -13077,7 +13015,6 @@ "integrity": "sha512-9kY+CygyYM6j02t5YFHbNz2FN5QmYGv9zAjVp4lCDjlCw7amdckXlEt/bjMhUIfj4ThGRE4gCUH5+yGnNuPo5A==", "dev": true, "license": "BSD-3-Clause", - "peer": true, "dependencies": { "ajv": "^8.0.1", "lodash.truncate": "^4.4.2", @@ -13105,6 +13042,7 @@ "integrity": "sha512-6A2rnmW5xZMdw11LYjhcI5846rt9pbLSabY5XPxo+XWdxwZaFEn47Go4NzFiHu9sNNmr/kXivP1vStfvMaK1GQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@alloc/quick-lru": "^5.2.0", "arg": "^5.0.2", @@ -13357,6 +13295,7 @@ "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "devOptional": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -13419,7 +13358,6 @@ "integrity": "sha512-jRKj0n0jXWo6kh62nA5TEh3+4igKDXLvzBJcPpiizP7oOolUrYIxmVBG9TOtHYFHoddUk6YvAkGeGoSVTXfQXQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -13537,6 +13475,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -13773,7 +13712,6 @@ "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "spdx-correct": "^3.0.0", "spdx-expression-parse": "^3.0.0" @@ -13827,6 +13765,7 @@ "integrity": "sha512-MFtjBYgzmSxmgA4RAfjIyXWpGe1oALnjgUTzzV7QLx/TKxCzjtMH6Fd9/eVK+5Fg1qNoz5VAwsmMs/NofrmJvw==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "lightningcss": "^1.32.0", "picomatch": "^4.0.4", @@ -14396,9 +14335,9 @@ "integrity": "sha512-uHNL8F72/Tf96xF3hOHnPDjkEyqXw7fNjcPJiUhth9sTQkcwUIoJMOdwm8/cs+j9kKVRJ4tgNYMHEBLylazp6g==" }, "node_modules/windmill-utils-internal": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/windmill-utils-internal/-/windmill-utils-internal-1.7.0.tgz", - "integrity": "sha512-K5kAiJKhavfGatmicbJyqT+KFspzFZK5Ou14pHj4Xg6Q2Z1a4ZgziBdrRNsaeAuBhB62yXWZA0OXDzg936Ymmw==", + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/windmill-utils-internal/-/windmill-utils-internal-1.8.2.tgz", + "integrity": "sha512-Otdn4iCE0QiOtMKPdHX5I89oECfZa3vmO2jdLdSETf7MBSv/gw9gvQm3oUQ29ymIHXLeuhQk/ebPD+WYolQJEA==", "license": "Apache 2.0" }, "node_modules/word-wrap": { @@ -14534,7 +14473,6 @@ "integrity": "sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw==", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "imurmurhash": "^0.1.4", "signal-exit": "^4.0.1" @@ -14730,7 +14668,6 @@ "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", "dev": true, "license": "ISC", - "peer": true, "engines": { "node": ">=10" } @@ -14750,6 +14687,7 @@ "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.27.tgz", "integrity": "sha512-OIDwaflOaq4wC6YlPBy2L6ceKeKuF7DeTxx+jPzv1FHn9tCZ0ZwSRnUBxD05E3yed46fv/FWJbvR+Ud7x0L7zw==", "license": "MIT", + "peer": true, "dependencies": { "lib0": "^0.2.99" }, @@ -14785,6 +14723,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.1.12.tgz", "integrity": "sha512-JInaHOamG8pt5+Ey8kGmdcAcg3OL9reK8ltczgHTAwNhMys/6ThXHityHxVV2p3fkw/c+MAvBHFVYHFZDmjMCQ==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/frontend/package.json b/frontend/package.json index 70d24aba68..fe0366eb62 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -169,7 +169,7 @@ "windmill-parser-wasm-wac": "1.668.6", "windmill-parser-wasm-yaml": "1.593.0", "windmill-sql-datatype-parser-wasm": "1.512.0", - "windmill-utils-internal": "^1.7.0", + "windmill-utils-internal": "1.8.2", "xterm": "^5.3.0", "xterm-readline": "^1.1.2", "y-monaco": "^0.1.4", diff --git a/frontend/src/lib/components/CompareWorkspaces.svelte b/frontend/src/lib/components/CompareWorkspaces.svelte index 8cfa311141..a2453add27 100644 --- a/frontend/src/lib/components/CompareWorkspaces.svelte +++ b/frontend/src/lib/components/CompareWorkspaces.svelte @@ -646,7 +646,7 @@ deploymentStatus[statusKey] = { status: 'deployed' } } else { deploymentStatus[statusKey] = { status: 'failed', error: result.error } - sendUserToast(`Failed to deploy ${statusKey}: ${result.error}`) + sendUserToast(`Failed to deploy ${statusKey}: ${result.error}`, 'error') } } @@ -696,7 +696,10 @@ if (!aIsFolder && bIsFolder) return 1 return 0 }) + const to = mergeIntoParent ? parent : current let anyFailed = false + // Datatables whose migrations deployed cleanly — candidates for a run prompt. + const deployedMigrationDatatables = new Set() for (const itemKey of sortedItems) { const deployable = deployableItems.find((d) => d.key === itemKey) @@ -705,16 +708,19 @@ continue } - const to = mergeIntoParent ? parent : current const from = mergeIntoParent ? current : parent await deploy(deployable.kind, deployable.path, to, from, itemKey) if (deploymentStatus[itemKey]?.status === 'failed') { anyFailed = true + } else if (deployable.kind === 'datatable_migration') { + deployedMigrationDatatables.add(deployable.path.split('/')[0]) } } deploying = false deselectAll() + await maybePromptRunMigrations(deployedMigrationDatatables, to) + // If every selected item deployed cleanly and the direction was // merge-into-parent, resolve any open deployment request for this fork. if (!anyFailed && mergeIntoParent) { @@ -750,6 +756,51 @@ onChanged?.() } + /** + * After a deploy, offer to run the migrations of every cloned datatable that + * received one. `forked_from` is set on the fork's datatable config only when + * the datatable was cloned into a separate database — shared-DB datatables + * have already had the schema change applied and must not be re-run. + */ + async function maybePromptRunMigrations( + deployedMigrationDatatables: Set, + runTargetWorkspace: string + ) { + if (deployedMigrationDatatables.size === 0) return + try { + const forkSettings = await WorkspaceService.getPublicSettings({ + workspace: currentWorkspaceId + }) + const datatables = forkSettings.datatable?.datatables ?? {} + const cloned = [...deployedMigrationDatatables].filter( + (dt) => datatables[dt]?.forked_from != null + ) + if (cloned.length === 0) return + runMigrationsDatatables = cloned.sort() + runMigrationsTargetWorkspace = runTargetWorkspace + runMigrationsModalOpen = true + } catch (e) { + console.error('Failed to determine cloned datatables for migration run prompt', e) + } + } + + async function runDeployedMigrations() { + runMigrationsModalOpen = false + for (const dt of runMigrationsDatatables) { + try { + const res = await WorkspaceService.runDatatableMigrations({ + workspace: runMigrationsTargetWorkspace, + datatableName: dt + }) + sendUserToast( + `Ran ${res.applied.length} migration${res.applied.length !== 1 ? 's' : ''} on ${dt}` + ) + } catch (e: any) { + sendUserToast(`Failed to run migrations on ${dt}: ${e.body ?? e.message ?? e}`, true) + } + } + } + function toggleKey(key: string) { if (selectedItems.includes(key)) { selectedItems = selectedItems.filter((i) => i !== key) @@ -930,6 +981,17 @@ let deploymentRequestPanel: DeploymentRequestPanel | undefined = $state(undefined) let hasOpenDeploymentRequest = $state(false) + // After deploying datatable migrations to a cloned (separate-DB) datatable, we + // offer to run them in the target workspace. Shared-DB datatables are skipped: + // the schema change is already physically applied, so re-running is redundant. + let runMigrationsModalOpen = $state(false) + let runMigrationsDatatables = $state([]) + let runMigrationsTargetWorkspace = $state('') + let runMigrationsTargetWorkspaceName = $derived( + $userWorkspaces.find((w) => w.id == runMigrationsTargetWorkspace)?.name ?? + runMigrationsTargetWorkspace + ) + /** Display labels for trigger/schedule kinds in the merge UI. */ const KIND_DISPLAY_NAMES: Record = { schedule: 'Schedule', @@ -942,7 +1004,8 @@ sqs_trigger: 'SQS trigger', gcp_trigger: 'GCP trigger', azure_trigger: 'Azure trigger', - email_trigger: 'Email trigger' + email_trigger: 'Email trigger', + datatable_migration: 'Data table migration' } // Human label for a diff kind, lowercased for inline use in the hidden-items @@ -1453,9 +1516,7 @@ />
-
- -
+ {#if pinnedItems.length > 0}
@@ -1536,6 +1597,26 @@
+ (runMigrationsModalOpen = false)} + > +
+

+ Run the deployed migrations in {runMigrationsTargetWorkspaceName} now? These data tables + use a separate database, so the schema changes won't apply until the migrations are run. +

+
    + {#each runMigrationsDatatables as dt (dt)} +
  • {dt}
    • + {/each} +
+
+ + { if (dbTableEditorState.alterTableKey && dbTableEditorAlterTableData.current) { let diff = diffTableEditorValues(dbTableEditorAlterTableData.current, values) - await dbSchemaOps.onAlter({ schema: selected.schemaKey, values: diff }) + // Reverse diff (new → old) so the migration's down undoes the alter. + let reverse = diffTableEditorValues(values, dbTableEditorAlterTableData.current) + await dbSchemaOps.onAlter({ schema: selected.schemaKey, values: diff, reverse }) } else { await dbSchemaOps.onCreate({ values, schema: selected.schemaKey }) } diff --git a/frontend/src/lib/components/DBManagerContent.svelte b/frontend/src/lib/components/DBManagerContent.svelte index d84bdb862e..c40a2f7428 100644 --- a/frontend/src/lib/components/DBManagerContent.svelte +++ b/frontend/src/lib/components/DBManagerContent.svelte @@ -20,6 +20,10 @@ import type { SelectedTable } from './DBManager.svelte' import { getDbFeatures } from './apps/components/display/dbtable/dbFeatures' import { resource } from 'runed' + import ConfirmationModal from './common/confirmationModal/ConfirmationModal.svelte' + import { createAsyncConfirmationModal } from './common/confirmationModal/asyncConfirmationModal.svelte' + import Portal from '$lib/components/Portal.svelte' + import { outOfOrderRunMessage } from './workspaceSettings/datatableMigrationUtils' interface Props { input?: DbInput @@ -52,6 +56,8 @@ let dbSchema: DBSchema | undefined = $derived(input && $dbSchemas[getDbSchemasPath(input)]) + const outOfOrderModal = createAsyncConfirmationModal() + function getDbSchemasPath(input: DbInput): string { switch (input.type) { case 'database': @@ -163,7 +169,13 @@ })} dbSchemaOps={dbSchemaOpsWithPreviewScripts({ input: _input, - workspace: $workspaceStore + workspace: $workspaceStore, + confirmRunOutOfOrder: (pending) => + outOfOrderModal.ask({ + title: 'Run migration out of order', + confirmationText: 'Run anyway', + children: outOfOrderRunMessage(pending) + }) })} initialTableKey={input.specificTable} initialSchemaKey={input.specificSchema} @@ -192,6 +204,7 @@ onData={(data) => { replResultData = data }} + onSchemaChange={() => refresh()} placeholderTableName={sortArray( Object.keys( dbSchema?.schema[ @@ -214,3 +227,9 @@ {/if} + + + + + diff --git a/frontend/src/lib/components/DBManagerDrawer.svelte b/frontend/src/lib/components/DBManagerDrawer.svelte index 6598103751..b303671fe8 100644 --- a/frontend/src/lib/components/DBManagerDrawer.svelte +++ b/frontend/src/lib/components/DBManagerDrawer.svelte @@ -16,6 +16,7 @@ Upload } from 'lucide-svelte' import DBManagerContent from './DBManagerContent.svelte' + import DataTableMigrationsButton from './workspaceSettings/DataTableMigrationsButton.svelte' import { resource } from 'runed' import { untrack } from 'svelte' import type { DbManagerUriState } from './dbManagerDrawerModel.svelte' @@ -105,6 +106,11 @@ return toSourceIdentifier(input.resourcePath) } + function refreshManager() { + dbManagerContent?.refresh() + dbManagerContent?.dbManager()?.dbTable()?.refresh() + } + async function handleExportSchema() { const source = currentSourceIdentifier() if (!source || !$workspaceStore) return @@ -201,6 +207,13 @@ {/key} {/if} {#snippet actions()} + {#if uriState.isDatatableInput && uriState.selectedDatatable && $workspaceStore} + + {/if} {#if enableImportExport} + /> +{#if applicableCount > 0} +
+

Datatable schema changes

+ {#if loading} +
+ Loading datatable diffs... +
+ {:else if error} +
Failed to load datatable diffs: {error}
+ {:else if diffs.length > 0} +
+ {#each diffs as diff} + + - {#if expandedDatatables.has(diff.datatableName)} -
- {#if diff.aheadChanges.length > 0} -
-
Fork changes (ahead)
- {#each diff.aheadChanges as change} -
- {#if change.kind === 'added'} - - {:else if change.kind === 'removed'} - - {:else} - - {/if} - {change.schemaName}. - {change.tableName} - {operationSummary(change)} - + {#each diff.aheadChanges as change} +
+ {#if change.kind === 'added'} + + {:else if change.kind === 'removed'} + + {:else} + + {/if} + {change.schemaName}. + {change.tableName} + {operationSummary(change)} + +
+ {/each}
- {/each} + {/if} + {#if diff.behindChanges.length > 0} +
+
+ Parent changes (behind) +
+ {#each diff.behindChanges as change} +
+ {#if change.kind === 'added'} + + {:else if change.kind === 'removed'} + + {:else} + + {/if} + {change.schemaName}. + {change.tableName} + {operationSummary(change)} + +
+ {/each} +
+ {/if}
{/if} - {#if diff.behindChanges.length > 0} -
-
- Parent changes (behind) -
- {#each diff.behindChanges as change} -
- {#if change.kind === 'added'} - - {:else if change.kind === 'removed'} - - {:else} - - {/if} - {change.schemaName}. - {change.tableName} - {operationSummary(change)} - -
- {/each} -
- {/if} -
- {/if} - - {/each} + + {/each} +
+ {:else} + No changes detected + {/if}
-{:else} - No changes detected {/if} @@ -580,3 +655,7 @@ >{migrationSql} + + + + diff --git a/frontend/src/lib/components/DdlMigrationGuard.svelte b/frontend/src/lib/components/DdlMigrationGuard.svelte new file mode 100644 index 0000000000..b6833521f8 --- /dev/null +++ b/frontend/src/lib/components/DdlMigrationGuard.svelte @@ -0,0 +1,159 @@ + + + + + +
+

+ This looks like a schema-changing (DDL) statement. Schema changes are best tracked as + migrations rather than run ad-hoc. Create a migration for it instead? +

+ 
{promptStatement ?? ''}
+
+ + +
+
+ + + migrationsModal?.openMigration(m.timestamp)} +/> + + diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index 78a75734c9..8563190a89 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -43,6 +43,7 @@ import { editorFontSize } from '$lib/editorFontSize.svelte' import { createHash as randomHash } from '$lib/editorLangUtils' import { workspaceStore } from '$lib/stores' + import DdlMigrationGuard from './DdlMigrationGuard.svelte' import { type Preview, ResourceService, @@ -222,6 +223,35 @@ let lang = $state(scriptLangToEditorLang(untrack(() => scriptLang))) + // On a postgres script targeting a datatable, DDL statements are intercepted + // on run (cmd+enter) and offered as migrations instead. + let datatableForMigrations = $derived( + scriptLang === 'postgresql' && + typeof args?.database === 'string' && + args.database.startsWith('datatable://') + ? args.database.slice('datatable://'.length).split('/')[0] + : undefined + ) + let ddlGuard = $state(undefined) + + // Run the DDL migration guard against the current code. Returns false when the + // user cancels (the run must be aborted); may rewrite the code (migrated + // statements stripped). Exported so run paths that bypass the Monaco + // Cmd+Enter binding (e.g. the Test button) can guard too. + export async function guardDdlBeforeRun(): Promise { + if (datatableForMigrations && ddlGuard) { + const res = await ddlGuard.guard(getCode()) + if (!res.proceed) return false + if (res.code !== getCode()) setCode(res.code) + } + return true + } + + async function runCmdEnterWithDdlGuard() { + if (!(await guardDdlBeforeRun())) return + cmdEnterAction?.() + } + let filePath = $state(computePath(untrack(() => path))) let initialPath: string | undefined = $state(untrack(() => path)) @@ -1639,7 +1669,8 @@ editor?.addCommand(KeyMod.CtrlCmd | KeyCode.Enter, function () { updateCode() - shouldBindKey && cmdEnterAction && cmdEnterAction() + if (!shouldBindKey || !cmdEnterAction) return + void runCmdEnterWithDdlGuard() }) editor?.addCommand(KeyMod.CtrlCmd | KeyMod.Shift | KeyCode.Digit7, function () { @@ -2209,6 +2240,13 @@ +{#if datatableForMigrations && $workspaceStore} + +{/if} {#if !editor}
diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index ac941d0bf0..bf5349b126 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -793,7 +793,17 @@ args = nargs } - export async function runTest(opts?: { cascade?: boolean }) { + export async function runTest(opts?: { cascade?: boolean; skipDdlGuard?: boolean }) { + // Intercept DDL statements (offer to turn them into data table migrations) + // on every run path, not just the editor's Cmd+Enter. `skipDdlGuard` is set + // by the Cmd+Enter action, which already guarded before calling us. + if (!opts?.skipDdlGuard) { + if ((await editor?.guardDdlBeforeRun()) === false) return + // The guard may have rewritten the code (migrated statements stripped); + // `editorCode` is kept in sync by the editor binding, so mirror the + // on:change handler and pull it into `code` before we run. + if (activeModuleTab === null) code = editorCode + } // When the caller forces a cascade choice (e.g. the canvas runnable // menu's "Run + trigger N downstream"), also flip the persistent // `cascadeDownstream` state so the split button's label/icon reflect @@ -2677,7 +2687,8 @@ } else { await inferModuleSchema() } - runTest() + // The Editor already ran the DDL guard before invoking this action. + runTest({ skipDdlGuard: true }) }} formatAction={async () => { if (activeModuleTab === null) { diff --git a/frontend/src/lib/components/SimpleEditor.svelte b/frontend/src/lib/components/SimpleEditor.svelte index 741d874f81..f1c4daa962 100644 --- a/frontend/src/lib/components/SimpleEditor.svelte +++ b/frontend/src/lib/components/SimpleEditor.svelte @@ -95,6 +95,7 @@ loadAsync = false, key, disabled = false, + readOnly = false, minHeight = 1000, renderLineHighlight = 'none', suggestion @@ -123,6 +124,9 @@ initialCursorPos?: IPosition key?: string disabled?: boolean + /** Read-only Monaco mode: not editable, but still scrollable/selectable + * (unlike `disabled`, which makes the editor non-interactive). */ + readOnly?: boolean minHeight?: number renderLineHighlight?: 'all' | 'line' | 'gutter' | 'none' suggestion?: string @@ -239,6 +243,9 @@ lineNumbers: $relativeLineNumbers ? 'relative' : 'on' }) }) + $effect(() => { + editor?.updateOptions({ readOnly }) + }) function onVimDisable() { vimDisposable?.dispose() @@ -342,6 +349,7 @@ ), model, ...(yPadding !== undefined ? { padding: { bottom: yPadding, top: yPadding } } : {}), + readOnly, renderLineHighlight, lineDecorationsWidth: 0, lineNumbersMinChars: 2, diff --git a/frontend/src/lib/components/SqlRepl.svelte b/frontend/src/lib/components/SqlRepl.svelte index 661b0ccd2c..6c8d90ba15 100644 --- a/frontend/src/lib/components/SqlRepl.svelte +++ b/frontend/src/lib/components/SqlRepl.svelte @@ -1,49 +1,3 @@ - - + +{#if !hideTrigger} + +{/if} + + + {#snippet headerLeft()} + + Each schema edit made in the database manager is captured here as a migration. Tracking schema + changes as migrations makes it easy to export data tables to other workspaces, and to + reproduce their schema when forking a workspace. + + {/snippet} + {#snippet headerRight()} + {#if enabled && canManage} + disableMigrations() + } + ]} + > + {#snippet buttonReplacement()} + + {/if} +
+ {:else} + {#if loadError} +
+ Could not read applied status from the data table: {loadError} +
+ {/if} +
+ {#if migrations.length === 0} +
+ No migrations yet + +
+ {:else} + {#each migrations as m (m.timestamp)} +
+
+ +
+ {/each} + {/if} +
+
+ + +
+ {/if} +
+ + + (newMigrationOpen = false)} + onSeeMigration={(m) => openMigration(m.timestamp)} +/> + + + {#if viewMigration} +
+ {viewMigration.timestamp} + + + + {#snippet content()} + + + + + {#if viewMigration?.code_down} + + {:else} +
No down migration
+ {/if} + + {/snippet} + +
+ {/if} + + + +
+

+ "{deleteTarget?.name}" is installed on the data table. Revert it first to undo its schema + change, or delete only the definition and leave the schema as-is — deleting without reverting + means it can no longer be reverted. +

+
+ + + +
+
+ + + + + diff --git a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte index af6fe29e58..f652ed8704 100644 --- a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte @@ -1,6 +1,12 @@ + + + +
+ + + + + {#snippet content()} + + + + +
+ + {#if enableDown} +
+ +
+ {/if} +
+ + {/snippet} + +
+ +
+
+ + + + + diff --git a/frontend/src/lib/components/workspaceSettings/datatableMigrationUtils.ts b/frontend/src/lib/components/workspaceSettings/datatableMigrationUtils.ts new file mode 100644 index 0000000000..ed046418f2 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/datatableMigrationUtils.ts @@ -0,0 +1,30 @@ +import { WorkspaceService, type DatatableMigrationWithStatus } from '$lib/gen' + +/** + * Migrations that are defined but not yet applied. A newly-created migration + * always gets the highest timestamp, so every pending migration is "earlier": + * running the new one on its own would apply it ahead of them (out of order). + */ +export function pendingMigrations( + migrations: DatatableMigrationWithStatus[] +): DatatableMigrationWithStatus[] { + return migrations.filter((m) => m.status !== 'ran') +} + +/** Fetch the data table's migration status and return the pending ones. */ +export async function fetchPendingMigrations( + workspace: string, + datatableName: string +): Promise { + const { migrations } = await WorkspaceService.getDatatableMigrationsStatus({ + workspace, + datatableName + }) + return pendingMigrations(migrations) +} + +/** Confirmation copy shown before running a just-created migration ahead of + * `count` still-pending earlier ones (mirrors the row-level Run warning). */ +export function outOfOrderRunMessage(count: number): string { + return `${count} earlier migration(s) have not been run yet. This migration might depend on them. Run it anyway?` +} diff --git a/frontend/src/lib/utils_deployable.ts b/frontend/src/lib/utils_deployable.ts index 5f49eb5c49..556ac0a3c7 100644 --- a/frontend/src/lib/utils_deployable.ts +++ b/frontend/src/lib/utils_deployable.ts @@ -40,6 +40,8 @@ export type Kind = | 'gcp_trigger' | 'azure_trigger' | 'email_trigger' + // Data table migration, diffed per `/_` path. + | 'datatable_migration' // Legacy generic kind used by the cross-workspace `DeployWorkspace` UI, // which carries the trigger sub-kind in `additionalInformation`. | 'trigger' diff --git a/frontend/src/lib/utils_workspace_deploy.ts b/frontend/src/lib/utils_workspace_deploy.ts index 112ed69e17..72400bb962 100644 --- a/frontend/src/lib/utils_workspace_deploy.ts +++ b/frontend/src/lib/utils_workspace_deploy.ts @@ -16,7 +16,8 @@ import { SqsTriggerService, UserService, VariableService, - WebsocketTriggerService + WebsocketTriggerService, + WorkspaceService } from '$lib/gen' import { fetchProtectionRulesForWorkspace, @@ -234,7 +235,11 @@ function makeProvider(): DeployProvider { getSchedule: (p) => ScheduleService.getSchedule(p), createSchedule: (p) => ScheduleService.createSchedule(p), updateSchedule: (p) => ScheduleService.updateSchedule(p), - deleteSchedule: (p) => ScheduleService.deleteSchedule(p) + deleteSchedule: (p) => ScheduleService.deleteSchedule(p), + // Datatable migrations + listDatatableMigrations: (p) => WorkspaceService.listDatatableMigrations(p), + upsertDatatableMigration: (p) => WorkspaceService.upsertDatatableMigration(p), + deleteDatatableMigration: (p) => WorkspaceService.deleteDatatableMigration(p) } } diff --git a/system_prompts/auto-generated/cli/cli-commands.md b/system_prompts/auto-generated/cli/cli-commands.md index 01579a737a..a74fdc6a17 100644 --- a/system_prompts/auto-generated/cli/cli-commands.md +++ b/system_prompts/auto-generated/cli/cli-commands.md @@ -77,6 +77,13 @@ datatable related commands - `datatable run ` - run a SQL query on a datatable - `-n --name ` - Datatable name (default: main) - `-s --silent` - Output only the final result as JSON. Useful for scripting. +- `datatable migrate` - manage datatable migrations + - `datatable migrate new ` - scaffold a new migration (.up.sql / .down.sql files) + - `-d --datatable ` - Target datatable (default: main) + - `datatable migrate up` - apply all pending migrations to the main datatable (or one via --datatable) + - `-d --datatable ` - Target datatable (default: main) + - `datatable migrate down` - roll back the most recent migration on the main datatable (or one via --datatable) + - `-d --datatable ` - Target datatable (default: main) - `datatable create [name:string]` - register a datatable database in the workspace (default: instance-backed 'main') so scripts can use datatable:// - `--resource ` - Back the datatable with an existing postgresql resource path instead of the instance database - `--force` - Allow adding to a workspace that already has datatables (fork metadata on existing ones is not preserved) @@ -345,19 +352,18 @@ Manage jobs (list, inspect, cancel) ### jobs -Pull completed and queued jobs from workspace - -**Arguments:** `[workspace:string]` - -**Options:** -- `-c, --completed-output ` - Completed jobs output file (default: completed_jobs.json) -- `-q, --queued-output ` - Queued jobs output file (default: queued_jobs.json) -- `--skip-worker-check` - Skip checking for active workers before export +Manage jobs (import/export) **Subcommands:** -- `jobs pull` -- `jobs push` +- `jobs pull [workspace:string]` - Pull completed and queued jobs from workspace + - `-c, --completed-output ` - Completed jobs output file (default: completed_jobs.json) + - `-q, --queued-output ` - Queued jobs output file (default: queued_jobs.json) + - `--skip-worker-check` - Skip checking for active workers before export +- `jobs push [workspace:string]` - Push completed and queued jobs to workspace + - `-c, --completed-file ` - Completed jobs input file (default: completed_jobs.json) + - `-q, --queued-file ` - Queued jobs input file (default: queued_jobs.json) + - `--skip-worker-check` - Skip checking for active workers before import ### lint diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 13776fe1e6..58ac1ac69d 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -2755,6 +2755,13 @@ datatable related commands - \`datatable run \` - run a SQL query on a datatable - \`-n --name \` - Datatable name (default: main) - \`-s --silent\` - Output only the final result as JSON. Useful for scripting. +- \`datatable migrate\` - manage datatable migrations + - \`datatable migrate new \` - scaffold a new migration (.up.sql / .down.sql files) + - \`-d --datatable \` - Target datatable (default: main) + - \`datatable migrate up\` - apply all pending migrations to the main datatable (or one via --datatable) + - \`-d --datatable \` - Target datatable (default: main) + - \`datatable migrate down\` - roll back the most recent migration on the main datatable (or one via --datatable) + - \`-d --datatable \` - Target datatable (default: main) - \`datatable create [name:string]\` - register a datatable database in the workspace (default: instance-backed 'main') so scripts can use datatable:// - \`--resource \` - Back the datatable with an existing postgresql resource path instead of the instance database - \`--force\` - Allow adding to a workspace that already has datatables (fork metadata on existing ones is not preserved) @@ -3023,19 +3030,18 @@ Manage jobs (list, inspect, cancel) ### jobs -Pull completed and queued jobs from workspace - -**Arguments:** \`[workspace:string]\` - -**Options:** -- \`-c, --completed-output \` - Completed jobs output file (default: completed_jobs.json) -- \`-q, --queued-output \` - Queued jobs output file (default: queued_jobs.json) -- \`--skip-worker-check\` - Skip checking for active workers before export +Manage jobs (import/export) **Subcommands:** -- \`jobs pull\` -- \`jobs push\` +- \`jobs pull [workspace:string]\` - Pull completed and queued jobs from workspace + - \`-c, --completed-output \` - Completed jobs output file (default: completed_jobs.json) + - \`-q, --queued-output \` - Queued jobs output file (default: queued_jobs.json) + - \`--skip-worker-check\` - Skip checking for active workers before export +- \`jobs push [workspace:string]\` - Push completed and queued jobs to workspace + - \`-c, --completed-file \` - Completed jobs input file (default: completed_jobs.json) + - \`-q, --queued-file \` - Queued jobs input file (default: queued_jobs.json) + - \`--skip-worker-check\` - Skip checking for active workers before import ### lint diff --git a/system_prompts/auto-generated/skills/cli-commands/SKILL.md b/system_prompts/auto-generated/skills/cli-commands/SKILL.md index cc314a5067..db04118c3f 100644 --- a/system_prompts/auto-generated/skills/cli-commands/SKILL.md +++ b/system_prompts/auto-generated/skills/cli-commands/SKILL.md @@ -82,6 +82,13 @@ datatable related commands - `datatable run ` - run a SQL query on a datatable - `-n --name ` - Datatable name (default: main) - `-s --silent` - Output only the final result as JSON. Useful for scripting. +- `datatable migrate` - manage datatable migrations + - `datatable migrate new ` - scaffold a new migration (.up.sql / .down.sql files) + - `-d --datatable ` - Target datatable (default: main) + - `datatable migrate up` - apply all pending migrations to the main datatable (or one via --datatable) + - `-d --datatable ` - Target datatable (default: main) + - `datatable migrate down` - roll back the most recent migration on the main datatable (or one via --datatable) + - `-d --datatable ` - Target datatable (default: main) - `datatable create [name:string]` - register a datatable database in the workspace (default: instance-backed 'main') so scripts can use datatable:// - `--resource ` - Back the datatable with an existing postgresql resource path instead of the instance database - `--force` - Allow adding to a workspace that already has datatables (fork metadata on existing ones is not preserved) @@ -350,19 +357,18 @@ Manage jobs (list, inspect, cancel) ### jobs -Pull completed and queued jobs from workspace - -**Arguments:** `[workspace:string]` - -**Options:** -- `-c, --completed-output ` - Completed jobs output file (default: completed_jobs.json) -- `-q, --queued-output ` - Queued jobs output file (default: queued_jobs.json) -- `--skip-worker-check` - Skip checking for active workers before export +Manage jobs (import/export) **Subcommands:** -- `jobs pull` -- `jobs push` +- `jobs pull [workspace:string]` - Pull completed and queued jobs from workspace + - `-c, --completed-output ` - Completed jobs output file (default: completed_jobs.json) + - `-q, --queued-output ` - Queued jobs output file (default: queued_jobs.json) + - `--skip-worker-check` - Skip checking for active workers before export +- `jobs push [workspace:string]` - Push completed and queued jobs to workspace + - `-c, --completed-file ` - Completed jobs input file (default: completed_jobs.json) + - `-q, --queued-file ` - Queued jobs input file (default: queued_jobs.json) + - `--skip-worker-check` - Skip checking for active workers before import ### lint diff --git a/system_prompts/generate.py b/system_prompts/generate.py index 98c03c4367..dac36ad97a 100644 --- a/system_prompts/generate.py +++ b/system_prompts/generate.py @@ -340,13 +340,54 @@ def extract_description(section: str) -> str | None: return ''.join(_unquote_js_string(p) for p in parts).strip() or None -def parse_command_block(content: str, file_path: Path | None = None) -> dict: +def extract_named_command_block(content: str, var_name: str) -> str | None: + """Return the chained-call body of `const = new Command() ...`, + from just after `new Command()` up to the next top-level statement. + + Returns None when the var isn't a *direct* `new Command()` (e.g. it's wrapped + in a helper call like `auditListOptions(new Command()...)`), so callers can + fall back to a looser match. + """ + m = re.search( + r'const\s+' + re.escape(var_name) + r'\s*=\s*new\s+Command\(\)' + r'([\s\S]*?)(?=\n(?:const|let|var|async|function|export)\b)', + content, + ) + return m.group(1) if m else None + + +def extract_exported_command_block(content: str) -> str | None: + """Return the chained-call body of the command that is `export default`ed. + + A command file may define helper `new Command()` groups (assigned to local + consts and mounted as nested subcommands via `.command("x", localCmd)`) + *before* the exported command. Anchoring on the first `new Command()` in the + file would merge those helpers into the top-level command, so resolve the + exported variable first and only then fall back to the first `new Command()` + (which covers inline/wrapped exports). + """ + export_match = re.search(r'export\s+default\s+(\w+)\s*;', content) + if export_match: + block = extract_named_command_block(content, export_match.group(1)) + if block is not None: + return block + command_match = re.search( + r'(?:const\s+command\s*=\s*)?new\s+Command\(\)([\s\S]*?)(?=export\s+default)', + content, + ) + return command_match.group(1) if command_match else None + + +def parse_command_block( + content: str, file_path: Path | None = None, block: str | None = None +) -> dict: """ Parse a Cliffy Command() definition block and extract metadata. Returns a dict with: description, options, subcommands, arguments, alias If file_path is provided, imported subcommands will be resolved by parsing - the imported files. + the imported files. `block` may be passed to parse a specific pre-extracted + command body (used to recurse into locally-defined nested command groups). """ result = { 'description': '', @@ -357,15 +398,11 @@ def parse_command_block(content: str, file_path: Path | None = None) -> dict: } # Find the command block - command_match = re.search( - r'(?:const\s+command\s*=\s*)?new\s+Command\(\)([\s\S]*?)(?=export\s+default)', - content - ) - if not command_match: + if block is None: + block = extract_exported_command_block(content) + if block is None: return result - block = command_match.group(1) - # Find where subcommands start first_subcommand_pos = block.find('.command(') if first_subcommand_pos == -1: @@ -451,12 +488,31 @@ def parse_command_block(content: str, file_path: Path | None = None) -> dict: 'name': cmd_name, 'description': imported_cmd.get('description', ''), 'arguments': imported_cmd.get('arguments', ''), - 'options': imported_cmd.get('options', []) + 'options': imported_cmd.get('options', []), + 'subcommands': imported_cmd.get('subcommands', []), }) continue except Exception as e: print(f" Warning: Could not parse imported command {second_arg}: {e}") cmd_desc = '' + elif second_arg and re.search( + r'const\s+' + re.escape(second_arg) + r'\s*=\s*new\s+Command\(\)', content + ): + # Locally-defined command group mounted as a subcommand + # (e.g. `.command("migrate", migrateCommand)`): recurse into its + # definition so its own subcommands/options are captured. + nested_block = extract_named_command_block(content, second_arg) + if nested_block is not None: + nested = parse_command_block(content, file_path, block=nested_block) + result['subcommands'].append({ + 'name': cmd_name, + 'description': nested.get('description', ''), + 'arguments': nested.get('arguments', ''), + 'options': nested.get('options', []), + 'subcommands': nested.get('subcommands', []), + }) + continue + cmd_desc = '' else: cmd_desc = '' @@ -628,6 +684,16 @@ def generate_cli_commands_markdown(cli_data: dict) -> str: for opt in sub['options']: md += f" - `{opt['flag']}` - {opt['description']}\n" + # Nested sub-subcommands (e.g. `datatable migrate new`) + for subsub in sub.get('subcommands', []): + ss_args = f" {subsub['arguments']}" if subsub.get('arguments') else "" + md += f" - `{cmd['name']} {sub_name} {subsub['name']}{ss_args}`" + if subsub.get('description'): + md += f" - {subsub['description']}" + md += "\n" + for opt in subsub.get('options', []): + md += f" - `{opt['flag']}` - {opt['description']}\n" + md += "\n" return md