diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 458055b1dd..9d543633b8 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -29123,6 +29123,7 @@ components: - folders_changed - schedules_changed - triggers_changed + - datatable_migrations_changed - conflicts properties: total_diffs: @@ -29161,6 +29162,9 @@ components: triggers_changed: type: integer description: Number of triggers with differences (sum across all trigger kinds) + datatable_migrations_changed: + type: integer + description: Number of data table migrations with differences conflicts: type: integer description: Number of items that are both ahead and behind (conflicts) diff --git a/cli/src/commands/workspace/merge.ts b/cli/src/commands/workspace/merge.ts index b8ef24f7d4..4483f24778 100644 --- a/cli/src/commands/workspace/merge.ts +++ b/cli/src/commands/workspace/merge.ts @@ -85,6 +85,10 @@ const provider: DeployProvider = { createSchedule: wmill.createSchedule, updateSchedule: wmill.updateSchedule, deleteSchedule: wmill.deleteSchedule, + // Datatable migrations + listDatatableMigrations: wmill.listDatatableMigrations, + upsertDatatableMigration: wmill.upsertDatatableMigration, + deleteDatatableMigration: wmill.deleteDatatableMigration, }; /** diff --git a/cli/windmill-utils-internal/src/deploy.ts b/cli/windmill-utils-internal/src/deploy.ts index 990fd47edd..81d4fcdb9f 100644 --- a/cli/windmill-utils-internal/src/deploy.ts +++ b/cli/windmill-utils-internal/src/deploy.ts @@ -30,7 +30,8 @@ export type DeployKind = | "sqs_trigger" | "gcp_trigger" | "azure_trigger" - | "email_trigger"; + | "email_trigger" + | "datatable_migration"; export const TRIGGER_KINDS = [ "http_trigger", @@ -213,6 +214,20 @@ export interface DeployProvider { requestBody: any; }): Promise; deleteSchedule(p: { workspace: string; path: string }): Promise; + // Datatable migrations — there is no per-migration GET endpoint, so the + // value/existence lookups list the workspace's migrations and filter by + // (datatable, timestamp). + listDatatableMigrations(p: { workspace: string }): Promise; + upsertDatatableMigration(p: { + workspace: string; + datatableName: string; + requestBody: any; + }): Promise; + deleteDatatableMigration(p: { + workspace: string; + datatableName: string; + timestamp: number; + }): Promise; } // --------------------------------------------------------------------------- @@ -224,6 +239,37 @@ export function folderName(path: string): string { return path.replace(/^f\//, ""); } +/** + * Parse a datatable-migration diff path `/_` into its + * parts. Mirrors the backend's `parse_datatable_migration_diff_path`: the + * datatable name is a validated path segment (no `/`), and the timestamp is the + * leading digits of the file part. Returns null when the path doesn't match. + */ +export function parseDatatableMigrationPath( + path: string +): { datatable: string; timestamp: number; name: string } | null { + const slash = path.indexOf("/"); + if (slash <= 0) return null; + const datatable = path.slice(0, slash); + const file = path.slice(slash + 1); + const m = file.match(/^(\d+)_(.*)$/); + if (!m) return null; + return { datatable, timestamp: Number(m[1]), name: m[2] }; +} + +/** Fetch one migration row from a workspace, or undefined if it no longer exists. */ +async function findDatatableMigration( + provider: DeployProvider, + workspace: string, + datatable: string, + timestamp: number +): Promise { + const migrations = await provider.listDatatableMigrations({ workspace }); + return migrations.find( + (m) => m.datatable === datatable && Number(m.timestamp) === timestamp + ); +} + /** * Strip operational state (`mode`, `enabled`) from a trigger/schedule payload * when deploying as an update; pass through unchanged on create. @@ -327,6 +373,16 @@ export async function checkItemExists( return provider.existsSchedule({ workspace, path }); } else if (isTriggerKind(kind)) { return provider.existsTriggerByKind(kind, { workspace, path }); + } else if (kind === "datatable_migration") { + const parsed = parseDatatableMigrationPath(path); + if (!parsed) return false; + const found = await findDatatableMigration( + provider, + workspace, + parsed.datatable, + parsed.timestamp + ); + return found !== undefined; } throw new Error(`Unknown kind: ${kind}`); } @@ -639,6 +695,34 @@ export async function deployItem( requestBody, }); } + } else if (kind === "datatable_migration") { + const parsed = parseDatatableMigrationPath(path); + if (!parsed) { + return { success: false, error: `Invalid migration path: ${path}` }; + } + const migration = await findDatatableMigration( + provider, + workspaceFrom, + parsed.datatable, + parsed.timestamp + ); + if (!migration) { + return { + success: false, + error: `Migration ${path} not found in ${workspaceFrom}`, + }; + } + // upsert is idempotent on (datatable, timestamp), covering create + update. + await provider.upsertDatatableMigration({ + workspace: workspaceTo, + datatableName: parsed.datatable, + requestBody: { + timestamp: Number(migration.timestamp), + name: migration.name, + code_up: migration.code_up, + code_down: migration.code_down ?? undefined, + }, + }); } else { throw new Error(`Unknown kind: ${kind}`); } @@ -686,6 +770,14 @@ export async function deleteItemInWorkspace( await provider.deleteSchedule({ workspace, path }); } else if (isTriggerKind(kind)) { await provider.deleteTriggerByKind(kind, { workspace, path }); + } else if (kind === "datatable_migration") { + const parsed = parseDatatableMigrationPath(path); + if (!parsed) throw new Error(`Invalid migration path: ${path}`); + await provider.deleteDatatableMigration({ + workspace, + datatableName: parsed.datatable, + timestamp: parsed.timestamp, + }); } else { throw new Error(`Deletion not supported for kind: ${kind}`); } @@ -767,6 +859,23 @@ export async function getItemValue( } else if (isTriggerKind(kind)) { const trigger = await provider.getTriggerValue(kind, { workspace, path }); return stripTriggerOrScheduleRuntimeFields(trigger); + } else if (kind === "datatable_migration") { + const parsed = parseDatatableMigrationPath(path); + if (parsed) { + const migration = await findDatatableMigration( + provider, + workspace, + parsed.datatable, + parsed.timestamp + ); + if (migration) { + return { + name: migration.name, + code_up: migration.code_up, + code_down: migration.code_down ?? null, + }; + } + } } } catch { // Item may not exist diff --git a/frontend/src/lib/components/CompareWorkspaces.svelte b/frontend/src/lib/components/CompareWorkspaces.svelte index 5b62e1bf22..d41f93e439 100644 --- a/frontend/src/lib/components/CompareWorkspaces.svelte +++ b/frontend/src/lib/components/CompareWorkspaces.svelte @@ -444,7 +444,10 @@ if (!aIsFolder && bIsFolder) return 1 return 0 }) + const to = mergeIntoParent ? parent : current let anyFailed = false + // Datatables whose migrations deployed cleanly — candidates for a run prompt. + const deployedMigrationDatatables = new Set() for (const itemKey of sortedItems) { const deployable = deployableItems.find((d) => d.key === itemKey) @@ -453,16 +456,19 @@ continue } - const to = mergeIntoParent ? parent : current const from = mergeIntoParent ? current : parent await deploy(deployable.kind, deployable.path, to, from, itemKey) if (deploymentStatus[itemKey]?.status === 'failed') { anyFailed = true + } else if (deployable.kind === 'datatable_migration') { + deployedMigrationDatatables.add(deployable.path.split('/')[0]) } } deploying = false deselectAll() + await maybePromptRunMigrations(deployedMigrationDatatables, to) + // If every selected item deployed cleanly and the direction was // merge-into-parent, close any open deployment request for this fork. if (!anyFailed && mergeIntoParent) { @@ -487,6 +493,51 @@ onChanged?.() } + /** + * After a deploy, offer to run the migrations of every cloned datatable that + * received one. `forked_from` is set on the fork's datatable config only when + * the datatable was cloned into a separate database — shared-DB datatables + * have already had the schema change applied and must not be re-run. + */ + async function maybePromptRunMigrations( + deployedMigrationDatatables: Set, + runTargetWorkspace: string + ) { + if (deployedMigrationDatatables.size === 0) return + try { + const forkSettings = await WorkspaceService.getPublicSettings({ + workspace: currentWorkspaceId + }) + const datatables = forkSettings.datatable?.datatables ?? {} + const cloned = [...deployedMigrationDatatables].filter( + (dt) => datatables[dt]?.forked_from != null + ) + if (cloned.length === 0) return + runMigrationsDatatables = cloned.sort() + runMigrationsTargetWorkspace = runTargetWorkspace + runMigrationsModalOpen = true + } catch (e) { + console.error('Failed to determine cloned datatables for migration run prompt', e) + } + } + + async function runDeployedMigrations() { + runMigrationsModalOpen = false + for (const dt of runMigrationsDatatables) { + try { + const res = await WorkspaceService.runDatatableMigrations({ + workspace: runMigrationsTargetWorkspace, + datatableName: dt + }) + sendUserToast( + `Ran ${res.applied.length} migration${res.applied.length !== 1 ? 's' : ''} on ${dt}` + ) + } catch (e: any) { + sendUserToast(`Failed to run migrations on ${dt}: ${e.body ?? e.message ?? e}`, true) + } + } + } + function toggleKey(key: string) { if (selectedItems.includes(key)) { selectedItems = selectedItems.filter((i) => i !== key) @@ -644,6 +695,17 @@ let deploymentRequestPanel: DeploymentRequestPanel | undefined = $state(undefined) let hasOpenDeploymentRequest = $state(false) + // After deploying datatable migrations to a cloned (separate-DB) datatable, we + // offer to run them in the target workspace. Shared-DB datatables are skipped: + // the schema change is already physically applied, so re-running is redundant. + let runMigrationsModalOpen = $state(false) + let runMigrationsDatatables = $state([]) + let runMigrationsTargetWorkspace = $state('') + let runMigrationsTargetWorkspaceName = $derived( + $userWorkspaces.find((w) => w.id == runMigrationsTargetWorkspace)?.name ?? + runMigrationsTargetWorkspace + ) + /** Display labels for trigger/schedule kinds in the merge UI. */ const KIND_DISPLAY_NAMES: Record = { schedule: 'Schedule', @@ -656,7 +718,8 @@ sqs_trigger: 'SQS trigger', gcp_trigger: 'GCP trigger', azure_trigger: 'Azure trigger', - email_trigger: 'Email trigger' + email_trigger: 'Email trigger', + datatable_migration: 'Data table migration' } @@ -1127,6 +1190,26 @@ + + (runMigrationsModalOpen = false)} + > +
+

+ Run the deployed migrations in {runMigrationsTargetWorkspaceName} now? These data tables + use a separate database, so the schema changes won't apply until the migrations are run. +

+
    + {#each runMigrationsDatatables as dt (dt)} +
  • {dt}
    • + {/each} +
+
+ {:else}
No comparison data available
diff --git a/frontend/src/lib/components/common/table/Row.svelte b/frontend/src/lib/components/common/table/Row.svelte index 8b24b1e23b..43bec92921 100644 --- a/frontend/src/lib/components/common/table/Row.svelte +++ b/frontend/src/lib/components/common/table/Row.svelte @@ -50,6 +50,7 @@ | 'azure_trigger' | 'email_trigger' | 'data_pipeline' + | 'datatable_migration' triggerKind?: string | undefined summary?: string | undefined path: string diff --git a/frontend/src/lib/components/common/table/RowIcon.svelte b/frontend/src/lib/components/common/table/RowIcon.svelte index fd08c77087..414ff773d8 100644 --- a/frontend/src/lib/components/common/table/RowIcon.svelte +++ b/frontend/src/lib/components/common/table/RowIcon.svelte @@ -53,6 +53,7 @@ | 'azure_trigger' | 'email_trigger' | 'data_pipeline' + | 'datatable_migration' /** For 'trigger' kind, specifies the specific trigger type (routes, schedules, etc.) */ triggerKind?: string | undefined size?: number @@ -121,6 +122,8 @@ {:else if effectiveKind === 'data_pipeline'} + {:else if effectiveKind === 'datatable_migration'} + {:else}
{/if} diff --git a/frontend/src/lib/utils_deployable.ts b/frontend/src/lib/utils_deployable.ts index 5f49eb5c49..556ac0a3c7 100644 --- a/frontend/src/lib/utils_deployable.ts +++ b/frontend/src/lib/utils_deployable.ts @@ -40,6 +40,8 @@ export type Kind = | 'gcp_trigger' | 'azure_trigger' | 'email_trigger' + // Data table migration, diffed per `/_` path. + | 'datatable_migration' // Legacy generic kind used by the cross-workspace `DeployWorkspace` UI, // which carries the trigger sub-kind in `additionalInformation`. | 'trigger' diff --git a/frontend/src/lib/utils_workspace_deploy.ts b/frontend/src/lib/utils_workspace_deploy.ts index bde0003882..616654b446 100644 --- a/frontend/src/lib/utils_workspace_deploy.ts +++ b/frontend/src/lib/utils_workspace_deploy.ts @@ -15,7 +15,8 @@ import { ScriptService, SqsTriggerService, VariableService, - WebsocketTriggerService + WebsocketTriggerService, + WorkspaceService } from '$lib/gen' import { existsTrigger, @@ -229,7 +230,11 @@ function makeProvider(): DeployProvider { getSchedule: (p) => ScheduleService.getSchedule(p), createSchedule: (p) => ScheduleService.createSchedule(p), updateSchedule: (p) => ScheduleService.updateSchedule(p), - deleteSchedule: (p) => ScheduleService.deleteSchedule(p) + deleteSchedule: (p) => ScheduleService.deleteSchedule(p), + // Datatable migrations + listDatatableMigrations: (p) => WorkspaceService.listDatatableMigrations(p), + upsertDatatableMigration: (p) => WorkspaceService.upsertDatatableMigration(p), + deleteDatatableMigration: (p) => WorkspaceService.deleteDatatableMigration(p) } }