feat: deploy and run datatable migrations on workspace merge

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

backend/windmill-api/openapi.yaml

@@ -29123,6 +29123,7 @@ components:
         - folders_changed
         - schedules_changed
         - triggers_changed
+        - datatable_migrations_changed
         - conflicts
       properties:
         total_diffs:
@@ -29161,6 +29162,9 @@ components:
         triggers_changed:
           type: integer
           description: Number of triggers with differences (sum across all trigger kinds)
+        datatable_migrations_changed:
+          type: integer
+          description: Number of data table migrations with differences
         conflicts:
           type: integer
           description: Number of items that are both ahead and behind (conflicts)

cli/src/commands/workspace/merge.ts

@@ -85,6 +85,10 @@ const provider: DeployProvider = {
   createSchedule: wmill.createSchedule,
   updateSchedule: wmill.updateSchedule,
   deleteSchedule: wmill.deleteSchedule,
+  // Datatable migrations
+  listDatatableMigrations: wmill.listDatatableMigrations,
+  upsertDatatableMigration: wmill.upsertDatatableMigration,
+  deleteDatatableMigration: wmill.deleteDatatableMigration,
 };
 
 /**

cli/windmill-utils-internal/src/deploy.ts

@@ -30,7 +30,8 @@ export type DeployKind =
   | "sqs_trigger"
   | "gcp_trigger"
   | "azure_trigger"
-  | "email_trigger";
+  | "email_trigger"
+  | "datatable_migration";
 
 export const TRIGGER_KINDS = [
   "http_trigger",
@@ -213,6 +214,20 @@ export interface DeployProvider {
     requestBody: any;
   }): Promise<any>;
   deleteSchedule(p: { workspace: string; path: string }): Promise<any>;
+  // Datatable migrations — there is no per-migration GET endpoint, so the
+  // value/existence lookups list the workspace's migrations and filter by
+  // (datatable, timestamp).
+  listDatatableMigrations(p: { workspace: string }): Promise<any[]>;
+  upsertDatatableMigration(p: {
+    workspace: string;
+    datatableName: string;
+    requestBody: any;
+  }): Promise<any>;
+  deleteDatatableMigration(p: {
+    workspace: string;
+    datatableName: string;
+    timestamp: number;
+  }): Promise<any>;
 }
 
 // ---------------------------------------------------------------------------
@@ -224,6 +239,37 @@ export function folderName(path: string): string {
   return path.replace(/^f\//, "");
 }
 
+/**
+ * Parse a datatable-migration diff path `<datatable>/<timestamp>_<name>` into its
+ * parts. Mirrors the backend's `parse_datatable_migration_diff_path`: the
+ * datatable name is a validated path segment (no `/`), and the timestamp is the
+ * leading digits of the file part. Returns null when the path doesn't match.
+ */
+export function parseDatatableMigrationPath(
+  path: string
+): { datatable: string; timestamp: number; name: string } | null {
+  const slash = path.indexOf("/");
+  if (slash <= 0) return null;
+  const datatable = path.slice(0, slash);
+  const file = path.slice(slash + 1);
+  const m = file.match(/^(\d+)_(.*)$/);
+  if (!m) return null;
+  return { datatable, timestamp: Number(m[1]), name: m[2] };
+}
+
+/** Fetch one migration row from a workspace, or undefined if it no longer exists. */
+async function findDatatableMigration(
+  provider: DeployProvider,
+  workspace: string,
+  datatable: string,
+  timestamp: number
+): Promise<any | undefined> {
+  const migrations = await provider.listDatatableMigrations({ workspace });
+  return migrations.find(
+    (m) => m.datatable === datatable && Number(m.timestamp) === timestamp
+  );
+}
+
 /**
  * Strip operational state (`mode`, `enabled`) from a trigger/schedule payload
  * when deploying as an update; pass through unchanged on create.
@@ -327,6 +373,16 @@ export async function checkItemExists(
     return provider.existsSchedule({ workspace, path });
   } else if (isTriggerKind(kind)) {
     return provider.existsTriggerByKind(kind, { workspace, path });
+  } else if (kind === "datatable_migration") {
+    const parsed = parseDatatableMigrationPath(path);
+    if (!parsed) return false;
+    const found = await findDatatableMigration(
+      provider,
+      workspace,
+      parsed.datatable,
+      parsed.timestamp
+    );
+    return found !== undefined;
   }
   throw new Error(`Unknown kind: ${kind}`);
 }
@@ -639,6 +695,34 @@ export async function deployItem(
           requestBody,
         });
       }
+    } else if (kind === "datatable_migration") {
+      const parsed = parseDatatableMigrationPath(path);
+      if (!parsed) {
+        return { success: false, error: `Invalid migration path: ${path}` };
+      }
+      const migration = await findDatatableMigration(
+        provider,
+        workspaceFrom,
+        parsed.datatable,
+        parsed.timestamp
+      );
+      if (!migration) {
+        return {
+          success: false,
+          error: `Migration ${path} not found in ${workspaceFrom}`,
+        };
+      }
+      // upsert is idempotent on (datatable, timestamp), covering create + update.
+      await provider.upsertDatatableMigration({
+        workspace: workspaceTo,
+        datatableName: parsed.datatable,
+        requestBody: {
+          timestamp: Number(migration.timestamp),
+          name: migration.name,
+          code_up: migration.code_up,
+          code_down: migration.code_down ?? undefined,
+        },
+      });
     } else {
       throw new Error(`Unknown kind: ${kind}`);
     }
@@ -686,6 +770,14 @@ export async function deleteItemInWorkspace(
       await provider.deleteSchedule({ workspace, path });
     } else if (isTriggerKind(kind)) {
       await provider.deleteTriggerByKind(kind, { workspace, path });
+    } else if (kind === "datatable_migration") {
+      const parsed = parseDatatableMigrationPath(path);
+      if (!parsed) throw new Error(`Invalid migration path: ${path}`);
+      await provider.deleteDatatableMigration({
+        workspace,
+        datatableName: parsed.datatable,
+        timestamp: parsed.timestamp,
+      });
     } else {
       throw new Error(`Deletion not supported for kind: ${kind}`);
     }
@@ -767,6 +859,23 @@ export async function getItemValue(
     } else if (isTriggerKind(kind)) {
       const trigger = await provider.getTriggerValue(kind, { workspace, path });
       return stripTriggerOrScheduleRuntimeFields(trigger);
+    } else if (kind === "datatable_migration") {
+      const parsed = parseDatatableMigrationPath(path);
+      if (parsed) {
+        const migration = await findDatatableMigration(
+          provider,
+          workspace,
+          parsed.datatable,
+          parsed.timestamp
+        );
+        if (migration) {
+          return {
+            name: migration.name,
+            code_up: migration.code_up,
+            code_down: migration.code_down ?? null,
+          };
+        }
+      }
     }
   } catch {
     // Item may not exist

frontend/src/lib/components/CompareWorkspaces.svelte

@@ -444,7 +444,10 @@
 			if (!aIsFolder && bIsFolder) return 1
 			return 0
 		})
+		const to = mergeIntoParent ? parent : current
 		let anyFailed = false
+		// Datatables whose migrations deployed cleanly — candidates for a run prompt.
+		const deployedMigrationDatatables = new Set<string>()
 		for (const itemKey of sortedItems) {
 			const deployable = deployableItems.find((d) => d.key === itemKey)
 
@@ -453,16 +456,19 @@
 				continue
 			}
 
-			const to = mergeIntoParent ? parent : current
 			const from = mergeIntoParent ? current : parent
 			await deploy(deployable.kind, deployable.path, to, from, itemKey)
 			if (deploymentStatus[itemKey]?.status === 'failed') {
 				anyFailed = true
+			} else if (deployable.kind === 'datatable_migration') {
+				deployedMigrationDatatables.add(deployable.path.split('/')[0])
 			}
 		}
 		deploying = false
 		deselectAll()
 
+		await maybePromptRunMigrations(deployedMigrationDatatables, to)
+
 		// If every selected item deployed cleanly and the direction was
 		// merge-into-parent, close any open deployment request for this fork.
 		if (!anyFailed && mergeIntoParent) {
@@ -487,6 +493,51 @@
 		onChanged?.()
 	}
 
+	/**
+	 * After a deploy, offer to run the migrations of every cloned datatable that
+	 * received one. `forked_from` is set on the fork's datatable config only when
+	 * the datatable was cloned into a separate database — shared-DB datatables
+	 * have already had the schema change applied and must not be re-run.
+	 */
+	async function maybePromptRunMigrations(
+		deployedMigrationDatatables: Set<string>,
+		runTargetWorkspace: string
+	) {
+		if (deployedMigrationDatatables.size === 0) return
+		try {
+			const forkSettings = await WorkspaceService.getPublicSettings({
+				workspace: currentWorkspaceId
+			})
+			const datatables = forkSettings.datatable?.datatables ?? {}
+			const cloned = [...deployedMigrationDatatables].filter(
+				(dt) => datatables[dt]?.forked_from != null
+			)
+			if (cloned.length === 0) return
+			runMigrationsDatatables = cloned.sort()
+			runMigrationsTargetWorkspace = runTargetWorkspace
+			runMigrationsModalOpen = true
+		} catch (e) {
+			console.error('Failed to determine cloned datatables for migration run prompt', e)
+		}
+	}
+
+	async function runDeployedMigrations() {
+		runMigrationsModalOpen = false
+		for (const dt of runMigrationsDatatables) {
+			try {
+				const res = await WorkspaceService.runDatatableMigrations({
+					workspace: runMigrationsTargetWorkspace,
+					datatableName: dt
+				})
+				sendUserToast(
+					`Ran ${res.applied.length} migration${res.applied.length !== 1 ? 's' : ''} on ${dt}`
+				)
+			} catch (e: any) {
+				sendUserToast(`Failed to run migrations on ${dt}: ${e.body ?? e.message ?? e}`, true)
+			}
+		}
+	}
+
 	function toggleKey(key: string) {
 		if (selectedItems.includes(key)) {
 			selectedItems = selectedItems.filter((i) => i !== key)
@@ -644,6 +695,17 @@
 	let deploymentRequestPanel: DeploymentRequestPanel | undefined = $state(undefined)
 	let hasOpenDeploymentRequest = $state(false)
 
+	// After deploying datatable migrations to a cloned (separate-DB) datatable, we
+	// offer to run them in the target workspace. Shared-DB datatables are skipped:
+	// the schema change is already physically applied, so re-running is redundant.
+	let runMigrationsModalOpen = $state(false)
+	let runMigrationsDatatables = $state<string[]>([])
+	let runMigrationsTargetWorkspace = $state('')
+	let runMigrationsTargetWorkspaceName = $derived(
+		$userWorkspaces.find((w) => w.id == runMigrationsTargetWorkspace)?.name ??
+			runMigrationsTargetWorkspace
+	)
+
 	/** Display labels for trigger/schedule kinds in the merge UI. */
 	const KIND_DISPLAY_NAMES: Record<string, string> = {
 		schedule: 'Schedule',
@@ -656,7 +718,8 @@
 		sqs_trigger: 'SQS trigger',
 		gcp_trigger: 'GCP trigger',
 		azure_trigger: 'Azure trigger',
-		email_trigger: 'Email trigger'
+		email_trigger: 'Email trigger',
+		datatable_migration: 'Data table migration'
 	}
 </script>
 
@@ -1127,6 +1190,26 @@
 			</ul>
 		</div>
 	</ConfirmationModal>
+
+	<ConfirmationModal
+		open={runMigrationsModalOpen}
+		title="Run datatable migrations?"
+		confirmationText="Run migrations"
+		onConfirmed={runDeployedMigrations}
+		onCanceled={() => (runMigrationsModalOpen = false)}
+	>
+		<div class="flex flex-col gap-2">
+			<p>
+				Run the deployed migrations in <b>{runMigrationsTargetWorkspaceName}</b> now? These data tables
+				use a separate database, so the schema changes won't apply until the migrations are run.
+			</p>
+			<ul class="list-disc pl-5 text-sm font-mono text-secondary">
+				{#each runMigrationsDatatables as dt (dt)}
+					<li>{dt}</li>
+				{/each}
+			</ul>
+		</div>
+	</ConfirmationModal>
 {:else}
 	<div class="flex items-center justify-center h-full">
 		<div class="text-gray-500">No comparison data available</div>

frontend/src/lib/components/common/table/Row.svelte

@@ -50,6 +50,7 @@
 			| 'azure_trigger'
 			| 'email_trigger'
 			| 'data_pipeline'
+			| 'datatable_migration'
 		triggerKind?: string | undefined
 		summary?: string | undefined
 		path: string

frontend/src/lib/components/common/table/RowIcon.svelte

@@ -53,6 +53,7 @@
 			| 'azure_trigger'
 			| 'email_trigger'
 			| 'data_pipeline'
+			| 'datatable_migration'
 		/** For 'trigger' kind, specifies the specific trigger type (routes, schedules, etc.) */
 		triggerKind?: string | undefined
 		size?: number
@@ -121,6 +122,8 @@
 		<Calendar {size} class="text-gray-400" />
 	{:else if effectiveKind === 'data_pipeline'}
 		<Workflow {size} class="text-indigo-500" />
+	{:else if effectiveKind === 'datatable_migration'}
+		<Database {size} class="text-violet-500" />
 	{:else}
 		<div style="width: {size}px;"></div>
 	{/if}

frontend/src/lib/utils_deployable.ts

@@ -40,6 +40,8 @@ export type Kind =
 	| 'gcp_trigger'
 	| 'azure_trigger'
 	| 'email_trigger'
+	// Data table migration, diffed per `<datatable>/<timestamp>_<name>` path.
+	| 'datatable_migration'
 	// Legacy generic kind used by the cross-workspace `DeployWorkspace` UI,
 	// which carries the trigger sub-kind in `additionalInformation`.
 	| 'trigger'

frontend/src/lib/utils_workspace_deploy.ts

@@ -15,7 +15,8 @@ import {
 	ScriptService,
 	SqsTriggerService,
 	VariableService,
-	WebsocketTriggerService
+	WebsocketTriggerService,
+	WorkspaceService
 } from '$lib/gen'
 import {
 	existsTrigger,
@@ -229,7 +230,11 @@ function makeProvider(): DeployProvider {
 		getSchedule: (p) => ScheduleService.getSchedule(p),
 		createSchedule: (p) => ScheduleService.createSchedule(p),
 		updateSchedule: (p) => ScheduleService.updateSchedule(p),
-		deleteSchedule: (p) => ScheduleService.deleteSchedule(p)
+		deleteSchedule: (p) => ScheduleService.deleteSchedule(p),
+		// Datatable migrations
+		listDatatableMigrations: (p) => WorkspaceService.listDatatableMigrations(p),
+		upsertDatatableMigration: (p) => WorkspaceService.upsertDatatableMigration(p),
+		deleteDatatableMigration: (p) => WorkspaceService.deleteDatatableMigration(p)
 	}
 }