feat: add datatable migrate up/down commands and post-push run prompt

cli/src/commands/datatable/datatable.ts

@@ -9,6 +9,10 @@ import { GlobalOptions } from "../../types.ts";
 import { runCatalogQuery } from "../../utils/catalog.ts";
 import { psql as psqlDatatable } from "./psql.ts";
 import { serve as serveDatatable } from "./serve.ts";
+import {
+  rollbackMigrations,
+  runMigrations,
+} from "../datatable_migrations.ts";
 
 const DEFAULT_DATATABLE_NAME = "main";
 
@@ -41,6 +45,27 @@ async function run(
   await runCatalogQuery(opts, "datatable", name, sql);
 }
 
+async function migrateUp(opts: GlobalOptions, name?: string) {
+  const workspace = await resolveWorkspace(opts);
+  await requireLogin(opts);
+  await runMigrations(workspace.workspaceId, name ?? DEFAULT_DATATABLE_NAME);
+}
+
+async function migrateDown(opts: GlobalOptions, name?: string) {
+  const workspace = await resolveWorkspace(opts);
+  await requireLogin(opts);
+  await rollbackMigrations(workspace.workspaceId, name ?? DEFAULT_DATATABLE_NAME);
+}
+
+const migrateCommand = new Command()
+  .description("apply or roll back datatable migrations")
+  .command("up", "apply all pending migrations to a datatable")
+  .arguments("[name:string]")
+  .action(migrateUp as any)
+  .command("down", "roll back the most recent migration on a datatable")
+  .arguments("[name:string]")
+  .action(migrateDown as any);
+
 async function serve(
   opts: GlobalOptions & { port?: number; host?: string; password?: string },
 ) {
@@ -69,6 +94,7 @@ const command = new Command()
     "Output only the final result as JSON. Useful for scripting.",
   )
   .action(run as any)
+  .command("migrate", migrateCommand)
   .command(
     "serve",
     "Serve all datatables as a Postgres-wire endpoint (psql, DBeaver, pgAdmin); the client picks the datatable via the database name in its connection string",

cli/src/commands/datatable_migrations.ts

@@ -5,9 +5,63 @@ import { colors } from "@cliffy/ansi/colors";
 import * as wmill from "../../gen/services.gen.ts";
 import type { DatatableMigration } from "../../gen/types.gen.ts";
 import { readTextFile, readTextFileSync } from "../utils/utils.ts";
+import { Confirm } from "@cliffy/prompt/confirm";
 
+// Migrations live under <cwd>/datatable_migrations/<datatable>/, one folder per
+// target data table, as `<timestamp>_<name>.up.sql` (and optional `.down.sql`).
 const MIGRATIONS_DIR = "datatable_migrations";
 
+/**
+ * Apply the workspace's pending migrations to a data table (forwards migrations
+ * recorded in `_wm_migrations`). Mirrors `wmill datatable migrate up`.
+ */
+export async function runMigrations(
+  workspace: string,
+  datatableName: string,
+): Promise<void> {
+  const result = await wmill.runDatatableMigrations({
+    workspace,
+    datatableName,
+  });
+  const applied = result.applied ?? [];
+  if (applied.length === 0) {
+    log.info(colors.gray(`No pending migrations to run on '${datatableName}'`));
+    return;
+  }
+  log.info(
+    colors.green(`Applied ${applied.length} migration(s) to '${datatableName}':`),
+  );
+  for (const m of applied) {
+    log.info(colors.gray(`  ${m.version} ${m.name}`));
+  }
+}
+
+/**
+ * Roll back the most recently applied migration on a data table (one step).
+ * Mirrors `wmill datatable migrate down`.
+ */
+export async function rollbackMigrations(
+  workspace: string,
+  datatableName: string,
+): Promise<void> {
+  const result = await wmill.rollbackDatatableMigrations({
+    workspace,
+    datatableName,
+  });
+  const rolledBack = result.rolled_back ?? [];
+  if (rolledBack.length === 0) {
+    log.info(
+      colors.gray(`No applied migrations to roll back on '${datatableName}'`),
+    );
+    return;
+  }
+  for (const m of rolledBack) {
+    log.info(
+      colors.green(`Rolled back migration ${m.version} ${m.name} on '${datatableName}'`),
+    );
+  }
+}
+
 function upFileName(m: { timestamp: number; name: string }): string {
   return `${m.timestamp}_${m.name}.up.sql`;
 }
@@ -28,6 +82,13 @@ function parseMigrationFileName(
   return { timestamp: Number(m[1]), name: m[2], kind: m[3] as "up" | "down" };
 }
 
+function sortMigrations(migrations: DatatableMigration[]): DatatableMigration[] {
+  return migrations.sort(
+    (a, b) =>
+      a.datatable.localeCompare(b.datatable) || a.timestamp - b.timestamp,
+  );
+}
+
 /**
  * Push the local <cwd>/datatable_migrations/ folder to the workspace, replacing
  * the remote set. Returns true if a push was performed, false if the folder is
@@ -35,47 +96,55 @@ function parseMigrationFileName(
  */
 export async function pushDatatableMigrations(
   workspace: string,
+  opts?: { yes?: boolean; jsonOutput?: boolean },
 ): Promise<boolean> {
   const localDir = path.join(process.cwd(), MIGRATIONS_DIR);
   if (!fs.existsSync(localDir)) {
     return false;
   }
 
-  // Group .up.sql / .down.sql files by their (timestamp, name) key.
-  const byKey = new Map<
-    string,
-    { timestamp: number; name: string; code_up?: string; code_down?: string }
-  >();
-  for (const file of fs.readdirSync(localDir)) {
-    const parsed = parseMigrationFileName(file);
-    if (!parsed) continue;
-    const key = `${parsed.timestamp}_${parsed.name}`;
-    const entry =
-      byKey.get(key) ?? { timestamp: parsed.timestamp, name: parsed.name };
-    const content = await readTextFile(path.join(localDir, file));
-    if (parsed.kind === "up") entry.code_up = content;
-    else entry.code_down = content;
-    byKey.set(key, entry);
-  }
-
+  // Each immediate subfolder is a target data table; read its migration files.
   const migrations: DatatableMigration[] = [];
-  for (const entry of byKey.values()) {
-    if (entry.code_up === undefined) {
-      log.warn(
-        colors.yellow(
-          `Skipping migration ${entry.timestamp}_${entry.name}: missing .up.sql file`,
-        ),
-      );
-      continue;
+  for (const datatable of fs.readdirSync(localDir)) {
+    const dtDir = path.join(localDir, datatable);
+    if (!fs.statSync(dtDir).isDirectory()) continue;
+
+    // Group .up.sql / .down.sql files by their (timestamp, name) key.
+    const byKey = new Map<
+      string,
+      { timestamp: number; name: string; code_up?: string; code_down?: string }
+    >();
+    for (const file of fs.readdirSync(dtDir)) {
+      const parsed = parseMigrationFileName(file);
+      if (!parsed) continue;
+      const key = `${parsed.timestamp}_${parsed.name}`;
+      const entry =
+        byKey.get(key) ?? { timestamp: parsed.timestamp, name: parsed.name };
+      const content = await readTextFile(path.join(dtDir, file));
+      if (parsed.kind === "up") entry.code_up = content;
+      else entry.code_down = content;
+      byKey.set(key, entry);
+    }
+
+    for (const entry of byKey.values()) {
+      if (entry.code_up === undefined) {
+        log.warn(
+          colors.yellow(
+            `Skipping ${datatable}/${entry.timestamp}_${entry.name}: missing .up.sql file`,
+          ),
+        );
+        continue;
+      }
+      migrations.push({
+        datatable,
+        timestamp: entry.timestamp,
+        name: entry.name,
+        code_up: entry.code_up,
+        ...(entry.code_down !== undefined ? { code_down: entry.code_down } : {}),
+      });
     }
-    migrations.push({
-      timestamp: entry.timestamp,
-      name: entry.name,
-      code_up: entry.code_up,
-      ...(entry.code_down !== undefined ? { code_down: entry.code_down } : {}),
-    });
   }
-  migrations.sort((a, b) => a.timestamp - b.timestamp);
+  sortMigrations(migrations);
 
   // Skip if the remote set is already identical.
   let remote: DatatableMigration[] = [];
@@ -94,17 +163,62 @@ export async function pushDatatableMigrations(
     requestBody: { migrations },
   });
   log.info(
-    colors.green(
-      `Pushed ${migrations.length} datatable migration(s)`,
-    ),
+    colors.green(`Pushed ${migrations.length} datatable migration(s)`),
   );
+
+  // Migrations newly introduced by this push (absent from the remote set before),
+  // keyed by (datatable, timestamp) since timestamps are unique only per table.
+  const remoteKeys = new Set(remote.map((m) => `${m.datatable}\0${m.timestamp}`));
+  const newMigrations = migrations.filter(
+    (m) => !remoteKeys.has(`${m.datatable}\0${m.timestamp}`),
+  );
+  if (newMigrations.length > 0) {
+    await offerToRunNewMigrations(workspace, newMigrations, opts);
+  }
+
   return true;
 }
 
 /**
- * Pull the workspace's datatable migrations into <cwd>/datatable_migrations/ as
- * `<timestamp>_<name>.up.sql` (and `.down.sql` when a down migration exists).
- * Files no longer present remotely are removed locally.
+ * After a push that introduced new migrations, list them and (interactively)
+ * offer to run them, equivalent to `wmill datatable migrate up` on each affected
+ * data table.
+ */
+async function offerToRunNewMigrations(
+  workspace: string,
+  newMigrations: DatatableMigration[],
+  opts?: { yes?: boolean; jsonOutput?: boolean },
+): Promise<void> {
+  log.info(colors.green("New migrations were pushed:"));
+  for (const m of newMigrations) {
+    log.info(colors.gray(`  ${m.datatable}: ${m.timestamp} ${m.name}`));
+  }
+
+  // Running migrations mutates the data tables, so skip the prompt in
+  // non-interactive contexts (--yes, --json, no TTY).
+  const interactive = !opts?.jsonOutput && !opts?.yes && !!process.stdin.isTTY;
+  if (!interactive) {
+    return;
+  }
+
+  const shouldRun = await Confirm.prompt({
+    message: "New migrations were pushed, run them?",
+    default: false,
+  });
+  if (!shouldRun) {
+    return;
+  }
+
+  for (const datatable of new Set(newMigrations.map((m) => m.datatable))) {
+    await runMigrations(workspace, datatable);
+  }
+}
+
+/**
+ * Pull the workspace's datatable migrations into
+ * <cwd>/datatable_migrations/<datatable>/ as `<timestamp>_<name>.up.sql` (and
+ * `.down.sql` when a down migration exists). Files no longer present remotely
+ * are removed locally.
  */
 export async function pullDatatableMigrations(
   workspace: string,
@@ -121,35 +235,41 @@ export async function pullDatatableMigrations(
   if (migrations.length === 0 && !fs.existsSync(localDir)) {
     return false;
   }
-  fs.mkdirSync(localDir, { recursive: true });
 
+  // Relative paths (`<datatable>/<file>`) that should exist after the pull.
   const known = new Set<string>();
   for (const m of migrations) {
+    const dtDir = path.join(localDir, m.datatable);
+    fs.mkdirSync(dtDir, { recursive: true });
     const up = upFileName(m);
-    known.add(up);
-    writeIfChanged(path.join(localDir, up), m.code_up);
+    known.add(`${m.datatable}/${up}`);
+    writeIfChanged(path.join(dtDir, up), m.code_up);
     if (m.code_down !== undefined && m.code_down !== null) {
       const down = downFileName(m);
-      known.add(down);
-      writeIfChanged(path.join(localDir, down), m.code_down);
+      known.add(`${m.datatable}/${down}`);
+      writeIfChanged(path.join(dtDir, down), m.code_down);
     }
   }
 
-  // Delete locally-orphaned migration files.
-  for (const file of fs.readdirSync(localDir)) {
-    if (!parseMigrationFileName(file)) continue;
-    if (!known.has(file)) {
-      try {
-        fs.unlinkSync(path.join(localDir, file));
-      } catch {
-        // ignore
+  // Delete locally-orphaned migration files across all datatable subfolders.
+  if (fs.existsSync(localDir)) {
+    for (const datatable of fs.readdirSync(localDir)) {
+      const dtDir = path.join(localDir, datatable);
+      if (!fs.statSync(dtDir).isDirectory()) continue;
+      for (const file of fs.readdirSync(dtDir)) {
+        if (!parseMigrationFileName(file)) continue;
+        if (!known.has(`${datatable}/${file}`)) {
+          try {
+            fs.unlinkSync(path.join(dtDir, file));
+          } catch {
+            // ignore
+          }
+        }
       }
     }
   }
 
-  log.info(
-    colors.green(`Pulled ${migrations.length} datatable migration(s)`),
-  );
+  log.info(colors.green(`Pulled ${migrations.length} datatable migration(s)`));
   return true;
 }
 

cli/src/commands/sync/sync.ts

@@ -4928,7 +4928,10 @@ export async function push(
       log.warn(`Failed to push shared UI folder: ${e}`);
     }
     try {
-      await pushDatatableMigrations(workspace.workspaceId);
+      await pushDatatableMigrations(workspace.workspaceId, {
+        yes: opts.yes,
+        jsonOutput: opts.jsonOutput,
+      });
     } catch (e) {
       log.warn(`Failed to push datatable migrations folder: ${e}`);
     }
@@ -4977,7 +4980,10 @@ export async function push(
       log.warn(`Failed to push shared UI folder: ${e}`);
     }
     try {
-      await pushDatatableMigrations(workspace.workspaceId);
+      await pushDatatableMigrations(workspace.workspaceId, {
+        yes: opts.yes,
+        jsonOutput: opts.jsonOutput,
+      });
     } catch (e) {
       log.warn(`Failed to push datatable migrations folder: ${e}`);
     }