feat: add datatable migrations management UI

This commit is contained in:
Diego Imbert
2026-06-17 15:18:09 +02:00
parent 6e1ce7fd4e
commit a18fa7ccd6
8 changed files with 773 additions and 1 deletions
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Int8"
]
},
"nullable": []
},
"hash": "5b67c3af6477d7029d118ac259a7535d4d962c328a060f33c07191ac3e033e82"
}
@@ -0,0 +1,19 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) VALUES ($1, $2, $3, $4, $5, $6)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Int8",
"Varchar",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "760908f44cafb500e4ba2515d1b030390d9510e73e9b7df347b697d6dc7aefe6"
}
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT MAX(timestamp) FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "max",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "95e370a82ef46d9310f77a99b437a20a5773113e290d69a016363543067baf14"
}
@@ -0,0 +1,41 @@
{
"db_name": "PostgreSQL",
"query": "SELECT timestamp, name, code_up, code_down FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 ORDER BY timestamp ASC",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "timestamp",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "code_up",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "code_down",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
false,
false,
true
]
},
"hash": "e5d8d46b9431033fa7572a880ee0e74e14afde3c02000a88e696d7b6adcad0d6"
}
@@ -139,6 +139,18 @@ pub fn workspaced_service() -> Router {
"/update_datatable_migrations",
post(update_datatable_migrations),
)
.route(
"/datatable_migrations_status/{datatable_name}",
get(datatable_migrations_status),
)
.route(
"/create_datatable_migration/{datatable_name}",
post(create_datatable_migration),
)
.route(
"/delete_datatable_migration/{datatable_name}/{timestamp}",
delete(delete_datatable_migration),
)
.route("/git_sync_enabled", get(get_git_sync_enabled))
.route("/edit_git_sync_config", post(edit_git_sync_config))
.route("/edit_git_sync_repository", post(edit_git_sync_repository))
@@ -2397,6 +2409,12 @@ struct RunDatatableMigrationsResult {
applied: Vec<AppliedMigration>,
}
#[derive(Deserialize)]
struct RunDatatableMigrationsQuery {
/// When set, only apply pending migrations up to and including this version.
up_to: Option<i64>,
}
/// Apply the workspace's pending data table migrations to a given data table.
/// Applied versions are tracked in the data table's own `_wm_migrations` table,
/// so only migrations not recorded there are run, in ascending `timestamp` order.
@@ -2404,6 +2422,7 @@ async fn run_datatable_migrations(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path((w_id, datatable_name)): Path<(String, String)>,
Query(query): Query<RunDatatableMigrationsQuery>,
) -> JsonResult<RunDatatableMigrationsResult> {
require_admin(authed.is_admin, &authed.username)?;
@@ -2459,6 +2478,10 @@ async fn run_datatable_migrations(
let mut applied = Vec::new();
for m in migrations {
// Migrations are ordered ascending, so once we pass `up_to` we're done.
if query.up_to.is_some_and(|up_to| m.timestamp > up_to) {
break;
}
if applied_versions.contains(&m.timestamp) {
continue;
}
@@ -2706,6 +2729,212 @@ async fn update_datatable_migrations(
))
}
#[derive(Serialize)]
#[serde(rename_all = "snake_case")]
enum DatatableMigrationRunStatus {
/// Recorded in the data table's `_wm_migrations` table.
Ran,
/// Defined but not yet applied.
NotRun,
/// Applied status could not be determined (connection failure).
Unknown,
}
#[derive(Serialize)]
struct DatatableMigrationWithStatus {
timestamp: i64,
name: String,
code_up: String,
#[serde(skip_serializing_if = "Option::is_none")]
code_down: Option<String>,
status: DatatableMigrationRunStatus,
}
#[derive(Serialize)]
struct DatatableMigrationsStatusResult {
migrations: Vec<DatatableMigrationWithStatus>,
/// Set when the applied status couldn't be read from the data table.
#[serde(skip_serializing_if = "Option::is_none")]
error: Option<String>,
}
/// Read the versions recorded in a data table's `_wm_migrations` table. A
/// missing table means nothing has been applied yet (empty set, not an error).
async fn read_applied_datatable_versions(
db: &DB,
w_id: &str,
datatable_name: &str,
) -> Result<HashSet<i64>> {
let db_resource = get_datatable_resource_from_db_unchecked(db, w_id, datatable_name).await?;
let pg_db: PgDatabase = serde_json::from_value(db_resource)
.map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?;
let (client, connection) = pg_db.connect(Some(db)).await?;
tokio::spawn(async move {
if let Err(e) = connection.await {
tracing::error!("Datatable connection error: {}", e);
}
});
match client
.query("SELECT version FROM _wm_migrations", &[])
.await
{
Ok(rows) => Ok(rows.iter().map(|row| row.get::<_, i64>(0)).collect()),
// 42P01 = undefined_table: the data table has never been migrated yet.
Err(e) if e.as_db_error().map(|d| d.code().code()) == Some("42P01") => Ok(HashSet::new()),
Err(e) => Err(Error::internal_err(format!(
"Failed to read _wm_migrations: {}",
e
))),
}
}
/// List a data table's migrations annotated with whether each has been applied.
async fn datatable_migrations_status(
_authed: ApiAuthed,
Extension(db): Extension<DB>,
Path((w_id, datatable_name)): Path<(String, String)>,
) -> JsonResult<DatatableMigrationsStatusResult> {
let defs = sqlx::query!(
"SELECT timestamp, name, code_up, code_down FROM datatable_migrations \
WHERE workspace_id = $1 AND datatable = $2 ORDER BY timestamp ASC",
&w_id,
&datatable_name,
)
.fetch_all(&db)
.await?;
let (applied, error) = match read_applied_datatable_versions(&db, &w_id, &datatable_name).await
{
Ok(set) => (Some(set), None),
Err(e) => (None, Some(e.to_string())),
};
let migrations = defs
.into_iter()
.map(|m| {
let status = match &applied {
Some(set) if set.contains(&m.timestamp) => DatatableMigrationRunStatus::Ran,
Some(_) => DatatableMigrationRunStatus::NotRun,
None => DatatableMigrationRunStatus::Unknown,
};
DatatableMigrationWithStatus {
timestamp: m.timestamp,
name: m.name,
code_up: m.code_up,
code_down: m.code_down,
status,
}
})
.collect();
Ok(Json(DatatableMigrationsStatusResult { migrations, error }))
}
#[derive(Deserialize)]
pub struct CreateDatatableMigration {
pub name: String,
pub code_up: String,
#[serde(default)]
pub code_down: Option<String>,
}
/// Create a single migration for a data table. The version is generated
/// server-side (current UTC `YYYYMMDDHHMMSS`), bumped past any existing version
/// so it stays unique and monotonically increasing.
async fn create_datatable_migration(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path((w_id, datatable_name)): Path<(String, String)>,
Json(payload): Json<CreateDatatableMigration>,
) -> JsonResult<DatatableMigration> {
require_admin(authed.is_admin, &authed.username)?;
let now_ts: i64 = Utc::now()
.format("%Y%m%d%H%M%S")
.to_string()
.parse()
.map_err(|e| Error::internal_err(format!("Failed to build migration version: {}", e)))?;
let max_existing: Option<i64> = sqlx::query_scalar!(
"SELECT MAX(timestamp) FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2",
&w_id,
&datatable_name,
)
.fetch_one(&db)
.await?;
let timestamp = match max_existing {
Some(m) if m >= now_ts => m + 1,
_ => now_ts,
};
sqlx::query!(
"INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) \
VALUES ($1, $2, $3, $4, $5, $6)",
&w_id,
&datatable_name,
timestamp,
&payload.name,
&payload.code_up,
payload.code_down.as_deref(),
)
.execute(&db)
.await?;
audit_log(
&db,
&authed,
"workspaces.create_datatable_migration",
ActionKind::Create,
&w_id,
Some(datatable_name.as_str()),
None,
)
.await?;
Ok(Json(DatatableMigration {
datatable: datatable_name,
timestamp,
name: payload.name,
code_up: payload.code_up,
code_down: payload.code_down,
}))
}
/// Delete a single migration definition from a data table.
async fn delete_datatable_migration(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path((w_id, datatable_name, timestamp)): Path<(String, String, i64)>,
) -> Result<String> {
require_admin(authed.is_admin, &authed.username)?;
sqlx::query!(
"DELETE FROM datatable_migrations \
WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3",
&w_id,
&datatable_name,
timestamp,
)
.execute(&db)
.await?;
audit_log(
&db,
&authed,
"workspaces.delete_datatable_migration",
ActionKind::Delete,
&w_id,
Some(datatable_name.as_str()),
None,
)
.await?;
Ok(format!(
"Deleted migration {} from {}",
timestamp, datatable_name
))
}
#[derive(Deserialize)]
pub struct EditGitSyncConfig {
pub git_sync_settings: Option<WorkspaceGitSyncSettings>,
+117
View File
@@ -4364,6 +4364,13 @@ paths:
required: true
schema:
type: string
- name: up_to
in: query
required: false
description: only apply pending migrations up to and including this version
schema:
type: integer
format: int64
responses:
"200":
description: applied migrations
@@ -4465,6 +4472,97 @@ paths:
schema:
type: string
/w/{workspace}/workspaces/datatable_migrations_status/{datatable_name}:
get:
summary: list a datatable's migrations with their applied status
operationId: getDatatableMigrationsStatus
tags:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: datatable_name
in: path
required: true
schema:
type: string
responses:
"200":
description: migrations with status
content:
application/json:
schema:
type: object
required: [migrations]
properties:
migrations:
type: array
items:
$ref: "#/components/schemas/DatatableMigrationWithStatus"
error:
type: string
/w/{workspace}/workspaces/create_datatable_migration/{datatable_name}:
post:
summary: create a single datatable migration (version generated server-side)
operationId: createDatatableMigration
tags:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: datatable_name
in: path
required: true
schema:
type: string
requestBody:
required: true
content:
application/json:
schema:
type: object
required: [name, code_up]
properties:
name:
type: string
code_up:
type: string
code_down:
type: string
responses:
"200":
description: created migration
content:
application/json:
schema:
$ref: "#/components/schemas/DatatableMigration"
/w/{workspace}/workspaces/delete_datatable_migration/{datatable_name}/{timestamp}:
delete:
summary: delete a single datatable migration definition
operationId: deleteDatatableMigration
tags:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: datatable_name
in: path
required: true
schema:
type: string
- name: timestamp
in: path
required: true
schema:
type: integer
format: int64
responses:
"200":
description: status
content:
text/plain:
schema:
type: string
/w/{workspace}/workspaces/create_pg_database:
post:
summary: create a new PostgreSQL database for a datatable
@@ -27808,6 +27906,25 @@ components:
type: string
code_down:
type: string
DatatableMigrationWithStatus:
type: object
required: [timestamp, name, code_up, status]
properties:
timestamp:
type: integer
format: int64
name:
type: string
code_up:
type: string
code_down:
type: string
status:
type: string
enum:
- ran
- not_run
- unknown
DataTableSchema:
type: object
required: [datatable_name, schemas]
@@ -0,0 +1,321 @@
<script lang="ts">
import { Button } from '../common'
import Modal2 from '../common/modal/Modal2.svelte'
import Tabs from '../common/tabs/Tabs.svelte'
import Tab from '../common/tabs/Tab.svelte'
import TabContent from '../common/tabs/TabContent.svelte'
import Toggle from '../Toggle.svelte'
import TextInput from '../text_input/TextInput.svelte'
import SimpleEditor from '../SimpleEditor.svelte'
import ConfirmationModal from '../common/confirmationModal/ConfirmationModal.svelte'
import { createAsyncConfirmationModal } from '../common/confirmationModal/asyncConfirmationModal.svelte'
import { ChevronDown, Play, Trash2, Plus, Undo2, Loader2 } from 'lucide-svelte'
import { WorkspaceService, type DatatableMigrationWithStatus } from '$lib/gen'
import { sendUserToast } from '$lib/toast'
let {
workspace,
datatable,
disabled = false
}: { workspace: string; datatable: string; disabled?: boolean } = $props()
let listOpen = $state(false)
let migrations = $state<DatatableMigrationWithStatus[]>([])
let loadError = $state<string | undefined>(undefined)
let loading = $state(false)
let busy = $state(false)
let newOpen = $state(false)
let newTab = $state('up')
let newName = $state('')
let newCodeUp = $state('')
let enableDown = $state(false)
let newCodeDown = $state('')
let creating = $state(false)
const confirmationModal = createAsyncConfirmationModal()
const hasPending = $derived(migrations.some((m) => m.status !== 'ran'))
const hasApplied = $derived(migrations.some((m) => m.status === 'ran'))
async function loadMigrations() {
loading = true
try {
const res = await WorkspaceService.getDatatableMigrationsStatus({
workspace,
datatableName: datatable
})
migrations = res.migrations
loadError = res.error
} catch (e) {
sendUserToast(`Failed to load migrations: ${e}`, true)
} finally {
loading = false
}
}
function openList() {
listOpen = true
loadMigrations()
}
async function runUpTo(upTo: number | undefined) {
busy = true
try {
const res = await WorkspaceService.runDatatableMigrations({
workspace,
datatableName: datatable,
upTo
})
sendUserToast(
res.applied.length > 0
? `Applied ${res.applied.length} migration(s)`
: 'No pending migrations to run'
)
await loadMigrations()
} catch (e) {
sendUserToast(`Failed to run migrations: ${e}`, true)
} finally {
busy = false
}
}
async function revertLast() {
busy = true
try {
const res = await WorkspaceService.rollbackDatatableMigrations({
workspace,
datatableName: datatable
})
sendUserToast(
res.rolled_back.length > 0
? `Rolled back ${res.rolled_back[0].name}`
: 'No applied migrations to roll back'
)
await loadMigrations()
} catch (e) {
sendUserToast(`Failed to revert migration: ${e}`, true)
} finally {
busy = false
}
}
async function deleteMigration(m: DatatableMigrationWithStatus) {
const body =
m.status === 'ran'
? `"${m.name}" is installed on the data table. Deleting its definition (including its down migration) means it can no longer be reverted and may leave the data table in a broken state. Delete anyway?`
: m.status === 'unknown'
? `The applied status of "${m.name}" is unknown. If it is installed on the data table, deleting it may leave the data table in a broken state. Delete anyway?`
: `Delete the definition of "${m.name}"? It has not been run, so this only removes the migration definition.`
const confirmed = await confirmationModal.ask({
title: 'Delete migration',
confirmationText: 'Delete',
children: body
})
if (!confirmed) return
busy = true
try {
await WorkspaceService.deleteDatatableMigration({
workspace,
datatableName: datatable,
timestamp: m.timestamp
})
await loadMigrations()
} catch (e) {
sendUserToast(`Failed to delete migration: ${e}`, true)
} finally {
busy = false
}
}
function resetNew() {
newName = ''
newCodeUp = ''
newCodeDown = ''
enableDown = false
newTab = 'up'
}
function openNew() {
resetNew()
newOpen = true
}
async function create(run: boolean) {
if (newName.trim() === '') {
sendUserToast('Migration name is required', true)
return
}
creating = true
try {
const created = await WorkspaceService.createDatatableMigration({
workspace,
datatableName: datatable,
requestBody: {
name: newName.trim(),
code_up: newCodeUp,
code_down: enableDown ? newCodeDown : undefined
}
})
if (run) {
await WorkspaceService.runDatatableMigrations({
workspace,
datatableName: datatable,
upTo: created.timestamp
})
}
newOpen = false
sendUserToast(run ? 'Migration created and run' : 'Migration created')
await loadMigrations()
} catch (e) {
sendUserToast(`Failed to create migration: ${e}`, true)
} finally {
creating = false
}
}
const statusColor = {
ran: 'bg-green-500',
not_run: 'bg-orange-500',
unknown: 'bg-gray-400'
}
const statusTitle = {
ran: 'Already ran',
not_run: 'Not run',
unknown: 'Status unknown'
}
</script>
<Button
variant="default"
size="sm"
{disabled}
startIcon={{ icon: ChevronDown }}
on:click={openList}
>
Migrations
</Button>
<Modal2 bind:isOpen={listOpen} title="Migrations {datatable}" fixedWidth="md" fixedHeight="lg">
<div class="flex flex-col gap-2 w-full grow min-h-0">
{#if loadError}
<div class="text-xs text-red-500">
Could not read applied status from the data table: {loadError}
</div>
{/if}
<div class="flex flex-col grow min-h-0 overflow-auto border rounded-md divide-y">
{#if loading}
<div class="flex items-center justify-center p-6 text-tertiary">
<Loader2 size={18} class="animate-spin" />
</div>
{:else if migrations.length === 0}
<div class="p-6 text-center text-sm text-tertiary">No migrations yet</div>
{:else}
{#each migrations as m (m.timestamp)}
<div class="flex items-center gap-3 px-3 py-2">
<div
class="shrink-0 w-2.5 h-2.5 rounded-full {statusColor[m.status]}"
title={statusTitle[m.status]}
></div>
<div class="flex flex-col min-w-0 grow">
<span class="text-sm text-primary truncate">{m.name}</span>
<span class="text-xs text-hint">{m.timestamp}</span>
</div>
<Button
variant="subtle"
size="xs"
iconOnly
startIcon={{ icon: Play }}
title="Run up to this migration"
disabled={busy || m.status === 'ran'}
on:click={() => runUpTo(m.timestamp)}
/>
<Button
variant="subtle"
size="xs"
iconOnly
color="red"
startIcon={{ icon: Trash2 }}
title="Delete migration"
disabled={busy}
on:click={() => deleteMigration(m)}
/>
</div>
{/each}
{/if}
</div>
<div class="flex justify-between gap-2 pt-2">
<Button variant="default" size="sm" startIcon={{ icon: Plus }} on:click={openNew}>New</Button>
<div class="flex gap-2">
<Button
variant="default"
size="sm"
startIcon={{ icon: Undo2 }}
disabled={busy || !hasApplied}
on:click={revertLast}
>
Revert last
</Button>
<Button
variant="accent"
size="sm"
startIcon={{ icon: Play }}
disabled={busy || !hasPending}
on:click={() => runUpTo(undefined)}
>
Run all
</Button>
</div>
</div>
</div>
</Modal2>
<Modal2 bind:isOpen={newOpen} title="New migration" fixedWidth="md" fixedHeight="lg">
<div class="flex flex-col gap-3 w-full grow min-h-0">
<TextInput
bind:value={newName}
inputProps={{ placeholder: 'Migration name (e.g. add_index_to_customers)' }}
/>
<Tabs bind:selected={newTab} class="grow min-h-0">
<Tab value="up" label="Up" />
<Tab value="down" label="Down" />
{#snippet content()}
<TabContent value="up" class="h-80">
<SimpleEditor class="h-full" lang="sql" bind:code={newCodeUp} />
</TabContent>
<TabContent value="down" class="h-80">
<div class="flex flex-col gap-2 h-full">
<Toggle
bind:checked={enableDown}
options={{ right: 'Enable down migration' }}
size="sm"
/>
{#if enableDown}
<div class="grow min-h-0">
<SimpleEditor class="h-full" lang="sql" bind:code={newCodeDown} />
</div>
{/if}
</div>
</TabContent>
{/snippet}
</Tabs>
<div class="flex justify-end pt-2">
<Button
variant="accent"
size="sm"
disabled={creating}
on:click={() => create(true)}
dropdownItems={[
{
label: 'Create without running',
onClick: () => create(false)
}
]}
>
Create and run
</Button>
</div>
</div>
</Modal2>
<ConfirmationModal {...confirmationModal.props} />
@@ -72,6 +72,7 @@
import CustomInstanceDbSelect from './CustomInstanceDbSelect.svelte'
import { Popover } from '../meltComponents'
import ExploreAssetButton from '../ExploreAssetButton.svelte'
import DataTableMigrationsButton from './DataTableMigrationsButton.svelte'
import { deepEqual } from 'fast-equals'
import { clone } from '$lib/utils'
import SettingsFooter from './SettingsFooter.svelte'
@@ -256,8 +257,13 @@
</div>
</Cell>
<Cell class="w-12">
<Cell class="whitespace-nowrap">
<div class="flex gap-2">
<DataTableMigrationsButton
workspace={$workspaceStore ?? ''}
datatable={dataTable.name}
disabled={!!dirtyMap[dataTable.name]}
/>
{#if dirtyMap[dataTable.name]}
<Popover
openOnHover