From a18fa7ccd6c0305c1f214bed89b0f95f2aa85d77 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Wed, 17 Jun 2026 15:18:09 +0200 Subject: [PATCH] feat: add datatable migrations management UI --- ...7535d4d962c328a060f33c07191ac3e033e82.json | 16 + ...030390d9510e73e9b7df347b697d6dc7aefe6.json | 19 ++ ...7a20a5773113e290d69a016363543067baf14.json | 23 ++ ...0e74e14afde3c02000a88e696d7b6adcad0d6.json | 41 +++ .../windmill-api-workspaces/src/workspaces.rs | 229 +++++++++++++ backend/windmill-api/openapi.yaml | 117 +++++++ .../DataTableMigrationsButton.svelte | 321 ++++++++++++++++++ .../DataTableSettings.svelte | 8 +- 8 files changed, 773 insertions(+), 1 deletion(-) create mode 100644 backend/.sqlx/query-5b67c3af6477d7029d118ac259a7535d4d962c328a060f33c07191ac3e033e82.json create mode 100644 backend/.sqlx/query-760908f44cafb500e4ba2515d1b030390d9510e73e9b7df347b697d6dc7aefe6.json create mode 100644 backend/.sqlx/query-95e370a82ef46d9310f77a99b437a20a5773113e290d69a016363543067baf14.json create mode 100644 backend/.sqlx/query-e5d8d46b9431033fa7572a880ee0e74e14afde3c02000a88e696d7b6adcad0d6.json create mode 100644 frontend/src/lib/components/workspaceSettings/DataTableMigrationsButton.svelte diff --git a/backend/.sqlx/query-5b67c3af6477d7029d118ac259a7535d4d962c328a060f33c07191ac3e033e82.json b/backend/.sqlx/query-5b67c3af6477d7029d118ac259a7535d4d962c328a060f33c07191ac3e033e82.json new file mode 100644 index 0000000000..12628305cc --- /dev/null +++ b/backend/.sqlx/query-5b67c3af6477d7029d118ac259a7535d4d962c328a060f33c07191ac3e033e82.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "5b67c3af6477d7029d118ac259a7535d4d962c328a060f33c07191ac3e033e82" +} diff --git a/backend/.sqlx/query-760908f44cafb500e4ba2515d1b030390d9510e73e9b7df347b697d6dc7aefe6.json b/backend/.sqlx/query-760908f44cafb500e4ba2515d1b030390d9510e73e9b7df347b697d6dc7aefe6.json new file mode 100644 index 0000000000..384686eb35 --- /dev/null +++ b/backend/.sqlx/query-760908f44cafb500e4ba2515d1b030390d9510e73e9b7df347b697d6dc7aefe6.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) VALUES ($1, $2, $3, $4, $5, $6)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "760908f44cafb500e4ba2515d1b030390d9510e73e9b7df347b697d6dc7aefe6" +} diff --git a/backend/.sqlx/query-95e370a82ef46d9310f77a99b437a20a5773113e290d69a016363543067baf14.json b/backend/.sqlx/query-95e370a82ef46d9310f77a99b437a20a5773113e290d69a016363543067baf14.json new file mode 100644 index 0000000000..eb94fe6ab5 --- /dev/null +++ b/backend/.sqlx/query-95e370a82ef46d9310f77a99b437a20a5773113e290d69a016363543067baf14.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT MAX(timestamp) FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "max", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "95e370a82ef46d9310f77a99b437a20a5773113e290d69a016363543067baf14" +} diff --git a/backend/.sqlx/query-e5d8d46b9431033fa7572a880ee0e74e14afde3c02000a88e696d7b6adcad0d6.json b/backend/.sqlx/query-e5d8d46b9431033fa7572a880ee0e74e14afde3c02000a88e696d7b6adcad0d6.json new file mode 100644 index 0000000000..d6c6a5c02f --- /dev/null +++ b/backend/.sqlx/query-e5d8d46b9431033fa7572a880ee0e74e14afde3c02000a88e696d7b6adcad0d6.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT timestamp, name, code_up, code_down FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 ORDER BY timestamp ASC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "timestamp", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "code_up", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "code_down", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true + ] + }, + "hash": "e5d8d46b9431033fa7572a880ee0e74e14afde3c02000a88e696d7b6adcad0d6" +} diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 1125c9ad5f..ad8df9fcfb 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -139,6 +139,18 @@ pub fn workspaced_service() -> Router { "/update_datatable_migrations", post(update_datatable_migrations), ) + .route( + "/datatable_migrations_status/{datatable_name}", + get(datatable_migrations_status), + ) + .route( + "/create_datatable_migration/{datatable_name}", + post(create_datatable_migration), + ) + .route( + "/delete_datatable_migration/{datatable_name}/{timestamp}", + delete(delete_datatable_migration), + ) .route("/git_sync_enabled", get(get_git_sync_enabled)) .route("/edit_git_sync_config", post(edit_git_sync_config)) .route("/edit_git_sync_repository", post(edit_git_sync_repository)) @@ -2397,6 +2409,12 @@ struct RunDatatableMigrationsResult { applied: Vec, } +#[derive(Deserialize)] +struct RunDatatableMigrationsQuery { + /// When set, only apply pending migrations up to and including this version. + up_to: Option, +} + /// Apply the workspace's pending data table migrations to a given data table. /// Applied versions are tracked in the data table's own `_wm_migrations` table, /// so only migrations not recorded there are run, in ascending `timestamp` order. @@ -2404,6 +2422,7 @@ async fn run_datatable_migrations( authed: ApiAuthed, Extension(db): Extension, Path((w_id, datatable_name)): Path<(String, String)>, + Query(query): Query, ) -> JsonResult { require_admin(authed.is_admin, &authed.username)?; @@ -2459,6 +2478,10 @@ async fn run_datatable_migrations( let mut applied = Vec::new(); for m in migrations { + // Migrations are ordered ascending, so once we pass `up_to` we're done. + if query.up_to.is_some_and(|up_to| m.timestamp > up_to) { + break; + } if applied_versions.contains(&m.timestamp) { continue; } @@ -2706,6 +2729,212 @@ async fn update_datatable_migrations( )) } +#[derive(Serialize)] +#[serde(rename_all = "snake_case")] +enum DatatableMigrationRunStatus { + /// Recorded in the data table's `_wm_migrations` table. + Ran, + /// Defined but not yet applied. + NotRun, + /// Applied status could not be determined (connection failure). + Unknown, +} + +#[derive(Serialize)] +struct DatatableMigrationWithStatus { + timestamp: i64, + name: String, + code_up: String, + #[serde(skip_serializing_if = "Option::is_none")] + code_down: Option, + status: DatatableMigrationRunStatus, +} + +#[derive(Serialize)] +struct DatatableMigrationsStatusResult { + migrations: Vec, + /// Set when the applied status couldn't be read from the data table. + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, +} + +/// Read the versions recorded in a data table's `_wm_migrations` table. A +/// missing table means nothing has been applied yet (empty set, not an error). +async fn read_applied_datatable_versions( + db: &DB, + w_id: &str, + datatable_name: &str, +) -> Result> { + let db_resource = get_datatable_resource_from_db_unchecked(db, w_id, datatable_name).await?; + let pg_db: PgDatabase = serde_json::from_value(db_resource) + .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?; + let (client, connection) = pg_db.connect(Some(db)).await?; + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::error!("Datatable connection error: {}", e); + } + }); + + match client + .query("SELECT version FROM _wm_migrations", &[]) + .await + { + Ok(rows) => Ok(rows.iter().map(|row| row.get::<_, i64>(0)).collect()), + // 42P01 = undefined_table: the data table has never been migrated yet. + Err(e) if e.as_db_error().map(|d| d.code().code()) == Some("42P01") => Ok(HashSet::new()), + Err(e) => Err(Error::internal_err(format!( + "Failed to read _wm_migrations: {}", + e + ))), + } +} + +/// List a data table's migrations annotated with whether each has been applied. +async fn datatable_migrations_status( + _authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, +) -> JsonResult { + let defs = sqlx::query!( + "SELECT timestamp, name, code_up, code_down FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 ORDER BY timestamp ASC", + &w_id, + &datatable_name, + ) + .fetch_all(&db) + .await?; + + let (applied, error) = match read_applied_datatable_versions(&db, &w_id, &datatable_name).await + { + Ok(set) => (Some(set), None), + Err(e) => (None, Some(e.to_string())), + }; + + let migrations = defs + .into_iter() + .map(|m| { + let status = match &applied { + Some(set) if set.contains(&m.timestamp) => DatatableMigrationRunStatus::Ran, + Some(_) => DatatableMigrationRunStatus::NotRun, + None => DatatableMigrationRunStatus::Unknown, + }; + DatatableMigrationWithStatus { + timestamp: m.timestamp, + name: m.name, + code_up: m.code_up, + code_down: m.code_down, + status, + } + }) + .collect(); + + Ok(Json(DatatableMigrationsStatusResult { migrations, error })) +} + +#[derive(Deserialize)] +pub struct CreateDatatableMigration { + pub name: String, + pub code_up: String, + #[serde(default)] + pub code_down: Option, +} + +/// Create a single migration for a data table. The version is generated +/// server-side (current UTC `YYYYMMDDHHMMSS`), bumped past any existing version +/// so it stays unique and monotonically increasing. +async fn create_datatable_migration( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, + Json(payload): Json, +) -> JsonResult { + require_admin(authed.is_admin, &authed.username)?; + + let now_ts: i64 = Utc::now() + .format("%Y%m%d%H%M%S") + .to_string() + .parse() + .map_err(|e| Error::internal_err(format!("Failed to build migration version: {}", e)))?; + let max_existing: Option = sqlx::query_scalar!( + "SELECT MAX(timestamp) FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2", + &w_id, + &datatable_name, + ) + .fetch_one(&db) + .await?; + let timestamp = match max_existing { + Some(m) if m >= now_ts => m + 1, + _ => now_ts, + }; + + sqlx::query!( + "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) \ + VALUES ($1, $2, $3, $4, $5, $6)", + &w_id, + &datatable_name, + timestamp, + &payload.name, + &payload.code_up, + payload.code_down.as_deref(), + ) + .execute(&db) + .await?; + + audit_log( + &db, + &authed, + "workspaces.create_datatable_migration", + ActionKind::Create, + &w_id, + Some(datatable_name.as_str()), + None, + ) + .await?; + + Ok(Json(DatatableMigration { + datatable: datatable_name, + timestamp, + name: payload.name, + code_up: payload.code_up, + code_down: payload.code_down, + })) +} + +/// Delete a single migration definition from a data table. +async fn delete_datatable_migration( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name, timestamp)): Path<(String, String, i64)>, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + + sqlx::query!( + "DELETE FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + &w_id, + &datatable_name, + timestamp, + ) + .execute(&db) + .await?; + + audit_log( + &db, + &authed, + "workspaces.delete_datatable_migration", + ActionKind::Delete, + &w_id, + Some(datatable_name.as_str()), + None, + ) + .await?; + + Ok(format!( + "Deleted migration {} from {}", + timestamp, datatable_name + )) +} + #[derive(Deserialize)] pub struct EditGitSyncConfig { pub git_sync_settings: Option, diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index fc23022483..df21148cd0 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -4364,6 +4364,13 @@ paths: required: true schema: type: string + - name: up_to + in: query + required: false + description: only apply pending migrations up to and including this version + schema: + type: integer + format: int64 responses: "200": description: applied migrations @@ -4465,6 +4472,97 @@ paths: schema: type: string + /w/{workspace}/workspaces/datatable_migrations_status/{datatable_name}: + get: + summary: list a datatable's migrations with their applied status + operationId: getDatatableMigrationsStatus + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + responses: + "200": + description: migrations with status + content: + application/json: + schema: + type: object + required: [migrations] + properties: + migrations: + type: array + items: + $ref: "#/components/schemas/DatatableMigrationWithStatus" + error: + type: string + + /w/{workspace}/workspaces/create_datatable_migration/{datatable_name}: + post: + summary: create a single datatable migration (version generated server-side) + operationId: createDatatableMigration + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [name, code_up] + properties: + name: + type: string + code_up: + type: string + code_down: + type: string + responses: + "200": + description: created migration + content: + application/json: + schema: + $ref: "#/components/schemas/DatatableMigration" + + /w/{workspace}/workspaces/delete_datatable_migration/{datatable_name}/{timestamp}: + delete: + summary: delete a single datatable migration definition + operationId: deleteDatatableMigration + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + - name: timestamp + in: path + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + /w/{workspace}/workspaces/create_pg_database: post: summary: create a new PostgreSQL database for a datatable @@ -27808,6 +27906,25 @@ components: type: string code_down: type: string + DatatableMigrationWithStatus: + type: object + required: [timestamp, name, code_up, status] + properties: + timestamp: + type: integer + format: int64 + name: + type: string + code_up: + type: string + code_down: + type: string + status: + type: string + enum: + - ran + - not_run + - unknown DataTableSchema: type: object required: [datatable_name, schemas] diff --git a/frontend/src/lib/components/workspaceSettings/DataTableMigrationsButton.svelte b/frontend/src/lib/components/workspaceSettings/DataTableMigrationsButton.svelte new file mode 100644 index 0000000000..5a68d43336 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/DataTableMigrationsButton.svelte @@ -0,0 +1,321 @@ + + + + + +
+ {#if loadError} +
+ Could not read applied status from the data table: {loadError} +
+ {/if} +
+ {#if loading} +
+ +
+ {:else if migrations.length === 0} +
No migrations yet
+ {:else} + {#each migrations as m (m.timestamp)} +
+
+
+ {m.name} + {m.timestamp} +
+
+ {/each} + {/if} +
+
+ +
+ + +
+
+
+
+ + +
+ + + + + {#snippet content()} + + + + +
+ + {#if enableDown} +
+ +
+ {/if} +
+
+ {/snippet} +
+
+ +
+
+
+ + diff --git a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte index 60794c6923..490f8fcb89 100644 --- a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte @@ -72,6 +72,7 @@ import CustomInstanceDbSelect from './CustomInstanceDbSelect.svelte' import { Popover } from '../meltComponents' import ExploreAssetButton from '../ExploreAssetButton.svelte' + import DataTableMigrationsButton from './DataTableMigrationsButton.svelte' import { deepEqual } from 'fast-equals' import { clone } from '$lib/utils' import SettingsFooter from './SettingsFooter.svelte' @@ -256,8 +257,13 @@ - +
+ {#if dirtyMap[dataTable.name]}