starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-25 16:02:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Datatable fork behavior

Browse Source
This commit is contained in:
Diego Imbert
2026-03-12 15:31:11 +01:00
parent 45a83cf427
commit d7150d05f2
7 changed files with 187 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/Cargo.lock
Generated
+3 -3
View File
@@ -16869,7 +16869,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-asset"
version = "1.653.0"
version = "1.654.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -16949,7 +16949,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql-asset"
version = "1.653.0"
version = "1.654.0"
dependencies = [
"anyhow",
"serde",
@@ -16979,7 +16979,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts-asset"
version = "1.653.0"
version = "1.654.0"
dependencies = [
"anyhow",
"serde-wasm-bindgen",
backend/windmill-api-workspaces/src/workspaces.rs
+103 -16
View File
@@ -41,8 +41,8 @@ use windmill_common::workspaces::GitRepositorySettings;
use windmill_common::workspaces::WorkspaceDeploymentUISettings;
use windmill_common::workspaces::{
check_user_against_rule, get_datatable_resource_from_db_unchecked, DataTable,
DataTableCatalogResourceType, DataTableDatabase, ProtectionRuleKind, ProtectionRules,
ProtectionRuleset, RuleCheckResult, WorkspaceGitSyncSettings,
DataTableCatalogResourceType, DataTableDatabase, DataTableForkBehavior, ProtectionRuleKind,
ProtectionRules, ProtectionRuleset, RuleCheckResult, WorkspaceGitSyncSettings,
};
use windmill_common::workspaces::{Ducklake, DucklakeCatalogResourceType};
use windmill_common::{
@@ -1394,18 +1394,37 @@ async fn fork_datatable(
Json(req): Json<ForkDatatableRequest>,
) -> Result<String> {
require_super_admin(&db, &authed.email).await?;
fork_datatable_inner(
&db,
&w_id,
&req.source_datatable_name,
&req.new_datatable_name,
&req.new_custom_instance_database_name,
req.include_data,
)
.await
}
/// Core logic for forking a single datatable: creates a new instance DB,
/// dumps the source schema (and optionally data), imports into the new DB,
/// and updates the target workspace's datatable config.
async fn fork_datatable_inner(
db: &DB,
w_id: &str,
source_datatable_name: &str,
new_datatable_name: &str,
new_custom_instance_database_name: &str,
include_data: bool,
) -> Result<String> {
// Resolve the source datatable to get the original instance DB name
let db_resource =
get_datatable_resource_from_db_unchecked(&db, &w_id, &req.source_datatable_name).await?;
get_datatable_resource_from_db_unchecked(db, w_id, source_datatable_name).await?;
let pg_db: PgDatabase = serde_json::from_value(db_resource)
.map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?;
// Interpolate $current_name with the original database name
let original_dbname = &pg_db.dbname;
let new_dbname = req
.new_custom_instance_database_name
.replace("$current_name", original_dbname);
let new_dbname = new_custom_instance_database_name.replace("$current_name", original_dbname);
// Create the new custom instance database
let wmill_pg_creds = PgDatabase::parse_uri(&get_database_url().await?.as_str().await)?;
@@ -1415,13 +1434,13 @@ async fn fork_datatable(
"SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_database WHERE datname = $1)",
&new_dbname
)
.fetch_one(&db)
.fetch_one(db)
.await?
.unwrap_or(false);
if !db_exists {
sqlx::query(&format!("CREATE DATABASE \"{}\"", &new_dbname))
.execute(&db)
.execute(db)
.await?;
}
@@ -1472,31 +1491,32 @@ async fn fork_datatable(
r#"UPDATE global_settings SET value = jsonb_set(value, '{databases}', (COALESCE(value->'databases', '{}'::jsonb) || to_jsonb($1::json))) WHERE name = 'custom_instance_pg_databases'"#,
serde_json::json!({ &new_dbname: status_json })
)
.execute(&db)
.execute(db)
.await?;
// Export the schema (and optionally data) from the source BEFORE updating the config
let dump = dump_datatable(db, w_id, source_datatable_name, !include_data).await?;
// Update the forked workspace's datatable config to point to the new database
let new_datatable = DataTable {
database: DataTableDatabase {
resource_type: DataTableCatalogResourceType::Instance,
resource_path: new_dbname.clone(),
},
fork_behavior: DataTableForkBehavior::default(),
};
let mut datatables = HashMap::new();
datatables.insert(req.new_datatable_name.clone(), new_datatable);
datatables.insert(new_datatable_name.to_string(), new_datatable);
let new_settings = DataTableSettings { datatables };
let config: serde_json::Value =
serde_json::to_value(new_settings).map_err(|err| Error::internal_err(err.to_string()))?;
// Export the schema from the source BEFORE updating the config (which points to the new empty DB)
let dump = dump_datatable(&db, &w_id, &req.source_datatable_name, !req.include_data).await?;
sqlx::query!(
"UPDATE workspace_settings SET datatable = $1 WHERE workspace_id = $2",
config,
&w_id
w_id
)
.execute(&db)
.execute(db)
.await?;
// Import the dumped schema into the new database
@@ -1504,10 +1524,67 @@ async fn fork_datatable(
Ok(format!(
"Forked datatable '{}' as '{}' with new database '{}'",
req.source_datatable_name, req.new_datatable_name, new_dbname
source_datatable_name, new_datatable_name, new_dbname
))
}
/// Fork all datatables from a source workspace into a target workspace,
/// respecting each datatable's configured `fork_behavior`.
async fn fork_all_datatables(
db: &DB,
source_workspace_id: &str,
target_workspace_id: &str,
) -> Result<()> {
let datatable_config = sqlx::query_scalar!(
"SELECT datatable FROM workspace_settings WHERE workspace_id = $1",
source_workspace_id
)
.fetch_one(db)
.await?;
let datatables: HashMap<String, DataTable> = match datatable_config {
Some(config) => {
let settings: DataTableSettings = serde_json::from_value(config)
.unwrap_or(DataTableSettings { datatables: HashMap::new() });
settings.datatables
}
None => return Ok(()),
};
for (name, dt) in &datatables {
match dt.fork_behavior {
DataTableForkBehavior::KeepOriginal => continue,
behavior => {
let include_data = behavior == DataTableForkBehavior::SchemaAndData;
let new_db_name = format!(
"__wmfork__{}__$current_name",
target_workspace_id.replace('-', "_")
);
if let Err(e) = fork_datatable_inner(
db,
target_workspace_id,
name,
name,
&new_db_name,
include_data,
)
.await
{
tracing::error!(
"Failed to fork datatable '{}' from '{}' to '{}': {}",
name,
source_workspace_id,
target_workspace_id,
e
);
}
}
}
}
Ok(())
}
/// Import a pg_dump output into a target database using tokio_postgres.
/// Filters out psql meta-commands (lines starting with `\`) and comment-only lines
/// that are not valid SQL but are included in pg_dump's plain-text output.
@@ -3746,6 +3823,16 @@ async fn create_workspace_fork(
)
.await?;
tx.commit().await?;
// Fork datatables after the transaction commits (creates external databases)
if let Err(e) = fork_all_datatables(&db, &parent_workspace_id, &forked_id).await {
tracing::error!(
"Failed to fork datatables for workspace '{}': {}",
&forked_id,
e
);
}
Ok(format!("Created forked workspace {}", &forked_id))
}
backend/windmill-api/openapi.yaml
+7
View File
@@ -23358,6 +23358,13 @@ components:
type: string
required:
- resource_type
fork_behavior:
type: string
enum:
- schema_only
- schema_and_data
- keep_original
default: schema_only
DataTableSchema:
type: object
backend/windmill-common/src/workspaces.rs
+20
View File
@@ -381,9 +381,29 @@ pub async fn check_user_against_rule(
Ok(RuleCheckResult::Allowed)
}
#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq)]
#[serde(rename_all = "snake_case")]
pub enum DataTableForkBehavior {
SchemaOnly,
SchemaAndData,
KeepOriginal,
}
impl Default for DataTableForkBehavior {
fn default() -> Self {
DataTableForkBehavior::SchemaOnly
}
}
#[derive(Deserialize, Serialize, Debug)]
pub struct DataTable {
pub database: DataTableDatabase,
#[serde(default, skip_serializing_if = "is_default_fork_behavior")]
pub fork_behavior: DataTableForkBehavior,
}
fn is_default_fork_behavior(v: &DataTableForkBehavior) -> bool {
*v == DataTableForkBehavior::SchemaOnly
}
#[derive(Deserialize, Serialize, Debug)]
frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte
+1 -1
View File
@@ -84,7 +84,7 @@
{#if open}
<div
transition:fadeFast|local
class={'fixed top-0 bottom-0 left-0 right-0 z-[5000]'}
class={'fixed top-0 bottom-0 left-0 right-0 z-[9999]'}
role="dialog"
{id}
>
frontend/src/lib/components/workspaceSettings/CreateWorkspace.svelte
-45
View File
@@ -188,51 +188,6 @@
forkCreationLoading = false
sendUserToast(`Successfully forked workspace ${$workspaceStore} as: wm-fork-${id}`)
// Check if the source workspace has a "main" datatable and offer to fork it
try {
const datatables = await WorkspaceService.listDataTables({
workspace: $workspaceStore!
})
if (datatables.includes('main')) {
const forkDatatableAction = async (includeData: boolean) => {
try {
await WorkspaceService.forkDatatable({
workspace: prefixed_id,
requestBody: {
source_datatable_name: 'main',
new_datatable_name: 'main',
new_custom_instance_database_name: `__wmfork__${prefixed_id.replace(/-/g, '_')}__$current_name`,
include_data: includeData
}
})
sendUserToast(
`Successfully forked "main" datatable${includeData ? ' with data' : ''} to workspace "${prefixed_id}"`
)
} catch (e) {
sendUserToast(`Failed to fork datatable: ${e?.body ?? e}`, 'error')
}
}
sendUserToast(
`Fork the main datatable?`,
'info',
[
{
label: 'Fork schema only',
callback: () => forkDatatableAction(false)
},
{
label: 'Fork schema and data',
callback: () => forkDatatableAction(true)
}
],
undefined,
30000
)
}
} catch {
// Silently ignore if we can't list datatables
}
} else {
sendUserToast('No workspace selected, cannot fork non-existent workspace', true)
}
frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte
+53 -4
View File
@@ -1,4 +1,6 @@
<script lang="ts" module>
export type DataTableForkBehavior = 'schema_only' | 'schema_and_data' | 'keep_original'
export type DataTableSettingsType = {
dataTables: {
name: string
@@ -6,6 +8,7 @@
resource_type: 'postgresql' | 'instance'
resource_path?: string | undefined
}
fork_behavior?: DataTableForkBehavior
}[]
}
@@ -15,7 +18,11 @@
const s: DataTableSettingsType = { dataTables: [] }
if (settings?.datatables) {
for (const [name, rest] of Object.entries(settings.datatables)) {
s.dataTables.push({ name, ...rest })
s.dataTables.push({
name,
...rest,
fork_behavior: (rest as any).fork_behavior ?? 'schema_only'
})
}
}
return s
@@ -33,8 +40,9 @@
throw dataTable.name + ' database cannot be called "windmill"'
s.datatables[dataTable.name] = {
database: dataTable.database
}
database: dataTable.database,
fork_behavior: dataTable.fork_behavior
} as any
}
return s
}
@@ -43,7 +51,7 @@
</script>
<script lang="ts">
import { Plus } from 'lucide-svelte'
import { Plus, Settings } from 'lucide-svelte'
import Button from '../common/button/Button.svelte'
@@ -72,6 +80,7 @@
import { deepEqual } from 'fast-equals'
import { clone } from '$lib/utils'
import SettingsFooter from './SettingsFooter.svelte'
import Label from '../Label.svelte'
type Props = {
dataTableSettings: DataTableSettingsType
@@ -252,6 +261,46 @@
</div>
</div>
</Cell>
<Cell class="w-12">
<Popover contentClasses="p-4 w-64" enableFlyTransition disableFocusTrap>
{#snippet trigger()}
<Button variant="default" iconOnly size="sm" startIcon={{ icon: Settings }} />
{/snippet}
{#snippet content()}
<Label
label="Fork behavior"
tooltip="Determines what happens to this datatable when the workspace is forked."
>
<Select
items={[
{ value: 'schema_only', label: 'Fork schema only' },
{ value: 'schema_and_data', label: 'Fork schema and data' },
{ value: 'keep_original', label: 'Keep original datatable' }
]}
bind:value={
() => dataTable.fork_behavior ?? 'schema_only',
(v) => {
if (v === 'schema_and_data') {
confirmationModal
.ask({
title: 'Fork schema and data',
children:
'This will copy ALL data when the workspace is forked, which may take a long time and use significant storage space. Are you sure?',
confirmationText: 'Confirm'
})
.then((confirmed) => {
if (confirmed) dataTable.fork_behavior = v
})
} else {
dataTable.fork_behavior = v
}
}
}
/>
</Label>
{/snippet}
</Popover>
</Cell>
<Cell class="w-12">
{#if dirtyMap[dataTable.name]}
<Popover