feat: add datatable clone UI with step-by-step confirmation modal

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Diego Imbert
2026-03-23 15:29:28 +01:00
co-authored by Claude Opus 4.5
parent 7605524841
commit 6cc7a9fcae
11 changed files with 402 additions and 61 deletions
@@ -1142,26 +1142,38 @@ async fn list_ducklakes(
Ok(Json(ducklakes))
}
#[derive(Serialize)]
struct DataTableListItem {
name: String,
resource_type: String,
}
async fn list_datatables(
_authed: ApiAuthed,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
) -> JsonResult<Vec<String>> {
let datatables = sqlx::query_scalar!(
r#"
SELECT jsonb_object_keys(ws.datatable->'datatables') AS datatable_name
FROM workspace_settings ws
WHERE ws.workspace_id = $1
"#,
) -> JsonResult<Vec<DataTableListItem>> {
let config = sqlx::query_scalar!(
"SELECT datatable->'datatables' FROM workspace_settings WHERE workspace_id = $1",
&w_id
)
.fetch_all(&db)
.await?
.into_iter()
.filter_map(|s| s)
.collect();
.fetch_one(&db)
.await?;
Ok(Json(datatables))
let items: Vec<DataTableListItem> = match config {
Some(val) => {
let map: HashMap<String, DataTable> = serde_json::from_value(val).unwrap_or_default();
map.into_iter()
.map(|(name, dt)| DataTableListItem {
name,
resource_type: dt.database.resource_type.as_ref().to_string(),
})
.collect()
}
None => vec![],
};
Ok(Json(items))
}
/// Compact column representation: "type" or "type?" for nullable, with "=default" suffix if has default
@@ -1464,6 +1476,11 @@ struct ForkPgDatabaseRequest {
source: String,
target: String,
fork_behavior: DataTableForkBehavior,
#[serde(default)]
target_override_dbname: Option<String>,
/// When true, CREATE DATABASE is run on the target server before dump/import
#[serde(default)]
create_target_db: bool,
}
async fn fork_pg_database(
@@ -1481,7 +1498,47 @@ async fn fork_pg_database(
let schema_only = req.fork_behavior != DataTableForkBehavior::SchemaAndData || *CLOUD_HOSTED;
let source_pg = resolve_pg_source(&db, &w_id, &req.source).await?;
let target_pg = resolve_pg_source(&db, &w_id, &req.target).await?;
let mut target_pg = resolve_pg_source(&db, &w_id, &req.target).await?;
if let Some(override_dbname) = &req.target_override_dbname {
target_pg.dbname = override_dbname.clone();
}
// Optionally create the target database before importing
if req.create_target_db {
let admin_pg = PgDatabase { dbname: "postgres".to_string(), ..target_pg.clone() };
let (client, connection) = admin_pg.connect().await?;
let join_handle = tokio::spawn(async move { connection.await });
let row = client
.query_one(
"SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_database WHERE datname = $1)",
&[&target_pg.dbname],
)
.await
.map_err(|e| {
Error::internal_err(format!("Failed to check database existence: {}", e))
})?;
let db_exists: bool = row.get(0);
if !db_exists {
client
.execute(&format!("CREATE DATABASE \"{}\"", &target_pg.dbname), &[])
.await
.map_err(|e| {
Error::internal_err(format!(
"Failed to create database '{}': {}",
target_pg.dbname, e
))
})?;
}
drop(client);
join_handle
.await
.map_err(|e| Error::internal_err(format!("join error: {}", e)))?
.map_err(|e| Error::internal_err(format!("tokio_postgres error: {}", e)))?;
}
let dump = pg_dump_database(&source_pg, schema_only).await?;
pg_import_dump(&target_pg, &dump).await?;
+14 -1
View File
@@ -3333,7 +3333,14 @@ paths:
schema:
type: array
items:
type: string
type: object
required: [name, resource_type]
properties:
name:
type: string
resource_type:
type: string
enum: [postgres, instance]
/w/{workspace}/workspaces/list_datatable_schemas:
get:
@@ -3435,6 +3442,12 @@ paths:
- schema_only
- schema_and_data
- keep_original
target_override_dbname:
type: string
description: "Optional override for the target database name"
create_target_db:
type: boolean
description: "When true, CREATE DATABASE is run on the target server before dump/import"
responses:
"200":
description: status
@@ -29,9 +29,10 @@
}: Props = $props()
let datatables = usePromise(() =>
WorkspaceService.listDataTables({ workspace: $workspaceStore ?? '' })
WorkspaceService.listDataTables({ workspace: $workspaceStore ?? '' }).then((d) =>
d.map((d) => d.name)
)
)
</script>
<div class={className}>
@@ -46,9 +47,6 @@
{onClear}
/>
{#if showSchemaExplorer && value && assetCanBeExplored({ kind: 'datatable', path: value })}
<ExploreAssetButton
class="mt-1 w-fit"
asset={{ kind: 'datatable', path: value }}
/>
<ExploreAssetButton class="mt-1 w-fit" asset={{ kind: 'datatable', path: value }} />
{/if}
</div>
+12 -20
View File
@@ -1570,7 +1570,9 @@
let customTsTypesData = resource([() => lang], async () => {
if (lang !== 'typescript') return undefined
let datatables = await WorkspaceService.listDataTables({ workspace: $workspaceStore ?? '' })
let datatables = (
await WorkspaceService.listDataTables({ workspace: $workspaceStore ?? '' })
).map((d) => d.name)
let ducklakes = await WorkspaceService.listDucklakes({ workspace: $workspaceStore ?? '' })
return { datatables, ducklakes }
})
@@ -1912,15 +1914,12 @@
})
})
let isTsWorkerInitialized = resource(
[() => lang, () => initialized],
async () => {
if (lang !== 'typescript' || !initialized) return false
// Use the stable model URI (computed once at mount), not filePath which changes on rename
await waitForWorkerInitialization(uri)
return true
}
)
let isTsWorkerInitialized = resource([() => lang, () => initialized], async () => {
if (lang !== 'typescript' || !initialized) return false
// Use the stable model URI (computed once at mount), not filePath which changes on rename
await waitForWorkerInitialization(uri)
return true
})
// Update SQL query type information in the TypeScript worker
// This enables TypeScript to show proper types for SQL template literals
@@ -1939,16 +1938,9 @@
updateSqlQueriesInWorker(uri, $state.snapshot(preparedAssetsSqlQueries))
}, 250)
watch(
[
() => preparedAssetsSqlQueries,
() => lang,
() => isTsWorkerInitialized.current
],
() => {
handleSqlTypingInTs()
}
)
watch([() => preparedAssetsSqlQueries, () => lang, () => isTsWorkerInitialized.current], () => {
handleSqlTypingInTs()
})
watch([() => customTsTypesData.current], setTypescriptCustomTypes)
</script>
+9 -2
View File
@@ -79,7 +79,14 @@
iconOnly?: boolean
validCode?: boolean
kind?: 'script' | 'trigger' | 'approval'
template?: 'pgsql' | 'mysql' | 'script' | 'docker' | 'powershell' | 'bunnative' | 'claudesandbox'
template?:
| 'pgsql'
| 'mysql'
| 'script'
| 'docker'
| 'powershell'
| 'bunnative'
| 'claudesandbox'
collabMode?: boolean
collabLive?: boolean
collabUsers?: { name: string }[]
@@ -773,7 +780,7 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS
itemName="data table"
loadItems={async () =>
(await WorkspaceService.listDataTables({ workspace: $workspaceStore ?? 'NO_W' })).map(
(path) => ({ path })
(d) => ({ path: d.name })
)}
>
{#snippet submission()}
@@ -121,9 +121,9 @@
if (datatableAsPgResource && resourceType === 'postgresql') {
try {
const datatables = await WorkspaceService.listDataTables({
const datatables = (await WorkspaceService.listDataTables({
workspace: $workspaceStore!
})
})).map(d => d.name)
for (const dt of datatables) {
nc.push({
value: `datatable://${dt}`,
@@ -61,7 +61,9 @@
})
let datatables = resource([], () =>
WorkspaceService.listDataTables({ workspace: $workspaceStore ?? '' })
WorkspaceService.listDataTables({ workspace: $workspaceStore ?? '' }).then((d) =>
d.map((d) => d.name)
)
)
let ducklakes = resource([], () =>
WorkspaceService.listDucklakes({ workspace: $workspaceStore ?? '' })
@@ -42,7 +42,9 @@
const datatables = resource<string[]>([], async () => {
if (!$workspaceStore) return []
try {
return await WorkspaceService.listDataTables({ workspace: $workspaceStore })
return (await WorkspaceService.listDataTables({ workspace: $workspaceStore })).map(
(d) => d.name
)
} catch (e) {
console.error('Failed to load datatables:', e)
return []
@@ -13,7 +13,7 @@ export function createDatatablesResource(getWorkspace: () => string | undefined)
const workspace = getWorkspace()
if (!workspace) return []
try {
return await WorkspaceService.listDataTables({ workspace })
return (await WorkspaceService.listDataTables({ workspace })).map((d) => d.name)
} catch (e) {
console.error('Failed to load datatables:', e)
return []
@@ -26,13 +26,16 @@
import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte'
import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte'
import { AI_PROVIDERS } from '$lib/components/copilot/lib'
import { LoaderCircle } from 'lucide-svelte'
import { LoaderCircle, Check, X, Loader2 } from 'lucide-svelte'
import PrefixedInput from '../PrefixedInput.svelte'
import TextInput from '../text_input/TextInput.svelte'
import { jobManager } from '$lib/services/JobManager'
import Alert from '../common/alert/Alert.svelte'
import { base } from '$lib/base'
import Label from '../Label.svelte'
import Select from '../select/Select.svelte'
import { resource } from 'runed'
import ConfirmationModal from '../common/confirmationModal/ConfirmationModal.svelte'
interface Props {
isFork?: boolean
@@ -51,6 +54,30 @@
let codeCompletionEnabled = $state(true)
let checking = $state(false)
let allDatatables = resource([], async () =>
$workspaceStore
? WorkspaceService.listDataTables({ workspace: $workspaceStore ?? '' })
: undefined
)
let datatableBehaviors: Record<string, 'schema_only' | 'schema_and_data' | 'keep_original'> =
$state({})
type ForkStep = {
label: string
status: 'pending' | 'running' | 'done' | 'error'
error?: string
}
type DatatableCloneJob = {
name: string
resourceType: string
behavior: 'schema_only' | 'schema_and_data'
steps: ForkStep[]
}
let cloneModalOpen = $state(false)
let currentCloneJob: DatatableCloneJob | undefined = $state(undefined)
let cloneQueue: DatatableCloneJob[] = $state([])
let cloneRunning = $state(false)
let workspaceColor: string | undefined = $state(undefined)
let colorEnabled = $state(false)
@@ -194,6 +221,62 @@
return
}
// Build clone queue for datatables that need cloning
const datatablesToClone = (allDatatables.current ?? [])
.filter((dt) => {
const behavior = datatableBehaviors[dt.name] ?? 'keep_original'
return behavior !== 'keep_original'
})
.map((dt) => {
const behavior = datatableBehaviors[dt.name] as 'schema_only' | 'schema_and_data'
const isInstance = dt.resource_type === 'instance'
const newDbName = isInstance
? `${prefixed_id.replace(/-/g, '_')}__${dt.name}`
: `${prefixed_id.replace(/-/g, '_')}__${dt.name}`
const steps: ForkStep[] = isInstance
? [
{
label: `CREATE DATABASE "${newDbName}" + grant permissions`,
status: 'pending'
},
{
label: `pg_dump → pg_import (${behavior === 'schema_only' ? 'schema only' : 'schema + data'})`,
status: 'pending'
}
]
: [
{
label: `CREATE DATABASE "${newDbName}" + pg_dump → pg_import (${behavior === 'schema_only' ? 'schema only' : 'schema + data'})`,
status: 'pending'
}
]
return {
name: dt.name,
resourceType: dt.resource_type,
behavior,
steps,
_newDbName: newDbName,
_isInstance: isInstance,
_sourceWorkspace: $workspaceStore!,
_targetWorkspace: prefixed_id
} as DatatableCloneJob & {
_newDbName: string
_isInstance: boolean
_sourceWorkspace: string
_targetWorkspace: string
}
})
if (datatablesToClone.length > 0) {
cloneQueue = datatablesToClone
currentCloneJob = cloneQueue[0]
cloneModalOpen = true
// Don't finish fork yet — the modal handles remaining steps
return
}
forkCreationLoading = false
sendUserToast(`Successfully forked workspace ${$workspaceStore} as: wm-fork-${id}`)
} else {
@@ -201,6 +284,103 @@
}
}
async function executeCloneJob(
job: DatatableCloneJob & {
_newDbName: string
_isInstance: boolean
_sourceWorkspace: string
_targetWorkspace: string
}
) {
cloneRunning = true
let stepIdx = 0
if (job._isInstance) {
// Step 1: CREATE DATABASE + grant permissions via setupCustomInstanceDb
job.steps[stepIdx].status = 'running'
try {
await SettingService.setupCustomInstanceDb({
name: job._newDbName,
requestBody: { tag: 'datatable' }
})
job.steps[stepIdx].status = 'done'
} catch (e: any) {
const msg = e?.body ?? e?.message ?? String(e)
if (msg.includes('already exists')) {
job.steps[stepIdx].status = 'done'
} else {
job.steps[stepIdx].status = 'error'
job.steps[stepIdx].error = msg
cloneRunning = false
return
}
}
stepIdx++
// Step 2: fork_pg_database (dump + import)
job.steps[stepIdx].status = 'running'
try {
await WorkspaceService.forkPgDatabase({
workspace: job._targetWorkspace,
requestBody: {
source: `datatable://${job.name}`,
target: `datatable://${job.name}`,
fork_behavior: job.behavior,
target_override_dbname: job._newDbName
}
})
job.steps[stepIdx].status = 'done'
} catch (e: any) {
job.steps[stepIdx].status = 'error'
job.steps[stepIdx].error = e?.body ?? e?.message ?? String(e)
cloneRunning = false
return
}
} else {
// Resource DB: CREATE DATABASE + dump/import in one call
job.steps[stepIdx].status = 'running'
try {
await WorkspaceService.forkPgDatabase({
workspace: job._targetWorkspace,
requestBody: {
source: `datatable://${job.name}`,
target: `datatable://${job.name}`,
fork_behavior: job.behavior,
target_override_dbname: job._newDbName,
create_target_db: true
}
})
job.steps[stepIdx].status = 'done'
} catch (e: any) {
job.steps[stepIdx].status = 'error'
job.steps[stepIdx].error = e?.body ?? e?.message ?? String(e)
cloneRunning = false
return
}
}
cloneRunning = false
}
function advanceCloneQueue() {
const idx = cloneQueue.indexOf(currentCloneJob!)
if (idx < cloneQueue.length - 1) {
currentCloneJob = cloneQueue[idx + 1]
} else {
// All done
cloneModalOpen = false
currentCloneJob = undefined
cloneQueue = []
forkCreationLoading = false
sendUserToast(`Successfully forked workspace ${$workspaceStore} as: wm-fork-${id}`)
usersWorkspaceStore.set(undefined)
WorkspaceService.listUserWorkspaces().then((ws) => {
usersWorkspaceStore.set(ws)
switchWorkspace(`wm-fork-${id}`)
})
}
}
async function createWorkspace(): Promise<void> {
await WorkspaceService.createWorkspace({
requestBody: {
@@ -467,6 +647,35 @@
{/if}
</div>
</Label>
{#if isFork && allDatatables.current && allDatatables.current.length > 0}
<Label label="Data table behavior">
<span class="text-xs text-secondary">
Choose how to handle each datatable when forking
</span>
<div class="border rounded-md divide-y">
{#each allDatatables.current as dt}
<div class="flex items-center gap-2 justify-between px-4 py-1.5">
<div class="flex flex-col">
<span class="text-xs font-medium">{dt.name}</span>
<span class="text-2xs text-tertiary">{dt.resource_type === 'instance' ? 'Instance DB' : 'Resource DB'}</span>
</div>
<Select
dropdownClass="max-w-96"
bind:value={
() => datatableBehaviors[dt.name] ?? 'keep_original',
(v) => (datatableBehaviors[dt.name] = v)
}
items={[
{ value: 'keep_original', label: 'Keep original' },
{ value: 'schema_only', label: 'Clone schema only' },
{ value: 'schema_and_data', label: 'Clone schema and data' }
]}
/>
</div>
{/each}
</div>
</Label>
{/if}
{#if !automateUsernameCreation}
<Label label="Your username in that workspace">
<TextInput
@@ -619,3 +828,66 @@
{/if}
</div>
</div>
{#if cloneModalOpen && currentCloneJob}
<ConfirmationModal
title="Clone datatable: {currentCloneJob.name}"
confirmationText={cloneRunning ? 'Running...' : currentCloneJob.steps.every(s => s.status === 'done') ? 'Next' : 'Start'}
open={cloneModalOpen}
loading={cloneRunning}
onConfirmed={async () => {
if (currentCloneJob!.steps.every(s => s.status === 'done')) {
advanceCloneQueue()
} else {
await executeCloneJob(currentCloneJob as any)
}
}}
onCanceled={() => {
cloneModalOpen = false
forkCreationLoading = false
}}
>
{#if currentCloneJob.behavior === 'schema_and_data'}
<Alert type="error" title="Heavy operation">
This will copy the <b>entire database</b> including all data.
The pg_dump output is temporarily stored on disk and may consume significant server disk space during the operation.
</Alert>
{:else}
<Alert type="info" title="Schema only">
This will copy the database schema only. All tables will be empty. This is a lightweight operation.
</Alert>
{/if}
{#if currentCloneJob.resourceType === 'instance'}
<p class="text-xs text-secondary mt-2">
This will run <code>CREATE DATABASE {currentCloneJob.steps[0]?.label.match(/"([^"]+)"/)?.[1] ?? ''}</code> on the Windmill PostgreSQL instance.
</p>
{:else}
<p class="text-xs text-secondary mt-2">
This will run <code>CREATE DATABASE</code> on the resource's PostgreSQL server.
</p>
{/if}
<div class="mt-4 flex flex-col gap-2">
{#each currentCloneJob.steps as step}
<div class="flex items-center gap-2 text-xs">
{#if step.status === 'done'}
<Check class="w-4 h-4 text-green-500" />
{:else if step.status === 'running'}
<Loader2 class="w-4 h-4 animate-spin text-blue-500" />
{:else if step.status === 'error'}
<X class="w-4 h-4 text-red-500" />
{:else}
<div class="w-4 h-4 rounded-full border border-gray-300"></div>
{/if}
<span class:text-tertiary={step.status === 'pending'} class:font-medium={step.status === 'running'}>
{step.label}
</span>
</div>
{#if step.error}
<p class="text-2xs text-red-500 ml-6">{step.error}</p>
{/if}
{/each}
</div>
</ConfirmationModal>
{/if}
@@ -128,7 +128,7 @@
() => $workspaceStore,
() =>
WorkspaceService.listDataTables({ workspace: $workspaceStore! }).then((d) =>
d.map((d) => ({ label: d == 'main' ? 'Main data table' : d, value: d }))
d.map((d) => d.name).map((d) => ({ label: d == 'main' ? 'Main data table' : d, value: d }))
)
)
let allVolumes = resource(
@@ -280,7 +280,8 @@
startIcon={{ icon: HardDriveIcon }}
on:click={() => volumesDrawer?.openDrawer()}
>
{allVolumes.current?.length ?? 0} {(allVolumes.current?.length ?? 0) === 1 ? 'volume' : 'volumes'}
{allVolumes.current?.length ?? 0}
{(allVolumes.current?.length ?? 0) === 1 ? 'volume' : 'volumes'}
</Button>
{/if}
{/snippet}
@@ -326,7 +327,8 @@
<VolumesDrawer
bind:this={volumesDrawer}
onExplore={async (name) => {
const storage = (await VolumeService.getVolumeStorage({ workspace: $workspaceStore! })) ?? undefined
const storage =
(await VolumeService.getVolumeStorage({ workspace: $workspaceStore! })) ?? undefined
s3FilePicker?.open({ s3: `volumes/${$workspaceStore}/${name}/`, storage })
}}
/>
@@ -355,8 +357,8 @@
<Tooltip>
<AssetGenericIcon assetKind={asset.kind} size="16px" class="!text-secondary" />
{#snippet text()}
{formatAssetKind(asset)}
{/snippet}
{formatAssetKind(asset)}
{/snippet}
</Tooltip>
</Cell>
<Cell class="flex flex-col">
@@ -370,18 +372,14 @@
</Cell>
<Cell class="w-24">
{#if assetCanBeExplored(asset, asset.metadata) && !$userStore?.operator}
<ExploreAssetButton
{asset}
{s3FilePicker}
_resourceMetadata={asset.metadata}
/>
<ExploreAssetButton {asset} {s3FilePicker} _resourceMetadata={asset.metadata} />
{/if}
{#if asset.kind === 'resource' && asset.metadata === undefined}
<Tooltip class={'w-24 flex items-center justify-center'}>
<AlertTriangle size={20} class="text-orange-600 dark:text-orange-500" />
{#snippet text()}
Could not find resource
{/snippet}
Could not find resource
{/snippet}
</Tooltip>
{/if}
</Cell>