diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 871d739eb3..83ab30c007 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -1142,26 +1142,38 @@ async fn list_ducklakes( Ok(Json(ducklakes)) } +#[derive(Serialize)] +struct DataTableListItem { + name: String, + resource_type: String, +} + async fn list_datatables( _authed: ApiAuthed, Extension(db): Extension, Path(w_id): Path, -) -> JsonResult> { - let datatables = sqlx::query_scalar!( - r#" - SELECT jsonb_object_keys(ws.datatable->'datatables') AS datatable_name - FROM workspace_settings ws - WHERE ws.workspace_id = $1 - "#, +) -> JsonResult> { + let config = sqlx::query_scalar!( + "SELECT datatable->'datatables' FROM workspace_settings WHERE workspace_id = $1", &w_id ) - .fetch_all(&db) - .await? - .into_iter() - .filter_map(|s| s) - .collect(); + .fetch_one(&db) + .await?; - Ok(Json(datatables)) + let items: Vec = match config { + Some(val) => { + let map: HashMap = serde_json::from_value(val).unwrap_or_default(); + map.into_iter() + .map(|(name, dt)| DataTableListItem { + name, + resource_type: dt.database.resource_type.as_ref().to_string(), + }) + .collect() + } + None => vec![], + }; + + Ok(Json(items)) } /// Compact column representation: "type" or "type?" for nullable, with "=default" suffix if has default @@ -1464,6 +1476,11 @@ struct ForkPgDatabaseRequest { source: String, target: String, fork_behavior: DataTableForkBehavior, + #[serde(default)] + target_override_dbname: Option, + /// When true, CREATE DATABASE is run on the target server before dump/import + #[serde(default)] + create_target_db: bool, } async fn fork_pg_database( @@ -1481,7 +1498,47 @@ async fn fork_pg_database( let schema_only = req.fork_behavior != DataTableForkBehavior::SchemaAndData || *CLOUD_HOSTED; let source_pg = resolve_pg_source(&db, &w_id, &req.source).await?; - let target_pg = resolve_pg_source(&db, &w_id, &req.target).await?; + let mut target_pg = resolve_pg_source(&db, &w_id, &req.target).await?; + + if let Some(override_dbname) = &req.target_override_dbname { + target_pg.dbname = override_dbname.clone(); + } + + // Optionally create the target database before importing + if req.create_target_db { + let admin_pg = PgDatabase { dbname: "postgres".to_string(), ..target_pg.clone() }; + let (client, connection) = admin_pg.connect().await?; + let join_handle = tokio::spawn(async move { connection.await }); + + let row = client + .query_one( + "SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_database WHERE datname = $1)", + &[&target_pg.dbname], + ) + .await + .map_err(|e| { + Error::internal_err(format!("Failed to check database existence: {}", e)) + })?; + let db_exists: bool = row.get(0); + + if !db_exists { + client + .execute(&format!("CREATE DATABASE \"{}\"", &target_pg.dbname), &[]) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to create database '{}': {}", + target_pg.dbname, e + )) + })?; + } + + drop(client); + join_handle + .await + .map_err(|e| Error::internal_err(format!("join error: {}", e)))? + .map_err(|e| Error::internal_err(format!("tokio_postgres error: {}", e)))?; + } let dump = pg_dump_database(&source_pg, schema_only).await?; pg_import_dump(&target_pg, &dump).await?; diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index ca096a91ef..3099242d57 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -3333,7 +3333,14 @@ paths: schema: type: array items: - type: string + type: object + required: [name, resource_type] + properties: + name: + type: string + resource_type: + type: string + enum: [postgres, instance] /w/{workspace}/workspaces/list_datatable_schemas: get: @@ -3435,6 +3442,12 @@ paths: - schema_only - schema_and_data - keep_original + target_override_dbname: + type: string + description: "Optional override for the target database name" + create_target_db: + type: boolean + description: "When true, CREATE DATABASE is run on the target server before dump/import" responses: "200": description: status diff --git a/frontend/src/lib/components/DatatablePicker.svelte b/frontend/src/lib/components/DatatablePicker.svelte index e533e63f93..e1741aaf4c 100644 --- a/frontend/src/lib/components/DatatablePicker.svelte +++ b/frontend/src/lib/components/DatatablePicker.svelte @@ -29,9 +29,10 @@ }: Props = $props() let datatables = usePromise(() => - WorkspaceService.listDataTables({ workspace: $workspaceStore ?? '' }) + WorkspaceService.listDataTables({ workspace: $workspaceStore ?? '' }).then((d) => + d.map((d) => d.name) + ) ) -
@@ -46,9 +47,6 @@ {onClear} /> {#if showSchemaExplorer && value && assetCanBeExplored({ kind: 'datatable', path: value })} - + {/if}
diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index 779bc2bf93..4d1b911806 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -1570,7 +1570,9 @@ let customTsTypesData = resource([() => lang], async () => { if (lang !== 'typescript') return undefined - let datatables = await WorkspaceService.listDataTables({ workspace: $workspaceStore ?? '' }) + let datatables = ( + await WorkspaceService.listDataTables({ workspace: $workspaceStore ?? '' }) + ).map((d) => d.name) let ducklakes = await WorkspaceService.listDucklakes({ workspace: $workspaceStore ?? '' }) return { datatables, ducklakes } }) @@ -1912,15 +1914,12 @@ }) }) - let isTsWorkerInitialized = resource( - [() => lang, () => initialized], - async () => { - if (lang !== 'typescript' || !initialized) return false - // Use the stable model URI (computed once at mount), not filePath which changes on rename - await waitForWorkerInitialization(uri) - return true - } - ) + let isTsWorkerInitialized = resource([() => lang, () => initialized], async () => { + if (lang !== 'typescript' || !initialized) return false + // Use the stable model URI (computed once at mount), not filePath which changes on rename + await waitForWorkerInitialization(uri) + return true + }) // Update SQL query type information in the TypeScript worker // This enables TypeScript to show proper types for SQL template literals @@ -1939,16 +1938,9 @@ updateSqlQueriesInWorker(uri, $state.snapshot(preparedAssetsSqlQueries)) }, 250) - watch( - [ - () => preparedAssetsSqlQueries, - () => lang, - () => isTsWorkerInitialized.current - ], - () => { - handleSqlTypingInTs() - } - ) + watch([() => preparedAssetsSqlQueries, () => lang, () => isTsWorkerInitialized.current], () => { + handleSqlTypingInTs() + }) watch([() => customTsTypesData.current], setTypescriptCustomTypes) diff --git a/frontend/src/lib/components/EditorBar.svelte b/frontend/src/lib/components/EditorBar.svelte index 4ddbb70b19..6ff3eeaee2 100644 --- a/frontend/src/lib/components/EditorBar.svelte +++ b/frontend/src/lib/components/EditorBar.svelte @@ -79,7 +79,14 @@ iconOnly?: boolean validCode?: boolean kind?: 'script' | 'trigger' | 'approval' - template?: 'pgsql' | 'mysql' | 'script' | 'docker' | 'powershell' | 'bunnative' | 'claudesandbox' + template?: + | 'pgsql' + | 'mysql' + | 'script' + | 'docker' + | 'powershell' + | 'bunnative' + | 'claudesandbox' collabMode?: boolean collabLive?: boolean collabUsers?: { name: string }[] @@ -773,7 +780,7 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS itemName="data table" loadItems={async () => (await WorkspaceService.listDataTables({ workspace: $workspaceStore ?? 'NO_W' })).map( - (path) => ({ path }) + (d) => ({ path: d.name }) )} > {#snippet submission()} diff --git a/frontend/src/lib/components/ResourcePicker.svelte b/frontend/src/lib/components/ResourcePicker.svelte index fddf9205f5..274c889180 100644 --- a/frontend/src/lib/components/ResourcePicker.svelte +++ b/frontend/src/lib/components/ResourcePicker.svelte @@ -121,9 +121,9 @@ if (datatableAsPgResource && resourceType === 'postgresql') { try { - const datatables = await WorkspaceService.listDataTables({ + const datatables = (await WorkspaceService.listDataTables({ workspace: $workspaceStore! - }) + })).map(d => d.name) for (const dt of datatables) { nc.push({ value: `datatable://${dt}`, diff --git a/frontend/src/lib/components/assets/AssetsDropdownButton.svelte b/frontend/src/lib/components/assets/AssetsDropdownButton.svelte index 351ee68331..3ed8690079 100644 --- a/frontend/src/lib/components/assets/AssetsDropdownButton.svelte +++ b/frontend/src/lib/components/assets/AssetsDropdownButton.svelte @@ -61,7 +61,9 @@ }) let datatables = resource([], () => - WorkspaceService.listDataTables({ workspace: $workspaceStore ?? '' }) + WorkspaceService.listDataTables({ workspace: $workspaceStore ?? '' }).then((d) => + d.map((d) => d.name) + ) ) let ducklakes = resource([], () => WorkspaceService.listDucklakes({ workspace: $workspaceStore ?? '' }) diff --git a/frontend/src/lib/components/raw_apps/RawAppDataTableDrawer.svelte b/frontend/src/lib/components/raw_apps/RawAppDataTableDrawer.svelte index b7bcb9a5ec..aa53999453 100644 --- a/frontend/src/lib/components/raw_apps/RawAppDataTableDrawer.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppDataTableDrawer.svelte @@ -42,7 +42,9 @@ const datatables = resource([], async () => { if (!$workspaceStore) return [] try { - return await WorkspaceService.listDataTables({ workspace: $workspaceStore }) + return (await WorkspaceService.listDataTables({ workspace: $workspaceStore })).map( + (d) => d.name + ) } catch (e) { console.error('Failed to load datatables:', e) return [] diff --git a/frontend/src/lib/components/raw_apps/datatableUtils.svelte.ts b/frontend/src/lib/components/raw_apps/datatableUtils.svelte.ts index e964e9bc5d..bdb93355c6 100644 --- a/frontend/src/lib/components/raw_apps/datatableUtils.svelte.ts +++ b/frontend/src/lib/components/raw_apps/datatableUtils.svelte.ts @@ -13,7 +13,7 @@ export function createDatatablesResource(getWorkspace: () => string | undefined) const workspace = getWorkspace() if (!workspace) return [] try { - return await WorkspaceService.listDataTables({ workspace }) + return (await WorkspaceService.listDataTables({ workspace })).map((d) => d.name) } catch (e) { console.error('Failed to load datatables:', e) return [] diff --git a/frontend/src/lib/components/workspaceSettings/CreateWorkspaceInner.svelte b/frontend/src/lib/components/workspaceSettings/CreateWorkspaceInner.svelte index 2e53a08baa..2c10794646 100644 --- a/frontend/src/lib/components/workspaceSettings/CreateWorkspaceInner.svelte +++ b/frontend/src/lib/components/workspaceSettings/CreateWorkspaceInner.svelte @@ -26,13 +26,16 @@ import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte' import { AI_PROVIDERS } from '$lib/components/copilot/lib' - import { LoaderCircle } from 'lucide-svelte' + import { LoaderCircle, Check, X, Loader2 } from 'lucide-svelte' import PrefixedInput from '../PrefixedInput.svelte' import TextInput from '../text_input/TextInput.svelte' import { jobManager } from '$lib/services/JobManager' import Alert from '../common/alert/Alert.svelte' import { base } from '$lib/base' import Label from '../Label.svelte' + import Select from '../select/Select.svelte' + import { resource } from 'runed' + import ConfirmationModal from '../common/confirmationModal/ConfirmationModal.svelte' interface Props { isFork?: boolean @@ -51,6 +54,30 @@ let codeCompletionEnabled = $state(true) let checking = $state(false) + let allDatatables = resource([], async () => + $workspaceStore + ? WorkspaceService.listDataTables({ workspace: $workspaceStore ?? '' }) + : undefined + ) + let datatableBehaviors: Record = + $state({}) + + type ForkStep = { + label: string + status: 'pending' | 'running' | 'done' | 'error' + error?: string + } + type DatatableCloneJob = { + name: string + resourceType: string + behavior: 'schema_only' | 'schema_and_data' + steps: ForkStep[] + } + let cloneModalOpen = $state(false) + let currentCloneJob: DatatableCloneJob | undefined = $state(undefined) + let cloneQueue: DatatableCloneJob[] = $state([]) + let cloneRunning = $state(false) + let workspaceColor: string | undefined = $state(undefined) let colorEnabled = $state(false) @@ -194,6 +221,62 @@ return } + // Build clone queue for datatables that need cloning + const datatablesToClone = (allDatatables.current ?? []) + .filter((dt) => { + const behavior = datatableBehaviors[dt.name] ?? 'keep_original' + return behavior !== 'keep_original' + }) + .map((dt) => { + const behavior = datatableBehaviors[dt.name] as 'schema_only' | 'schema_and_data' + const isInstance = dt.resource_type === 'instance' + const newDbName = isInstance + ? `${prefixed_id.replace(/-/g, '_')}__${dt.name}` + : `${prefixed_id.replace(/-/g, '_')}__${dt.name}` + + const steps: ForkStep[] = isInstance + ? [ + { + label: `CREATE DATABASE "${newDbName}" + grant permissions`, + status: 'pending' + }, + { + label: `pg_dump → pg_import (${behavior === 'schema_only' ? 'schema only' : 'schema + data'})`, + status: 'pending' + } + ] + : [ + { + label: `CREATE DATABASE "${newDbName}" + pg_dump → pg_import (${behavior === 'schema_only' ? 'schema only' : 'schema + data'})`, + status: 'pending' + } + ] + + return { + name: dt.name, + resourceType: dt.resource_type, + behavior, + steps, + _newDbName: newDbName, + _isInstance: isInstance, + _sourceWorkspace: $workspaceStore!, + _targetWorkspace: prefixed_id + } as DatatableCloneJob & { + _newDbName: string + _isInstance: boolean + _sourceWorkspace: string + _targetWorkspace: string + } + }) + + if (datatablesToClone.length > 0) { + cloneQueue = datatablesToClone + currentCloneJob = cloneQueue[0] + cloneModalOpen = true + // Don't finish fork yet — the modal handles remaining steps + return + } + forkCreationLoading = false sendUserToast(`Successfully forked workspace ${$workspaceStore} as: wm-fork-${id}`) } else { @@ -201,6 +284,103 @@ } } + async function executeCloneJob( + job: DatatableCloneJob & { + _newDbName: string + _isInstance: boolean + _sourceWorkspace: string + _targetWorkspace: string + } + ) { + cloneRunning = true + let stepIdx = 0 + + if (job._isInstance) { + // Step 1: CREATE DATABASE + grant permissions via setupCustomInstanceDb + job.steps[stepIdx].status = 'running' + try { + await SettingService.setupCustomInstanceDb({ + name: job._newDbName, + requestBody: { tag: 'datatable' } + }) + job.steps[stepIdx].status = 'done' + } catch (e: any) { + const msg = e?.body ?? e?.message ?? String(e) + if (msg.includes('already exists')) { + job.steps[stepIdx].status = 'done' + } else { + job.steps[stepIdx].status = 'error' + job.steps[stepIdx].error = msg + cloneRunning = false + return + } + } + stepIdx++ + + // Step 2: fork_pg_database (dump + import) + job.steps[stepIdx].status = 'running' + try { + await WorkspaceService.forkPgDatabase({ + workspace: job._targetWorkspace, + requestBody: { + source: `datatable://${job.name}`, + target: `datatable://${job.name}`, + fork_behavior: job.behavior, + target_override_dbname: job._newDbName + } + }) + job.steps[stepIdx].status = 'done' + } catch (e: any) { + job.steps[stepIdx].status = 'error' + job.steps[stepIdx].error = e?.body ?? e?.message ?? String(e) + cloneRunning = false + return + } + } else { + // Resource DB: CREATE DATABASE + dump/import in one call + job.steps[stepIdx].status = 'running' + try { + await WorkspaceService.forkPgDatabase({ + workspace: job._targetWorkspace, + requestBody: { + source: `datatable://${job.name}`, + target: `datatable://${job.name}`, + fork_behavior: job.behavior, + target_override_dbname: job._newDbName, + create_target_db: true + } + }) + job.steps[stepIdx].status = 'done' + } catch (e: any) { + job.steps[stepIdx].status = 'error' + job.steps[stepIdx].error = e?.body ?? e?.message ?? String(e) + cloneRunning = false + return + } + } + + cloneRunning = false + } + + function advanceCloneQueue() { + const idx = cloneQueue.indexOf(currentCloneJob!) + if (idx < cloneQueue.length - 1) { + currentCloneJob = cloneQueue[idx + 1] + } else { + // All done + cloneModalOpen = false + currentCloneJob = undefined + cloneQueue = [] + forkCreationLoading = false + sendUserToast(`Successfully forked workspace ${$workspaceStore} as: wm-fork-${id}`) + usersWorkspaceStore.set(undefined) + WorkspaceService.listUserWorkspaces().then((ws) => { + usersWorkspaceStore.set(ws) + switchWorkspace(`wm-fork-${id}`) + }) + } + } + async function createWorkspace(): Promise { await WorkspaceService.createWorkspace({ requestBody: { @@ -467,6 +647,35 @@ {/if} + {#if isFork && allDatatables.current && allDatatables.current.length > 0} +