diff --git a/backend/migrations/20251121144144_datatables.down.sql b/backend/migrations/20251121144144_datatables.down.sql new file mode 100644 index 0000000000..9051d5198f --- /dev/null +++ b/backend/migrations/20251121144144_datatables.down.sql @@ -0,0 +1,4 @@ +-- Add down migration script here + +ALTER TABLE workspace_settings +DROP COLUMN datatable; \ No newline at end of file diff --git a/backend/migrations/20251121144144_datatables.up.sql b/backend/migrations/20251121144144_datatables.up.sql new file mode 100644 index 0000000000..6feea09429 --- /dev/null +++ b/backend/migrations/20251121144144_datatables.up.sql @@ -0,0 +1,4 @@ +-- Add up migration script here + +ALTER TABLE workspace_settings +ADD COLUMN datatable JSONB; diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 051f7d985f..876254d23c 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -2077,6 +2077,8 @@ paths: $ref: "#/components/schemas/LargeFileStorage" ducklake: $ref: "#/components/schemas/DucklakeSettings" + datatable: + $ref: "#/components/schemas/DataTableSettings" git_sync: $ref: "#/components/schemas/WorkspaceGitSyncSettings" deploy_ui: @@ -2789,6 +2791,32 @@ paths: application/json: schema: {} + /w/{workspace}/workspaces/edit_datatable_config: + post: + summary: edit datatable settings + operationId: editDataTableConfig + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: DataTable settings + required: true + content: + application/json: + schema: + type: object + required: [settings] + properties: + settings: + $ref: "#/components/schemas/DataTableSettings" + responses: + "200": + description: status + content: + application/json: + schema: {} + /w/{workspace}/workspaces/edit_git_sync_config: post: summary: edit workspace git sync settings @@ -19428,6 +19456,29 @@ components: required: - path + DataTableSettings: + type: object + required: [datatables] + properties: + datatables: + type: object + additionalProperties: + type: object + required: [database] + properties: + database: + type: object + properties: + resource_type: + type: string + enum: + - postgresql + - instance + resource_path: + type: string + required: + - resource_type + DynamicInputData: type: object properties: diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index c16f7ab604..b956a7b295 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -42,7 +42,9 @@ use windmill_common::worker::{to_raw_value, CLOUD_HOSTED}; use windmill_common::workspaces::GitRepositorySettings; #[cfg(feature = "enterprise")] use windmill_common::workspaces::WorkspaceDeploymentUISettings; -use windmill_common::workspaces::WorkspaceGitSyncSettings; +use windmill_common::workspaces::{ + DataTable, DataTableCatalogResourceType, WorkspaceGitSyncSettings, +}; use windmill_common::workspaces::{Ducklake, DucklakeCatalogResourceType}; use windmill_common::{ error::{Error, JsonResult, Result}, @@ -124,6 +126,7 @@ pub fn workspaced_service() -> Router { ) .route("/edit_ducklake_config", post(edit_ducklake_config)) .route("/list_ducklakes", get(list_ducklakes)) + .route("/edit_datatable_config", post(edit_datatable_config)) .route("/edit_git_sync_config", post(edit_git_sync_config)) .route("/edit_git_sync_repository", post(edit_git_sync_repository)) .route( @@ -327,12 +330,6 @@ struct LargeFileStorageWithSecondary { #[serde(default)] secondary_storage: HashMap, } - -#[derive(Deserialize, Serialize, Debug)] -pub struct DucklakeSettings { - pub ducklakes: HashMap, -} - #[derive(Deserialize, Debug)] struct EditLargeFileStorageConfig { large_file_storage: Option, @@ -343,6 +340,21 @@ struct EditDucklakeConfig { settings: DucklakeSettings, } +#[derive(Deserialize, Serialize, Debug)] +pub struct DucklakeSettings { + pub ducklakes: HashMap, +} + +#[derive(Deserialize, Debug)] +struct EditDataTableConfig { + settings: DataTableSettings, +} + +#[derive(Deserialize, Serialize, Debug)] +pub struct DataTableSettings { + pub datatables: HashMap, +} + #[derive(Deserialize)] struct CreateWorkspace { id: String, @@ -1247,6 +1259,101 @@ async fn edit_ducklake_config( Ok(format!("Edit ducklake config for workspace {}", &w_id)) } +async fn edit_datatable_config( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + ApiAuthed { is_admin, username, email, .. }: ApiAuthed, + Json(new_config): Json, +) -> Result { + require_admin(is_admin, &username)?; + let is_superadmin = require_super_admin(&db, &email).await.is_ok(); + + let mut tx = db.begin().await?; + + let args_for_audit = format!("{:?}", new_config.settings); + audit_log( + &mut *tx, + &authed, + "workspaces.edit_datatable_config", + ActionKind::Update, + &w_id, + Some(&authed.email), + Some([("datatable", args_for_audit.as_str())].into()), + ) + .await?; + + // Check that all database resources exist to prevent + // exploiting the shared property to see any resource + for dl in new_config.settings.datatables.values() { + if dl.database.resource_type == DataTableCatalogResourceType::Instance { + continue; + } + let database_res = sqlx::query_scalar!( + "SELECT 1 FROM resource WHERE workspace_id = $1 AND path = $2", + &w_id, + &dl.database.resource_path + ) + .fetch_optional(&mut *tx) + .await? + .flatten(); + + if database_res.is_none() { + return Err(Error::BadRequest(format!( + "Data table database resource {} not found in workspace {}", + dl.database.resource_path, &w_id + ))); + } + } + + // Check that non-superadmins are not abusing Instance catalogs + if !is_superadmin { + let old_datatables = sqlx::query_scalar!( + r#" + SELECT ws.datatable->'datatables' AS datatable_name + FROM workspace_settings ws + WHERE ws.workspace_id = $1 + "#, + &w_id + ) + .fetch_one(&db) + .await? + .unwrap_or(serde_json::Value::Null); + let old_datatables: HashMap = + serde_json::from_value(old_datatables).unwrap_or_default(); + for (name, dt) in new_config.settings.datatables.iter() { + if dt.database.resource_type == DataTableCatalogResourceType::Instance { + let old_dt = old_datatables.get(name); + if old_dt.is_none() + || old_dt.unwrap().database.resource_type + != DataTableCatalogResourceType::Instance + || old_dt.unwrap().database.resource_path != dt.database.resource_path + { + return Err(Error::BadRequest( + "Only superadmins can create or modify data tables with Instance catalogs" + .to_string(), + )); + } + } + } + } + + let config: serde_json::Value = serde_json::to_value(new_config.settings) + .map_err(|err| Error::internal_err(err.to_string()))?; + + sqlx::query!( + "UPDATE workspace_settings SET datatable = $1 WHERE workspace_id = $2", + config, + &w_id + ) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + Ok(format!("Edit datatable config for workspace {}", &w_id)) +} + #[derive(Deserialize)] pub struct EditGitSyncConfig { #[cfg(feature = "enterprise")] diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 20a5d42c71..75d731c3e0 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -149,6 +149,27 @@ pub async fn get_team_plan_status(_db: &crate::DB, _w_id: &str) -> Result - export type DataTablesSettingsStruct = { + export type DataTableSettingsType = { dataTables: { name: string database: { - resource_type: 'postgresql' | 'mysql' | 'instance' - resource_path: string | undefined + resource_type: 'postgresql' | 'instance' + resource_path?: string | undefined } }[] } + export function convertDataTableSettingsFromBackend( + settings: GetSettingsResponse['datatable'] + ): DataTableSettingsType { + const s: DataTableSettingsType = { dataTables: [] } + if (settings?.datatables) { + for (const [name, rest] of Object.entries(settings.datatables)) { + s.dataTables.push({ name, ...rest }) + } + } + return s + } + export function convertDataTableSettingsToBackend( + settings: DataTableSettingsType + ): NonNullable { + const s: GetSettingsResponse['datatable'] = { datatables: {} } + for (const dataTable of settings.dataTables) { + const database = dataTable.database + if (dataTable.name in s.datatables) + throw 'Settings contain duplicate dataTable name: ' + dataTable.name + if (!database.resource_path) throw 'No resource selected for ' + dataTable.name + if (database.resource_type === 'instance' && database.resource_path === 'windmill') + throw dataTable.name + ' database cannot be called "windmill"' + + s.datatables[dataTable.name] = { + database: dataTable.database + } + } + return s + } + let DEFAULT_DATATABLE_DB_NAME = 'datatable_db' @@ -31,6 +61,17 @@ import { isCustomInstanceDbEnabled } from './utils.svelte' import { random_adj } from '../random_positive_adjetive' import { sendUserToast } from '$lib/toast' + import { SettingService, WorkspaceService, type GetSettingsResponse } from '$lib/gen' + import { workspaceStore } from '$lib/stores' + import { createAsyncConfirmationModal } from '../common/confirmationModal/asyncConfirmationModal.svelte' + import ConfirmationModal from '../common/confirmationModal/ConfirmationModal.svelte' + import { resource } from 'runed' + + type Props = { + dataTableSettings: DataTableSettingsType + } + + let { dataTableSettings = $bindable() }: Props = $props() let tableHeadNames = ['Name', 'Database', '', ''] as const let tableHeadTooltips: Partial> = { @@ -38,9 +79,7 @@ Database: 'The database where the data is stored.' } - let tempSettings: DataTablesSettingsStruct = $state({ - dataTables: [] - }) + let tempSettings: DataTableSettingsType = $derived(dataTableSettings) function removeDataTable(index: number) { tempSettings.dataTables.splice(index, 1) @@ -58,15 +97,38 @@ } }) } + const instanceCatalogStatuses = resource([], SettingService.getCustomInstanceDbStatus) async function onSave() { try { + if ( + $isCustomInstanceDbEnabled && + dataTableSettings.dataTables.some( + (d) => + d.database.resource_type === 'instance' && + !instanceCatalogStatuses.current?.[d.database.resource_path ?? '']?.success + ) + ) { + let confirm = await confirmationModal.ask({ + title: 'Some instance catalogs are not setup', + children: 'Are you sure you want to save without setting them up ?', + confirmationText: 'Save anyway' + }) + if (!confirm) return + } + const settings = convertDataTableSettingsToBackend(tempSettings) + await WorkspaceService.editDataTableConfig({ + workspace: $workspaceStore!, + requestBody: { settings } + }) + dataTableSettings = tempSettings sendUserToast('Data table settings saved successfully') } catch (e) { sendUserToast(e, true) console.error('Error saving data table settings', e) } } + let confirmationModal = createAsyncConfirmationModal()
@@ -74,7 +136,7 @@
Data tables
Store relational data out of the box. Interact with a fully managed PostgreSQL database - directly from the windmill SDK. + directly from the Windmill SDK.
@@ -179,3 +241,5 @@ + + diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index 52e56d6db1..652313c5a1 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -59,7 +59,10 @@ import UnsavedConfirmationModal from '$lib/components/common/confirmationModal/UnsavedConfirmationModal.svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' import CollapseLink from '$lib/components/CollapseLink.svelte' - import DataTableSettings from '$lib/components/workspaceSettings/DataTableSettings.svelte' + import DataTableSettings, { + convertDataTableSettingsFromBackend, + type DataTableSettingsType + } from '$lib/components/workspaceSettings/DataTableSettings.svelte' let slackInitialPath: string = $state('') let slackScriptPath: string = $state('') @@ -111,9 +114,9 @@ secondaryStorage: undefined }) - let ducklakeSettings: DucklakeSettingsType = $state({ - ducklakes: [] - }) + let dataTableSettings: DataTableSettingsType = $state({ dataTables: [] }) + + let ducklakeSettings: DucklakeSettingsType = $state({ ducklakes: [] }) let ducklakeSavedSettings: DucklakeSettingsType = $state(untrack(() => ducklakeSettings)) let workspaceDefaultAppPath: string | undefined = $state(undefined) @@ -307,6 +310,7 @@ !!$enterpriseLicense ) initialS3ResourceSettings = clone(s3ResourceSettings) + dataTableSettings = convertDataTableSettingsFromBackend(settings.datatable) ducklakeSettings = convertDucklakeSettingsFromBackend(settings.ducklake) ducklakeSavedSettings = clone(ducklakeSettings) @@ -1114,7 +1118,7 @@ }} /> {:else if tab == 'windmill_data_tables'} - + {:else if tab == 'windmill_lfs'}