From 9a5ec223bc7dd1b5c4e58a9cc883a5d4e5d9237b Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 14 Oct 2026 14:35:08 +0200 Subject: [PATCH] feat: workspace settings as one document, applied server-side (#11740) * feat: add the workspace settings document, applied server-side * feat: add wmill workspace get-settings and apply-settings * fix: warn on unknown settings keys and refuse ee-only ones in the dry run Co-Authored-By: Claude Opus 5.5 * test: stub the settings document endpoint in the push settings unit tests Co-Authored-By: Claude Opus 5.5 * fix: refuse mistyped settings and keep 4xx status on a partial settings apply Reject non-boolean switches and non-string Slack credentials instead of coercing them into destructive writes, preserve the failing endpoint's status when wrapping a partial-apply error, keep the stored Slack secret when an older CLI pushes one encrypted under another workspace's key, print the plan's warnings in apply-settings, and state the admin check load_settings_document leaves to its caller. Co-Authored-By: Claude Opus 5.5 * fix: compare handler arguments verbatim and type-check scalar settings A handler's extra_args is stored as sent and its nulls reach the job as arguments, so an added or removed null argument is now a change. Scalar settings are type-checked in the plan so a dry run never reports a change the endpoint then refuses, and an error handler's omitted mute flags compare as the false the endpoint stores. Co-Authored-By: Claude Opus 5.5 * fix: require a 32-bit integer rate limit in the settings plan Co-Authored-By: Claude Opus 5.5 * feat: data table permissions and protection rules in the settings document The settings document gains datatable_permissions, each data table's roles named by their Postgres role name and written by catalog id, and protection_rules, keyed by rule name. Both are managed only when stated and stated as a whole. The dev workspace lock stays with the dev workspace setting. wmill sync carries datatable_permissions only with includeDatatablePermissions (or --include-datatable-permissions), since tenants and roles are specific to a workspace and an instance. protection_rules stay out of settings.yaml: protection-rules.yaml syncs them. Co-Authored-By: Claude Opus 5.5 * fix: validate the default data table role and plan permissions against a replaced datatable Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- ...c687681b5d36ff1918ed9e495c5596f2ba9f2.json | 22 + .../windmill-api-workspaces/src/workspaces.rs | 188 ++ backend/windmill-api/openapi.yaml | 92 + backend/windmill-api/src/lib.rs | 1 + .../src/workspace_settings_document.rs | 1784 +++++++++++++++++ backend/windmill-api/src/workspaces.rs | 11 +- backend/windmill-api/src/workspaces_export.rs | 137 +- backend/windmill-common/src/error.rs | 16 +- backend/windmill-common/src/variables.rs | 4 + .../windmill-common/src/workspace_crypt.rs | 40 +- backend/windmill-common/src/workspaces.rs | 16 +- cli/src/commands/init/template.ts | 3 + cli/src/commands/sync/pull.ts | 4 +- cli/src/commands/sync/sync.ts | 21 + cli/src/commands/workspace/settings.ts | 75 + cli/src/commands/workspace/workspace.ts | 19 +- cli/src/core/conf.ts | 5 + cli/src/core/settings.ts | 96 +- cli/src/guidance/skills.gen.ts | 9 + cli/src/types.ts | 7 +- ...orkspace_settings_auto_invite_unit.test.ts | 4 + ...h_workspace_settings_identity_unit.test.ts | 4 + .../auto-generated/cli/cli-commands.md | 9 + system_prompts/auto-generated/prompts.ts | 9 + .../skills/cli-commands/SKILL.md | 9 + 25 files changed, 2449 insertions(+), 136 deletions(-) create mode 100644 backend/.sqlx/query-0d04ec377bef28416a5721a43d2c687681b5d36ff1918ed9e495c5596f2ba9f2.json create mode 100644 backend/windmill-api/src/workspace_settings_document.rs create mode 100644 cli/src/commands/workspace/settings.ts diff --git a/backend/.sqlx/query-0d04ec377bef28416a5721a43d2c687681b5d36ff1918ed9e495c5596f2ba9f2.json b/backend/.sqlx/query-0d04ec377bef28416a5721a43d2c687681b5d36ff1918ed9e495c5596f2ba9f2.json new file mode 100644 index 0000000000..3f4a5c7992 --- /dev/null +++ b/backend/.sqlx/query-0d04ec377bef28416a5721a43d2c687681b5d36ff1918ed9e495c5596f2ba9f2.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT slack_oauth_client_secret FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "slack_oauth_client_secret", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "0d04ec377bef28416a5721a43d2c687681b5d36ff1918ed9e495c5596f2ba9f2" +} diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index c2d6d79038..19327440b0 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -1765,6 +1765,43 @@ async fn set_slack_oauth_config( "Both client ID and client secret are required".to_string(), )); } + // A settings.yaml pulled by `wmill sync` holds the secret encrypted, and a CLI that + // predates the settings document pushes it here as it reads it. One encrypted under another + // workspace's key keeps the stored secret, as the settings document does, so that CLI's push + // does not abort over it. + let mut req = req; + if let Some(ciphertext) = req + .slack_oauth_client_secret + .strip_prefix(windmill_common::variables::SETTINGS_ENCRYPTED_PREFIX) + { + match windmill_common::variables::build_crypt(&db, &w_id) + .await? + .decrypt_str(ciphertext) + { + Ok(plain) => req.slack_oauth_client_secret = plain, + Err(_) => { + let stored = sqlx::query_scalar!( + "SELECT slack_oauth_client_secret FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_optional(&db) + .await? + .flatten(); + tracing::warn!( + workspace_id = %w_id, + "Slack client secret left unchanged: it is encrypted with another workspace's key" + ); + match stored { + Some(stored) => req.slack_oauth_client_secret = stored, + None => { + return Ok("Slack OAuth config left unchanged: the client secret is \ + encrypted with another workspace's key" + .to_string()) + } + } + } + } + } let mut tx = db.begin().await?; @@ -14852,3 +14889,154 @@ async fn set_ws_specific( body.value, body.item_kind, body.path )) } + +/// A setting the workspace settings document writes, each through the endpoint that owns it, +/// so applying a document validates, audits and has the side effects of the matching UI save. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SettingsEdit { + Webhook, + AutoInvite, + InstanceGroups, + ErrorHandler, + SuccessHandler, + LargeFileStorage, + GitSync, + DefaultScripts, + DefaultApp, + Color, + OperatorSettings, + Datatable, + SlackCommand, + SetSlackOAuth, + DeleteSlackOAuth, + Ducklake, + DbtWarehouses, + DeployUi, + GuestAccess, + GuestJwtKey, + PublicAppRateLimit, + AddAdminsAndDevelopersToForks, + /// Body: `{ datatable, permissioned, default_role, roles: [{ id, tenants }] }`, role ids. + DatatablePermissions, + CreateProtectionRule, + /// Body: the update request plus `rule_name`, the rule it updates. + UpdateProtectionRule, + /// Body: `{ name }`. + DeleteProtectionRule, +} + +fn settings_edit_body( + edit: SettingsEdit, + body: serde_json::Value, +) -> Result> { + serde_json::from_value(body) + .map(Json) + .map_err(|e| Error::BadRequest(format!("Invalid value for {edit:?}: {e}"))) +} + +/// Returns the endpoint's JSON response, or `null` for one that answers with prose. +pub async fn apply_settings_edit( + authed: ApiAuthed, + db: DB, + w_id: String, + edit: SettingsEdit, + body: serde_json::Value, +) -> Result { + use SettingsEdit::*; + let a = authed.clone(); + let (d, p) = (Extension(db), Path(w_id)); + match edit { + Webhook => edit_webhook(a, d, p, authed, settings_edit_body(edit, body)?).await?, + AutoInvite => edit_auto_invite(a, d, p, settings_edit_body(edit, body)?).await?, + InstanceGroups => edit_instance_groups(a, d, p, settings_edit_body(edit, body)?).await?, + ErrorHandler => { + edit_error_handler(a, d, p, authed, settings_edit_body(edit, body)?).await? + } + SuccessHandler => edit_success_handler(a, d, p, settings_edit_body(edit, body)?).await?, + LargeFileStorage => { + edit_large_file_storage_config(a, d, p, authed, settings_edit_body(edit, body)?).await? + } + GitSync => edit_git_sync_config(a, d, p, authed, settings_edit_body(edit, body)?).await?, + DefaultScripts => { + edit_default_scripts(a, d, p, authed, settings_edit_body(edit, body)?).await? + } + #[cfg(feature = "enterprise")] + DefaultApp => edit_default_app(a, d, p, authed, settings_edit_body(edit, body)?).await?, + #[cfg(not(feature = "enterprise"))] + DefaultApp => edit_default_app(a, d, p, settings_edit_body(edit, body)?).await?, + Color => change_workspace_color(a, p, d, settings_edit_body(edit, body)?).await?, + OperatorSettings => { + update_operator_settings(a, p, d, settings_edit_body(edit, body)?).await? + } + Datatable => { + let Json(result) = + edit_datatable_config(a, d, p, authed, settings_edit_body(edit, body)?).await?; + return serde_json::to_value(result).map_err(|e| Error::internal_err(e.to_string())); + } + SlackCommand => { + edit_slack_command(a, d, p, authed, settings_edit_body(edit, body)?).await? + } + SetSlackOAuth => set_slack_oauth_config(a, d, p, settings_edit_body(edit, body)?).await?, + DeleteSlackOAuth => delete_slack_oauth_config(a, d, p).await?, + Ducklake => edit_ducklake_config(a, d, p, authed, settings_edit_body(edit, body)?).await?, + DbtWarehouses => { + edit_dbt_warehouses(a, d, p, authed, settings_edit_body(edit, body)?).await? + } + #[cfg(feature = "enterprise")] + DeployUi => edit_deploy_ui_config(a, d, p, authed, settings_edit_body(edit, body)?).await?, + #[cfg(not(feature = "enterprise"))] + DeployUi => edit_deploy_ui_config(a, d, p).await?, + GuestAccess => edit_guest_access(a, d, p, settings_edit_body(edit, body)?).await?, + GuestJwtKey => edit_guest_jwt_key(a, d, p, settings_edit_body(edit, body)?).await?, + PublicAppRateLimit => { + edit_public_app_rate_limit(d, p, a, settings_edit_body(edit, body)?).await? + } + AddAdminsAndDevelopersToForks => { + edit_add_admins_and_developers_to_forks(a, d, p, settings_edit_body(edit, body)?) + .await? + } + #[cfg(all(feature = "private", feature = "enterprise"))] + DatatablePermissions => { + #[derive(Deserialize)] + struct Body { + datatable: String, + #[serde(flatten)] + req: crate::datatable_permissions_ee::SetDatatablePermissions, + } + let Json(Body { datatable, req }) = settings_edit_body(edit, body)?; + crate::datatable_permissions_oss::set_datatable_permissions( + a, + d, + Path((p.0, datatable)), + Json(req), + ) + .await? + } + #[cfg(not(all(feature = "private", feature = "enterprise")))] + DatatablePermissions => { + crate::datatable_permissions_oss::set_datatable_permissions(a).await? + } + CreateProtectionRule => { + create_protection_rule(a, d, p, settings_edit_body(edit, body)?).await? + } + UpdateProtectionRule => { + #[derive(Deserialize)] + struct Body { + rule_name: String, + #[serde(flatten)] + req: UpdateProtectionRuleRequest, + } + let Json(Body { rule_name, req }) = settings_edit_body(edit, body)?; + update_protection_rule(a, d, Path((p.0, rule_name)), Json(req)).await? + } + DeleteProtectionRule => { + #[derive(Deserialize)] + struct Body { + name: String, + } + let Json(Body { name }) = settings_edit_body(edit, body)?; + delete_protection_rule(a, d, Path((p.0, name))).await? + } + }; + Ok(serde_json::Value::Null) +} diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index fabded991b..e41bfc5013 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -5184,6 +5184,98 @@ paths: - has_instance_ai_config - uses_instance_ai_config + /w/{workspace}/workspaces/settings_document: + get: + summary: get the workspace settings document + description: >- + The workspace's settings in the layout of the settings.yaml that `wmill sync` writes. + Admin only. + operationId: getWorkspaceSettingsDocument + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: secrets + description: >- + How secrets are written: `redacted` (the default) replaces each set secret with + `__SENSITIVE_AND_UNCHANGED__`; `encrypted` writes it encrypted with the workspace key, + prefixed with `$encrypted:`. + in: query + schema: + type: string + enum: [redacted, encrypted] + responses: + "200": + description: the settings document + content: + application/json: + schema: + type: object + additionalProperties: true + post: + summary: apply a workspace settings document + description: >- + Applies a settings document, each setting through the endpoint that owns it. Admin only. + operationId: applyWorkspaceSettingsDocument + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + document: + description: >- + The settings to apply, in the layout of the document. A secret given as + `__SENSITIVE_AND_UNCHANGED__` keeps its stored value; one prefixed with + `$encrypted:` is decrypted with the workspace key. + type: object + additionalProperties: true + replace: + description: >- + Clear the settings `wmill sync push` clears when settings.yaml omits them. + Otherwise a setting the document omits is left as it is. + type: boolean + dry_run: + description: Only compute the changes, without applying them. + type: boolean + required: + - document + responses: + "200": + description: the changes, applied unless dry_run + content: + application/json: + schema: + type: object + properties: + changes: + type: array + items: + type: object + properties: + key: + type: string + before: {} + after: {} + required: + - key + - before + - after + applied: + type: boolean + warnings: + type: array + items: + type: string + required: + - changes + - applied + /w/{workspace}/workspaces/get_copilot_settings_state: get: summary: get copilot settings state diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 92291bd1ea..9163256997 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -209,6 +209,7 @@ mod workspaces; #[cfg(feature = "private")] pub mod workspaces_ee; mod workspaces_export; +mod workspace_settings_document; #[cfg(feature = "mcp")] mod mcp_tools; diff --git a/backend/windmill-api/src/workspace_settings_document.rs b/backend/windmill-api/src/workspace_settings_document.rs new file mode 100644 index 0000000000..eb4b0dd7f8 --- /dev/null +++ b/backend/windmill-api/src/workspace_settings_document.rs @@ -0,0 +1,1784 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2026 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! The workspace settings document: the workspace's settings in the layout of the `settings.yaml` +//! that `wmill sync` writes, read and applied as one. `wmill sync push`, the settings page's YAML +//! mode and the AI tools all apply through [`apply_settings_document`], so the file layout and +//! the rules for applying it exist once. + +use axum::{ + extract::{Path, Query}, + Extension, Json, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Map, Value}; +use sqlx::PgConnection; +use std::collections::BTreeMap; +use windmill_api_workspaces::workspaces::{apply_settings_edit, SettingsEdit}; +use windmill_common::{ + datatable_roles::{ + read_role_catalog, DatatableRoleCluster, ADMIN_DATATABLE_ROLE, NO_DEFAULT_DATATABLE_ROLE, + }, + error::{Error, JsonResult, Result}, + utils::require_admin, + variables::build_crypt, + workspaces::{ + DataTable, DataTablePermissions, ProtectionRuleKind, ProtectionRules, + DATATABLE_TENANT_WILDCARD, DEV_WORKSPACE_LOCK_RULE_NAME, + }, + DB, +}; + +use crate::db::ApiAuthed; + +/// Stands for a secret a reader may not see. Sent back unchanged, it keeps the stored value. +pub const SENSITIVE_UNCHANGED: &str = "__SENSITIVE_AND_UNCHANGED__"; +use windmill_common::variables::SETTINGS_ENCRYPTED_PREFIX as ENCRYPTED_PREFIX; + +const SECRET_KEYS: &[&str] = &["slack_oauth_client_secret"]; + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, Default)] +#[serde(rename_all = "snake_case")] +pub enum SecretMode { + /// Secrets as [`SENSITIVE_UNCHANGED`], for the UI and AI tools. + #[default] + Redacted, + /// Secrets encrypted with the workspace key, deterministically so that an unchanged secret + /// exports to the same text and a synced file does not change on every pull. + Encrypted, + Plain, +} + +#[derive(sqlx::FromRow)] +struct SettingsRow { + auto_invite: Option, + webhook: Option, + error_handler: Option, + success_handler: Option, + ai_config: Option, + large_file_storage: Option, + git_sync: Option, + default_app: Option, + default_scripts: Option, + name: Option, + mute_critical_alerts: Option, + color: Option, + operator_settings: Option, + datatable: Option, + slack_team_id: Option, + slack_name: Option, + slack_command_script: Option, + slack_oauth_client_id: Option, + slack_oauth_client_secret: Option, + ducklake: Option, + dbt_warehouses: Option, + deploy_ui: Option, + teams_command_script: Option, + public_app_execution_limit_per_minute: Option, + guest_access_enabled: Option, + guest_jwt_public_key: Option, + guest_jwt_jwks_url: Option, + error_handler_fallback_to_instance_alerts: Option, + add_admins_and_developers_to_forks: Option, +} + +/// Server-owned git sync state (the webhook secret and hook id, the synced sha, the admin +/// automatic pulls run as, what the credential check observed) never leaves the server: a +/// re-imported workspace must not inherit another install's hook, sync state or pull identity, +/// and in a synced file it would churn for no reason. +pub(crate) fn redact_git_sync(git_sync: Option) -> Option { + let mut git_sync = git_sync?; + if let Some(repos) = git_sync + .get_mut("repositories") + .and_then(|r| r.as_array_mut()) + { + for repo in repos { + if let Some(auto_pull) = repo.get_mut("auto_pull").and_then(|a| a.as_object_mut()) { + for field in [ + "webhook_secret", + "webhook_id", + "webhook_error", + "last_synced_sha", + "last_pull_status", + "enabled_by", + ] { + auto_pull.remove(field); + } + } + if let Some(repo) = repo.as_object_mut() { + repo.remove("credential"); + } + } + } + Some(git_sync) +} + +/// Role names by catalog id, per cluster: the document names a data table role, the stored +/// permissions its catalog id, which differs from one instance to the next. `None` for a catalog +/// that could not be read, as on an edition without data table roles. +#[derive(Default)] +struct RoleNames { + instance: Option>, + external_instance: Option>, +} + +impl RoleNames { + async fn load(db: &DB) -> Self { + let read = |cluster| async move { + read_role_catalog(db, cluster) + .await + .ok() + .map(|c| c.into_iter().map(|(id, r)| (id, r.name)).collect()) + }; + Self { + instance: read(DatatableRoleCluster::Instance).await, + external_instance: read(DatatableRoleCluster::ExternalInstance).await, + } + } + + fn of(&self, cluster: DatatableRoleCluster) -> Option<&BTreeMap> { + match cluster { + DatatableRoleCluster::Instance => self.instance.as_ref(), + DatatableRoleCluster::ExternalInstance => self.external_instance.as_ref(), + } + } +} + +/// The cluster whose roles a data table entry can be put under: only an entry that owns a +/// database on a cluster Windmill manages, and is not a clone, which its origin governs. +fn role_cluster_of(entry: &Value) -> Option { + let entry: DataTable = serde_json::from_value(entry.clone()).ok()?; + if entry.governed_by.is_some() { + return None; + } + DatatableRoleCluster::of(entry.database?.resource_type) +} + +/// Each data table's stored `permissions`, with roles named rather than given by catalog id. A +/// role the catalog no longer has keeps its id, so the document still shows it. +fn datatable_permissions_document(datatable: Option<&Value>, roles: &RoleNames) -> Value { + let mut doc = Map::new(); + let entries = datatable + .and_then(|d| d.get("datatables")) + .and_then(Value::as_object); + for (name, entry) in entries.into_iter().flatten() { + let Some(permissions) = entry + .get("permissions") + .filter(|p| !p.is_null()) + .and_then(|p| serde_json::from_value::(p.clone()).ok()) + else { + continue; + }; + let names = role_cluster_of(entry).and_then(|c| roles.of(c)); + let name_of = |id: &str| { + names + .and_then(|n| n.get(id)) + .cloned() + .unwrap_or_else(|| id.to_string()) + }; + let default_role = match permissions.default_role() { + NO_DEFAULT_DATATABLE_ROLE => Value::Null, + id => Value::String(name_of(id)), + }; + let roles: Map = permissions + .roles + .iter() + .map(|(id, t)| (name_of(id), json!(t.tenants))) + .collect(); + doc.insert( + name.clone(), + json!({ "default_role": default_role, "roles": roles }), + ); + } + Value::Object(doc) +} + +/// Protection rules by name, their rules in a fixed order so a reordered file is no change. The +/// dev workspace lock is left out: the dev workspace setting creates and removes it. +async fn protection_rules_document(conn: &mut PgConnection, w_id: &str) -> Result { + let rows = sqlx::query_as::<_, (String, i32, Vec, Vec)>( + "SELECT name, rules, bypass_groups, bypass_users FROM workspace_protection_rule + WHERE workspace_id = $1 AND name <> $2 ORDER BY name", + ) + .bind(w_id) + .bind(DEV_WORKSPACE_LOCK_RULE_NAME) + .fetch_all(&mut *conn) + .await?; + Ok(Value::Object( + rows.into_iter() + .map(|(name, bits, bypass_groups, bypass_users)| { + let rules = ProtectionRules::from_bits_truncate(bits); + let rule = json!({ + "rules": rule_kinds(rules), + "bypass_groups": bypass_groups, + "bypass_users": bypass_users, + }); + (name, rule) + }) + .collect(), + )) +} + +fn rule_kinds(rules: ProtectionRules) -> Value { + json!(rules.kinds()) +} + +/// Reads no permission itself: the caller must have checked that the requester is an admin of +/// `w_id`, since `SecretMode::Plain` returns the workspace's secrets in plain text. +pub async fn load_settings_document( + conn: &mut PgConnection, + db: &DB, + w_id: &str, + secrets: SecretMode, +) -> Result> { + let row = sqlx::query_as::<_, SettingsRow>( + r#"SELECT + auto_invite, webhook, error_handler, success_handler, ai_config, large_file_storage, + git_sync, default_app, default_scripts, workspace.name AS name, mute_critical_alerts, + color, operator_settings, datatable, slack_team_id, slack_name, slack_command_script, + slack_oauth_client_id, slack_oauth_client_secret, ducklake, dbt_warehouses, deploy_ui, + teams_command_script, public_app_execution_limit_per_minute, guest_access_enabled, + guest_jwt_public_key, guest_jwt_jwks_url, error_handler_fallback_to_instance_alerts, + add_admins_and_developers_to_forks + FROM workspace_settings + LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id + WHERE workspace_id = $1"#, + ) + .bind(w_id) + .fetch_one(&mut *conn) + .await?; + + let slack_oauth_client_secret = match (row.slack_oauth_client_secret, secrets) { + (Some(s), SecretMode::Redacted) if !s.is_empty() => Some(SENSITIVE_UNCHANGED.to_string()), + (Some(s), SecretMode::Encrypted) if !s.is_empty() => { + let mc = build_crypt(db, w_id).await?; + Some(format!( + "{ENCRYPTED_PREFIX}{}", + mc.encrypt_str_deterministic(&s) + )) + } + (s, _) => s, + }; + + let has_permissions = row + .datatable + .as_ref() + .and_then(|d| d.get("datatables")) + .and_then(Value::as_object) + .is_some_and(|e| { + e.values() + .any(|e| e.get("permissions").is_some_and(|p| !p.is_null())) + }); + let role_names = if has_permissions { + RoleNames::load(db).await + } else { + RoleNames::default() + }; + let datatable_permissions = datatable_permissions_document(row.datatable.as_ref(), &role_names); + let protection_rules = protection_rules_document(&mut *conn, w_id).await?; + + // In the order `wmill sync pull` has always written settings.yaml. A key that predates the + // document and is not marked `always` is written only when set, as it always was; a key + // added since is always written, so a pulled file states it and from then on manages it. + let entries: [(&str, Option, bool); 31] = [ + ("auto_invite", row.auto_invite, false), + ("webhook", row.webhook.map(Value::String), false), + ("error_handler", row.error_handler, true), + ("success_handler", row.success_handler, true), + ("ai_config", row.ai_config, false), + ("large_file_storage", row.large_file_storage, false), + ("git_sync", redact_git_sync(row.git_sync), false), + ("default_app", row.default_app.map(Value::String), false), + ("default_scripts", row.default_scripts, false), + ("name", Some(json!(row.name.unwrap_or_default())), true), + ( + "mute_critical_alerts", + row.mute_critical_alerts.map(Value::Bool), + false, + ), + ("color", row.color.map(Value::String), false), + ("operator_settings", row.operator_settings, false), + ( + "datatable", + windmill_common::workspaces::strip_datatable_permissions(row.datatable), + false, + ), + ("slack_team_id", row.slack_team_id.map(Value::String), false), + ("slack_name", row.slack_name.map(Value::String), false), + ( + "slack_command_script", + row.slack_command_script.map(Value::String), + false, + ), + ( + "slack_oauth_client_id", + row.slack_oauth_client_id.map(Value::String), + true, + ), + ( + "slack_oauth_client_secret", + slack_oauth_client_secret.map(Value::String), + true, + ), + ("ducklake", row.ducklake, true), + ("dbt_warehouses", row.dbt_warehouses, true), + ("deploy_ui", row.deploy_ui, true), + ( + "teams_command_script", + row.teams_command_script.map(Value::String), + true, + ), + ( + "public_app_execution_limit_per_minute", + row.public_app_execution_limit_per_minute.map(|n| json!(n)), + true, + ), + ( + "guest_access_enabled", + Some(json!(row.guest_access_enabled.unwrap_or(false))), + true, + ), + ( + "guest_jwt_public_key", + row.guest_jwt_public_key.map(Value::String), + true, + ), + ( + "guest_jwt_jwks_url", + row.guest_jwt_jwks_url.map(Value::String), + true, + ), + ( + "error_handler_fallback_to_instance_alerts", + Some(json!(row + .error_handler_fallback_to_instance_alerts + .unwrap_or(false))), + true, + ), + ( + "add_admins_and_developers_to_forks", + Some(json!(row + .add_admins_and_developers_to_forks + .unwrap_or(false))), + true, + ), + ("datatable_permissions", Some(datatable_permissions), true), + ("protection_rules", Some(protection_rules), true), + ]; + let mut doc = Map::new(); + for (key, value, always) in entries { + if always || value.is_some() { + doc.insert(key.to_string(), value.unwrap_or(Value::Null)); + } + } + Ok(doc) +} + +#[derive(Deserialize)] +pub struct GetSettingsDocumentQuery { + #[serde(default)] + secrets: SecretMode, +} + +pub async fn get_settings_document( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Query(q): Query, +) -> JsonResult> { + require_admin(authed.is_admin, &authed.username)?; + if q.secrets == SecretMode::Plain { + return Err(Error::BadRequest( + "Plain secrets are only exported by the workspace archive".to_string(), + )); + } + let mut conn = db.acquire().await?; + Ok(Json( + load_settings_document(&mut conn, &db, &w_id, q.secrets).await?, + )) +} + +#[derive(Deserialize)] +pub struct ApplySettingsDocument { + document: Map, + /// Clear the settings the document omits that `wmill sync push` has always cleared when + /// settings.yaml omits them. Otherwise a setting the document omits is left as it is. + #[serde(default)] + replace: bool, + #[serde(default)] + dry_run: bool, +} + +#[derive(Serialize, Debug)] +pub struct SettingsChange { + key: String, + before: Value, + after: Value, +} + +#[derive(Serialize, Debug, Default)] +pub struct ApplySettingsDocumentResult { + changes: Vec, + applied: bool, + #[serde(skip_serializing_if = "Vec::is_empty")] + warnings: Vec, +} + +/// Where one planned write goes: a settings endpoint of the workspaces crate, or one of the few +/// that live in this crate. +#[derive(Debug)] +enum Target { + Edit(SettingsEdit), + AiConfig, + MuteCriticalAlerts, + TeamsCommand, +} + +#[derive(Debug)] +struct Write { + target: Target, + body: Value, + /// The document keys this write applies, for error messages. + keys: Vec, +} + +fn is_null(v: Option<&Value>) -> bool { + v.is_none_or(Value::is_null) +} + +/// The endpoints drop a null member rather than store it, so a document stating one would +/// otherwise differ from the stored value on every apply. A handler's `extra_args` is stored +/// verbatim and its nulls are passed to the job as arguments, so it is compared as is. +fn without_null_members(v: &Value) -> Value { + match v { + Value::Object(m) => Value::Object( + m.iter() + .filter(|(_, v)| !v.is_null()) + .map(|(k, v)| { + let v = if k == "extra_args" { + v.clone() + } else { + without_null_members(v) + }; + (k.clone(), v) + }) + .collect(), + ), + Value::Array(a) => Value::Array(a.iter().map(without_null_members).collect()), + v => v.clone(), + } +} + +fn same(a: Option<&Value>, b: Option<&Value>) -> bool { + match (a, b) { + (a, b) if is_null(a) && is_null(b) => true, + (Some(a), Some(b)) => without_null_members(a) == without_null_members(b), + _ => false, + } +} + +/// Switches the endpoint stores as `false` when cleared, compared as `false` when unset. +const FALSE_WHEN_UNSET: &[&str] = &[ + "mute_critical_alerts", + "guest_access_enabled", + "add_admins_and_developers_to_forks", + "error_handler_fallback_to_instance_alerts", +]; + +fn null_as_false(v: Option) -> Option { + v.map(|v| if v.is_null() { Value::Bool(false) } else { v }) +} + +fn redact(key: &str, v: Value) -> Value { + if SECRET_KEYS.contains(&key) && !v.is_null() { + json!(SENSITIVE_UNCHANGED) + } else { + v + } +} + +/// The auto-invite object without the instance group keys, which a separate endpoint owns. +fn domain_invite(v: Option<&Value>) -> Value { + let mut m = v.and_then(Value::as_object).cloned().unwrap_or_default(); + m.remove("instance_groups"); + m.remove("instance_groups_roles"); + Value::Object(m) +} + +struct Plan { + current: Map, + desired: Map, + replace: bool, + changes: Vec, + writes: Vec, + warnings: Vec, +} + +impl Plan { + /// The document's value for a key that predates the document: under `replace`, an omitted + /// one counts as `null`, which clears it. + fn legacy(&self, key: &str) -> Option { + match self.desired.get(key) { + Some(v) => Some(v.clone()), + None if self.replace => Some(Value::Null), + None => None, + } + } + + /// The document's value for a key that is only managed when present. + fn managed(&self, key: &str) -> Option { + self.desired.get(key).cloned() + } + + fn differs(&self, key: &str, desired: &Value) -> bool { + let current = self.current.get(key).cloned(); + let current = if FALSE_WHEN_UNSET.contains(&key) { + null_as_false(Some(current.unwrap_or(Value::Null))) + } else { + current + }; + !same(current.as_ref(), Some(desired)) + } + + fn record(&mut self, key: &str, after: Value) { + let before = self.current.get(key).cloned().unwrap_or(Value::Null); + self.record_change(key, before, after); + } + + fn record_change(&mut self, key: &str, before: Value, after: Value) { + self.changes.push(SettingsChange { + key: key.to_string(), + before: redact(key, before), + after: redact(key, after), + }); + } + + /// A write applying the keys recorded since the previous one. + fn write(&mut self, target: Target, body: Value) { + let from = self.writes.iter().map(|w| w.keys.len()).sum::(); + let keys = self.changes[from..].iter().map(|c| c.key.clone()).collect(); + self.writes.push(Write { target, body, keys }); + } + + /// The common case: one key, one endpoint, a body built from the desired value. + fn simple( + &mut self, + key: &str, + desired: Option, + target: Target, + body: impl FnOnce(Value) -> Value, + ) { + if let Some(desired) = desired { + if self.differs(key, &desired) { + self.record(key, desired.clone()); + self.write(target, body(desired)); + } + } + } +} + +/// The plain value of a secret the document states, or `None` to keep the stored one: for the +/// placeholder, and for a value encrypted under another workspace's key, which a repo synced to +/// several workspaces carries. That one is reported rather than failing every other setting. +fn resolve_secret( + key: &str, + v: Value, + mc: &windmill_common::variables::WorkspaceCrypt, + warnings: &mut Vec, +) -> Option { + match v.as_str() { + Some(SENSITIVE_UNCHANGED) => None, + Some(s) if s.starts_with(ENCRYPTED_PREFIX) => { + match mc.decrypt_str(&s[ENCRYPTED_PREFIX.len()..]) { + Ok(plain) => Some(Value::String(plain)), + Err(_) => { + warnings.push(format!( + "{key} was left unchanged: it is encrypted with another workspace's key. \ + Give this workspace its own settings file, export secrets in plain text \ + (plainSecrets), or share the workspace key (includeKey)." + )); + None + } + } + } + _ => Some(v), + } +} + +const KNOWN_KEYS: &[&str] = &[ + "auto_invite", + "webhook", + "error_handler", + "success_handler", + "ai_config", + "large_file_storage", + "git_sync", + "default_app", + "default_scripts", + "name", + "mute_critical_alerts", + "color", + "operator_settings", + "datatable", + "slack_team_id", + "slack_name", + "slack_command_script", + "slack_oauth_client_id", + "slack_oauth_client_secret", + "ducklake", + "dbt_warehouses", + "deploy_ui", + "teams_command_script", + "public_app_execution_limit_per_minute", + "guest_access_enabled", + "guest_jwt_public_key", + "guest_jwt_jwks_url", + "error_handler_fallback_to_instance_alerts", + "add_admins_and_developers_to_forks", + "datatable_permissions", + "protection_rules", +]; + +fn string_list(v: Option<&Value>, what: &str) -> Result { + match v { + None | Some(Value::Null) => Ok(json!([])), + Some(Value::Array(a)) if a.iter().all(Value::is_string) => Ok(Value::Array(a.clone())), + _ => Err(Error::BadRequest(format!( + "{what} must be a list of strings" + ))), + } +} + +/// `datatable_permissions` as the endpoint stores it: an omitted default role is `admin`, and +/// `admin` is always one of the roles. A data table stated as null is not under roles, as an +/// omitted one is. +fn normalize_datatable_permissions(v: &Value) -> Result { + let entries = match v { + Value::Null => return Ok(json!({})), + Value::Object(m) => m, + _ => { + return Err(Error::BadRequest( + "datatable_permissions must map data tables to their roles".to_string(), + )) + } + }; + let mut out = Map::new(); + for (name, entry) in entries { + if entry.is_null() { + continue; + } + let what = format!("datatable_permissions.{name}"); + let entry = entry + .as_object() + .ok_or_else(|| Error::BadRequest(format!("{what} must be an object or null")))?; + let default_role = match entry.get("default_role") { + None => json!(ADMIN_DATATABLE_ROLE), + Some(r) if r.is_null() || r.is_string() => r.clone(), + Some(_) => { + return Err(Error::BadRequest(format!( + "{what}.default_role must be a role name or null" + ))) + } + }; + let mut roles = Map::new(); + match entry.get("roles") { + None | Some(Value::Null) => {} + Some(Value::Object(r)) => { + for (role, tenants) in r { + let tenants = string_list(Some(tenants), &format!("{what}.roles.{role}"))?; + // The endpoint's own check, here so a dry run refuses it too. + for tenant in tenants.as_array().into_iter().flatten() { + let tenant = tenant.as_str().unwrap_or_default(); + let valid = tenant == DATATABLE_TENANT_WILDCARD + || matches!(tenant.split_once('/'), + Some(("u" | "g" | "f", rest)) if !rest.is_empty()); + if !valid { + return Err(Error::BadRequest(format!( + "Invalid tenant '{tenant}' in {what}.roles.{role}: expected \ + 'u/', 'g/', 'f/' or '*'" + ))); + } + } + roles.insert(role.clone(), tenants); + } + } + Some(_) => { + return Err(Error::BadRequest(format!( + "{what}.roles must map role names to their tenants" + ))) + } + } + roles + .entry(ADMIN_DATATABLE_ROLE.to_string()) + .or_insert(json!([])); + if let Some(role) = default_role.as_str() { + if !roles.contains_key(role) { + return Err(Error::BadRequest(format!( + "The default role '{role}' of {what} is not among its roles" + ))); + } + } + out.insert( + name.clone(), + json!({ "default_role": default_role, "roles": roles }), + ); + } + Ok(Value::Object(out)) +} + +/// `protection_rules` as it is stored: rules in their fixed order, the bypass lists stated. The +/// dev workspace lock is dropped with a warning, since the dev workspace setting owns it. +fn normalize_protection_rules(v: &Value, warnings: &mut Vec) -> Result { + let rules = match v { + Value::Null => return Ok(json!({})), + Value::Object(m) => m, + _ => { + return Err(Error::BadRequest( + "protection_rules must map rule names to rules".to_string(), + )) + } + }; + let mut out = Map::new(); + for (name, rule) in rules { + if name == DEV_WORKSPACE_LOCK_RULE_NAME { + warnings.push(format!( + "protection_rules.{name} was left unchanged: the dev workspace setting manages it" + )); + continue; + } + if rule.is_null() { + continue; + } + let what = format!("protection_rules.{name}"); + let rule = rule + .as_object() + .ok_or_else(|| Error::BadRequest(format!("{what} must be an object or null")))?; + let kinds: Vec = + serde_json::from_value(rule.get("rules").cloned().unwrap_or(json!([]))) + .map_err(|e| Error::BadRequest(format!("Invalid {what}.rules: {e}")))?; + out.insert( + name.clone(), + json!({ + "rules": rule_kinds(ProtectionRules::from(&kinds)), + "bypass_groups": string_list(rule.get("bypass_groups"), &format!("{what}.bypass_groups"))?, + "bypass_users": string_list(rule.get("bypass_users"), &format!("{what}.bypass_users"))?, + }), + ); + } + Ok(Value::Object(out)) +} + +async fn plan(db: &DB, w_id: &str, desired: Map, replace: bool) -> Result { + let current = { + let mut conn = db.acquire().await?; + load_settings_document(&mut conn, db, w_id, SecretMode::Plain).await? + }; + let mc = build_crypt(db, w_id).await?; + let roles = if desired + .get("datatable_permissions") + .is_some_and(|v| !v.is_null()) + { + RoleNames::load(db).await + } else { + RoleNames::default() + }; + plan_document(current, desired, replace, &mc, &roles) +} + +fn plan_document( + current: Map, + mut desired: Map, + replace: bool, + mc: &windmill_common::variables::WorkspaceCrypt, + roles: &RoleNames, +) -> Result { + let mut warnings = vec![]; + let unknown: Vec<&str> = desired + .keys() + .map(String::as_str) + .filter(|k| !KNOWN_KEYS.contains(k)) + .collect(); + if !unknown.is_empty() { + warnings.push(format!("Ignored unknown settings: {}", unknown.join(", "))); + } + for key in ["auto_invite", "error_handler", "success_handler"] { + if desired + .get(key) + .is_some_and(|v| !v.is_null() && !v.is_object()) + { + return Err(Error::BadRequest(format!( + "{key} must be an object or null" + ))); + } + } + // The bodies built below coerce a mistyped value to `false` or to a delete, so a wrong type + // must be refused here rather than reach an endpoint as a destructive write. + let typed = |key: &str, v: Option<&Value>, ok: fn(&Value) -> bool, ty: &str| match v { + Some(v) if !v.is_null() && !ok(v) => { + Err(Error::BadRequest(format!("{key} must be {ty} or null"))) + } + _ => Ok(()), + }; + for key in FALSE_WHEN_UNSET { + typed(key, desired.get(*key), Value::is_boolean, "a boolean")?; + } + // The rest are refused by their endpoint anyway, but only after the writes before them + // went through, and after a dry run reported them as changes. + for key in [ + "slack_oauth_client_id", + "slack_oauth_client_secret", + "webhook", + "color", + "default_app", + "slack_command_script", + "teams_command_script", + "guest_jwt_public_key", + "guest_jwt_jwks_url", + ] { + typed(key, desired.get(key), Value::is_string, "a string")?; + } + typed( + "public_app_execution_limit_per_minute", + desired.get("public_app_execution_limit_per_minute"), + |v| v.as_i64().is_some_and(|n| i32::try_from(n).is_ok()), + "a 32-bit integer", + )?; + for (parent, key, ok, ty) in [ + ( + "auto_invite", + "enabled", + Value::is_boolean as fn(&Value) -> bool, + "a boolean", + ), + ("auto_invite", "operator", Value::is_boolean, "a boolean"), + ("auto_invite", "mode", Value::is_string, "a string"), + ( + "error_handler", + "muted_on_cancel", + Value::is_boolean, + "a boolean", + ), + ( + "error_handler", + "muted_on_user_path", + Value::is_boolean, + "a boolean", + ), + ] { + let v = desired.get(parent).and_then(|p| p.get(key)); + typed(&format!("{parent}.{key}"), v, ok, ty)?; + } + if let Some(v) = desired.get("datatable_permissions") { + let v = normalize_datatable_permissions(v)?; + desired.insert("datatable_permissions".to_string(), v); + } + if let Some(v) = desired.get("protection_rules") { + let v = normalize_protection_rules(v, &mut warnings)?; + desired.insert("protection_rules".to_string(), v); + } + // The endpoint stores an omitted mute flag as `false`. + if let Some(handler) = desired + .get_mut("error_handler") + .and_then(Value::as_object_mut) + { + for key in ["muted_on_cancel", "muted_on_user_path"] { + if is_null(handler.get(key)) { + handler.insert(key.to_string(), Value::Bool(false)); + } + } + } + for key in SECRET_KEYS { + if let Some(v) = desired.remove(*key) { + if let Some(v) = resolve_secret(key, v, mc, &mut warnings) { + desired.insert(key.to_string(), v); + } else if let Some(cur) = current.get(*key) { + desired.insert(key.to_string(), cur.clone()); + } + } + } + + let stated: Vec = desired.keys().cloned().collect(); + let mut p = Plan { current, desired, replace, changes: vec![], writes: vec![], warnings }; + + let webhook = p.legacy("webhook"); + p.simple( + "webhook", + webhook, + Target::Edit(SettingsEdit::Webhook), + |v| json!({ "webhook": v }), + ); + + if let Some(auto_invite) = p.legacy("auto_invite") { + let desired_domain = domain_invite(Some(&auto_invite)); + if desired_domain != domain_invite(p.current.get("auto_invite")) { + if let Some(mode) = desired_domain.get("mode").and_then(Value::as_str) { + if mode != "invite" && mode != "add" { + return Err(Error::BadRequest( + r#"Invalid value for auto_invite.mode. Valid values are "invite" and "add""# + .to_string(), + )); + } + } + let mut after = if auto_invite.is_null() { + Value::Null + } else { + desired_domain.clone() + }; + if let (Some(after), Some(cur)) = ( + after.as_object_mut(), + p.current.get("auto_invite").and_then(Value::as_object), + ) { + for k in ["instance_groups", "instance_groups_roles"] { + if let Some(v) = cur.get(k) { + after.insert(k.to_string(), v.clone()); + } + } + } + p.record("auto_invite", after); + let enabled = desired_domain.get("enabled").and_then(Value::as_bool) == Some(true); + let body = if enabled { + json!({ + "operator": desired_domain.get("operator").and_then(Value::as_bool).unwrap_or(false), + "invite_all": true, + "auto_add": desired_domain.get("mode").and_then(Value::as_str) == Some("add"), + }) + } else { + json!({}) + }; + p.write(Target::Edit(SettingsEdit::AutoInvite), body); + } + // Clearing an instance group removes the members it granted, so only a document that + // states the groups manages them. + if let Some(groups) = auto_invite.get("instance_groups").filter(|g| !g.is_null()) { + let roles = auto_invite + .get("instance_groups_roles") + .filter(|r| !r.is_null()) + .cloned() + .unwrap_or(json!({})); + let cur = p.current.get("auto_invite"); + let cur_groups = cur.and_then(|c| c.get("instance_groups")); + let cur_roles = cur + .and_then(|c| c.get("instance_groups_roles")) + .filter(|r| !r.is_null()) + .cloned() + .unwrap_or(json!({})); + if !same(Some(groups), cur_groups) || roles != cur_roles { + p.record( + "auto_invite.instance_groups", + json!({ "instance_groups": groups, "instance_groups_roles": roles }), + ); + p.write( + Target::Edit(SettingsEdit::InstanceGroups), + json!({ "groups": groups, "roles": roles }), + ); + } + } + } + + let ai_config = p.legacy("ai_config"); + p.simple("ai_config", ai_config, Target::AiConfig, |v| { + if v.is_null() { + json!({}) + } else { + v + } + }); + + let fallback = null_as_false(p.managed("error_handler_fallback_to_instance_alerts")); + let fallback_changes = fallback + .as_ref() + .is_some_and(|f| p.differs("error_handler_fallback_to_instance_alerts", f)); + let error_handler = p.legacy("error_handler"); + let error_handler_changes = error_handler + .as_ref() + .is_some_and(|e| p.differs("error_handler", e)); + if error_handler_changes || fallback_changes { + let handler = if error_handler_changes { + error_handler.clone().unwrap_or(Value::Null) + } else { + p.current + .get("error_handler") + .cloned() + .unwrap_or(Value::Null) + }; + if error_handler_changes { + p.record("error_handler", handler.clone()); + } + if fallback_changes { + p.record( + "error_handler_fallback_to_instance_alerts", + fallback.clone().unwrap_or(Value::Null), + ); + } + p.write( + Target::Edit(SettingsEdit::ErrorHandler), + json!({ + "path": handler.get("path"), + "extra_args": handler.get("extra_args"), + "muted_on_cancel": handler.get("muted_on_cancel").and_then(Value::as_bool).unwrap_or(false), + "muted_on_user_path": handler.get("muted_on_user_path").and_then(Value::as_bool).unwrap_or(false), + "fallback_to_instance_alerts": if fallback_changes { fallback.and_then(|f| f.as_bool()) } else { None }, + }), + ); + } + + let success_handler = p.legacy("success_handler"); + p.simple( + "success_handler", + success_handler, + Target::Edit(SettingsEdit::SuccessHandler), + |v| json!({ "path": v.get("path"), "extra_args": v.get("extra_args") }), + ); + + let lfs = p.legacy("large_file_storage"); + p.simple( + "large_file_storage", + lfs, + Target::Edit(SettingsEdit::LargeFileStorage), + |v| json!({ "large_file_storage": v }), + ); + + let git_sync = p.legacy("git_sync"); + p.simple( + "git_sync", + git_sync, + Target::Edit(SettingsEdit::GitSync), + |v| json!({ "git_sync_settings": v }), + ); + + let default_scripts = p.legacy("default_scripts"); + p.simple( + "default_scripts", + default_scripts, + Target::Edit(SettingsEdit::DefaultScripts), + |v| v, + ); + + let default_app = p.legacy("default_app"); + p.simple( + "default_app", + default_app, + Target::Edit(SettingsEdit::DefaultApp), + |v| json!({ "default_app_path": v }), + ); + + // The display name is never applied: settings.yaml is shared across a repo's branches, so + // applying it would let one workspace's name overwrite another's. + + let mute = null_as_false(p.legacy("mute_critical_alerts")); + p.simple( + "mute_critical_alerts", + mute, + Target::MuteCriticalAlerts, + |v| json!({ "mute_critical_alerts": v }), + ); + + // `sync pull` omits the key for a workspace without a color, so an absent or null color + // means "not managed", never "clear". + let color = p.managed("color").filter(|c| !c.is_null()); + p.simple( + "color", + color, + Target::Edit(SettingsEdit::Color), + |v| json!({ "color": v }), + ); + + if let Some(desired) = p.managed("operator_settings").filter(|o| !o.is_null()) { + // The endpoint keeps the stored value of a write right the document omits, so an + // older document must not count as differing on those keys. + let mut compared = desired.clone(); + if let (Some(compared), Some(cur)) = ( + compared.as_object_mut(), + p.current + .get("operator_settings") + .and_then(Value::as_object), + ) { + for key in [ + "builder_flows", + "builder_apps", + "manage_schedules", + "manage_triggers", + ] { + if !compared.contains_key(key) { + if let Some(v) = cur.get(key) { + compared.insert(key.to_string(), v.clone()); + } + } + } + } + if p.differs("operator_settings", &compared) { + p.record("operator_settings", compared); + p.write(Target::Edit(SettingsEdit::OperatorSettings), desired); + } + } + + let datatable = p.legacy("datatable"); + p.simple( + "datatable", + datatable, + Target::Edit(SettingsEdit::Datatable), + |v| json!({ "settings": if v.is_null() { json!({ "datatables": {} }) } else { v } }), + ); + + // Stated as a whole: a data table the map omits is not under roles, a rule it omits is + // deleted. + if let Some(desired) = p.managed("datatable_permissions") { + plan_datatable_permissions(&mut p, desired, roles)?; + } + if let Some(desired) = p.managed("protection_rules") { + plan_protection_rules(&mut p, desired); + } + + let slack_command = p.legacy("slack_command_script"); + p.simple( + "slack_command_script", + slack_command, + Target::Edit(SettingsEdit::SlackCommand), + |v| json!({ "slack_command_script": v }), + ); + + let (client_id, client_secret) = ( + p.legacy("slack_oauth_client_id"), + p.legacy("slack_oauth_client_secret"), + ); + if client_id.is_some() || client_secret.is_some() { + let id = client_id.unwrap_or_else(|| { + p.current + .get("slack_oauth_client_id") + .cloned() + .unwrap_or(Value::Null) + }); + let secret = client_secret.unwrap_or_else(|| { + p.current + .get("slack_oauth_client_secret") + .cloned() + .unwrap_or(Value::Null) + }); + let id_changes = p.differs("slack_oauth_client_id", &id); + let secret_changes = p.differs("slack_oauth_client_secret", &secret); + if id_changes || secret_changes { + if id_changes { + p.record("slack_oauth_client_id", id.clone()); + } + if secret_changes { + p.record("slack_oauth_client_secret", secret.clone()); + } + let set = |v: &Value| v.as_str().is_some_and(|s| !s.is_empty()); + if set(&id) && set(&secret) { + p.write( + Target::Edit(SettingsEdit::SetSlackOAuth), + json!({ "slack_oauth_client_id": id, "slack_oauth_client_secret": secret }), + ); + } else { + p.write(Target::Edit(SettingsEdit::DeleteSlackOAuth), Value::Null); + } + } + } + + // Settings added to the document after `wmill sync` shipped are managed only when the + // document states them, so a settings.yaml written before them never clears them. + let ducklake = p.managed("ducklake"); + p.simple( + "ducklake", + ducklake, + Target::Edit(SettingsEdit::Ducklake), + |v| json!({ "settings": if v.is_null() { json!({ "ducklakes": {} }) } else { v } }), + ); + + let dbt = p.managed("dbt_warehouses"); + p.simple( + "dbt_warehouses", + dbt, + Target::Edit(SettingsEdit::DbtWarehouses), + |v| json!({ "dbt_warehouses": v }), + ); + + let deploy_ui = p.managed("deploy_ui"); + p.simple( + "deploy_ui", + deploy_ui, + Target::Edit(SettingsEdit::DeployUi), + |v| json!({ "deploy_ui_settings": v }), + ); + + let teams = p.managed("teams_command_script"); + p.simple( + "teams_command_script", + teams, + Target::TeamsCommand, + |v| json!({ "teams_command_script": v }), + ); + + let rate = p.managed("public_app_execution_limit_per_minute"); + p.simple( + "public_app_execution_limit_per_minute", + rate, + Target::Edit(SettingsEdit::PublicAppRateLimit), + |v| json!({ "public_app_execution_limit_per_minute": v }), + ); + + let guest_access = null_as_false(p.managed("guest_access_enabled")); + p.simple( + "guest_access_enabled", + guest_access, + Target::Edit(SettingsEdit::GuestAccess), + |v| json!({ "guest_access_enabled": v.as_bool().unwrap_or(false) }), + ); + + let (public_key, jwks_url) = ( + p.managed("guest_jwt_public_key"), + p.managed("guest_jwt_jwks_url"), + ); + if public_key.is_some() || jwks_url.is_some() { + // At most one of the two is set, so stating one clears the other. + let other = |stated: &Option, key: &str| { + if stated.as_ref().is_some_and(|v| !v.is_null()) { + Value::Null + } else { + p.current.get(key).cloned().unwrap_or(Value::Null) + } + }; + let (public_key, jwks_url) = ( + public_key + .clone() + .unwrap_or_else(|| other(&jwks_url, "guest_jwt_public_key")), + jwks_url + .clone() + .unwrap_or_else(|| other(&public_key, "guest_jwt_jwks_url")), + ); + let key_changes = p.differs("guest_jwt_public_key", &public_key); + let url_changes = p.differs("guest_jwt_jwks_url", &jwks_url); + if key_changes || url_changes { + if key_changes { + p.record("guest_jwt_public_key", public_key.clone()); + } + if url_changes { + p.record("guest_jwt_jwks_url", jwks_url.clone()); + } + p.write( + Target::Edit(SettingsEdit::GuestJwtKey), + json!({ "public_key": public_key, "jwks_url": jwks_url }), + ); + } + } + + let forks = null_as_false(p.managed("add_admins_and_developers_to_forks")); + p.simple( + "add_admins_and_developers_to_forks", + forks, + Target::Edit(SettingsEdit::AddAdminsAndDevelopersToForks), + |v| json!({ "add_admins_and_developers_to_forks": v.as_bool().unwrap_or(false) }), + ); + + // As `wmill sync push` always did: a document whose stated settings all match changes + // nothing, so a settings.yaml pulled before a setting was turned on in the UI does not + // clear it just for omitting it. + let changes_stated = p.changes.iter().any(|c| { + stated + .iter() + .any(|k| c.key == *k || c.key.starts_with(&format!("{k}."))) + }); + if replace && !changes_stated { + p.changes.clear(); + p.writes.clear(); + } + + // Refused here rather than by the endpoint, so a dry run already says so. + let unavailable: Vec = p + .writes + .iter() + .filter(|w| !available_in_this_edition(&w.target)) + .flat_map(|w| w.keys.clone()) + .collect(); + if !unavailable.is_empty() { + return Err(Error::BadRequest(format!( + "Only available on Windmill Enterprise Edition: {}", + unavailable.join(", ") + ))); + } + + Ok(p) +} + +fn object_of(v: Option<&Value>) -> Map { + v.and_then(Value::as_object).cloned().unwrap_or_default() +} + +fn plan_datatable_permissions(p: &mut Plan, desired: Value, roles: &RoleNames) -> Result<()> { + let desired = object_of(Some(&desired)); + let current = object_of(p.current.get("datatable_permissions")); + // The data tables as they will be once the `datatable` write before this one has gone through. + let entries = object_of( + p.legacy("datatable") + .as_ref() + .or(p.current.get("datatable")) + .and_then(|d| d.get("datatables")), + ); + let names: std::collections::BTreeSet<&String> = desired.keys().chain(current.keys()).collect(); + for name in names { + let (before, after) = (current.get(name), desired.get(name)); + if same(before, after) { + continue; + } + let entry = entries.get(name); + // A removed data table takes its roles with it. + if after.is_none() && entry.is_none() { + continue; + } + let body = match after { + None => json!({ "datatable": name, "permissioned": false }), + Some(after) => { + let cluster = entry.and_then(role_cluster_of).ok_or_else(|| { + Error::BadRequest(format!( + "Data table '{name}' cannot be put under roles: only a data table this \ + workspace governs, on a cluster Windmill manages, can" + )) + })?; + let ids: BTreeMap<&str, &str> = roles + .of(cluster) + .ok_or_else(|| { + Error::BadRequest( + "Data table roles are unavailable on this instance".to_string(), + ) + })? + .iter() + .map(|(id, name)| (name.as_str(), id.as_str())) + .collect(); + let id_of = |role: &str| -> Result { + if role == ADMIN_DATATABLE_ROLE { + return Ok(role.to_string()); + } + ids.get(role).map(|id| id.to_string()).ok_or_else(|| { + Error::BadRequest(format!( + "'{role}' is not a data table role of the {} cluster", + cluster.as_str() + )) + }) + }; + let default_role = match after.get("default_role").and_then(Value::as_str) { + None => NO_DEFAULT_DATATABLE_ROLE.to_string(), + Some(role) => id_of(role)?, + }; + let roles = object_of(after.get("roles")) + .iter() + .map(|(role, tenants)| Ok(json!({ "id": id_of(role)?, "tenants": tenants }))) + .collect::>>()?; + json!({ + "datatable": name, + "permissioned": true, + "default_role": default_role, + "roles": roles, + }) + } + }; + p.record_change( + &format!("datatable_permissions.{name}"), + before.cloned().unwrap_or(Value::Null), + after.cloned().unwrap_or(Value::Null), + ); + p.write(Target::Edit(SettingsEdit::DatatablePermissions), body); + } + Ok(()) +} + +fn plan_protection_rules(p: &mut Plan, desired: Value) { + let desired = object_of(Some(&desired)); + let current = object_of(p.current.get("protection_rules")); + let names: std::collections::BTreeSet<&String> = desired.keys().chain(current.keys()).collect(); + for name in names { + let (before, after) = (current.get(name), desired.get(name)); + if same(before, after) { + continue; + } + p.record_change( + &format!("protection_rules.{name}"), + before.cloned().unwrap_or(Value::Null), + after.cloned().unwrap_or(Value::Null), + ); + let rule = |r: &Value, name_key: &str| { + json!({ + name_key: name, + "rules": r.get("rules"), + "bypass_groups": r.get("bypass_groups"), + "bypass_users": r.get("bypass_users"), + }) + }; + match (before, after) { + (None, Some(after)) => p.write( + Target::Edit(SettingsEdit::CreateProtectionRule), + rule(after, "name"), + ), + (Some(_), Some(after)) => p.write( + Target::Edit(SettingsEdit::UpdateProtectionRule), + rule(after, "rule_name"), + ), + _ => p.write( + Target::Edit(SettingsEdit::DeleteProtectionRule), + json!({ "name": name }), + ), + } + } +} + +fn available_in_this_edition(target: &Target) -> bool { + match target { + Target::MuteCriticalAlerts + | Target::Edit(SettingsEdit::DefaultApp) + | Target::Edit(SettingsEdit::DeployUi) => cfg!(feature = "enterprise"), + Target::Edit(SettingsEdit::DatatablePermissions) => { + cfg!(all(feature = "private", feature = "enterprise")) + } + Target::TeamsCommand | Target::Edit(SettingsEdit::InstanceGroups) => { + cfg!(feature = "private") + } + _ => true, + } +} + +async fn execute(authed: &ApiAuthed, db: &DB, w_id: &str, write: Write) -> Result { + let (a, d, p) = ( + authed.clone(), + Extension(db.clone()), + Path(w_id.to_string()), + ); + let body_err = |e: serde_json::Error| { + Error::BadRequest(format!("Invalid value for {:?}: {e}", write.target)) + }; + match write.target { + Target::Edit(SettingsEdit::AutoInvite) => { + match apply_settings_edit( + a.clone(), + db.clone(), + w_id.to_string(), + SettingsEdit::AutoInvite, + write.body.clone(), + ) + .await + { + // Inviting every domain is refused on cloud, where only the workspace's own + // domain can be auto-invited. + Err(_) if write.body.get("invite_all") == Some(&json!(true)) => { + let mut body = write.body; + body["invite_all"] = json!(false); + apply_settings_edit( + a, + db.clone(), + w_id.to_string(), + SettingsEdit::AutoInvite, + body, + ) + .await + } + r => r, + } + } + Target::Edit(edit) => { + apply_settings_edit(a, db.clone(), w_id.to_string(), edit, write.body).await + } + Target::AiConfig => { + let body = serde_json::from_value(write.body).map_err(body_err)?; + let _ = + crate::workspaces::edit_copilot_config(a, d, p, authed.clone(), Json(body)).await?; + Ok(Value::Null) + } + #[cfg(feature = "enterprise")] + Target::MuteCriticalAlerts => { + let body = serde_json::from_value(write.body).map_err(body_err)?; + crate::workspaces::mute_critical_alerts(d, p, a, Json(body)).await?; + Ok(Value::Null) + } + #[cfg(not(feature = "enterprise"))] + Target::MuteCriticalAlerts => Err(crate::workspaces::mute_critical_alerts().await), + #[cfg(feature = "private")] + Target::TeamsCommand => { + let body = serde_json::from_value(write.body).map_err(body_err)?; + crate::teams_ee::edit_teams_command(a, d, p, authed.clone(), Json(body)).await?; + Ok(Value::Null) + } + #[cfg(not(feature = "private"))] + Target::TeamsCommand => { + crate::teams_oss::edit_teams_command().await?; + Ok(Value::Null) + } + } +} + +/// Applies `document` as a list of writes, each through the endpoint that owns the setting, in +/// the order `wmill sync push` has always written them. The writes are not one transaction: a +/// failing one stops the apply, and the error names the settings already written. +pub async fn apply_settings_document( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> JsonResult { + require_admin(authed.is_admin, &authed.username)?; + let p = plan(&db, &w_id, req.document, req.replace).await?; + let mut result = ApplySettingsDocumentResult { + changes: p.changes, + warnings: p.warnings, + ..Default::default() + }; + if req.dry_run || p.writes.is_empty() { + return Ok(Json(result)); + } + let mut written: Vec = vec![]; + for write in p.writes { + let keys = write.keys.clone(); + let response = execute(&authed, &db, &w_id, write).await.map_err(|e| { + if written.is_empty() { + return e; + } + Error::Generic( + e.status_code(), + format!( + "{e} (settings already applied before this error: {})", + written.join(", ") + ), + ) + })?; + if let Some(stranded) = response + .get("stranded_references") + .and_then(Value::as_array) + .filter(|s| !s.is_empty()) + { + let refs = stranded + .iter() + .filter_map(|r| { + Some(format!( + "{}/{}", + r.get("workspace_id")?.as_str()?, + r.get("datatable")?.as_str()? + )) + }) + .collect::>() + .join(", "); + result.warnings.push(format!( + "The removed data tables governed data tables in other workspaces, which no longer resolve: {refs}. A superadmin can point them somewhere else." + )); + } + written.extend(keys); + } + result.applied = true; + Ok(Json(result)) +} + +#[cfg(test)] +mod tests { + use super::*; + use windmill_common::variables::WorkspaceCrypt; + + fn doc(v: Value) -> Map { + v.as_object().unwrap().clone() + } + + fn keys(p: &Plan) -> Vec<&str> { + p.changes.iter().map(|c| c.key.as_str()).collect() + } + + fn current() -> Map { + doc(json!({ + "name": "ws", + "webhook": "https://a", + "large_file_storage": { "type": "S3Storage", "s3_resource_path": "f/s3" }, + "slack_oauth_client_id": "cid", + "slack_oauth_client_secret": "sekret", + "guest_jwt_public_key": "PEM", + "guest_jwt_jwks_url": null, + "public_app_execution_limit_per_minute": 5, + })) + } + + #[test] + fn replace_clears_an_omitted_setting_only_when_a_stated_one_changes() { + let mc = WorkspaceCrypt::new("k"); + let unchanged = doc(json!({ "name": "other", "webhook": "https://a" })); + let p = plan_document(current(), unchanged, true, &mc, &RoleNames::default()).unwrap(); + assert!(p.writes.is_empty(), "{:?}", keys(&p)); + + let changed = doc(json!({ "webhook": "https://b" })); + let p = plan_document(current(), changed, true, &mc, &RoleNames::default()).unwrap(); + assert_eq!( + keys(&p), + [ + "webhook", + "large_file_storage", + "slack_oauth_client_id", + "slack_oauth_client_secret" + ] + ); + assert!(!keys(&p).contains(&"public_app_execution_limit_per_minute")); + } + + #[test] + fn secrets_keep_their_value_for_the_placeholder_and_a_foreign_key() { + let mc = WorkspaceCrypt::new("k"); + let ours = format!( + "{ENCRYPTED_PREFIX}{}", + mc.encrypt_str_deterministic("sekret") + ); + let theirs = format!( + "{ENCRYPTED_PREFIX}{}", + WorkspaceCrypt::new("other").encrypt_str_deterministic("x") + ); + for secret in [SENSITIVE_UNCHANGED.to_string(), ours, theirs.clone()] { + let p = plan_document( + current(), + doc(json!({ "slack_oauth_client_secret": secret })), + false, + &mc, + &RoleNames::default(), + ) + .unwrap(); + assert!(p.changes.is_empty(), "{secret}: {:?}", keys(&p)); + assert_eq!(p.warnings.len(), usize::from(secret == theirs)); + } + } + + #[test] + fn unset_switches_compare_as_false() { + let mc = WorkspaceCrypt::new("k"); + let d = doc(json!({ "mute_critical_alerts": false, "guest_access_enabled": null })); + assert!( + plan_document(current(), d, false, &mc, &RoleNames::default()) + .unwrap() + .changes + .is_empty() + ); + } + + #[test] + fn stating_one_guest_key_clears_the_other() { + let mc = WorkspaceCrypt::new("k"); + let d = doc(json!({ "guest_jwt_jwks_url": "https://jwks" })); + let p = plan_document(current(), d, false, &mc, &RoleNames::default()).unwrap(); + assert_eq!(p.writes.len(), 1); + assert_eq!( + p.writes[0].body, + json!({ "public_key": null, "jwks_url": "https://jwks" }) + ); + } + + #[test] + fn mistyped_values_are_refused_rather_than_coerced() { + let mc = WorkspaceCrypt::new("k"); + for d in [ + json!({ "guest_access_enabled": "true" }), + json!({ "add_admins_and_developers_to_forks": 1 }), + json!({ "slack_oauth_client_id": 123 }), + json!({ "error_handler": { "path": "f/x", "muted_on_cancel": "yes" } }), + json!({ "auto_invite": { "enabled": "true" } }), + json!({ "webhook": 123 }), + json!({ "public_app_execution_limit_per_minute": "5" }), + json!({ "public_app_execution_limit_per_minute": 1.5 }), + json!({ "public_app_execution_limit_per_minute": 2147483648i64 }), + ] { + assert!( + plan_document(current(), doc(d.clone()), false, &mc, &RoleNames::default()) + .is_err(), + "{d}" + ); + } + } + + #[test] + fn a_null_handler_argument_is_a_change() { + let mc = WorkspaceCrypt::new("k"); + let mut cur = current(); + cur.insert( + "error_handler".into(), + json!({ "path": "f/x", "extra_args": {}, "muted_on_cancel": false, "muted_on_user_path": false }), + ); + let d = doc(json!({ "error_handler": { "path": "f/x", "extra_args": { "input": null } } })); + let p = plan_document(cur.clone(), d, false, &mc, &RoleNames::default()).unwrap(); + assert_eq!(keys(&p), ["error_handler"]); + + let d = doc(json!({ "error_handler": { "path": "f/x", "extra_args": {} } })); + let p = plan_document(cur, d, false, &mc, &RoleNames::default()).unwrap(); + assert!(p.changes.is_empty(), "{:?}", keys(&p)); + } + + #[cfg(all(feature = "private", feature = "enterprise"))] + #[test] + fn datatable_roles_are_named_in_the_document_and_written_by_id() { + let mc = WorkspaceCrypt::new("k"); + let roles = RoleNames { + instance: Some([("r1".to_string(), "readers".to_string())].into()), + external_instance: None, + }; + let mut cur = current(); + cur.insert( + "datatable".into(), + json!({ "datatables": { "main": { "database": { "resource_type": "instance", "resource_path": "main" } } } }), + ); + cur.insert( + "datatable_permissions".into(), + json!({ "main": { "default_role": "admin", "roles": { "admin": [], "readers": ["g/x"] } } }), + ); + + let d = doc( + json!({ "datatable_permissions": { "main": { "default_role": "readers", "roles": { "readers": ["g/x", "u/bob"] } } } }), + ); + let p = plan_document(cur.clone(), d, false, &mc, &roles).unwrap(); + assert_eq!(keys(&p), ["datatable_permissions.main"]); + assert_eq!( + p.writes[0].body, + json!({ "datatable": "main", "permissioned": true, "default_role": "r1", + "roles": [{ "id": "r1", "tenants": ["g/x", "u/bob"] }, { "id": "admin", "tenants": [] }] }) + ); + + let d = doc(json!({ "datatable_permissions": {} })); + let p = plan_document(cur.clone(), d, false, &mc, &roles).unwrap(); + assert_eq!( + p.writes[0].body, + json!({ "datatable": "main", "permissioned": false }) + ); + + let d = doc(json!({ "datatable_permissions": { "main": { "roles": { "writers": [] } } } })); + assert!(plan_document(cur.clone(), d, false, &mc, &roles).is_err()); + + let d = doc(json!({ "datatable_permissions": { "main": { "default_role": "readers" } } })); + assert!(plan_document(cur.clone(), d, false, &mc, &roles).is_err()); + + // Under replace an omitted `datatable` is cleared, taking its tables' roles with it. + let d = doc(json!({ "datatable_permissions": {} })); + let p = plan_document(cur.clone(), d, true, &mc, &roles).unwrap(); + assert!(!p + .writes + .iter() + .any(|w| matches!(w.target, Target::Edit(SettingsEdit::DatatablePermissions)))); + let d = doc(json!({ "datatable_permissions": { "main": {} } })); + assert!(plan_document(cur, d, true, &mc, &roles).is_err()); + } + + #[test] + fn protection_rules_are_diffed_by_name() { + let mc = WorkspaceCrypt::new("k"); + let mut cur = current(); + cur.insert( + "protection_rules".into(), + json!({ "old": { "rules": ["DisableWorkspaceForking"], "bypass_groups": [], "bypass_users": [] } }), + ); + let d = doc(json!({ "protection_rules": { + "new": { "rules": ["RestrictPublicRunSharing", "DisableDirectDeployment"] }, + "dev_workspace_lock": { "rules": [] }, + } })); + let p = plan_document(cur, d, false, &mc, &RoleNames::default()).unwrap(); + assert_eq!(keys(&p), ["protection_rules.new", "protection_rules.old"]); + assert_eq!( + p.writes[0].body, + json!({ "name": "new", "rules": ["DisableDirectDeployment", "RestrictPublicRunSharing"], + "bypass_groups": [], "bypass_users": [] }) + ); + assert!(matches!( + p.writes[1].target, + Target::Edit(SettingsEdit::DeleteProtectionRule) + )); + assert_eq!(p.warnings.len(), 1); + } +} diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 186cfbc291..e5b5cf3e72 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -61,6 +61,11 @@ pub fn workspaced_service() -> Router { ) .route("/tarball", get(crate::workspaces_export::tarball_workspace)) .route("/edit_copilot_config", post(edit_copilot_config)) + .route( + "/settings_document", + get(crate::workspace_settings_document::get_settings_document) + .post(crate::workspace_settings_document::apply_settings_document), + ) .route("/get_copilot_info", get(get_copilot_info)) .route("/critical_alerts", get(get_critical_alerts)) .route( @@ -85,7 +90,7 @@ pub fn workspaced_service() -> Router { router } -async fn edit_copilot_config( +pub(crate) async fn edit_copilot_config( authed: ApiAuthed, Extension(db): Extension, Path(w_id): Path, @@ -190,7 +195,7 @@ async fn edit_copilot_config( } #[derive(Serialize)] -struct EditCopilotConfigResponse { +pub(crate) struct EditCopilotConfigResponse { effective_ai_config: AIConfig, has_instance_ai_config: bool, uses_instance_ai_config: bool, @@ -321,7 +326,7 @@ pub struct MuteCriticalAlertRequest { } #[cfg(feature = "enterprise")] -async fn mute_critical_alerts( +pub(crate) async fn mute_critical_alerts( Extension(db): Extension, Path(w_id): Path, authed: ApiAuthed, diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 3c3e965248..bff261093a 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -359,6 +359,9 @@ pub(crate) struct ArchiveQueryParams { include_users: Option, include_groups: Option, include_settings: Option, + /// Opt-in, with `settings_version=v2`: state the data tables' roles in settings.json. Their + /// tenants name this workspace's users and groups, and their roles this instance's catalog. + include_datatable_permissions: Option, include_key: Option, include_workspace_dependencies: Option, /// Default `false`: data table migrations ship in the tarball unless the repo @@ -513,49 +516,6 @@ struct SimplifiedGroup { admins: Vec, } -// V2 format: New grouped format -#[derive(Serialize)] -struct SimplifiedSettings { - #[serde(skip_serializing_if = "Option::is_none")] - auto_invite: Option, - #[serde(skip_serializing_if = "Option::is_none")] - webhook: Option, - // Always serialize (including as `null`) so that `wmill sync pull` emits - // these fields in settings.yaml unconditionally. Makes round-trip - // bijective: YAML is the source of truth, absence/null = "clear remote", - // mirroring every other workspace setting. - error_handler: Option, - success_handler: Option, - #[serde(skip_serializing_if = "Option::is_none")] - ai_config: Option, - #[serde(skip_serializing_if = "Option::is_none")] - large_file_storage: Option, - #[serde(skip_serializing_if = "Option::is_none")] - git_sync: Option, - #[serde(skip_serializing_if = "Option::is_none")] - default_app: Option, - #[serde(skip_serializing_if = "Option::is_none")] - default_scripts: Option, - name: String, - #[serde(skip_serializing_if = "Option::is_none")] - mute_critical_alerts: Option, - #[serde(skip_serializing_if = "Option::is_none")] - color: Option, - #[serde(skip_serializing_if = "Option::is_none")] - operator_settings: Option, - #[serde(skip_serializing_if = "Option::is_none")] - datatable: Option, - #[serde(skip_serializing_if = "Option::is_none")] - slack_team_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - slack_name: Option, - #[serde(skip_serializing_if = "Option::is_none")] - slack_command_script: Option, - // Always serialize (see note above on error_handler / success_handler). - slack_oauth_client_id: Option, - slack_oauth_client_secret: Option, -} - // V1 format: Legacy flat format for backward compatibility (matches main branch exactly) #[derive(Serialize)] struct SimplifiedSettingsLegacy { @@ -602,7 +562,6 @@ struct SettingsRow { auto_invite: Option, webhook: Option, error_handler: Option, - success_handler: Option, ai_config: Option, large_file_storage: Option, git_sync: Option, @@ -616,8 +575,6 @@ struct SettingsRow { slack_team_id: Option, slack_name: Option, slack_command_script: Option, - slack_oauth_client_id: Option, - slack_oauth_client_secret: Option, } pub(crate) async fn tarball_workspace( @@ -638,6 +595,7 @@ pub(crate) async fn tarball_workspace( include_users, include_groups, include_settings, + include_datatable_permissions, include_key, include_workspace_dependencies, skip_datatable_migrations, @@ -1593,7 +1551,6 @@ pub(crate) async fn tarball_workspace( auto_invite, webhook, error_handler, - success_handler, ai_config, large_file_storage, git_sync, @@ -1606,9 +1563,7 @@ pub(crate) async fn tarball_workspace( datatable, slack_team_id, slack_name, - slack_command_script, - slack_oauth_client_id, - slack_oauth_client_secret + slack_command_script FROM workspace_settings LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id WHERE workspace_id = $1"#, @@ -1617,73 +1572,23 @@ pub(crate) async fn tarball_workspace( .fetch_one(&mut *tx) .await?; - // Use v2 format only if explicitly requested, otherwise use v1 (legacy) for backward compatibility - // Server-owned state (the HMAC webhook secret + hook id/error, the - // synced-sha / last-pull status, the admin automatic pulls run as, and what - // the credential check observed) must never leave the server: keep it out of - // export archives and synced repos, and don't let a re-imported workspace - // inherit another install's hook/sync state or pull identity. - fn redact_git_sync_for_export(git_sync: Option) -> Option { - let mut git_sync = git_sync?; - if let Some(repos) = git_sync - .get_mut("repositories") - .and_then(|r| r.as_array_mut()) - { - for repo in repos { - if let Some(auto_pull) = - repo.get_mut("auto_pull").and_then(|a| a.as_object_mut()) - { - for field in [ - "webhook_secret", - "webhook_id", - "webhook_error", - "last_synced_sha", - "last_pull_status", - "enabled_by", - ] { - auto_pull.remove(field); - } - } - // What this install observed about its own credential: a token - // id and expiry, and a `checked_at` that moves on its own. - // None of it describes the workspace, and in a git-synced - // `wmill.yaml` it would churn the file for no reason. - if let Some(repo) = repo.as_object_mut() { - repo.remove("credential"); - } - } - } - Some(git_sync) - } - let settings_str = if settings_version.as_deref() == Some("v2") { - let settings = SimplifiedSettings { - auto_invite: row.auto_invite, - webhook: row.webhook, - error_handler: row.error_handler, - success_handler: row.success_handler, - ai_config: row.ai_config, - large_file_storage: row.large_file_storage, - git_sync: redact_git_sync_for_export(row.git_sync), - default_app: row.default_app, - default_scripts: row.default_scripts, - name: row.name.clone().unwrap_or_default(), - mute_critical_alerts: row.mute_critical_alerts, - color: row.color.clone(), - operator_settings: row.operator_settings.clone(), - datatable: windmill_common::workspaces::strip_datatable_permissions( - row.datatable.clone(), - ), - slack_team_id: row.slack_team_id.clone(), - slack_name: row.slack_name.clone(), - slack_command_script: row.slack_command_script.clone(), - slack_oauth_client_id: row.slack_oauth_client_id.clone(), - slack_oauth_client_secret: row.slack_oauth_client_secret.clone(), + let secrets = if export_plain_secrets { + crate::workspace_settings_document::SecretMode::Plain + } else { + crate::workspace_settings_document::SecretMode::Encrypted }; - serde_json::to_value(settings) - .map(|v| serde_json::to_string_pretty(&v).ok()) - .ok() - .flatten() + let mut doc = crate::workspace_settings_document::load_settings_document( + &mut *tx, &db, &w_id, secrets, + ) + .await?; + // A repo syncs its rules through protection-rules.yaml, so settings.yaml leaving + // them out keeps one writer for them. + doc.remove("protection_rules"); + if !include_datatable_permissions.unwrap_or(false) { + doc.remove("datatable_permissions"); + } + serde_json::to_string_pretty(&doc).ok() } else { // V1 (legacy) format: convert JSONB to flat fields (matches main branch exactly) let (auto_invite_enabled, auto_invite_as, auto_invite_mode) = @@ -1730,7 +1635,7 @@ pub(crate) async fn tarball_workspace( error_handler_muted_on_cancel, ai_config: row.ai_config, large_file_storage: row.large_file_storage, - git_sync: redact_git_sync_for_export(row.git_sync), + git_sync: crate::workspace_settings_document::redact_git_sync(row.git_sync), default_app: row.default_app, default_scripts: row.default_scripts, name: row.name.unwrap_or_default(), diff --git a/backend/windmill-common/src/error.rs b/backend/windmill-common/src/error.rs index a54e9eb399..dd8ec49f92 100644 --- a/backend/windmill-common/src/error.rs +++ b/backend/windmill-common/src/error.rs @@ -289,9 +289,9 @@ pub fn error_source_chain(e: &dyn std::error::Error) -> String { msg } -impl IntoResponse for Error { - fn into_response(self) -> axum::response::Response { - let status = match self { +impl Error { + pub fn status_code(&self) -> axum::http::StatusCode { + match self { Self::NotFound(_) => axum::http::StatusCode::NOT_FOUND, Self::NotAuthorized(_) => axum::http::StatusCode::UNAUTHORIZED, Self::RequireAdmin(_) | Self::PermissionDenied(_) => axum::http::StatusCode::FORBIDDEN, @@ -301,9 +301,15 @@ impl IntoResponse for Error { | Self::JsonErr(_) | Self::QuotaExceeded(_) => axum::http::StatusCode::BAD_REQUEST, Self::BadGateway(_) => axum::http::StatusCode::BAD_GATEWAY, - Self::Generic(status_code, _) => status_code, + Self::Generic(status_code, _) => *status_code, _ => axum::http::StatusCode::INTERNAL_SERVER_ERROR, - }; + } + } +} + +impl IntoResponse for Error { + fn into_response(self) -> axum::response::Response { + let status = self.status_code(); let e = &self; diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index b7a2f19d86..ac9c43269a 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -373,6 +373,10 @@ pub async fn get_secret_value_as_admin( Ok(r) } +/// Marks a workspace setting's secret written into the settings document encrypted with the +/// workspace key, as `wmill sync pull` writes it. +pub const SETTINGS_ENCRYPTED_PREFIX: &str = "$encrypted:"; + pub fn encrypt(mc: &WorkspaceCrypt, value: &str) -> String { mc.encrypt_str(value) } diff --git a/backend/windmill-common/src/workspace_crypt.rs b/backend/windmill-common/src/workspace_crypt.rs index 6bdea28391..b8114c3abb 100644 --- a/backend/windmill-common/src/workspace_crypt.rs +++ b/backend/windmill-common/src/workspace_crypt.rs @@ -21,6 +21,7 @@ use aes_gcm::aead::{Aead, AeadCore, KeyInit, OsRng}; use aes_gcm::{Aes256Gcm, Key, Nonce}; use base64::Engine; use hkdf::Hkdf; +use hmac::{Hmac, Mac}; use magic_crypt::{MagicCrypt256, MagicCryptTrait}; use sha2::Sha256; @@ -36,6 +37,7 @@ const NARROW_VARIABLE_VALUE_WIDTH: usize = 15000; pub const MAX_BYTES_OVERHEAD: usize = V2_MAGIC.len() + NONCE_LEN + BLOCK + TAG_LEN; const AEAD_KEY_INFO: &[u8] = b"windmill workspace secret aes-256-gcm v1"; +const SYNTHETIC_NONCE_KEY_INFO: &[u8] = b"windmill workspace secret synthetic nonce v1"; const B64: base64::engine::GeneralPurpose = base64::engine::general_purpose::STANDARD; @@ -79,6 +81,7 @@ impl CipherFormat { pub struct WorkspaceCrypt { legacy: MagicCrypt256, aead: Aes256Gcm, + synthetic_nonce_key: [u8; 32], } /// Code that drives `magic_crypt` directly on this cipher (app link secrets) keeps the legacy @@ -99,9 +102,14 @@ impl WorkspaceCrypt { Hkdf::::new(None, crypt_key.as_bytes()) .expand(AEAD_KEY_INFO, &mut aead_key) .expect("32 bytes is a valid HKDF-SHA256 output length"); + let mut synthetic_nonce_key = [0u8; 32]; + Hkdf::::new(None, crypt_key.as_bytes()) + .expand(SYNTHETIC_NONCE_KEY_INFO, &mut synthetic_nonce_key) + .expect("32 bytes is a valid HKDF-SHA256 output length"); Self { legacy: magic_crypt::new_magic_crypt!(crypt_key, 256), aead: Aes256Gcm::new(Key::::from_slice(&aead_key)), + synthetic_nonce_key, } } @@ -132,6 +140,18 @@ impl WorkspaceCrypt { } } + /// A v2 value whose nonce is an HMAC of the plaintext, so the same plaintext always gives + /// the same ciphertext. For values written into synced files, where a fresh nonce on every + /// export would show as a change. It reveals that two values are equal, nothing more; it + /// decrypts with [`Self::decrypt_str`] like any v2 value. + pub fn encrypt_str_deterministic(&self, plaintext: &str) -> String { + let mut mac = as Mac>::new_from_slice(&self.synthetic_nonce_key) + .expect("HMAC accepts any key length"); + mac.update(plaintext.as_bytes()); + let tag = mac.finalize().into_bytes(); + B64.encode(self.seal_with_nonce(plaintext.as_bytes(), Nonce::from_slice(&tag[..NONCE_LEN]))) + } + pub fn decrypt_str(&self, ciphertext: &str) -> Result { let bytes = B64 .decode(ciphertext) @@ -175,18 +195,21 @@ impl WorkspaceCrypt { } fn seal(&self, plaintext: &[u8]) -> Vec { - let nonce = Aes256Gcm::generate_nonce(&mut OsRng); + self.seal_with_nonce(plaintext, &Aes256Gcm::generate_nonce(&mut OsRng)) + } + + fn seal_with_nonce(&self, plaintext: &[u8], nonce: &Nonce<::NonceSize>) -> Vec { let pad = BLOCK - plaintext.len() % BLOCK; let mut padded = Vec::with_capacity(plaintext.len() + pad); padded.extend_from_slice(plaintext); padded.resize(plaintext.len() + pad, pad as u8); let sealed = self .aead - .encrypt(&nonce, padded.as_slice()) + .encrypt(nonce, padded.as_slice()) .expect("AES-GCM encryption fails only past 64 GiB of plaintext"); let mut out = Vec::with_capacity(V2_MAGIC.len() + NONCE_LEN + sealed.len()); out.extend_from_slice(&V2_MAGIC); - out.extend_from_slice(&nonce); + out.extend_from_slice(nonce); out.extend_from_slice(&sealed); out } @@ -234,6 +257,17 @@ mod tests { } } + #[test] + fn deterministic_is_stable_per_plaintext_and_key() { + let wc = WorkspaceCrypt::new(KEY); + let a = wc.encrypt_str_deterministic("secret"); + assert_eq!(a, wc.encrypt_str_deterministic("secret")); + assert_ne!(a, wc.encrypt_str_deterministic("secret2")); + assert_ne!(a, WorkspaceCrypt::new("other-key").encrypt_str_deterministic("secret")); + assert!(wc.is_v2(&a)); + assert_eq!(wc.decrypt_str(&a).unwrap(), "secret"); + } + #[test] fn legacy_cbc_still_decrypts() { let wc = WorkspaceCrypt::new(KEY); diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 6c8bda7b6d..367aa9146e 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -144,6 +144,15 @@ impl From<&Vec> for ProtectionRules { } } +impl ProtectionRules { + /// The rules set, in declaration order. + pub fn kinds(self) -> Vec { + ::iter() + .filter(|k| self.contains(k.flag())) + .collect() + } +} + #[derive(Serialize, Deserialize, Debug, Default)] pub struct WorkspaceGitSyncSettings { #[serde(skip_serializing_if = "Option::is_none")] @@ -1547,9 +1556,10 @@ pub struct DataTable { /// which is how data tables behaved before roles existed. Only meaningful on a terminal entry; /// a pointer is governed by what it points at. /// - /// Never leaves the instance: stripped from the workspace export and ignored on import, since - /// tenants are workspace-scoped names and syncing them would make repo write access a second - /// door onto the access decision. + /// Stripped from the `datatable` setting wherever it is read or written as a whole. It leaves + /// the instance only as the settings document's `datatable_permissions`, which `wmill sync` + /// carries only when the repo opts in: tenants are workspace-scoped names, and syncing them + /// makes repo write access a second door onto the access decision. #[serde(default, skip_serializing_if = "Option::is_none")] pub permissions: Option, } diff --git a/cli/src/commands/init/template.ts b/cli/src/commands/init/template.ts index 540d8c7978..1f78c13148 100644 --- a/cli/src/commands/init/template.ts +++ b/cli/src/commands/init/template.ts @@ -109,6 +109,9 @@ export const CONFIG_REFERENCE: ConfigOption[] = [ commented: true, templateValue: "true" }, { name: "includeSettings", type: "boolean", default: "false", description: "Include workspace settings in sync", commented: true, templateValue: "true" }, + { name: "includeDatatablePermissions", type: "boolean", default: "false", + description: "With includeSettings, also sync each data table's roles and who may connect as them", + commented: true, templateValue: "true" }, { name: "includeKey", type: "boolean", default: "false", description: "Include encryption key in sync", commented: true, templateValue: "true" }, diff --git a/cli/src/commands/sync/pull.ts b/cli/src/commands/sync/pull.ts index d75dc81912..51709b7d73 100644 --- a/cli/src/commands/sync/pull.ts +++ b/cli/src/commands/sync/pull.ts @@ -86,7 +86,8 @@ export async function downloadZip( skipWorkspaceDependencies?: boolean, skipDatatableMigrations?: boolean, defaultTs?: "bun" | "deno", - syncBehavior?: string + syncBehavior?: string, + includeDatatablePermissions?: boolean, ): Promise { const requestHeaders = new Headers(); requestHeaders.set("Authorization", "Bearer " + workspace.token); @@ -112,6 +113,7 @@ export async function downloadZip( }&skip_secrets=${skipSecrets ?? false}&include_schedules=${includeSchedules ?? false }&include_triggers=${includeTriggers ?? false}&include_users=${includeUsers ?? false }&include_groups=${includeGroups ?? false}&include_settings=${includeSettings ?? false + }&include_datatable_permissions=${includeDatatablePermissions ?? false }&include_key=${includeKey ?? false}&include_workspace_dependencies=${includeWorkspaceDependenciesValue}&skip_datatable_migrations=${skipDatatableMigrations ?? false}&default_ts=${defaultTs ?? "bun"}&skip_resource_types=${skipResourceTypes ?? false}&settings_version=v2&preserve_extra_perms=true&sync_behavior_version=${syncBehavior ?? "v0"}`; const baseUrl = workspace.remote + "api/w/" + workspace.workspaceId + "/workspaces/tarball?"; diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 5ba8888332..cf58347cfc 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -14,6 +14,7 @@ import { colors } from "@cliffy/ansi/colors"; import { Command } from "@cliffy/command"; import { Confirm } from "@cliffy/prompt/confirm"; import * as log from "../../core/log.ts"; +import { SETTINGS_MANAGED_ONLY_WHEN_PRESENT } from "../../core/settings.ts"; import * as path from "node:path"; import { sep as SEP } from "node:path"; import { @@ -2576,6 +2577,14 @@ const isWorkspaceSettingsFile = (p: string) => function stripUnappliedSettingsFields(local: any, remote: any) { delete local?.name; delete remote?.name; + for (const key of ["operator_settings", ...SETTINGS_MANAGED_ONLY_WHEN_PRESENT]) { + if (local?.[key] === undefined) delete remote?.[key]; + } + // push clears an omitted mute_critical_alerts to false + if (local?.mute_critical_alerts == null && remote?.mute_critical_alerts === false) { + delete local?.mute_critical_alerts; + delete remote.mute_critical_alerts; + } if (local?.color == null) { delete local?.color; delete remote?.color; @@ -3843,6 +3852,7 @@ export async function pull( opts.skipDatatableMigrations, opts.defaultTs, opts.syncBehavior, + opts.includeDatatablePermissions, ); const remote = ZipFSElement( @@ -4946,6 +4956,7 @@ export async function push( opts.skipDatatableMigrations, opts.defaultTs, opts.syncBehavior, + opts.includeDatatablePermissions, ))!, !opts.json, opts.defaultTs ?? "bun", @@ -6049,6 +6060,7 @@ export async function push( (opts.yes ?? false) || !process.stdin.isTTY, skipReencrypt: opts.skipReencryptOnKeyChange, }, + includeDatatablePermissions: opts.includeDatatablePermissions, defaultTs: opts.defaultTs, }, ); @@ -6187,6 +6199,7 @@ export async function push( (opts.yes ?? false) || !process.stdin.isTTY, skipReencrypt: opts.skipReencryptOnKeyChange, }, + includeDatatablePermissions: opts.includeDatatablePermissions, defaultTs: opts.defaultTs, }, ); @@ -6815,6 +6828,10 @@ const command = new Command() .option("--include-users", "Include syncing users") .option("--include-groups", "Include syncing groups") .option("--include-settings", "Include syncing workspace settings") + .option( + "--include-datatable-permissions", + "With --include-settings, also sync who may connect to each data table as which role", + ) .option("--include-key", "Include workspace encryption key") .option( "--keep-deleted", @@ -6878,6 +6895,10 @@ const command = new Command() .option("--include-users", "Include syncing users") .option("--include-groups", "Include syncing groups") .option("--include-settings", "Include syncing workspace settings") + .option( + "--include-datatable-permissions", + "With --include-settings, also sync who may connect to each data table as which role", + ) .option("--include-key", "Include workspace encryption key") .option( "--skip-reencrypt-on-key-change", diff --git a/cli/src/commands/workspace/settings.ts b/cli/src/commands/workspace/settings.ts new file mode 100644 index 0000000000..11aca145fa --- /dev/null +++ b/cli/src/commands/workspace/settings.ts @@ -0,0 +1,75 @@ +import { colors } from "@cliffy/ansi/colors"; +import { Confirm } from "@cliffy/prompt/confirm"; +import { stringify as yamlStringify } from "yaml"; +import { GlobalOptions } from "../../types.ts"; +import * as log from "../../core/log.ts"; +import { requireLogin } from "../../core/auth.ts"; +import { resolveWorkspace } from "../../core/context.ts"; +import { yamlParseFile } from "../../utils/yaml.ts"; +import * as wmill from "../../../gen/services.gen.ts"; + +export async function getSettings( + opts: GlobalOptions & { encrypted?: boolean; json?: boolean }, +) { + await requireLogin(opts); + const workspace = await resolveWorkspace(opts); + const document = await wmill.getWorkspaceSettingsDocument({ + workspace: workspace.workspaceId, + secrets: opts.encrypted ? "encrypted" : "redacted", + }); + console.log( + opts.json ? JSON.stringify(document, null, 2) : yamlStringify(document), + ); +} + +export async function applySettings( + opts: GlobalOptions & { dryRun?: boolean; replace?: boolean; yes?: boolean }, + file: string, +) { + await requireLogin(opts); + const workspace = await resolveWorkspace(opts); + const document = await yamlParseFile(file); + if (!document || typeof document !== "object" || Array.isArray(document)) { + throw new Error(`${file} must hold a settings document (a mapping of setting to value)`); + } + const apply = (dry_run: boolean) => + wmill.applyWorkspaceSettingsDocument({ + workspace: workspace.workspaceId, + requestBody: { document, replace: opts.replace ?? false, dry_run }, + }); + + const plan = await apply(true); + for (const warning of plan.warnings ?? []) { + log.warn(warning); + } + if (plan.changes.length === 0) { + log.info(colors.green("Workspace settings are already up to date")); + return; + } + for (const change of plan.changes) { + log.info( + `${colors.bold(change.key)}: ${JSON.stringify(change.before)} -> ${JSON.stringify(change.after)}`, + ); + } + if (opts.dryRun) { + return; + } + if ( + !opts.yes && + !(await Confirm.prompt({ + message: `Apply ${plan.changes.length} change(s) to workspace ${workspace.workspaceId}?`, + default: true, + })) + ) { + return; + } + const result = await apply(false); + for (const warning of result.warnings ?? []) { + log.warn(warning); + } + log.info( + colors.bold.underline.green( + `Applied ${result.changes.length} change(s) to workspace ${workspace.workspaceId}`, + ), + ); +} diff --git a/cli/src/commands/workspace/workspace.ts b/cli/src/commands/workspace/workspace.ts index 9e12afe54f..caac17136a 100644 --- a/cli/src/commands/workspace/workspace.ts +++ b/cli/src/commands/workspace/workspace.ts @@ -18,6 +18,7 @@ import { requireLogin } from "../../core/auth.ts"; import { createWorkspaceFork, deleteWorkspaceFork } from "./fork.ts"; import { mergeWorkspaces } from "./merge.ts"; import { connectSlack, disconnectSlack } from "./slack.ts"; +import { applySettings, getSettings } from "./settings.ts"; import * as wmill from "../../../gen/services.gen.ts"; @@ -905,6 +906,22 @@ Arguments (omit both to be prompted interactively): .description( "Clear slack_team_id / slack_name on the active workspace (marks the workspace as disconnected). Does NOT remove the bot token variable/resource/folder/group — delete those from the local sync folder and run 'wmill sync push' to tear them down. Does NOT remove the workspace-level OAuth override — set slack_oauth_client_id/_secret to '' in settings.yaml and push." ) - .action(disconnectSlack as any); + .action(disconnectSlack as any) + .command("get-settings") + .description( + "Print the active workspace's settings in the layout of the settings.yaml that 'wmill sync' writes. Secrets show as __SENSITIVE_AND_UNCHANGED__ unless --encrypted. Admin only." + ) + .option("--encrypted", "Write secrets encrypted with the workspace key, as 'wmill sync pull' does") + .option("--json", "Output as JSON") + .action(getSettings as any) + .command("apply-settings") + .description( + "Apply a settings document (YAML or JSON, in the layout of settings.yaml) to the active workspace. Only the settings the file states are changed, unless --replace; a setting set to null is cleared. A secret left as __SENSITIVE_AND_UNCHANGED__ keeps its value. Shows the changes and asks before applying. Admin only." + ) + .arguments("") + .option("--dry-run", "Only show the changes") + .option("--replace", "Also clear the settings the file omits, as 'wmill sync push' does") + .option("-y --yes", "Apply without asking") + .action(applySettings as any); export default command; diff --git a/cli/src/core/conf.ts b/cli/src/core/conf.ts index 1ad0b69c1b..b010b1f8e3 100644 --- a/cli/src/core/conf.ts +++ b/cli/src/core/conf.ts @@ -89,6 +89,9 @@ export interface SyncOptions { includeUsers?: boolean; includeGroups?: boolean; includeSettings?: boolean; + /** With includeSettings: settings.yaml also states who may connect to each data table as which + * role. Off by default, since tenants and roles are specific to a workspace and an instance. */ + includeDatatablePermissions?: boolean; includeKey?: boolean; skipReencryptOnKeyChange?: boolean; skipBranchValidation?: boolean; @@ -388,6 +391,7 @@ export const DEFAULT_SYNC_OPTIONS: Readonly< | "includeUsers" | "includeGroups" | "includeSettings" + | "includeDatatablePermissions" | "includeKey" | "nonDottedPaths" | "syncBehavior" @@ -410,6 +414,7 @@ export const DEFAULT_SYNC_OPTIONS: Readonly< includeUsers: false, includeGroups: false, includeSettings: false, + includeDatatablePermissions: false, includeKey: false, skipWorkspaceDependencies: false, skipDatatableMigrations: false, diff --git a/cli/src/core/settings.ts b/cli/src/core/settings.ts index 75dfa3b8da..502d0037bb 100644 --- a/cli/src/core/settings.ts +++ b/cli/src/core/settings.ts @@ -58,8 +58,35 @@ export interface SimplifiedSettings { slack_command_script?: string; slack_oauth_client_id?: string; slack_oauth_client_secret?: string; + ducklake?: any; + dbt_warehouses?: any; + deploy_ui?: any; + teams_command_script?: string | null; + public_app_execution_limit_per_minute?: number | null; + guest_access_enabled?: boolean; + guest_jwt_public_key?: string | null; + guest_jwt_jwks_url?: string | null; + error_handler_fallback_to_instance_alerts?: boolean; + add_admins_and_developers_to_forks?: boolean; + datatable_permissions?: Record | null; } +/** Settings the server manages only when settings.yaml states them: a file written before + * they existed must neither clear them nor show as differing from the workspace. */ +export const SETTINGS_MANAGED_ONLY_WHEN_PRESENT = [ + "datatable_permissions", + "ducklake", + "dbt_warehouses", + "deploy_ui", + "teams_command_script", + "public_app_execution_limit_per_minute", + "guest_access_enabled", + "guest_jwt_public_key", + "guest_jwt_jwks_url", + "error_handler_fallback_to_instance_alerts", + "add_admins_and_developers_to_forks", +] as const; + // Legacy settings interface for reading old settings.yaml files interface LegacySimplifiedSettings { auto_invite_enabled?: boolean; @@ -108,6 +135,9 @@ export function migrateToGroupedFormat(settings: any): SimplifiedSettings { if (settings.slack_command_script !== undefined) result.slack_command_script = settings.slack_command_script; if (settings.slack_oauth_client_id !== undefined) result.slack_oauth_client_id = settings.slack_oauth_client_id; if (settings.slack_oauth_client_secret !== undefined) result.slack_oauth_client_secret = settings.slack_oauth_client_secret; + for (const key of SETTINGS_MANAGED_ONLY_WHEN_PRESENT) { + if (settings[key] !== undefined) (result as any)[key] = settings[key]; + } // Handle auto_invite: check if already grouped or needs migration if (settings.auto_invite && typeof settings.auto_invite === "object") { @@ -173,10 +203,37 @@ export async function pushWorkspaceSettings( workspace: string, _path: string, settings: SimplifiedSettings | undefined, - localSettings: SimplifiedSettings | any + localSettings: SimplifiedSettings | any, + opts: { includeDatatablePermissions?: boolean } = {}, ) { + if (localSettings?.protection_rules !== undefined) { + log.warn( + "Ignoring protection_rules in settings.yaml: protection-rules.yaml syncs them (wmill protection-rules push)." + ); + } // Migrate local settings from legacy format if needed localSettings = migrateToGroupedFormat(localSettings); + if (!opts.includeDatatablePermissions) { + delete localSettings.datatable_permissions; + } + + if (await applySettingsDocument(workspace, localSettings)) { + return; + } + // The per-setting endpoints of a server without the settings document know nothing of + // these keys, nor of a secret it would have to decrypt or keep. + for (const key of SETTINGS_MANAGED_ONLY_WHEN_PRESENT) { + delete (localSettings as any)[key]; + } + const secret = localSettings.slack_oauth_client_secret; + const slackSecretUnreadable = + typeof secret === "string" && + (secret.startsWith("$encrypted:") || secret === "__SENSITIVE_AND_UNCHANGED__"); + if (slackSecretUnreadable) { + log.warn( + "Leaving the Slack OAuth client and secret unchanged: this server cannot read an encrypted or redacted secret. Upgrade it, or write the secret in plain text." + ); + } try { const remoteSettings = await wmill.getSettings({ @@ -456,8 +513,9 @@ export async function pushWorkspaceSettings( // every other setting): both present → upsert; anything else → delete. // Pull always emits both fields as null when remote is NULL. if ( - localSettings.slack_oauth_client_id != settings.slack_oauth_client_id || - localSettings.slack_oauth_client_secret != settings.slack_oauth_client_secret + !slackSecretUnreadable && + (localSettings.slack_oauth_client_id != settings.slack_oauth_client_id || + localSettings.slack_oauth_client_secret != settings.slack_oauth_client_secret) ) { log.debug(`Updating slack oauth config...`); if ( @@ -477,6 +535,38 @@ export async function pushWorkspaceSettings( } } +/** Applies settings.yaml server-side, which owns the rules for what an omitted or null setting + * means. Returns false on a server that predates the settings document. */ +async function applySettingsDocument( + workspace: string, + document: SimplifiedSettings, +): Promise { + let result; + try { + result = await wmill.applyWorkspaceSettingsDocument({ + workspace, + requestBody: { document: document as unknown as Record, replace: true }, + }); + } catch (err: any) { + // A missing route answers 404 with no body; a handler's own 404 carries a message. + if (err?.status === 404 && !err?.body) { + return false; + } + throw new Error(`Failed to apply workspace settings: ${err?.body ?? err}`); + } + if (result.changes.length > 0) { + log.debug( + `Updated workspace settings: ${result.changes.map((c) => c.key).join(", ")}`, + ); + } else { + log.debug(`Workspace settings are up to date`); + } + for (const warning of result.warnings ?? []) { + log.warn(warning); + } + return true; +} + export interface PushWorkspaceKeyOptions { // True when no prompt may be shown (e.g. `--yes` was passed or stdin is not a // TTY). In that case the re-encryption decision is taken from `skipReencrypt` diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 57e5eca4e9..e71d5e96fc 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -8889,6 +8889,7 @@ sync local with a remote workspaces or the opposite (push or pull) - \`--include-users\` - Include syncing users - \`--include-groups\` - Include syncing groups - \`--include-settings\` - Include syncing workspace settings + - \`--include-datatable-permissions\` - With --include-settings, also sync who may connect to each data table as which role - \`--include-key\` - Include workspace encryption key - \`--keep-deleted\` - Do not delete local files for items that no longer exist on the remote workspace. Only adds and updates. - \`--skip-branch-validation\` - Skip git branch validation and prompts @@ -8920,6 +8921,7 @@ sync local with a remote workspaces or the opposite (push or pull) - \`--include-users\` - Include syncing users - \`--include-groups\` - Include syncing groups - \`--include-settings\` - Include syncing workspace settings + - \`--include-datatable-permissions\` - With --include-settings, also sync who may connect to each data table as which role - \`--include-key\` - Include workspace encryption key - \`--skip-reencrypt-on-key-change\` - When the pushed encryption key differs from the remote, do NOT re-encrypt existing remote secrets. Only safe if they are already encrypted with the new key (e.g. workspace/instance migration). Default is to re-encrypt. - \`--keep-deleted\` - Do not delete remote items that no longer exist locally. Only adds and updates. @@ -9113,6 +9115,13 @@ Arguments (omit both to be prompted interactively): - \`--team-id \` - Slack team id - \`--team-name \` - Slack team name - \`workspace disconnect-slack\` - Clear slack_team_id / slack_name on the active workspace (marks the workspace as disconnected). Does NOT remove the bot token variable/resource/folder/group — delete those from the local sync folder and run 'wmill sync push' to tear them down. Does NOT remove the workspace-level OAuth override — set slack_oauth_client_id/_secret to '' in settings.yaml and push. +- \`workspace get-settings\` - Print the active workspace's settings in the layout of the settings.yaml that 'wmill sync' writes. Secrets show as __SENSITIVE_AND_UNCHANGED__ unless --encrypted. Admin only. + - \`--encrypted\` - Write secrets encrypted with the workspace key, as 'wmill sync pull' does + - \`--json\` - Output as JSON +- \`workspace apply-settings \` - Apply a settings document (YAML or JSON, in the layout of settings.yaml) to the active workspace. Only the settings the file states are changed, unless --replace; a setting set to null is cleared. A secret left as __SENSITIVE_AND_UNCHANGED__ keeps its value. Shows the changes and asks before applying. Admin only. + - \`--dry-run\` - Only show the changes + - \`--replace\` - Also clear the settings the file omits, as 'wmill sync push' does + - \`-y --yes\` - Apply without asking diff --git a/cli/src/types.ts b/cli/src/types.ts index 866b2be21e..b908b5a7ba 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -187,6 +187,8 @@ export interface PushObjOptions { wsSpecific?: boolean; /** encryption_key push: non-interactive flag and explicit re-encryption choice */ keyPushOpts?: PushWorkspaceKeyOptions; + /** settings push: whether settings.yaml manages the data tables' roles */ + includeDatatablePermissions?: boolean; /** TypeScript runtime a bare `.ts` denotes, for raw-app runnables */ defaultTs?: "bun" | "deno"; /** schedule push into a fork: the file's `enabled` is the parent's */ @@ -218,6 +220,7 @@ export async function pushObj( permissionedAsContext, wsSpecific, keyPushOpts, + includeDatatablePermissions, defaultTs, enabledOwnedByParent, } = opts; @@ -285,7 +288,9 @@ export async function pushObj( } else if (typeEnding === "workspace_dependencies") { await pushWorkspaceDependencies(workspace, p, befObj, newObj); } else if (typeEnding === "settings") { - await pushWorkspaceSettings(workspace, p, befObj, newObj); + await pushWorkspaceSettings(workspace, p, befObj, newObj, { + includeDatatablePermissions, + }); } else if (typeEnding === "encryption_key") { await pushWorkspaceKey(workspace, p, befObj, newObj, keyPushOpts); } else { diff --git a/cli/test/push_workspace_settings_auto_invite_unit.test.ts b/cli/test/push_workspace_settings_auto_invite_unit.test.ts index f4564c25b2..d04b7f4337 100644 --- a/cli/test/push_workspace_settings_auto_invite_unit.test.ts +++ b/cli/test/push_workspace_settings_auto_invite_unit.test.ts @@ -21,6 +21,10 @@ const remoteAutoInvite = { // Every wmill.* call reachable from pushWorkspaceSettings is stubbed so the // function runs without a backend. mockServices({ + // A server without the settings document, so the per-setting endpoints run. + applyWorkspaceSettingsDocument: async () => { + throw { status: 404 }; + }, getSettings: async () => ({ auto_invite: remoteAutoInvite }), getWorkspaceName: async () => "phoenix", changeWorkspaceName: async () => {}, diff --git a/cli/test/push_workspace_settings_identity_unit.test.ts b/cli/test/push_workspace_settings_identity_unit.test.ts index d11b5efac4..16b3bd34b4 100644 --- a/cli/test/push_workspace_settings_identity_unit.test.ts +++ b/cli/test/push_workspace_settings_identity_unit.test.ts @@ -17,6 +17,10 @@ let remoteWebhook: string | undefined = undefined; // Every wmill.* call reachable from pushWorkspaceSettings is stubbed so the // function runs without a backend; only the three we assert on record calls. mockServices({ + // A server without the settings document, so the per-setting endpoints run. + applyWorkspaceSettingsDocument: async () => { + throw { status: 404 }; + }, getSettings: async (_a: { workspace: string }) => ({ webhook: remoteWebhook, color: remoteColor, diff --git a/system_prompts/auto-generated/cli/cli-commands.md b/system_prompts/auto-generated/cli/cli-commands.md index 16e0407f5f..a51e592bec 100644 --- a/system_prompts/auto-generated/cli/cli-commands.md +++ b/system_prompts/auto-generated/cli/cli-commands.md @@ -618,6 +618,7 @@ sync local with a remote workspaces or the opposite (push or pull) - `--include-users` - Include syncing users - `--include-groups` - Include syncing groups - `--include-settings` - Include syncing workspace settings + - `--include-datatable-permissions` - With --include-settings, also sync who may connect to each data table as which role - `--include-key` - Include workspace encryption key - `--keep-deleted` - Do not delete local files for items that no longer exist on the remote workspace. Only adds and updates. - `--skip-branch-validation` - Skip git branch validation and prompts @@ -649,6 +650,7 @@ sync local with a remote workspaces or the opposite (push or pull) - `--include-users` - Include syncing users - `--include-groups` - Include syncing groups - `--include-settings` - Include syncing workspace settings + - `--include-datatable-permissions` - With --include-settings, also sync who may connect to each data table as which role - `--include-key` - Include workspace encryption key - `--skip-reencrypt-on-key-change` - When the pushed encryption key differs from the remote, do NOT re-encrypt existing remote secrets. Only safe if they are already encrypted with the new key (e.g. workspace/instance migration). Default is to re-encrypt. - `--keep-deleted` - Do not delete remote items that no longer exist locally. Only adds and updates. @@ -842,6 +844,13 @@ Arguments (omit both to be prompted interactively): - `--team-id ` - Slack team id - `--team-name ` - Slack team name - `workspace disconnect-slack` - Clear slack_team_id / slack_name on the active workspace (marks the workspace as disconnected). Does NOT remove the bot token variable/resource/folder/group — delete those from the local sync folder and run 'wmill sync push' to tear them down. Does NOT remove the workspace-level OAuth override — set slack_oauth_client_id/_secret to '' in settings.yaml and push. +- `workspace get-settings` - Print the active workspace's settings in the layout of the settings.yaml that 'wmill sync' writes. Secrets show as __SENSITIVE_AND_UNCHANGED__ unless --encrypted. Admin only. + - `--encrypted` - Write secrets encrypted with the workspace key, as 'wmill sync pull' does + - `--json` - Output as JSON +- `workspace apply-settings ` - Apply a settings document (YAML or JSON, in the layout of settings.yaml) to the active workspace. Only the settings the file states are changed, unless --replace; a setting set to null is cleared. A secret left as __SENSITIVE_AND_UNCHANGED__ keeps its value. Shows the changes and asks before applying. Admin only. + - `--dry-run` - Only show the changes + - `--replace` - Also clear the settings the file omits, as 'wmill sync push' does + - `-y --yes` - Apply without asking diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 0fa0dde4c9..83355cd6b0 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -4262,6 +4262,7 @@ sync local with a remote workspaces or the opposite (push or pull) - \`--include-users\` - Include syncing users - \`--include-groups\` - Include syncing groups - \`--include-settings\` - Include syncing workspace settings + - \`--include-datatable-permissions\` - With --include-settings, also sync who may connect to each data table as which role - \`--include-key\` - Include workspace encryption key - \`--keep-deleted\` - Do not delete local files for items that no longer exist on the remote workspace. Only adds and updates. - \`--skip-branch-validation\` - Skip git branch validation and prompts @@ -4293,6 +4294,7 @@ sync local with a remote workspaces or the opposite (push or pull) - \`--include-users\` - Include syncing users - \`--include-groups\` - Include syncing groups - \`--include-settings\` - Include syncing workspace settings + - \`--include-datatable-permissions\` - With --include-settings, also sync who may connect to each data table as which role - \`--include-key\` - Include workspace encryption key - \`--skip-reencrypt-on-key-change\` - When the pushed encryption key differs from the remote, do NOT re-encrypt existing remote secrets. Only safe if they are already encrypted with the new key (e.g. workspace/instance migration). Default is to re-encrypt. - \`--keep-deleted\` - Do not delete remote items that no longer exist locally. Only adds and updates. @@ -4486,6 +4488,13 @@ Arguments (omit both to be prompted interactively): - \`--team-id \` - Slack team id - \`--team-name \` - Slack team name - \`workspace disconnect-slack\` - Clear slack_team_id / slack_name on the active workspace (marks the workspace as disconnected). Does NOT remove the bot token variable/resource/folder/group — delete those from the local sync folder and run 'wmill sync push' to tear them down. Does NOT remove the workspace-level OAuth override — set slack_oauth_client_id/_secret to '' in settings.yaml and push. +- \`workspace get-settings\` - Print the active workspace's settings in the layout of the settings.yaml that 'wmill sync' writes. Secrets show as __SENSITIVE_AND_UNCHANGED__ unless --encrypted. Admin only. + - \`--encrypted\` - Write secrets encrypted with the workspace key, as 'wmill sync pull' does + - \`--json\` - Output as JSON +- \`workspace apply-settings \` - Apply a settings document (YAML or JSON, in the layout of settings.yaml) to the active workspace. Only the settings the file states are changed, unless --replace; a setting set to null is cleared. A secret left as __SENSITIVE_AND_UNCHANGED__ keeps its value. Shows the changes and asks before applying. Admin only. + - \`--dry-run\` - Only show the changes + - \`--replace\` - Also clear the settings the file omits, as 'wmill sync push' does + - \`-y --yes\` - Apply without asking diff --git a/system_prompts/auto-generated/skills/cli-commands/SKILL.md b/system_prompts/auto-generated/skills/cli-commands/SKILL.md index 74973e102c..4c67c57d9a 100644 --- a/system_prompts/auto-generated/skills/cli-commands/SKILL.md +++ b/system_prompts/auto-generated/skills/cli-commands/SKILL.md @@ -623,6 +623,7 @@ sync local with a remote workspaces or the opposite (push or pull) - `--include-users` - Include syncing users - `--include-groups` - Include syncing groups - `--include-settings` - Include syncing workspace settings + - `--include-datatable-permissions` - With --include-settings, also sync who may connect to each data table as which role - `--include-key` - Include workspace encryption key - `--keep-deleted` - Do not delete local files for items that no longer exist on the remote workspace. Only adds and updates. - `--skip-branch-validation` - Skip git branch validation and prompts @@ -654,6 +655,7 @@ sync local with a remote workspaces or the opposite (push or pull) - `--include-users` - Include syncing users - `--include-groups` - Include syncing groups - `--include-settings` - Include syncing workspace settings + - `--include-datatable-permissions` - With --include-settings, also sync who may connect to each data table as which role - `--include-key` - Include workspace encryption key - `--skip-reencrypt-on-key-change` - When the pushed encryption key differs from the remote, do NOT re-encrypt existing remote secrets. Only safe if they are already encrypted with the new key (e.g. workspace/instance migration). Default is to re-encrypt. - `--keep-deleted` - Do not delete remote items that no longer exist locally. Only adds and updates. @@ -847,6 +849,13 @@ Arguments (omit both to be prompted interactively): - `--team-id ` - Slack team id - `--team-name ` - Slack team name - `workspace disconnect-slack` - Clear slack_team_id / slack_name on the active workspace (marks the workspace as disconnected). Does NOT remove the bot token variable/resource/folder/group — delete those from the local sync folder and run 'wmill sync push' to tear them down. Does NOT remove the workspace-level OAuth override — set slack_oauth_client_id/_secret to '' in settings.yaml and push. +- `workspace get-settings` - Print the active workspace's settings in the layout of the settings.yaml that 'wmill sync' writes. Secrets show as __SENSITIVE_AND_UNCHANGED__ unless --encrypted. Admin only. + - `--encrypted` - Write secrets encrypted with the workspace key, as 'wmill sync pull' does + - `--json` - Output as JSON +- `workspace apply-settings ` - Apply a settings document (YAML or JSON, in the layout of settings.yaml) to the active workspace. Only the settings the file states are changed, unless --replace; a setting set to null is cleared. A secret left as __SENSITIVE_AND_UNCHANGED__ keeps its value. Shows the changes and asks before applying. Admin only. + - `--dry-run` - Only show the changes + - `--replace` - Also clear the settings the file omits, as 'wmill sync push' does + - `-y --yes` - Apply without asking