feat: wmill sync workspace settings (#3425)

* feat: wmill sync workspace settings

* fix: build

* fix: nit

backend/.sqlx/query-8626f698fd20f2da77edcd9912a6f840f49002353d5c900a25a3024d7634a89c.json

@@ -0,0 +1,100 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n                -- slack_team_id, \n                -- slack_name, \n                -- slack_command_script, \n                -- CASE WHEN slack_email = 'missing@email.xyz' THEN NULL ELSE slack_email END AS slack_email,\n                auto_invite_domain IS NOT NULL AS \"auto_invite_enabled!\",\n                CASE WHEN auto_invite_operator IS TRUE THEN 'operator' ELSE 'developer' END AS \"auto_invite_as!\", \n                CASE WHEN auto_add IS TRUE THEN 'add' ELSE 'invite' END AS \"auto_invite_mode!\", \n                webhook, \n                deploy_to, \n                error_handler, \n                openai_resource_path, \n                code_completion_enabled, \n                error_handler_extra_args, \n                error_handler_muted_on_cancel, \n                large_file_storage, \n                git_sync, \n                default_app,\n                default_scripts \n            FROM workspace_settings\n            WHERE workspace_id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "auto_invite_enabled!",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 1,
+        "name": "auto_invite_as!",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "auto_invite_mode!",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 3,
+        "name": "webhook",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 4,
+        "name": "deploy_to",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 5,
+        "name": "error_handler",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 6,
+        "name": "openai_resource_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 7,
+        "name": "code_completion_enabled",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 8,
+        "name": "error_handler_extra_args",
+        "type_info": "Json"
+      },
+      {
+        "ordinal": 9,
+        "name": "error_handler_muted_on_cancel",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 10,
+        "name": "large_file_storage",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 11,
+        "name": "git_sync",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 12,
+        "name": "default_app",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 13,
+        "name": "default_scripts",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null,
+      null,
+      null,
+      true,
+      true,
+      true,
+      true,
+      false,
+      true,
+      false,
+      true,
+      true,
+      true,
+      true
+    ]
+  },
+  "hash": "8626f698fd20f2da77edcd9912a6f840f49002353d5c900a25a3024d7634a89c"
+}

backend/windmill-api/openapi.yaml

@@ -1287,9 +1287,12 @@ paths:
                     $ref: "#/components/schemas/WorkspaceGitSyncSettings"
                   default_app:
                     type: string
+                  default_scripts:
+                    $ref: "#/components/schemas/WorkspaceDefaultScripts"
                 required:
                   - code_completion_enabled
                   - automatic_billing
+                  - error_handler_muted_on_cancel
 
   /w/{workspace}/workspaces/get_deploy_to:
     get:
@@ -9894,9 +9897,6 @@ components:
           additionalProperties:
             type: string
 
-        
-
-            
     GitRepositorySettings:
       type: object
       properties:

backend/windmill-api/src/workspaces.rs

@@ -60,7 +60,7 @@ use crate::oauth2_ee::InstanceEvent;
 use crate::variables::{decrypt, encrypt};
 use hyper::{header, StatusCode};
 use serde::{Deserialize, Serialize};
-use serde_json::{json, Map};
+use serde_json::{json, Map, Value};
 use sqlx::{FromRow, Postgres, Transaction};
 use tempfile::TempDir;
 use tokio::fs::File;
@@ -2155,6 +2155,7 @@ struct ArchiveQueryParams {
     include_schedules: Option<bool>,
     include_users: Option<bool>,
     include_groups: Option<bool>,
+    include_settings: Option<bool>,
     default_ts: Option<String>,
 }
 
@@ -2229,6 +2230,28 @@ struct SimplifiedGroup {
     admins: Vec<String>,
 }
 
+#[derive(Serialize)]
+struct SimplifiedSettings {
+    // slack_team_id: Option<String>,
+    // slack_name: Option<String>,
+    // slack_command_script: Option<String>,
+    // slack_email: Option<String>,
+    auto_invite_enabled: bool,
+    auto_invite_as: String,
+    auto_invite_mode: String,
+    webhook: Option<String>,
+    deploy_to: Option<String>,
+    error_handler: Option<String>,
+    error_handler_extra_args: Option<Value>,
+    error_handler_muted_on_cancel: bool,
+    openai_resource_path: Option<String>,
+    code_completion_enabled: bool,
+    large_file_storage: Option<Value>,
+    git_sync: Option<Value>,
+    default_app: Option<String>,
+    default_scripts: Option<Value>,
+}
+
 async fn tarball_workspace(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
@@ -2244,6 +2267,7 @@ async fn tarball_workspace(
         include_schedules,
         include_users,
         include_groups,
+        include_settings,
         default_ts,
     }): Query<ArchiveQueryParams>,
 ) -> Result<([(headers::HeaderName, String); 2], impl IntoResponse)> {
@@ -2560,6 +2584,39 @@ async fn tarball_workspace(
         }
     }
 
+    if include_settings.unwrap_or(false) {
+        let settings = sqlx::query_as!(
+            SimplifiedSettings,
+            r#"SELECT
+                -- slack_team_id, 
+                -- slack_name, 
+                -- slack_command_script, 
+                -- CASE WHEN slack_email = 'missing@email.xyz' THEN NULL ELSE slack_email END AS slack_email,
+                auto_invite_domain IS NOT NULL AS "auto_invite_enabled!",
+                CASE WHEN auto_invite_operator IS TRUE THEN 'operator' ELSE 'developer' END AS "auto_invite_as!", 
+                CASE WHEN auto_add IS TRUE THEN 'add' ELSE 'invite' END AS "auto_invite_mode!", 
+                webhook, 
+                deploy_to, 
+                error_handler, 
+                openai_resource_path, 
+                code_completion_enabled, 
+                error_handler_extra_args, 
+                error_handler_muted_on_cancel, 
+                large_file_storage, 
+                git_sync, 
+                default_app,
+                default_scripts 
+            FROM workspace_settings
+            WHERE workspace_id = $1"#,
+            &w_id
+        ).fetch_one(&mut *tx).await?;
+
+        let settings_str = &to_string_without_metadata(&settings, true, None).unwrap();
+        archive
+            .write_to_archive(&settings_str, "settings.json")
+            .await?;
+    }
+
     archive.finish().await?;
 
     let file = tokio::fs::File::open(&file_path).await?;

cli/conf.ts

@@ -14,6 +14,7 @@ export interface SyncOptions {
   includeSchedules?: boolean;
   includeUsers?: boolean;
   includeGroups?: boolean;
+  includeSettings?: boolean;
   message?: string;
   includes?: string[];
   extraIncludes?: string[];

cli/deps.ts

@@ -1,6 +1,6 @@
 // windmill
-export { setClient } from "https://deno.land/x/windmill@v1.283.0/mod.ts";
-export * from "https://deno.land/x/windmill@v1.283.0/windmill-api/index.ts";
+export { setClient } from "https://deno.land/x/windmill@v1.291.4/mod.ts";
+export * from "https://deno.land/x/windmill@v1.291.4/windmill-api/index.ts";
 export { SEP } from "https://deno.land/std@0.201.0/path/separator.ts";
 // cliffy
 export { Command } from "https://deno.land/x/cliffy@v1.0.0-rc.3/command/mod.ts";

cli/pull.ts

@@ -13,6 +13,7 @@ export async function downloadZip(
   includeSchedules?: boolean,
   includeUsers?: boolean,
   includeGroups?: boolean,
+  includeSettings?: boolean,
   defaultTs?: "bun" | "deno"
 ): Promise<JSZip | undefined> {
   const requestHeaders: HeadersInit = new Headers();
@@ -38,7 +39,9 @@ export async function downloadZip(
         includeSchedules ?? false
       }&include_users=${includeUsers ?? false}&include_groups=${
         includeGroups ?? false
-      }&default_ts=${defaultTs ?? "deno"}`,
+      }&include_settings=${includeSettings ?? false}&default_ts=${
+        defaultTs ?? "deno"
+      }`,
     {
       headers: requestHeaders,
       method: "GET",

cli/settings.ts

@@ -0,0 +1,201 @@
+import { WorkspaceService, log } from "./deps.ts";
+import { isSuperset } from "./types.ts";
+import { deepEqual } from "./utils.ts";
+
+interface SimplifiedSettings {
+  // slack_team_id?: string;
+  // slack_name?: string;
+  // slack_command_script?: string;
+  // slack_email?: string;
+  auto_invite_enabled: boolean;
+  auto_invite_as: string;
+  auto_invite_mode: string;
+  webhook?: string;
+  deploy_to?: string;
+  error_handler?: string;
+  error_handler_extra_args?: any;
+  error_handler_muted_on_cancel?: boolean;
+  openai_resource_path?: string;
+  code_completion_enabled: boolean;
+  large_file_storage?: any;
+  git_sync?: any;
+  default_app?: string;
+  default_scripts?: any;
+}
+
+export async function pushWorkspaceSettings(
+  workspace: string,
+  _path: string,
+  settings: SimplifiedSettings | undefined,
+  localSettings: SimplifiedSettings
+) {
+  try {
+    const remoteSettings = await WorkspaceService.getSettings({
+      workspace,
+    });
+    settings = {
+      // slack_team_id: remoteSettings.slack_team_id,
+      // slack_name: remoteSettings.slack_name,
+      // slack_command_script: remoteSettings.slack_command_script,
+      // slack_email: remoteSettings.slack_email,
+      auto_invite_enabled: remoteSettings.auto_invite_domain !== null,
+      auto_invite_as: remoteSettings.auto_invite_operator
+        ? "operator"
+        : "developer",
+      auto_invite_mode: remoteSettings.auto_add ? "add" : "invite",
+      webhook: remoteSettings.webhook,
+      deploy_to: remoteSettings.deploy_to,
+      error_handler: remoteSettings.error_handler,
+      error_handler_extra_args: remoteSettings.error_handler_extra_args,
+      error_handler_muted_on_cancel:
+        remoteSettings.error_handler_muted_on_cancel,
+      openai_resource_path: remoteSettings.openai_resource_path,
+      code_completion_enabled: remoteSettings.code_completion_enabled,
+      large_file_storage: remoteSettings.large_file_storage,
+      git_sync: remoteSettings.git_sync,
+      default_app: remoteSettings.default_app,
+      default_scripts: remoteSettings.default_scripts,
+    };
+  } catch (err) {
+    throw new Error(`Failed to get workspace settings: ${err}`);
+  }
+
+  if (isSuperset(localSettings, settings)) {
+    log.debug(`Workspace settings are up to date`);
+    return;
+  }
+  log.debug(`Workspace settings are not up-to-date, updating...`);
+  if (localSettings.webhook !== settings.webhook) {
+    log.debug(`Updateing webhook...`);
+    await WorkspaceService.editWebhook({
+      workspace,
+      requestBody: {
+        webhook: localSettings.webhook,
+      },
+    });
+  }
+  if (
+    localSettings.auto_invite_as !== settings.auto_invite_as ||
+    localSettings.auto_invite_enabled !== settings.auto_invite_enabled ||
+    localSettings.auto_invite_mode !== settings.auto_invite_mode
+  ) {
+    log.debug(`Updating auto invite...`);
+
+    if (!["operator", "developer"].includes(settings.auto_invite_as)) {
+      throw new Error(
+        `Invalid value for auto_invite_as. Valid values are "operator" and "developer"`
+      );
+    }
+
+    if (!["add", "invite"].includes(settings.auto_invite_mode)) {
+      throw new Error(
+        `Invalid value for auto_invite_mode. Valid values are "invite" and "add"`
+      );
+    }
+    try {
+      await WorkspaceService.editAutoInvite({
+        workspace,
+        requestBody: localSettings.auto_invite_enabled
+          ? {
+              operator: localSettings.auto_invite_as === "operator",
+              invite_all: true,
+              auto_add: localSettings.auto_invite_mode === "add",
+            }
+          : {},
+      });
+    } catch (_) {
+      // on cloud
+      log.debug(
+        `Auto invite is not possible on cloud, only auto-inviting same domain...`
+      );
+      await WorkspaceService.editAutoInvite({
+        workspace,
+        requestBody: localSettings.auto_invite_enabled
+          ? {
+              operator: localSettings.auto_invite_as === "operator",
+              invite_all: false,
+              auto_add: localSettings.auto_invite_mode === "add",
+            }
+          : {},
+      });
+    }
+  }
+  if (
+    localSettings.openai_resource_path !== settings.openai_resource_path ||
+    localSettings.code_completion_enabled !== settings.code_completion_enabled
+  ) {
+    log.debug(`Updating openai settings...`);
+    await WorkspaceService.editCopilotConfig({
+      workspace,
+      requestBody: {
+        openai_resource_path: localSettings.openai_resource_path,
+        code_completion_enabled: localSettings.code_completion_enabled,
+      },
+    });
+  }
+  if (
+    localSettings.error_handler !== settings.error_handler ||
+    !deepEqual(
+      localSettings.error_handler_extra_args,
+      settings.error_handler_extra_args
+    ) ||
+    localSettings.error_handler_muted_on_cancel !==
+      settings.error_handler_muted_on_cancel
+  ) {
+    log.debug(`Updating error handler...`);
+    await WorkspaceService.editErrorHandler({
+      workspace,
+      requestBody: {
+        error_handler: localSettings.error_handler,
+        error_handler_extra_args: localSettings.error_handler_extra_args,
+        error_handler_muted_on_cancel:
+          localSettings.error_handler_muted_on_cancel,
+      },
+    });
+  }
+  if (localSettings.deploy_to !== settings.deploy_to) {
+    log.debug(`Updating deploy to...`);
+    await WorkspaceService.editDeployTo({
+      workspace,
+      requestBody: {
+        deploy_to: localSettings.deploy_to,
+      },
+    });
+  }
+  if (
+    !deepEqual(localSettings.large_file_storage, settings.large_file_storage)
+  ) {
+    log.debug(`Updating large file storage...`);
+    await WorkspaceService.editLargeFileStorageConfig({
+      workspace,
+      requestBody: {
+        large_file_storage: localSettings.large_file_storage,
+      },
+    });
+  }
+  if (!deepEqual(localSettings.git_sync, settings.git_sync)) {
+    log.debug(`Updating git sync...`);
+    await WorkspaceService.editWorkspaceGitSyncConfig({
+      workspace,
+      requestBody: {
+        git_sync_settings: localSettings.git_sync,
+      },
+    });
+  }
+  if (!deepEqual(localSettings.default_scripts, settings.default_scripts)) {
+    log.debug(`Updating default scripts...`);
+    await WorkspaceService.editDefaultScripts({
+      workspace,
+      requestBody: localSettings.default_scripts,
+    });
+  }
+  if (localSettings.default_app !== settings.default_app) {
+    log.debug(`Updating default app...`);
+    await WorkspaceService.editWorkspaceDefaultApp({
+      workspace,
+      requestBody: {
+        default_app_path: localSettings.default_app,
+      },
+    });
+  }
+}

cli/sync.ts

@@ -22,6 +22,8 @@ import {
   ScheduleService,
   SEP,
   gitignore_parser,
+  UserService,
+  GroupService,
 } from "./deps.ts";
 import {
   getTypeStrFromPath,
@@ -42,6 +44,7 @@ import {
 import { handleFile } from "./script.ts";
 import { deepEqual } from "./utils.ts";
 import { SyncOptions, mergeConfigWithConfigFile } from "./conf.ts";
+import { removePathPrefix } from "./types.ts";
 
 type DynFSElement = {
   isDirectory: boolean;
@@ -323,6 +326,7 @@ export async function elementsToMap(
     if (!skips.includeSchedules && path.endsWith(".schedule" + ext)) continue;
     if (!skips.includeUsers && path.endsWith(".user" + ext)) continue;
     if (!skips.includeGroups && path.endsWith(".group" + ext)) continue;
+    if (!skips.includeSettings && path === "settings" + ext) continue;
     if (skips.skipResources && path.endsWith(".resource" + ext)) continue;
     if (skips.skipVariables && path.endsWith(".variable" + ext)) continue;
 
@@ -363,6 +367,7 @@ interface Skips {
   includeSchedules?: boolean | undefined;
   includeUsers?: boolean | undefined;
   includeGroups?: boolean | undefined;
+  includeSettings?: boolean | undefined;
 }
 
 async function compareDynFSElement(
@@ -464,8 +469,10 @@ function getOrderFromPath(p: string) {
     return 8;
   } else if (typ == "group") {
     return 9;
-  } else {
+  } else if (typ == "settings") {
     return 10;
+  } else {
+    return 11;
   }
 }
 
@@ -485,7 +492,7 @@ const isNotWmillFile = (p: string, isDirectory: boolean) => {
 
   try {
     const typ = getTypeStrFromPath(p);
-    if (typ == "resource-type") {
+    if (typ == "resource-type" || typ == "settings") {
       return p.includes(SEP);
     } else {
       return (
@@ -598,6 +605,7 @@ async function pull(opts: GlobalOptions & SyncOptions) {
       opts.includeSchedules,
       opts.includeUsers,
       opts.includeGroups,
+      opts.includeSettings,
       opts.defaultTs
     ))!,
     !opts.json
@@ -811,6 +819,7 @@ async function push(opts: GlobalOptions & SyncOptions) {
       opts.includeSchedules,
       opts.includeUsers,
       opts.includeGroups,
+      opts.includeSettings,
       opts.defaultTs
     ))!,
     !opts.json
@@ -1005,6 +1014,34 @@ async function push(opts: GlobalOptions & SyncOptions) {
               path: removeSuffix(change.path, ".variable.json"),
             });
             break;
+          case "user": {
+            const users = await UserService.listUsers({
+              workspace: workspaceId,
+            });
+
+            const email = removeSuffix(
+              removePathPrefix(change.path, "users"),
+              ".user.json"
+            );
+            const user = users.find((u) => u.email === email);
+            if (!user) {
+              throw new Error(`User ${email} not found`);
+            }
+            await UserService.deleteUser({
+              workspace: workspaceId,
+              username: user.username,
+            });
+            break;
+          }
+          case "group":
+            await GroupService.deleteGroup({
+              workspace: workspaceId,
+              name: removeSuffix(
+                removePathPrefix(change.path, "groups"),
+                ".group.json"
+              ),
+            });
+            break;
           default:
             break;
         }
@@ -1056,6 +1093,7 @@ const command = new Command()
   .option("--include-schedules", "Include syncing  schedules")
   .option("--include-users", "Include syncing users")
   .option("--include-groups", "Include syncing groups")
+  .option("--include-settings", "Include syncing workspace settings")
   .option(
     "-i --includes <patterns:file[]>",
     "Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string)"
@@ -1092,6 +1130,7 @@ const command = new Command()
   .option("--include-schedules", "Include syncing schedules")
   .option("--include-users", "Include syncing users")
   .option("--include-groups", "Include syncing groups")
+  .option("--include-settings", "Include syncing workspace settings")
   .option(
     "-i --includes <patterns:file[]>",
     "Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string)"

cli/types.ts

@@ -14,6 +14,7 @@ import { deepEqual } from "./utils.ts";
 import { pushSchedule } from "./schedule.ts";
 import { pushWorkspaceUser } from "./user.ts";
 import { pushGroup } from "./user.ts";
+import { pushWorkspaceSettings } from "./settings.ts";
 
 export interface DifferenceCreate {
   type: "CREATE";
@@ -128,6 +129,8 @@ export async function pushObj(
     await pushWorkspaceUser(workspace, p, befObj, newObj);
   } else if (typeEnding === "group") {
     await pushGroup(workspace, p, befObj, newObj);
+  } else if (typeEnding === "settings") {
+    await pushWorkspaceSettings(workspace, p, befObj, newObj);
   } else {
     throw new Error(
       `The item ${p} has an unrecognized type ending ${typeEnding}`
@@ -163,7 +166,8 @@ export function getTypeStrFromPath(
   | "app"
   | "schedule"
   | "user"
-  | "group" {
+  | "group"
+  | "settings" {
   if (p.includes(".flow" + path.sep)) {
     return "flow";
   }
@@ -193,7 +197,8 @@ export function getTypeStrFromPath(
     typeEnding === "app" ||
     typeEnding === "schedule" ||
     typeEnding === "user" ||
-    typeEnding === "group"
+    typeEnding === "group" ||
+    typeEnding === "settings"
   ) {
     return typeEnding;
   } else {

cli/user.ts

@@ -141,6 +141,10 @@ export async function pushWorkspaceUser(
     //ignore
   }
 
+  if (user && user.username !== localUser.username) {
+    throw new Error("Username cannot be changed");
+  }
+
   if (user) {
     if (isSuperset(localUser, user)) {
       log.debug(`User ${email} is up to date`);