From bbce38490a93ea062f61bda5a55bfd3825f8476b Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Fri, 15 Mar 2024 20:07:48 +0100 Subject: [PATCH] feat: wmill sync workspace settings (#3425) * feat: wmill sync workspace settings * fix: build * fix: nit --- ...6f840f49002353d5c900a25a3024d7634a89c.json | 100 +++++++++ backend/windmill-api/openapi.yaml | 6 +- backend/windmill-api/src/workspaces.rs | 59 ++++- cli/conf.ts | 1 + cli/deps.ts | 4 +- cli/pull.ts | 5 +- cli/settings.ts | 201 ++++++++++++++++++ cli/sync.ts | 43 +++- cli/types.ts | 9 +- cli/user.ts | 4 + 10 files changed, 421 insertions(+), 11 deletions(-) create mode 100644 backend/.sqlx/query-8626f698fd20f2da77edcd9912a6f840f49002353d5c900a25a3024d7634a89c.json create mode 100644 cli/settings.ts diff --git a/backend/.sqlx/query-8626f698fd20f2da77edcd9912a6f840f49002353d5c900a25a3024d7634a89c.json b/backend/.sqlx/query-8626f698fd20f2da77edcd9912a6f840f49002353d5c900a25a3024d7634a89c.json new file mode 100644 index 0000000000..d122120b9a --- /dev/null +++ b/backend/.sqlx/query-8626f698fd20f2da77edcd9912a6f840f49002353d5c900a25a3024d7634a89c.json @@ -0,0 +1,100 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n -- slack_team_id, \n -- slack_name, \n -- slack_command_script, \n -- CASE WHEN slack_email = 'missing@email.xyz' THEN NULL ELSE slack_email END AS slack_email,\n auto_invite_domain IS NOT NULL AS \"auto_invite_enabled!\",\n CASE WHEN auto_invite_operator IS TRUE THEN 'operator' ELSE 'developer' END AS \"auto_invite_as!\", \n CASE WHEN auto_add IS TRUE THEN 'add' ELSE 'invite' END AS \"auto_invite_mode!\", \n webhook, \n deploy_to, \n error_handler, \n openai_resource_path, \n code_completion_enabled, \n error_handler_extra_args, \n error_handler_muted_on_cancel, \n large_file_storage, \n git_sync, \n default_app,\n default_scripts \n FROM workspace_settings\n WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "auto_invite_enabled!", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "auto_invite_as!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "auto_invite_mode!", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "webhook", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "deploy_to", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "error_handler", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "openai_resource_path", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "code_completion_enabled", + "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "error_handler_extra_args", + "type_info": "Json" + }, + { + "ordinal": 9, + "name": "error_handler_muted_on_cancel", + "type_info": "Bool" + }, + { + "ordinal": 10, + "name": "large_file_storage", + "type_info": "Jsonb" + }, + { + "ordinal": 11, + "name": "git_sync", + "type_info": "Jsonb" + }, + { + "ordinal": 12, + "name": "default_app", + "type_info": "Varchar" + }, + { + "ordinal": 13, + "name": "default_scripts", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + null, + null, + true, + true, + true, + true, + false, + true, + false, + true, + true, + true, + true + ] + }, + "hash": "8626f698fd20f2da77edcd9912a6f840f49002353d5c900a25a3024d7634a89c" +} diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 04fbddf31e..a33d7886fb 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1287,9 +1287,12 @@ paths: $ref: "#/components/schemas/WorkspaceGitSyncSettings" default_app: type: string + default_scripts: + $ref: "#/components/schemas/WorkspaceDefaultScripts" required: - code_completion_enabled - automatic_billing + - error_handler_muted_on_cancel /w/{workspace}/workspaces/get_deploy_to: get: @@ -9894,9 +9897,6 @@ components: additionalProperties: type: string - - - GitRepositorySettings: type: object properties: diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 26202390ac..8eb637ffc3 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -60,7 +60,7 @@ use crate::oauth2_ee::InstanceEvent; use crate::variables::{decrypt, encrypt}; use hyper::{header, StatusCode}; use serde::{Deserialize, Serialize}; -use serde_json::{json, Map}; +use serde_json::{json, Map, Value}; use sqlx::{FromRow, Postgres, Transaction}; use tempfile::TempDir; use tokio::fs::File; @@ -2155,6 +2155,7 @@ struct ArchiveQueryParams { include_schedules: Option, include_users: Option, include_groups: Option, + include_settings: Option, default_ts: Option, } @@ -2229,6 +2230,28 @@ struct SimplifiedGroup { admins: Vec, } +#[derive(Serialize)] +struct SimplifiedSettings { + // slack_team_id: Option, + // slack_name: Option, + // slack_command_script: Option, + // slack_email: Option, + auto_invite_enabled: bool, + auto_invite_as: String, + auto_invite_mode: String, + webhook: Option, + deploy_to: Option, + error_handler: Option, + error_handler_extra_args: Option, + error_handler_muted_on_cancel: bool, + openai_resource_path: Option, + code_completion_enabled: bool, + large_file_storage: Option, + git_sync: Option, + default_app: Option, + default_scripts: Option, +} + async fn tarball_workspace( authed: ApiAuthed, Extension(user_db): Extension, @@ -2244,6 +2267,7 @@ async fn tarball_workspace( include_schedules, include_users, include_groups, + include_settings, default_ts, }): Query, ) -> Result<([(headers::HeaderName, String); 2], impl IntoResponse)> { @@ -2560,6 +2584,39 @@ async fn tarball_workspace( } } + if include_settings.unwrap_or(false) { + let settings = sqlx::query_as!( + SimplifiedSettings, + r#"SELECT + -- slack_team_id, + -- slack_name, + -- slack_command_script, + -- CASE WHEN slack_email = 'missing@email.xyz' THEN NULL ELSE slack_email END AS slack_email, + auto_invite_domain IS NOT NULL AS "auto_invite_enabled!", + CASE WHEN auto_invite_operator IS TRUE THEN 'operator' ELSE 'developer' END AS "auto_invite_as!", + CASE WHEN auto_add IS TRUE THEN 'add' ELSE 'invite' END AS "auto_invite_mode!", + webhook, + deploy_to, + error_handler, + openai_resource_path, + code_completion_enabled, + error_handler_extra_args, + error_handler_muted_on_cancel, + large_file_storage, + git_sync, + default_app, + default_scripts + FROM workspace_settings + WHERE workspace_id = $1"#, + &w_id + ).fetch_one(&mut *tx).await?; + + let settings_str = &to_string_without_metadata(&settings, true, None).unwrap(); + archive + .write_to_archive(&settings_str, "settings.json") + .await?; + } + archive.finish().await?; let file = tokio::fs::File::open(&file_path).await?; diff --git a/cli/conf.ts b/cli/conf.ts index 420030262e..30439bf217 100644 --- a/cli/conf.ts +++ b/cli/conf.ts @@ -14,6 +14,7 @@ export interface SyncOptions { includeSchedules?: boolean; includeUsers?: boolean; includeGroups?: boolean; + includeSettings?: boolean; message?: string; includes?: string[]; extraIncludes?: string[]; diff --git a/cli/deps.ts b/cli/deps.ts index a410f09b5b..a23003e0f3 100644 --- a/cli/deps.ts +++ b/cli/deps.ts @@ -1,6 +1,6 @@ // windmill -export { setClient } from "https://deno.land/x/windmill@v1.283.0/mod.ts"; -export * from "https://deno.land/x/windmill@v1.283.0/windmill-api/index.ts"; +export { setClient } from "https://deno.land/x/windmill@v1.291.4/mod.ts"; +export * from "https://deno.land/x/windmill@v1.291.4/windmill-api/index.ts"; export { SEP } from "https://deno.land/std@0.201.0/path/separator.ts"; // cliffy export { Command } from "https://deno.land/x/cliffy@v1.0.0-rc.3/command/mod.ts"; diff --git a/cli/pull.ts b/cli/pull.ts index 091e06989b..5c99b9cc83 100644 --- a/cli/pull.ts +++ b/cli/pull.ts @@ -13,6 +13,7 @@ export async function downloadZip( includeSchedules?: boolean, includeUsers?: boolean, includeGroups?: boolean, + includeSettings?: boolean, defaultTs?: "bun" | "deno" ): Promise { const requestHeaders: HeadersInit = new Headers(); @@ -38,7 +39,9 @@ export async function downloadZip( includeSchedules ?? false }&include_users=${includeUsers ?? false}&include_groups=${ includeGroups ?? false - }&default_ts=${defaultTs ?? "deno"}`, + }&include_settings=${includeSettings ?? false}&default_ts=${ + defaultTs ?? "deno" + }`, { headers: requestHeaders, method: "GET", diff --git a/cli/settings.ts b/cli/settings.ts new file mode 100644 index 0000000000..a9ff102eff --- /dev/null +++ b/cli/settings.ts @@ -0,0 +1,201 @@ +import { WorkspaceService, log } from "./deps.ts"; +import { isSuperset } from "./types.ts"; +import { deepEqual } from "./utils.ts"; + +interface SimplifiedSettings { + // slack_team_id?: string; + // slack_name?: string; + // slack_command_script?: string; + // slack_email?: string; + auto_invite_enabled: boolean; + auto_invite_as: string; + auto_invite_mode: string; + webhook?: string; + deploy_to?: string; + error_handler?: string; + error_handler_extra_args?: any; + error_handler_muted_on_cancel?: boolean; + openai_resource_path?: string; + code_completion_enabled: boolean; + large_file_storage?: any; + git_sync?: any; + default_app?: string; + default_scripts?: any; +} + +export async function pushWorkspaceSettings( + workspace: string, + _path: string, + settings: SimplifiedSettings | undefined, + localSettings: SimplifiedSettings +) { + try { + const remoteSettings = await WorkspaceService.getSettings({ + workspace, + }); + settings = { + // slack_team_id: remoteSettings.slack_team_id, + // slack_name: remoteSettings.slack_name, + // slack_command_script: remoteSettings.slack_command_script, + // slack_email: remoteSettings.slack_email, + auto_invite_enabled: remoteSettings.auto_invite_domain !== null, + auto_invite_as: remoteSettings.auto_invite_operator + ? "operator" + : "developer", + auto_invite_mode: remoteSettings.auto_add ? "add" : "invite", + webhook: remoteSettings.webhook, + deploy_to: remoteSettings.deploy_to, + error_handler: remoteSettings.error_handler, + error_handler_extra_args: remoteSettings.error_handler_extra_args, + error_handler_muted_on_cancel: + remoteSettings.error_handler_muted_on_cancel, + openai_resource_path: remoteSettings.openai_resource_path, + code_completion_enabled: remoteSettings.code_completion_enabled, + large_file_storage: remoteSettings.large_file_storage, + git_sync: remoteSettings.git_sync, + default_app: remoteSettings.default_app, + default_scripts: remoteSettings.default_scripts, + }; + } catch (err) { + throw new Error(`Failed to get workspace settings: ${err}`); + } + + if (isSuperset(localSettings, settings)) { + log.debug(`Workspace settings are up to date`); + return; + } + log.debug(`Workspace settings are not up-to-date, updating...`); + if (localSettings.webhook !== settings.webhook) { + log.debug(`Updateing webhook...`); + await WorkspaceService.editWebhook({ + workspace, + requestBody: { + webhook: localSettings.webhook, + }, + }); + } + if ( + localSettings.auto_invite_as !== settings.auto_invite_as || + localSettings.auto_invite_enabled !== settings.auto_invite_enabled || + localSettings.auto_invite_mode !== settings.auto_invite_mode + ) { + log.debug(`Updating auto invite...`); + + if (!["operator", "developer"].includes(settings.auto_invite_as)) { + throw new Error( + `Invalid value for auto_invite_as. Valid values are "operator" and "developer"` + ); + } + + if (!["add", "invite"].includes(settings.auto_invite_mode)) { + throw new Error( + `Invalid value for auto_invite_mode. Valid values are "invite" and "add"` + ); + } + try { + await WorkspaceService.editAutoInvite({ + workspace, + requestBody: localSettings.auto_invite_enabled + ? { + operator: localSettings.auto_invite_as === "operator", + invite_all: true, + auto_add: localSettings.auto_invite_mode === "add", + } + : {}, + }); + } catch (_) { + // on cloud + log.debug( + `Auto invite is not possible on cloud, only auto-inviting same domain...` + ); + await WorkspaceService.editAutoInvite({ + workspace, + requestBody: localSettings.auto_invite_enabled + ? { + operator: localSettings.auto_invite_as === "operator", + invite_all: false, + auto_add: localSettings.auto_invite_mode === "add", + } + : {}, + }); + } + } + if ( + localSettings.openai_resource_path !== settings.openai_resource_path || + localSettings.code_completion_enabled !== settings.code_completion_enabled + ) { + log.debug(`Updating openai settings...`); + await WorkspaceService.editCopilotConfig({ + workspace, + requestBody: { + openai_resource_path: localSettings.openai_resource_path, + code_completion_enabled: localSettings.code_completion_enabled, + }, + }); + } + if ( + localSettings.error_handler !== settings.error_handler || + !deepEqual( + localSettings.error_handler_extra_args, + settings.error_handler_extra_args + ) || + localSettings.error_handler_muted_on_cancel !== + settings.error_handler_muted_on_cancel + ) { + log.debug(`Updating error handler...`); + await WorkspaceService.editErrorHandler({ + workspace, + requestBody: { + error_handler: localSettings.error_handler, + error_handler_extra_args: localSettings.error_handler_extra_args, + error_handler_muted_on_cancel: + localSettings.error_handler_muted_on_cancel, + }, + }); + } + if (localSettings.deploy_to !== settings.deploy_to) { + log.debug(`Updating deploy to...`); + await WorkspaceService.editDeployTo({ + workspace, + requestBody: { + deploy_to: localSettings.deploy_to, + }, + }); + } + if ( + !deepEqual(localSettings.large_file_storage, settings.large_file_storage) + ) { + log.debug(`Updating large file storage...`); + await WorkspaceService.editLargeFileStorageConfig({ + workspace, + requestBody: { + large_file_storage: localSettings.large_file_storage, + }, + }); + } + if (!deepEqual(localSettings.git_sync, settings.git_sync)) { + log.debug(`Updating git sync...`); + await WorkspaceService.editWorkspaceGitSyncConfig({ + workspace, + requestBody: { + git_sync_settings: localSettings.git_sync, + }, + }); + } + if (!deepEqual(localSettings.default_scripts, settings.default_scripts)) { + log.debug(`Updating default scripts...`); + await WorkspaceService.editDefaultScripts({ + workspace, + requestBody: localSettings.default_scripts, + }); + } + if (localSettings.default_app !== settings.default_app) { + log.debug(`Updating default app...`); + await WorkspaceService.editWorkspaceDefaultApp({ + workspace, + requestBody: { + default_app_path: localSettings.default_app, + }, + }); + } +} diff --git a/cli/sync.ts b/cli/sync.ts index a890387067..72bbff4800 100644 --- a/cli/sync.ts +++ b/cli/sync.ts @@ -22,6 +22,8 @@ import { ScheduleService, SEP, gitignore_parser, + UserService, + GroupService, } from "./deps.ts"; import { getTypeStrFromPath, @@ -42,6 +44,7 @@ import { import { handleFile } from "./script.ts"; import { deepEqual } from "./utils.ts"; import { SyncOptions, mergeConfigWithConfigFile } from "./conf.ts"; +import { removePathPrefix } from "./types.ts"; type DynFSElement = { isDirectory: boolean; @@ -323,6 +326,7 @@ export async function elementsToMap( if (!skips.includeSchedules && path.endsWith(".schedule" + ext)) continue; if (!skips.includeUsers && path.endsWith(".user" + ext)) continue; if (!skips.includeGroups && path.endsWith(".group" + ext)) continue; + if (!skips.includeSettings && path === "settings" + ext) continue; if (skips.skipResources && path.endsWith(".resource" + ext)) continue; if (skips.skipVariables && path.endsWith(".variable" + ext)) continue; @@ -363,6 +367,7 @@ interface Skips { includeSchedules?: boolean | undefined; includeUsers?: boolean | undefined; includeGroups?: boolean | undefined; + includeSettings?: boolean | undefined; } async function compareDynFSElement( @@ -464,8 +469,10 @@ function getOrderFromPath(p: string) { return 8; } else if (typ == "group") { return 9; - } else { + } else if (typ == "settings") { return 10; + } else { + return 11; } } @@ -485,7 +492,7 @@ const isNotWmillFile = (p: string, isDirectory: boolean) => { try { const typ = getTypeStrFromPath(p); - if (typ == "resource-type") { + if (typ == "resource-type" || typ == "settings") { return p.includes(SEP); } else { return ( @@ -598,6 +605,7 @@ async function pull(opts: GlobalOptions & SyncOptions) { opts.includeSchedules, opts.includeUsers, opts.includeGroups, + opts.includeSettings, opts.defaultTs ))!, !opts.json @@ -811,6 +819,7 @@ async function push(opts: GlobalOptions & SyncOptions) { opts.includeSchedules, opts.includeUsers, opts.includeGroups, + opts.includeSettings, opts.defaultTs ))!, !opts.json @@ -1005,6 +1014,34 @@ async function push(opts: GlobalOptions & SyncOptions) { path: removeSuffix(change.path, ".variable.json"), }); break; + case "user": { + const users = await UserService.listUsers({ + workspace: workspaceId, + }); + + const email = removeSuffix( + removePathPrefix(change.path, "users"), + ".user.json" + ); + const user = users.find((u) => u.email === email); + if (!user) { + throw new Error(`User ${email} not found`); + } + await UserService.deleteUser({ + workspace: workspaceId, + username: user.username, + }); + break; + } + case "group": + await GroupService.deleteGroup({ + workspace: workspaceId, + name: removeSuffix( + removePathPrefix(change.path, "groups"), + ".group.json" + ), + }); + break; default: break; } @@ -1056,6 +1093,7 @@ const command = new Command() .option("--include-schedules", "Include syncing schedules") .option("--include-users", "Include syncing users") .option("--include-groups", "Include syncing groups") + .option("--include-settings", "Include syncing workspace settings") .option( "-i --includes ", "Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string)" @@ -1092,6 +1130,7 @@ const command = new Command() .option("--include-schedules", "Include syncing schedules") .option("--include-users", "Include syncing users") .option("--include-groups", "Include syncing groups") + .option("--include-settings", "Include syncing workspace settings") .option( "-i --includes ", "Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string)" diff --git a/cli/types.ts b/cli/types.ts index 972b918887..7d5af19ceb 100644 --- a/cli/types.ts +++ b/cli/types.ts @@ -14,6 +14,7 @@ import { deepEqual } from "./utils.ts"; import { pushSchedule } from "./schedule.ts"; import { pushWorkspaceUser } from "./user.ts"; import { pushGroup } from "./user.ts"; +import { pushWorkspaceSettings } from "./settings.ts"; export interface DifferenceCreate { type: "CREATE"; @@ -128,6 +129,8 @@ export async function pushObj( await pushWorkspaceUser(workspace, p, befObj, newObj); } else if (typeEnding === "group") { await pushGroup(workspace, p, befObj, newObj); + } else if (typeEnding === "settings") { + await pushWorkspaceSettings(workspace, p, befObj, newObj); } else { throw new Error( `The item ${p} has an unrecognized type ending ${typeEnding}` @@ -163,7 +166,8 @@ export function getTypeStrFromPath( | "app" | "schedule" | "user" - | "group" { + | "group" + | "settings" { if (p.includes(".flow" + path.sep)) { return "flow"; } @@ -193,7 +197,8 @@ export function getTypeStrFromPath( typeEnding === "app" || typeEnding === "schedule" || typeEnding === "user" || - typeEnding === "group" + typeEnding === "group" || + typeEnding === "settings" ) { return typeEnding; } else { diff --git a/cli/user.ts b/cli/user.ts index 5050c1fa83..f1cd6c75c6 100644 --- a/cli/user.ts +++ b/cli/user.ts @@ -141,6 +141,10 @@ export async function pushWorkspaceUser( //ignore } + if (user && user.username !== localUser.username) { + throw new Error("Username cannot be changed"); + } + if (user) { if (isSuperset(localUser, user)) { log.debug(`User ${email} is up to date`);