From 69c8574113ca2b46460fb74590920cd75039af3c Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 3 Nov 2024 02:37:19 +0100 Subject: [PATCH] feat(cli): instance yaml files inside the instance prefix folders (#4627) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(cli): instance yaml files inside the instance folders * fix: add opt-in for save instance yamls inside prefixed folders * chore: change baseUrl not needed to opts.baseUrl * do not reassign global const path * set prefix from instance --------- Co-authored-by: Wítallo <15958770+witalloliveira@users.noreply.github.com> --- cli/instance.ts | 43 +++++++++------- cli/settings.ts | 127 ++++++++++++++++++++++++++++++++---------------- cli/user.ts | 72 +++++++++++++++++++-------- 3 files changed, 163 insertions(+), 79 deletions(-) diff --git a/cli/instance.ts b/cli/instance.ts index 1e86282d1a..fb495c714f 100644 --- a/cli/instance.ts +++ b/cli/instance.ts @@ -184,6 +184,7 @@ export type InstanceSyncOptions = { folderPerInstance?: boolean; yes?: boolean; prefix?: string; + prefixSettings?: boolean; }; export async function pickInstance( @@ -265,21 +266,22 @@ async function instancePull(opts: GlobalOptions & InstanceSyncOptions) { log.info("Pulling instance-level changes"); log.info(`remote (${instance.name}) -> local`); + opts.prefix = opts.prefix ?? instance.prefix; let uChanges = 0; if (!opts.skipUsers) { - uChanges = (await pullInstanceUsers(true)) as number; + uChanges = (await pullInstanceUsers(opts, true)) as number; } let sChanges = 0; if (!opts.skipSettings) { - sChanges = (await pullInstanceSettings(true)) as number; + sChanges = (await pullInstanceSettings(opts, true)) as number; } let cChanges = 0; if (!opts.skipConfigs) { - cChanges = (await pullInstanceConfigs(true)) as number; + cChanges = (await pullInstanceConfigs(opts, true)) as number; } let gChanges = 0; if (!opts.skipGroups) { - gChanges = (await pullInstanceGroups(true)) as number; + gChanges = (await pullInstanceGroups(opts, true)) as number; } const totalChanges = uChanges + sChanges + cChanges + gChanges; @@ -295,16 +297,16 @@ async function instancePull(opts: GlobalOptions & InstanceSyncOptions) { if (confirm) { if (!opts.skipUsers && uChanges > 0) { - await pullInstanceUsers(); + await pullInstanceUsers(opts); } if (!opts.skipSettings && sChanges > 0) { - await pullInstanceSettings(); + await pullInstanceSettings(opts); } if (!opts.skipConfigs && cChanges > 0) { - await pullInstanceConfigs(); + await pullInstanceConfigs(opts); } if (!opts.skipGroups && gChanges > 0) { - await pullInstanceGroups(); + await pullInstanceGroups(opts); } } } else { @@ -400,19 +402,19 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) { let uChanges = 0; if (!opts.skipUsers) { - uChanges = (await pushInstanceUsers(true)) as number; + uChanges = (await pushInstanceUsers(opts, true)) as number; } let sChanges = 0; if (!opts.skipSettings) { - sChanges = (await pushInstanceSettings(true, opts.baseUrl)) as number; + sChanges = (await pushInstanceSettings(opts, true)) as number; } let cChanges = 0; if (!opts.skipConfigs) { - cChanges = (await pushInstanceConfigs(true)) as number; + cChanges = (await pushInstanceConfigs(opts, true)) as number; } let gChanges = 0; if (!opts.skipGroups) { - gChanges = (await pushInstanceGroups(true)) as number; + gChanges = (await pushInstanceGroups(opts, true)) as number; } const totalChanges = uChanges + sChanges + cChanges + gChanges; @@ -428,16 +430,16 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) { if (confirm) { if (!opts.skipUsers && uChanges > 0) { - await pushInstanceUsers(); + await pushInstanceUsers(opts); } if (!opts.skipSettings && sChanges > 0) { - await pushInstanceSettings(false, opts.baseUrl); + await pushInstanceSettings(opts); } if (!opts.skipConfigs && cChanges > 0) { - await pushInstanceConfigs(); + await pushInstanceConfigs(opts); } if (!opts.skipGroups && gChanges > 0) { - await pushInstanceGroups(); + await pushInstanceGroups(opts); } } } else { @@ -709,7 +711,10 @@ const command = new Command() "--prefix ", "Prefix of the local workspaces to pull, used to create the folders when using --include-workspaces" ) - + .option( + "--prefix-settings", + "Store instance yamls inside prefixed folders when using --prefix and --folder-per-instance" + ) .action(instancePull as any) .command("push") .description( @@ -730,6 +735,10 @@ const command = new Command() "--prefix ", "Prefix of the local workspaces folders to push" ) + .option( + "--prefix-settings", + "Store instance yamls inside prefixed folders when using --prefix and --folder-per-instance" + ) .action(instancePush as any) .command("whoami") .description("Display information about the currently logged-in user") diff --git a/cli/settings.ts b/cli/settings.ts index f9dbac1fa6..d5feeca100 100644 --- a/cli/settings.ts +++ b/cli/settings.ts @@ -3,8 +3,8 @@ import { Confirm } from "./deps.ts"; import { colors } from "./deps.ts"; import { yamlParseFile } from "./deps.ts"; import { log } from "./deps.ts"; -import { compareInstanceObjects } from "./instance.ts"; -import { isSuperset } from "./types.ts"; +import { compareInstanceObjects, InstanceSyncOptions } from "./instance.ts"; +import { isSuperset, GlobalOptions } from "./types.ts"; import { deepEqual } from "./utils.ts"; import * as wmill from "./gen/services.gen.ts"; import { Config, GlobalSetting } from "./gen/types.gen.ts"; @@ -113,10 +113,10 @@ export async function pushWorkspaceSettings( workspace, requestBody: localSettings.auto_invite_enabled ? { - operator: localSettings.auto_invite_as === "operator", - invite_all: true, - auto_add: localSettings.auto_invite_mode === "add", - } + operator: localSettings.auto_invite_as === "operator", + invite_all: true, + auto_add: localSettings.auto_invite_mode === "add", + } : {}, }); } catch (_) { @@ -128,10 +128,10 @@ export async function pushWorkspaceSettings( workspace, requestBody: localSettings.auto_invite_enabled ? { - operator: localSettings.auto_invite_as === "operator", - invite_all: false, - auto_add: localSettings.auto_invite_mode === "add", - } + operator: localSettings.auto_invite_as === "operator", + invite_all: false, + auto_add: localSettings.auto_invite_mode === "add", + } : {}, }); } @@ -156,7 +156,7 @@ export async function pushWorkspaceSettings( settings.error_handler_extra_args ) || localSettings.error_handler_muted_on_cancel !== - settings.error_handler_muted_on_cancel + settings.error_handler_muted_on_cancel ) { log.debug(`Updating error handler...`); await wmill.editErrorHandler({ @@ -262,30 +262,41 @@ export async function pushWorkspaceKey( const INSTANCE_SETTINGS_PATH = "instance_settings.yaml"; -export async function readInstanceSettings() { +export async function readInstanceSettings( + opts: GlobalOptions & InstanceSyncOptions +) { let localSettings: GlobalSetting[] = []; + let path = INSTANCE_SETTINGS_PATH; + if (opts.prefix && opts.folderPerInstance && opts.prefixSettings) { + path = `${opts.prefix}/${INSTANCE_SETTINGS_PATH}`; + } + try { - localSettings = (await yamlParseFile(INSTANCE_SETTINGS_PATH)) as GlobalSetting[]; + localSettings = (await yamlParseFile(path)) as GlobalSetting[]; } catch { - log.warn(`No ${INSTANCE_SETTINGS_PATH} found`); + log.warn(`No ${path} found`); } return localSettings; } - import { decrypt, encrypt } from "./local_encryption.ts"; -const SENSITIVE_FIELD: string[] = ["license_key", "jwt_secret"] +const SENSITIVE_FIELD: string[] = ["license_key", "jwt_secret"]; -async function processInstanceSettings(settings: GlobalSetting[], mode: "encode" | "decode"): Promise { +async function processInstanceSettings( + settings: GlobalSetting[], + mode: "encode" | "decode" +): Promise { const encKey = process.env.WMILL_INSTANCE_LOCAL_ENCRYPTION_KEY; if (encKey) { - const res: GlobalSetting[] = [] + const res: GlobalSetting[] = []; for (const s of settings) { if (SENSITIVE_FIELD.includes(s.name) && typeof s.value === "string") { - res.push(await processField(s, "value", encKey, mode) as GlobalSetting); + res.push( + (await processField(s, "value", encKey, mode)) as GlobalSetting + ); } else if (s.name == "oauths") { if (typeof s.value === "object") { const oauths = s.value as { [key: string]: any }; @@ -303,24 +314,40 @@ async function processInstanceSettings(settings: GlobalSetting[], mode: "encode" } return res; } else { - log.warn("No encryption key found, skipping encryption. Recommend setting WMILL_INSTANCE_LOCAL_ENCRYPTION_KEY"); + log.warn( + "No encryption key found, skipping encryption. Recommend setting WMILL_INSTANCE_LOCAL_ENCRYPTION_KEY" + ); } return settings; } -async function processField(obj: { [key: string]: any }, field: string, encKey: string, mode: "encode" | "decode"): Promise<{ [key: string]: any }> { +async function processField( + obj: { [key: string]: any }, + field: string, + encKey: string, + mode: "encode" | "decode" +): Promise<{ [key: string]: any }> { return { ...obj, - [field]: mode === "encode" ? await encrypt(obj[field], encKey) : await decrypt(obj[field], encKey) as any, - } + [field]: + mode === "encode" + ? await encrypt(obj[field], encKey) + : ((await decrypt(obj[field], encKey)) as any), + }; } -export async function pullInstanceSettings(preview = false) { +export async function pullInstanceSettings( + opts: GlobalOptions & InstanceSyncOptions, + preview = false +) { const remoteSettings = await wmill.listGlobalSettings(); if (preview) { - const localSettings: GlobalSetting[] = await readInstanceSettings(); - const processedSettings = await processInstanceSettings(remoteSettings, "encode"); + const localSettings: GlobalSetting[] = await readInstanceSettings(opts); + const processedSettings = await processInstanceSettings( + remoteSettings, + "encode" + ); return compareInstanceObjects( processedSettings, localSettings, @@ -330,7 +357,10 @@ export async function pullInstanceSettings(preview = false) { } else { log.info("Pulling settings from instance"); - const processedSettings = await processInstanceSettings(remoteSettings, "encode"); + const processedSettings = await processInstanceSettings( + remoteSettings, + "encode" + ); await Deno.writeTextFile( INSTANCE_SETTINGS_PATH, yamlStringify(processedSettings) @@ -341,19 +371,19 @@ export async function pullInstanceSettings(preview = false) { } export async function pushInstanceSettings( - preview: boolean = false, - baseUrl?: string + opts: GlobalOptions & InstanceSyncOptions, + preview: boolean = false ) { const remoteSettings = await wmill.listGlobalSettings(); - let localSettings: GlobalSetting[] = await readInstanceSettings(); + let localSettings: GlobalSetting[] = await readInstanceSettings(opts); localSettings = await processInstanceSettings(localSettings, "decode"); - if (baseUrl) { + if (opts.baseUrl) { localSettings = localSettings.filter((s) => s.name !== "base_url"); localSettings.push({ name: "base_url", //@ts-ignore - value: baseUrl, + value: opts.baseUrl, }); } @@ -404,18 +434,30 @@ export async function pushInstanceSettings( } } -export async function readLocalConfigs() { +const INSTANCE_CONFIGS_PATH = "instance_configs.yaml"; + +export async function readLocalConfigs( + opts: GlobalOptions & InstanceSyncOptions +) { let localConfigs: Config[] = []; + let path = INSTANCE_CONFIGS_PATH; + if (opts.prefix && opts.folderPerInstance && opts.prefixSettings) { + path = `${opts.prefix}/${INSTANCE_CONFIGS_PATH}`; + } + try { - localConfigs = (await yamlParseFile("instance_configs.yaml")) as Config[]; + localConfigs = (await yamlParseFile(path)) as Config[]; } catch { - log.warn("No instance_configs.yaml found"); + log.warn(`No ${path} found`); } return localConfigs; } -export async function pullInstanceConfigs(preview = false) { +export async function pullInstanceConfigs( + opts: GlobalOptions & InstanceSyncOptions, + preview = false +) { const remoteConfigs = (await wmill.listConfigs()).map((x) => { return { ...x, @@ -424,7 +466,7 @@ export async function pullInstanceConfigs(preview = false) { }); if (preview) { - const localConfigs: Config[] = await readLocalConfigs(); + const localConfigs: Config[] = await readLocalConfigs(opts); return compareInstanceObjects( remoteConfigs, @@ -436,22 +478,25 @@ export async function pullInstanceConfigs(preview = false) { log.info("Pulling configs from instance"); await Deno.writeTextFile( - "instance_configs.yaml", + INSTANCE_CONFIGS_PATH, yamlStringify(remoteConfigs as any) ); - log.info(colors.green("Configs written to instance_configs.yaml")); + log.info(colors.green(`Configs written to ${INSTANCE_CONFIGS_PATH}`)); } } -export async function pushInstanceConfigs(preview: boolean = false) { +export async function pushInstanceConfigs( + opts: GlobalOptions & InstanceSyncOptions, + preview: boolean = false +) { const remoteConfigs = (await wmill.listConfigs()).map((x) => { return { ...x, name: removeWorkerPrefix(x.name), }; }); - const localConfigs = await readLocalConfigs(); + const localConfigs = await readLocalConfigs(opts); if (preview) { return compareInstanceObjects( diff --git a/cli/user.ts b/cli/user.ts index 2ddeb997c9..c955113600 100644 --- a/cli/user.ts +++ b/cli/user.ts @@ -6,7 +6,7 @@ import { removeType, removePathPrefix, } from "./types.ts"; -import { compareInstanceObjects } from "./instance.ts"; +import { compareInstanceObjects, InstanceSyncOptions } from "./instance.ts"; import { colors, Command, @@ -386,47 +386,71 @@ export async function pushGroup( } } -export async function pullInstanceUsers(preview: boolean = false) { +const INSTANCE_USERS_PATH = "instance_users.yaml"; + +export async function pullInstanceUsers( + opts: GlobalOptions & InstanceSyncOptions, + preview: boolean = false +) { const remoteUsers = await wmill.globalUsersExport(); if (preview) { - const localUsers: ExportedUser[] = await readInstanceUsers(); + const localUsers: ExportedUser[] = await readInstanceUsers(opts); return compareInstanceObjects(remoteUsers, localUsers, "email", "user"); } else { log.info("Pulling users from instance..."); await Deno.writeTextFile( - "instance_users.yaml", + INSTANCE_USERS_PATH, yamlStringify(remoteUsers as any) ); - log.info(colors.green("Users written to instance_users.yaml")); + log.info(colors.green(`Users written to ${INSTANCE_USERS_PATH}`)); } } -export async function readInstanceUsers() { +export async function readInstanceUsers( + opts: GlobalOptions & InstanceSyncOptions +) { let localUsers: ExportedUser[] = []; + + let path = INSTANCE_USERS_PATH; + if (opts.prefix && opts.folderPerInstance && opts.prefixSettings) { + path = `${opts.prefix}/${INSTANCE_USERS_PATH}`; + } + try { - localUsers = (await yamlParseFile("instance_users.yaml")) as ExportedUser[]; + localUsers = (await yamlParseFile(path)) as ExportedUser[]; } catch { - log.warn("No instance_users.yaml file found"); + log.warn(`No ${path} file found`); } return localUsers; } -export async function readInstanceGroups() { +const INSTANCE_GROUPS_PATH = "instance_groups.yaml"; + +export async function readInstanceGroups( + opts: GlobalOptions & InstanceSyncOptions +) { let localGroups: InstanceGroup[] = []; + + let path = INSTANCE_GROUPS_PATH; + if (opts.prefix && opts.folderPerInstance && opts.prefixSettings) { + path = `${opts.prefix}/${INSTANCE_GROUPS_PATH}`; + } + try { - localGroups = (await yamlParseFile( - "instance_groups.yaml" - )) as ExportedInstanceGroup[]; + localGroups = (await yamlParseFile(path)) as ExportedInstanceGroup[]; } catch { - log.warn("No instance_groups.yaml file found"); + log.warn(`No ${path} file found`); } return localGroups; } -export async function pushInstanceUsers(preview: boolean = false) { +export async function pushInstanceUsers( + opts: GlobalOptions & InstanceSyncOptions, + preview: boolean = false +) { const remoteUsers = await wmill.globalUsersExport(); - const localUsers: ExportedUser[] = await readInstanceUsers(); + const localUsers: ExportedUser[] = await readInstanceUsers(opts); if (preview) { return compareInstanceObjects(localUsers, remoteUsers, "email", "user"); @@ -440,27 +464,33 @@ export async function pushInstanceUsers(preview: boolean = false) { } } -export async function pullInstanceGroups(preview = false) { +export async function pullInstanceGroups( + opts: GlobalOptions & InstanceSyncOptions, + preview = false +) { const remoteGroups = await wmill.exportInstanceGroups(); if (preview) { - const localGroups = await readInstanceGroups(); + const localGroups = await readInstanceGroups(opts); return compareInstanceObjects(remoteGroups, localGroups, "name", "group"); } else { log.info("Pulling groups from instance..."); await Deno.writeTextFile( - "instance_groups.yaml", + INSTANCE_GROUPS_PATH, yamlStringify(remoteGroups as any) ); - log.info(colors.green("Groups written to instance_groups.yaml")); + log.info(colors.green(`Groups written to ${INSTANCE_GROUPS_PATH}`)); } } -export async function pushInstanceGroups(preview: boolean = false) { +export async function pushInstanceGroups( + opts: GlobalOptions & InstanceSyncOptions, + preview: boolean = false +) { const remoteGroups = await wmill.exportInstanceGroups(); - const localGroups = await readInstanceGroups(); + const localGroups = await readInstanceGroups(opts); if (preview) { return compareInstanceObjects(localGroups, remoteGroups, "name", "group");