feat(cli): instance yaml files inside the instance prefix folders (#4627)

* feat(cli): instance yaml files inside the instance folders

* fix: add opt-in for save instance yamls inside prefixed folders

* chore: change baseUrl not needed to opts.baseUrl

* do not reassign global const path

* set prefix from instance

---------

Co-authored-by: Wítallo <15958770+witalloliveira@users.noreply.github.com>

cli/instance.ts

@@ -184,6 +184,7 @@ export type InstanceSyncOptions = {
   folderPerInstance?: boolean;
   yes?: boolean;
   prefix?: string;
+  prefixSettings?: boolean;
 };
 
 export async function pickInstance(
@@ -265,21 +266,22 @@ async function instancePull(opts: GlobalOptions & InstanceSyncOptions) {
   log.info("Pulling instance-level changes");
   log.info(`remote (${instance.name}) -> local`);
 
+  opts.prefix = opts.prefix ?? instance.prefix;
   let uChanges = 0;
   if (!opts.skipUsers) {
-    uChanges = (await pullInstanceUsers(true)) as number;
+    uChanges = (await pullInstanceUsers(opts, true)) as number;
   }
   let sChanges = 0;
   if (!opts.skipSettings) {
-    sChanges = (await pullInstanceSettings(true)) as number;
+    sChanges = (await pullInstanceSettings(opts, true)) as number;
   }
   let cChanges = 0;
   if (!opts.skipConfigs) {
-    cChanges = (await pullInstanceConfigs(true)) as number;
+    cChanges = (await pullInstanceConfigs(opts, true)) as number;
   }
   let gChanges = 0;
   if (!opts.skipGroups) {
-    gChanges = (await pullInstanceGroups(true)) as number;
+    gChanges = (await pullInstanceGroups(opts, true)) as number;
   }
 
   const totalChanges = uChanges + sChanges + cChanges + gChanges;
@@ -295,16 +297,16 @@ async function instancePull(opts: GlobalOptions & InstanceSyncOptions) {
 
     if (confirm) {
       if (!opts.skipUsers && uChanges > 0) {
-        await pullInstanceUsers();
+        await pullInstanceUsers(opts);
       }
       if (!opts.skipSettings && sChanges > 0) {
-        await pullInstanceSettings();
+        await pullInstanceSettings(opts);
       }
       if (!opts.skipConfigs && cChanges > 0) {
-        await pullInstanceConfigs();
+        await pullInstanceConfigs(opts);
       }
       if (!opts.skipGroups && gChanges > 0) {
-        await pullInstanceGroups();
+        await pullInstanceGroups(opts);
       }
     }
   } else {
@@ -400,19 +402,19 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) {
 
   let uChanges = 0;
   if (!opts.skipUsers) {
-    uChanges = (await pushInstanceUsers(true)) as number;
+    uChanges = (await pushInstanceUsers(opts, true)) as number;
   }
   let sChanges = 0;
   if (!opts.skipSettings) {
-    sChanges = (await pushInstanceSettings(true, opts.baseUrl)) as number;
+    sChanges = (await pushInstanceSettings(opts, true)) as number;
   }
   let cChanges = 0;
   if (!opts.skipConfigs) {
-    cChanges = (await pushInstanceConfigs(true)) as number;
+    cChanges = (await pushInstanceConfigs(opts, true)) as number;
   }
   let gChanges = 0;
   if (!opts.skipGroups) {
-    gChanges = (await pushInstanceGroups(true)) as number;
+    gChanges = (await pushInstanceGroups(opts, true)) as number;
   }
 
   const totalChanges = uChanges + sChanges + cChanges + gChanges;
@@ -428,16 +430,16 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) {
 
     if (confirm) {
       if (!opts.skipUsers && uChanges > 0) {
-        await pushInstanceUsers();
+        await pushInstanceUsers(opts);
       }
       if (!opts.skipSettings && sChanges > 0) {
-        await pushInstanceSettings(false, opts.baseUrl);
+        await pushInstanceSettings(opts);
       }
       if (!opts.skipConfigs && cChanges > 0) {
-        await pushInstanceConfigs();
+        await pushInstanceConfigs(opts);
       }
       if (!opts.skipGroups && gChanges > 0) {
-        await pushInstanceGroups();
+        await pushInstanceGroups(opts);
       }
     }
   } else {
@@ -709,7 +711,10 @@ const command = new Command()
     "--prefix <prefix:string>",
     "Prefix of the local workspaces to pull, used to create the folders when using --include-workspaces"
   )
-
+  .option(
+    "--prefix-settings",
+    "Store instance yamls inside prefixed folders when using --prefix and --folder-per-instance"
+  )
   .action(instancePull as any)
   .command("push")
   .description(
@@ -730,6 +735,10 @@ const command = new Command()
     "--prefix <prefix:string>",
     "Prefix of the local workspaces folders to push"
   )
+  .option(
+    "--prefix-settings",
+    "Store instance yamls inside prefixed folders when using --prefix and --folder-per-instance"
+  )
   .action(instancePush as any)
   .command("whoami")
   .description("Display information about the currently logged-in user")

cli/settings.ts

@@ -3,8 +3,8 @@ import { Confirm } from "./deps.ts";
 import { colors } from "./deps.ts";
 import { yamlParseFile } from "./deps.ts";
 import { log } from "./deps.ts";
-import { compareInstanceObjects } from "./instance.ts";
-import { isSuperset } from "./types.ts";
+import { compareInstanceObjects, InstanceSyncOptions } from "./instance.ts";
+import { isSuperset, GlobalOptions } from "./types.ts";
 import { deepEqual } from "./utils.ts";
 import * as wmill from "./gen/services.gen.ts";
 import { Config, GlobalSetting } from "./gen/types.gen.ts";
@@ -113,10 +113,10 @@ export async function pushWorkspaceSettings(
         workspace,
         requestBody: localSettings.auto_invite_enabled
           ? {
-            operator: localSettings.auto_invite_as === "operator",
-            invite_all: true,
-            auto_add: localSettings.auto_invite_mode === "add",
-          }
+              operator: localSettings.auto_invite_as === "operator",
+              invite_all: true,
+              auto_add: localSettings.auto_invite_mode === "add",
+            }
           : {},
       });
     } catch (_) {
@@ -128,10 +128,10 @@ export async function pushWorkspaceSettings(
         workspace,
         requestBody: localSettings.auto_invite_enabled
           ? {
-            operator: localSettings.auto_invite_as === "operator",
-            invite_all: false,
-            auto_add: localSettings.auto_invite_mode === "add",
-          }
+              operator: localSettings.auto_invite_as === "operator",
+              invite_all: false,
+              auto_add: localSettings.auto_invite_mode === "add",
+            }
           : {},
       });
     }
@@ -156,7 +156,7 @@ export async function pushWorkspaceSettings(
       settings.error_handler_extra_args
     ) ||
     localSettings.error_handler_muted_on_cancel !==
-    settings.error_handler_muted_on_cancel
+      settings.error_handler_muted_on_cancel
   ) {
     log.debug(`Updating error handler...`);
     await wmill.editErrorHandler({
@@ -262,30 +262,41 @@ export async function pushWorkspaceKey(
 
 const INSTANCE_SETTINGS_PATH = "instance_settings.yaml";
 
-export async function readInstanceSettings() {
+export async function readInstanceSettings(
+  opts: GlobalOptions & InstanceSyncOptions
+) {
   let localSettings: GlobalSetting[] = [];
 
+  let path = INSTANCE_SETTINGS_PATH;
+  if (opts.prefix && opts.folderPerInstance && opts.prefixSettings) {
+    path = `${opts.prefix}/${INSTANCE_SETTINGS_PATH}`;
+  }
+
   try {
-    localSettings = (await yamlParseFile(INSTANCE_SETTINGS_PATH)) as GlobalSetting[];
+    localSettings = (await yamlParseFile(path)) as GlobalSetting[];
   } catch {
-    log.warn(`No ${INSTANCE_SETTINGS_PATH} found`);
+    log.warn(`No ${path} found`);
   }
   return localSettings;
 }
 
-
 import { decrypt, encrypt } from "./local_encryption.ts";
 
-const SENSITIVE_FIELD: string[] = ["license_key", "jwt_secret"]
+const SENSITIVE_FIELD: string[] = ["license_key", "jwt_secret"];
 
-async function processInstanceSettings(settings: GlobalSetting[], mode: "encode" | "decode"): Promise<GlobalSetting[]> {
+async function processInstanceSettings(
+  settings: GlobalSetting[],
+  mode: "encode" | "decode"
+): Promise<GlobalSetting[]> {
   const encKey = process.env.WMILL_INSTANCE_LOCAL_ENCRYPTION_KEY;
   if (encKey) {
-    const res: GlobalSetting[] = []
+    const res: GlobalSetting[] = [];
 
     for (const s of settings) {
       if (SENSITIVE_FIELD.includes(s.name) && typeof s.value === "string") {
-        res.push(await processField(s, "value", encKey, mode) as GlobalSetting);
+        res.push(
+          (await processField(s, "value", encKey, mode)) as GlobalSetting
+        );
       } else if (s.name == "oauths") {
         if (typeof s.value === "object") {
           const oauths = s.value as { [key: string]: any };
@@ -303,24 +314,40 @@ async function processInstanceSettings(settings: GlobalSetting[], mode: "encode"
     }
     return res;
   } else {
-    log.warn("No encryption key found, skipping encryption. Recommend setting WMILL_INSTANCE_LOCAL_ENCRYPTION_KEY");
+    log.warn(
+      "No encryption key found, skipping encryption. Recommend setting WMILL_INSTANCE_LOCAL_ENCRYPTION_KEY"
+    );
   }
   return settings;
 }
 
-async function processField(obj: { [key: string]: any }, field: string, encKey: string, mode: "encode" | "decode"): Promise<{ [key: string]: any }> {
+async function processField(
+  obj: { [key: string]: any },
+  field: string,
+  encKey: string,
+  mode: "encode" | "decode"
+): Promise<{ [key: string]: any }> {
   return {
     ...obj,
-    [field]: mode === "encode" ? await encrypt(obj[field], encKey) : await decrypt(obj[field], encKey) as any,
-  }
+    [field]:
+      mode === "encode"
+        ? await encrypt(obj[field], encKey)
+        : ((await decrypt(obj[field], encKey)) as any),
+  };
 }
 
-export async function pullInstanceSettings(preview = false) {
+export async function pullInstanceSettings(
+  opts: GlobalOptions & InstanceSyncOptions,
+  preview = false
+) {
   const remoteSettings = await wmill.listGlobalSettings();
 
   if (preview) {
-    const localSettings: GlobalSetting[] = await readInstanceSettings();
-    const processedSettings = await processInstanceSettings(remoteSettings, "encode");
+    const localSettings: GlobalSetting[] = await readInstanceSettings(opts);
+    const processedSettings = await processInstanceSettings(
+      remoteSettings,
+      "encode"
+    );
     return compareInstanceObjects(
       processedSettings,
       localSettings,
@@ -330,7 +357,10 @@ export async function pullInstanceSettings(preview = false) {
   } else {
     log.info("Pulling settings from instance");
 
-    const processedSettings = await processInstanceSettings(remoteSettings, "encode");
+    const processedSettings = await processInstanceSettings(
+      remoteSettings,
+      "encode"
+    );
     await Deno.writeTextFile(
       INSTANCE_SETTINGS_PATH,
       yamlStringify(processedSettings)
@@ -341,19 +371,19 @@ export async function pullInstanceSettings(preview = false) {
 }
 
 export async function pushInstanceSettings(
-  preview: boolean = false,
-  baseUrl?: string
+  opts: GlobalOptions & InstanceSyncOptions,
+  preview: boolean = false
 ) {
   const remoteSettings = await wmill.listGlobalSettings();
-  let localSettings: GlobalSetting[] = await readInstanceSettings();
+  let localSettings: GlobalSetting[] = await readInstanceSettings(opts);
   localSettings = await processInstanceSettings(localSettings, "decode");
 
-  if (baseUrl) {
+  if (opts.baseUrl) {
     localSettings = localSettings.filter((s) => s.name !== "base_url");
     localSettings.push({
       name: "base_url",
       //@ts-ignore
-      value: baseUrl,
+      value: opts.baseUrl,
     });
   }
 
@@ -404,18 +434,30 @@ export async function pushInstanceSettings(
   }
 }
 
-export async function readLocalConfigs() {
+const INSTANCE_CONFIGS_PATH = "instance_configs.yaml";
+
+export async function readLocalConfigs(
+  opts: GlobalOptions & InstanceSyncOptions
+) {
   let localConfigs: Config[] = [];
 
+  let path = INSTANCE_CONFIGS_PATH;
+  if (opts.prefix && opts.folderPerInstance && opts.prefixSettings) {
+    path = `${opts.prefix}/${INSTANCE_CONFIGS_PATH}`;
+  }
+
   try {
-    localConfigs = (await yamlParseFile("instance_configs.yaml")) as Config[];
+    localConfigs = (await yamlParseFile(path)) as Config[];
   } catch {
-    log.warn("No instance_configs.yaml found");
+    log.warn(`No ${path} found`);
   }
   return localConfigs;
 }
 
-export async function pullInstanceConfigs(preview = false) {
+export async function pullInstanceConfigs(
+  opts: GlobalOptions & InstanceSyncOptions,
+  preview = false
+) {
   const remoteConfigs = (await wmill.listConfigs()).map((x) => {
     return {
       ...x,
@@ -424,7 +466,7 @@ export async function pullInstanceConfigs(preview = false) {
   });
 
   if (preview) {
-    const localConfigs: Config[] = await readLocalConfigs();
+    const localConfigs: Config[] = await readLocalConfigs(opts);
 
     return compareInstanceObjects(
       remoteConfigs,
@@ -436,22 +478,25 @@ export async function pullInstanceConfigs(preview = false) {
     log.info("Pulling configs from instance");
 
     await Deno.writeTextFile(
-      "instance_configs.yaml",
+      INSTANCE_CONFIGS_PATH,
       yamlStringify(remoteConfigs as any)
     );
 
-    log.info(colors.green("Configs written to instance_configs.yaml"));
+    log.info(colors.green(`Configs written to ${INSTANCE_CONFIGS_PATH}`));
   }
 }
 
-export async function pushInstanceConfigs(preview: boolean = false) {
+export async function pushInstanceConfigs(
+  opts: GlobalOptions & InstanceSyncOptions,
+  preview: boolean = false
+) {
   const remoteConfigs = (await wmill.listConfigs()).map((x) => {
     return {
       ...x,
       name: removeWorkerPrefix(x.name),
     };
   });
-  const localConfigs = await readLocalConfigs();
+  const localConfigs = await readLocalConfigs(opts);
 
   if (preview) {
     return compareInstanceObjects(

cli/user.ts

@@ -6,7 +6,7 @@ import {
   removeType,
   removePathPrefix,
 } from "./types.ts";
-import { compareInstanceObjects } from "./instance.ts";
+import { compareInstanceObjects, InstanceSyncOptions } from "./instance.ts";
 import {
   colors,
   Command,
@@ -386,47 +386,71 @@ export async function pushGroup(
   }
 }
 
-export async function pullInstanceUsers(preview: boolean = false) {
+const INSTANCE_USERS_PATH = "instance_users.yaml";
+
+export async function pullInstanceUsers(
+  opts: GlobalOptions & InstanceSyncOptions,
+  preview: boolean = false
+) {
   const remoteUsers = await wmill.globalUsersExport();
 
   if (preview) {
-    const localUsers: ExportedUser[] = await readInstanceUsers();
+    const localUsers: ExportedUser[] = await readInstanceUsers(opts);
     return compareInstanceObjects(remoteUsers, localUsers, "email", "user");
   } else {
     log.info("Pulling users from instance...");
     await Deno.writeTextFile(
-      "instance_users.yaml",
+      INSTANCE_USERS_PATH,
       yamlStringify(remoteUsers as any)
     );
-    log.info(colors.green("Users written to instance_users.yaml"));
+    log.info(colors.green(`Users written to ${INSTANCE_USERS_PATH}`));
   }
 }
 
-export async function readInstanceUsers() {
+export async function readInstanceUsers(
+  opts: GlobalOptions & InstanceSyncOptions
+) {
   let localUsers: ExportedUser[] = [];
+
+  let path = INSTANCE_USERS_PATH;
+  if (opts.prefix && opts.folderPerInstance && opts.prefixSettings) {
+    path = `${opts.prefix}/${INSTANCE_USERS_PATH}`;
+  }
+
   try {
-    localUsers = (await yamlParseFile("instance_users.yaml")) as ExportedUser[];
+    localUsers = (await yamlParseFile(path)) as ExportedUser[];
   } catch {
-    log.warn("No instance_users.yaml file found");
+    log.warn(`No ${path} file found`);
   }
   return localUsers;
 }
 
-export async function readInstanceGroups() {
+const INSTANCE_GROUPS_PATH = "instance_groups.yaml";
+
+export async function readInstanceGroups(
+  opts: GlobalOptions & InstanceSyncOptions
+) {
   let localGroups: InstanceGroup[] = [];
+
+  let path = INSTANCE_GROUPS_PATH;
+  if (opts.prefix && opts.folderPerInstance && opts.prefixSettings) {
+    path = `${opts.prefix}/${INSTANCE_GROUPS_PATH}`;
+  }
+
   try {
-    localGroups = (await yamlParseFile(
-      "instance_groups.yaml"
-    )) as ExportedInstanceGroup[];
+    localGroups = (await yamlParseFile(path)) as ExportedInstanceGroup[];
   } catch {
-    log.warn("No instance_groups.yaml file found");
+    log.warn(`No ${path} file found`);
   }
   return localGroups;
 }
 
-export async function pushInstanceUsers(preview: boolean = false) {
+export async function pushInstanceUsers(
+  opts: GlobalOptions & InstanceSyncOptions,
+  preview: boolean = false
+) {
   const remoteUsers = await wmill.globalUsersExport();
-  const localUsers: ExportedUser[] = await readInstanceUsers();
+  const localUsers: ExportedUser[] = await readInstanceUsers(opts);
 
   if (preview) {
     return compareInstanceObjects(localUsers, remoteUsers, "email", "user");
@@ -440,27 +464,33 @@ export async function pushInstanceUsers(preview: boolean = false) {
   }
 }
 
-export async function pullInstanceGroups(preview = false) {
+export async function pullInstanceGroups(
+  opts: GlobalOptions & InstanceSyncOptions,
+  preview = false
+) {
   const remoteGroups = await wmill.exportInstanceGroups();
 
   if (preview) {
-    const localGroups = await readInstanceGroups();
+    const localGroups = await readInstanceGroups(opts);
     return compareInstanceObjects(remoteGroups, localGroups, "name", "group");
   } else {
     log.info("Pulling groups from instance...");
 
     await Deno.writeTextFile(
-      "instance_groups.yaml",
+      INSTANCE_GROUPS_PATH,
       yamlStringify(remoteGroups as any)
     );
 
-    log.info(colors.green("Groups written to instance_groups.yaml"));
+    log.info(colors.green(`Groups written to ${INSTANCE_GROUPS_PATH}`));
   }
 }
 
-export async function pushInstanceGroups(preview: boolean = false) {
+export async function pushInstanceGroups(
+  opts: GlobalOptions & InstanceSyncOptions,
+  preview: boolean = false
+) {
   const remoteGroups = await wmill.exportInstanceGroups();
-  const localGroups = await readInstanceGroups();
+  const localGroups = await readInstanceGroups(opts);
 
   if (preview) {
     return compareInstanceObjects(localGroups, remoteGroups, "name", "group");