feat!(cli): Folders support & Less Tarball nesting (#1040)

* Basic folder support

* Proper Folder Support + deps.ts

* Upgrade Versions

* Add folder meta to tarball

* Remove tarball folders

* Minor fixes

* Fix typo

* Remove extra_perms check

* Use new endpoint

* Use new untar location

* Fix shrinking files

backend/windmill-api/src/workspaces.rs

@@ -8,6 +8,7 @@
 
 use crate::{
     db::{UserDB, DB},
+    folders::Folder,
     resources::{Resource, ResourceType},
     users::{Authed, WorkspaceInvite},
     utils::require_super_admin,
@@ -700,6 +701,22 @@ async fn tarball_workspace(
     let file = File::create(&file_path).await?;
     let mut a = tokio_tar::Builder::new(file);
 
+    {
+        let folders = sqlx::query_as::<_, Folder>("SELECT * FROM folder WHERE workspace_id = $1")
+            .bind(&w_id)
+            .fetch_all(&db)
+            .await?;
+
+        for folder in folders {
+            write_to_archive(
+                serde_json::to_string_pretty(&folder).unwrap(),
+                format!("f/{}/folder.meta.json", folder.name),
+                &mut a,
+            )
+            .await?;
+        }
+    }
+
     {
         let scripts = sqlx::query_as::<_, Script>(
             "SELECT * FROM script as o WHERE workspace_id = $1 AND archived = false
@@ -717,12 +734,7 @@ async fn tarball_workspace(
                 ScriptLang::Go => "go",
                 ScriptLang::Bash => "sh",
             };
-            write_to_archive(
-                script.content,
-                format!("scripts/{}.{}", script.path, ext),
-                &mut a,
-            )
-            .await?;
+            write_to_archive(script.content, format!("{}.{}", script.path, ext), &mut a).await?;
 
             let lock = script
                 .lock
@@ -738,12 +750,7 @@ async fn tarball_workspace(
                 lock,
             };
             let metadata_str = serde_json::to_string_pretty(&metadata).unwrap();
-            write_to_archive(
-                metadata_str,
-                format!("scripts/{}.script.json", script.path),
-                &mut a,
-            )
-            .await?;
+            write_to_archive(metadata_str, format!("{}.script.json", script.path), &mut a).await?;
         }
     }
 
@@ -760,7 +767,7 @@ async fn tarball_workspace(
             let resource_str = serde_json::to_string_pretty(&resource).unwrap();
             write_to_archive(
                 resource_str,
-                format!("resources/{}.resource.json", resource.path),
+                format!("{}.resource.json", resource.path),
                 &mut a,
             )
             .await?;
@@ -780,7 +787,7 @@ async fn tarball_workspace(
             let resource_str = serde_json::to_string_pretty(&resource_type).unwrap();
             write_to_archive(
                 resource_str,
-                format!("resource_types/{}.resource-type.json", resource_type.name),
+                format!("{}.resource-type.json", resource_type.name),
                 &mut a,
             )
             .await?;
@@ -797,7 +804,7 @@ async fn tarball_workspace(
 
         for flow in flows {
             let flow_str = serde_json::to_string_pretty(&flow).unwrap();
-            write_to_archive(flow_str, format!("flows/{}.flow.json", flow.path), &mut a).await?;
+            write_to_archive(flow_str, format!("{}.flow.json", flow.path), &mut a).await?;
         }
     }
 
@@ -811,12 +818,7 @@ async fn tarball_workspace(
 
         for var in variables {
             let flow_str = serde_json::to_string_pretty(&var).unwrap();
-            write_to_archive(
-                flow_str,
-                format!("variables/{}.variable.json", var.path),
-                &mut a,
-            )
-            .await?;
+            write_to_archive(flow_str, format!("{}.variable.json", var.path), &mut a).await?;
         }
     }
     a.into_inner().await?;

cli/context.ts

@@ -1,6 +1,5 @@
 // deno-lint-ignore-file no-explicit-any
-import { colors } from "https://deno.land/x/cliffy@v0.25.4/ansi/colors.ts";
-import { setClient } from "https://deno.land/x/windmill@v1.50.0/mod.ts";
+import { colors, setClient } from "./deps.ts";
 import { tryGetLoginInfo } from "./login.ts";
 import { GlobalOptions } from "./types.ts";
 import {
@@ -16,29 +15,47 @@ export type Context = {
   token: string;
 };
 
-export async function resolveWorkspace(
+async function tryResolveWorkspace(
   opts: GlobalOptions,
-): Promise<Workspace> {
+): Promise<
+  { isError: false; value: Workspace } | { isError: true; error: string }
+> {
   const cache = (opts as any).__secret_workspace;
   if (cache) return cache;
 
   if (opts.workspace) {
     const e = await getWorkspaceByName(opts.workspace);
     if (!e) {
-      console.log(colors.red.underline("Given workspace does not exist."));
-      return Deno.exit(-1);
+      return {
+        isError: true,
+        error: colors.red.underline("Given workspace does not exist."),
+      };
     }
     (opts as any).__secret_workspace = e;
-    return e;
+    return { isError: false, value: e };
   }
 
   const defaultWorkspace = await getActiveWorkspace(opts);
   if (!defaultWorkspace) {
-    console.log(colors.red.underline("No workspace given and no default set."));
-    return Deno.exit(-3);
+    return {
+      isError: true,
+      error: colors.red.underline("No workspace given and no default set."),
+    };
   }
 
-  return defaultWorkspace;
+  return { isError: false, value: defaultWorkspace };
+}
+
+export async function resolveWorkspace(
+  opts: GlobalOptions,
+): Promise<Workspace> {
+  const res = await tryResolveWorkspace(opts);
+  if (res.isError) {
+    console.log(res.error);
+    return Deno.exit(-1);
+  } else {
+    return res.value;
+  }
 }
 
 export async function requireLogin(opts: GlobalOptions) {
@@ -51,3 +68,56 @@ export async function requireLogin(opts: GlobalOptions) {
 
   setClient(token, workspace.remote.substring(0, workspace.remote.length - 1));
 }
+
+export async function tryResolveVersion(
+  opts: GlobalOptions,
+): Promise<number | undefined> {
+  if ((opts as any).__cache_version) {
+    return (opts as any).__cache_version;
+  }
+
+  const workspaceRes = await tryResolveWorkspace(opts);
+  if (workspaceRes.isError) return undefined;
+
+  const response = await fetch(
+    new URL(new URL(workspaceRes.value.remote).origin + "/api/version"),
+  );
+  const version = await response.text();
+  try {
+    return Number.parseInt(
+      version.split("-", 1)[0].replaceAll(".", "").replace("v", ""),
+    );
+  } catch {
+    return undefined;
+  }
+}
+
+export async function validatePath(
+  opts: GlobalOptions,
+  path: string,
+): Promise<boolean> {
+  const backendVersion = await tryResolveVersion(opts);
+  if (path.startsWith("f")) {
+    if (!backendVersion || backendVersion >= 1550) {
+      return true;
+    }
+    console.log(
+      `Attempting to use folders, but the current remote does not have support. Remote version is ${backendVersion} but folders are supported from 1560.`,
+    );
+    return false;
+  }
+
+  if (
+    !(path.startsWith("g") ||
+      path.startsWith("u"))
+  ) {
+    console.log(
+      colors.red(
+        "Given remote path looks invalid. Remote paths are typically of the form <u|g|f>/<username|group|folder>/...",
+      ),
+    );
+    return false;
+  }
+
+  return true;
+}

cli/deps.ts

@@ -0,0 +1,32 @@
+// windmill
+export { setClient } from "https://deno.land/x/windmill@v1.56.0/mod.ts";
+export * from "https://deno.land/x/windmill@v1.56.0/windmill-api/index.t";
+
+// cliffy
+export { Command } from "https://deno.land/x/cliffy@v0.25.6/command/command.ts";
+export { Table } from "https://deno.land/x/cliffy@v0.25.6/table/table.ts";
+export { colors } from "https://deno.land/x/cliffy@v0.25.6/ansi/colors.ts";
+export { Secret } from "https://deno.land/x/cliffy@v0.25.6/prompt/secret.ts";
+export { Select } from "https://deno.land/x/cliffy@v0.25.6/prompt/select.ts";
+export { Confirm } from "https://deno.land/x/cliffy@v0.25.6/prompt/confirm.ts";
+export { Input } from "https://deno.land/x/cliffy@v0.25.6/prompt/input.ts";
+export {
+  DenoLandProvider,
+  UpgradeCommand,
+} from "https://deno.land/x/cliffy@v0.25.6/command/upgrade/mod.ts";
+
+// std
+export { Untar } from "https://deno.land/std@0.170.0/archive/untar.ts";
+export * as path from "https://deno.land/std@0.170.0/path/mod.ts";
+export { ensureDir } from "https://deno.land/std@0.170.0/fs/ensure_dir.ts";
+export {
+  copy,
+  readAll,
+  readerFromStreamReader,
+} from "https://deno.land/std@0.170.0/streams/mod.ts";
+export { DelimiterStream } from "https://deno.land/std@0.170.0/streams/mod.ts";
+
+// other
+export { getAvailablePort } from "https://deno.land/x/port@1.0.0/mod.ts";
+export { default as dir } from "https://deno.land/x/dir@1.5.1/mod.ts";
+export { passwordGenerator } from "https://deno.land/x/password_generator@latest/mod.ts"; // TODO: I think the version is called latest, but it's still pinned.

cli/examples/f/my_folder/folder.meta.json

@@ -0,0 +1,6 @@
+{
+  "owners": [
+    "u/kai",
+    "u/test"
+  ]
+}

cli/flow.ts

@@ -1,28 +1,21 @@
 // deno-lint-ignore-file no-explicit-any
-import { Command } from "https://deno.land/x/cliffy@v0.25.4/command/command.ts";
-import {
-  FlowService,
-  JobService,
-} from "https://deno.land/x/windmill@v1.50.0/mod.ts";
 import { GlobalOptions } from "./types.ts";
 import {
+  colors,
+  Command,
   Flow,
+  FlowService,
+  JobService,
   OpenFlow,
-} from "https://deno.land/x/windmill@v1.50.0/windmill-api/index.ts";
-import { colors } from "https://deno.land/x/cliffy@v0.25.4/ansi/colors.ts";
-import { requireLogin, resolveWorkspace } from "./context.ts";
-import { Table } from "https://deno.land/x/cliffy@v0.25.4/table/table.ts";
+  Table,
+} from "./deps.ts";
+import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
 import { resolve, track_job } from "./script.ts";
 
 type Options = GlobalOptions;
 
 async function push(opts: Options, filePath: string, remotePath: string) {
-  if (!(remotePath.startsWith("g") || remotePath.startsWith("u"))) {
-    console.log(
-      colors.red(
-        "Given remote path looks invalid. Remote paths are typicall of the form <u|g>/<username|group>/...",
-      ),
-    );
+  if (!await validatePath(opts, remotePath)) {
     return;
   }
   const workspace = await resolveWorkspace(opts);

cli/folder.ts

@@ -0,0 +1,90 @@
+import { colors, Command, Folder, FolderService } from "./deps.ts";
+import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
+import { GlobalOptions } from "./types.ts";
+
+async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
+  const workspace = await resolveWorkspace(opts);
+  await requireLogin(opts);
+
+  if (!await validatePath(opts, remotePath)) {
+    return;
+  }
+
+  const fstat = await Deno.stat(filePath);
+  if (!fstat.isFile) {
+    throw new Error("file path must refer to a file.");
+  }
+
+  console.log(colors.bold.yellow("Pushing resource..."));
+
+  await pushFolder(workspace.workspaceId, filePath, remotePath);
+  console.log(colors.bold.underline.green("Resource successfully pushed"));
+}
+
+type FolderFile = {
+  owners: Array<string> | undefined;
+  extra_perms: Record<string, boolean> | undefined;
+};
+
+export async function pushFolder(
+  workspace: string,
+  filePath: string,
+  remotePath: string,
+) {
+  if (remotePath.startsWith("/")) {
+    remotePath = remotePath.substring(1);
+  }
+  if (remotePath.startsWith("f/")) {
+    remotePath = remotePath.substring(2);
+  }
+  const data: FolderFile = JSON.parse(await Deno.readTextFile(filePath));
+  let optFolder: Folder | undefined;
+  try {
+    optFolder = await FolderService.getFolder({ workspace, name: remotePath });
+  } catch {
+    optFolder = undefined;
+  }
+
+  if (optFolder) {
+    // for (const [k, v] of Object.entries(optFolder.extra_perms)) {
+    //   if (!data.extra_perms || data.extra_perms[k] !== v) {
+    //     console.log(colors.red.underline.bold(`Extra Perms missmatch on ${k}`));
+    //     return;
+    //   }
+    // }
+
+    console.log(colors.yellow("Updating existing folder..."));
+    await FolderService.updateFolder({
+      workspace,
+      name: remotePath,
+      requestBody: {
+        extra_perms: data.extra_perms,
+        owners: data.owners,
+      },
+    });
+  } else {
+    console.log(colors.yellow("Creating new folder..."));
+    await FolderService.createFolder({
+      workspace,
+      requestBody: {
+        name: remotePath,
+        extra_perms: data.extra_perms,
+        owners: data.owners,
+      },
+    });
+
+    // HACK: Workaround backend automatically adding current user to folder.
+    await pushFolder(workspace, filePath, remotePath);
+  }
+}
+
+const command = new Command()
+  .description("resource related commands")
+  .command(
+    "push",
+    "push a local folder spec. This overrides any remote versions.",
+  )
+  .arguments("<file_path:string> <remote_path:string>")
+  .action(push as any);
+
+export default command;

cli/hub.ts

@@ -1,4 +1,4 @@
-import { Command } from "https://deno.land/x/cliffy@v0.25.4/command/command.ts";
+import { Command } from "./deps.ts";
 import { requireLogin, resolveWorkspace } from "./context.ts";
 import { pushResourceTypeDef } from "./resource-type.ts";
 import { GlobalOptions } from "./types.ts";

cli/login.ts

@@ -1,8 +1,5 @@
-import { Select } from "https://deno.land/x/cliffy@v0.25.4/prompt/select.ts";
 import { GlobalOptions } from "./types.ts";
-import { colors } from "https://deno.land/x/cliffy@v0.25.4/ansi/colors.ts";
-import { getAvailablePort } from "https://deno.land/x/port@1.0.0/mod.ts";
-import { Secret } from "https://deno.land/x/cliffy@v0.25.4/prompt/secret.ts";
+import { colors, getAvailablePort, Secret, Select } from "./deps.ts";
 
 export async function loginInteractive(remote: string) {
   let token: string | undefined;

cli/main.ts

@@ -1,8 +1,4 @@
-import { Command } from "https://deno.land/x/cliffy@v0.25.4/command/mod.ts";
-import {
-  DenoLandProvider,
-  UpgradeCommand,
-} from "https://deno.land/x/cliffy@v0.25.4/command/upgrade/mod.ts";
+import { Command, DenoLandProvider, UpgradeCommand } from "./deps.ts";
 import flow from "./flow.ts";
 import script from "./script.ts";
 import workspace from "./workspace.ts";
@@ -12,6 +8,9 @@ import variable from "./variable.ts";
 import push from "./push.ts";
 import pull from "./pull.ts";
 import hub from "./hub.ts";
+// import folder from "./folder.ts";
+import { tryResolveVersion } from "./context.ts";
+import { GlobalOptions } from "./types.ts";
 
 const VERSION = "v1.55.0";
 
@@ -36,6 +35,17 @@ const command = new Command()
   .command("push", push)
   .command("pull", pull)
   .command("hub", hub)
+  // .command("folder", folder)
+  .command("version", "Show version information")
+  .action(async (opts) => {
+    console.log("CLI build against " + VERSION);
+    const backendVersion = await tryResolveVersion(opts as GlobalOptions);
+    if (backendVersion) {
+      console.log("Backend Version: " + backendVersion);
+    } else {
+      console.log("Cannot resolve Backend Version");
+    }
+  })
   .command(
     "upgrade",
     new UpgradeCommand({

cli/pull.ts

@@ -1,16 +1,16 @@
 // deno-lint-ignore-file no-explicit-any
-import { Untar } from "https://deno.land/std@0.162.0/archive/tar.ts";
-import { Command } from "https://deno.land/x/cliffy@v0.25.4/command/command.ts";
-import {
-  copy,
-  readerFromStreamReader,
-} from "https://deno.land/std@0.162.0/streams/mod.ts";
 import { resolveWorkspace } from "./context.ts";
 import { GlobalOptions } from "./types.ts";
-import * as path from "https://deno.land/std@0.162.0/path/mod.ts";
-import { colors } from "https://deno.land/x/cliffy@v0.25.4/ansi/colors.ts";
-import { ensureDir } from "https://deno.land/std@0.162.0/fs/ensure_dir.ts";
-import { Confirm } from "https://deno.land/x/cliffy@v0.25.4/prompt/confirm.ts";
+import {
+  colors,
+  Command,
+  Confirm,
+  copy,
+  ensureDir,
+  path,
+  readerFromStreamReader,
+  Untar,
+} from "./deps.ts";
 
 async function pull(opts: GlobalOptions & { override: boolean }, dir: string) {
   const workspace = await resolveWorkspace(opts);
@@ -29,8 +29,12 @@ async function pull(opts: GlobalOptions & { override: boolean }, dir: string) {
   );
 
   if (!tarResponse.ok) {
-    console.log(colors.red("Failed to request tarball from API"));
-    console.log(colors.red(await tarResponse.text()));
+    console.log(
+      colors.red(
+        "Failed to request tarball from API " + tarResponse.statusText,
+      ),
+    );
+    console.log(await tarResponse.text());
     return;
   }
 
@@ -71,7 +75,8 @@ async function pull(opts: GlobalOptions & { override: boolean }, dir: string) {
       }
     }
     const file = await Deno.open(filePath, { write: true, create: true });
-    await copy(entry, file);
+    const len = await copy(entry, file);
+    await file.truncate(len);
     file.close();
   }
   console.log(colors.green("Done. Wrote all files to disk."));

cli/push.ts

@@ -1,7 +1,5 @@
 // deno-lint-ignore-file no-explicit-any
-import { colors } from "https://deno.land/x/cliffy@v0.25.4/ansi/colors.ts";
-import { Command } from "https://deno.land/x/cliffy@v0.25.4/command/command.ts";
-import * as path from "https://deno.land/std@0.162.0/path/mod.ts";
+import { colors, Command, path } from "./deps.ts";
 import { requireLogin, resolveWorkspace } from "./context.ts";
 import { pushFlow } from "./flow.ts";
 import { pushResource } from "./resource.ts";
@@ -9,34 +7,73 @@ import { findContentFile, pushScript } from "./script.ts";
 import { GlobalOptions } from "./types.ts";
 import { pushVariable } from "./variable.ts";
 import { pushResourceType } from "./resource-type.ts";
+import { pushFolder } from "./folder.ts";
 
 type Candidate = {
   path: string;
-  group: boolean;
-  groupOrUsername: string;
+  namespaceKind: "user" | "group" | "folder";
+  namespaceName: string;
 };
-async function findCandidateFiles(dir: string): Promise<Candidate[]> {
-  const candidates: Candidate[] = [];
-  if (dir.startsWith(".")) return [];
+
+type ResourceTypeCandidate = {
+  path: string;
+};
+
+type FolderCandidate = {
+  path: string;
+  namespaceName: string;
+};
+
+async function findCandidateFiles(
+  dir: string,
+): Promise<
+  {
+    normal: Candidate[];
+    resourceTypes: ResourceTypeCandidate[];
+    folders: FolderCandidate[];
+  }
+> {
+  dir = path.resolve(dir);
+  if (path.dirname(dir).startsWith(".")) {
+    return { normal: [], resourceTypes: [], folders: [] };
+  }
+  const normalCandidates: Candidate[] = [];
+  const resourceTypeCandidates: ResourceTypeCandidate[] = [];
+  const folderCandidates: FolderCandidate[] = [];
   for await (const e of Deno.readDir(dir)) {
     if (e.isDirectory) {
-      if (e.name == "u" || e.name == "g") {
+      if (e.name == "u" || e.name == "g" || e.name == "f") { // TODO: Check version for f
         const newDir = dir + (dir.endsWith("/") ? "" : "/") + e.name;
         for await (const e2 of Deno.readDir(newDir)) {
           if (e2.isDirectory) {
-            if (e2.name.startsWith(".")) return [];
-            const groupOrUserName = e2.name;
+            if (e2.name.startsWith(".")) continue;
+            const namespaceName = e2.name;
             const stack: string[] = [];
-            stack.push(newDir + "/" + groupOrUserName + "/");
+            {
+              const path = newDir + "/" + namespaceName + "/";
+              stack.push(path);
+              try {
+                await Deno.stat(path + "folder.meta.json");
+                folderCandidates.push({
+                  namespaceName,
+                  path: path + "folder.meta.json",
+                });
+              } catch {}
+            }
 
             while (stack.length > 0) {
               const dir2 = stack.pop()!;
               for await (const e3 of Deno.readDir(dir2)) {
                 if (e3.isFile) {
-                  candidates.push({
+                  if (e3.name === "folder.meta.json") continue;
+                  normalCandidates.push({
                     path: dir2 + e3.name,
-                    group: e.name == "g",
-                    groupOrUsername: groupOrUserName,
+                    namespaceKind: e.name == "g"
+                      ? "group"
+                      : e.name == "u"
+                      ? "user"
+                      : "folder",
+                    namespaceName: namespaceName,
                   });
                 } else {
                   stack.push(dir2 + e3.name + "/");
@@ -51,21 +88,27 @@ async function findCandidateFiles(dir: string): Promise<Candidate[]> {
             "Including organizational folder " + e.name + " in push!",
           ),
         );
-        candidates.push(...(await findCandidateFiles(path.join(dir, e.name))));
+        const { normal, resourceTypes, folders } = await findCandidateFiles(
+          path.join(dir, e.name),
+        );
+        normalCandidates.push(...normal);
+        resourceTypeCandidates.push(...resourceTypes);
+        folderCandidates.push(...folders);
       }
     } else {
       // handle root files
       if (e.name.endsWith(".resource-type.json")) {
-        candidates.push({
-          group: false,
-          groupOrUsername: "",
+        resourceTypeCandidates.push({
           path: dir + (dir.endsWith("/") ? "" : "/") + e.name,
         });
-        console.log(candidates);
       }
     }
   }
-  return candidates;
+  return {
+    normal: normalCandidates,
+    folders: folderCandidates,
+    resourceTypes: resourceTypeCandidates,
+  };
 }
 
 async function push(opts: GlobalOptions, dir?: string) {
@@ -74,16 +117,55 @@ async function push(opts: GlobalOptions, dir?: string) {
   await requireLogin(opts);
 
   console.log(colors.blue("Searching Directory..."));
-  const candidates: Candidate[] = await findCandidateFiles(dir);
-  console.log(colors.blue("Found " + candidates.length + " candidates"));
-  for (const candidate of candidates) {
+  const { normal, resourceTypes, folders } = await findCandidateFiles(dir);
+  console.log(
+    colors.blue(
+      "Found " + (normal.length + resourceTypes.length + folders.length) +
+        " candidates",
+    ),
+  );
+  for (const resourceType of resourceTypes) {
+    const fileName = resourceType.path.substring(
+      resourceType.path.lastIndexOf("/") + 1,
+    );
+    const fileNameParts = fileName.split(".");
+    // invalid file names, like my.cool.script.script.json. Not valid.
+    if (fileNameParts.length != 3) {
+      console.log(
+        colors.yellow("invalid file name found at " + resourceType.path),
+      );
+      continue;
+    }
+
+    // filter out non-json files. Note that we filter out script contents above, so this is really an error.
+    if (fileNameParts.at(-1) != "json") {
+      console.log(colors.yellow("non-JSON file found at " + resourceType.path));
+      continue;
+    }
+
+    console.log("pushing resource type " + fileNameParts.at(-3)!);
+    await pushResourceType(
+      workspace.workspaceId,
+      resourceType.path,
+      fileNameParts.at(-3)!,
+    );
+  }
+  for (const folder of folders) {
+    await pushFolder(
+      workspace.workspaceId,
+      folder.path,
+      "f/" + folder.namespaceName,
+    );
+  }
+  for (const candidate of normal) {
     // full file name. No leading /. includes .type.json
     const fileName = candidate.path.substring(
       candidate.path.lastIndexOf("/") + 1,
     );
     // figure out just the path after ...../u|g/username|group/ (in extra dir)
     const dirParts = candidate.path.split("/").filter((x) => x.length > 0);
-    const gIndex = dirParts.findIndex((x) => x == "u" || x == "g");
+    // TODO: check version for folder
+    const gIndex = dirParts.findIndex((x) => x == "u" || x == "g" || x == "f");
     const extraDir = dirParts.slice(gIndex + 2, -1).join("/");
 
     // file name parts has .json (hopefully) at -1, type at -2, and the actual name at -3. Dots in names are not allowed.
@@ -116,22 +198,13 @@ async function push(opts: GlobalOptions, dir?: string) {
     // get the type & filter it for valid ones.
     const type = fileNameParts.at(-2);
     if (type == "resource-type") {
-      if (candidate.group == false && candidate.groupOrUsername == "") {
-        console.log("pushing resource type " + fileNameParts.at(-3)!);
-        await pushResourceType(
-          workspace.workspaceId,
-          candidate.path,
-          fileNameParts.at(-3)!,
-        );
-      } else {
-        console.log(
-          colors.yellow(
-            "Found resource type file at " +
-              candidate.path +
-              " this appears to be inside a path folder. Resource types are not addressed by path. Place them at the root or inside only an organizational folder. Ignoring this file!",
-          ),
-        );
-      }
+      console.log(
+        colors.yellow(
+          "Found resource type file at " +
+            candidate.path +
+            " this appears to be inside a path folder. Resource types are not addressed by path. Place them at the root or inside only an organizational folder. Ignoring this file!",
+        ),
+      );
       continue;
     }
 
@@ -150,8 +223,10 @@ async function push(opts: GlobalOptions, dir?: string) {
     }
 
     // create the remotePath for the API
-    const remotePath = (candidate.group ? "g/" : "u/") +
-      candidate.groupOrUsername +
+    const remotePath = (candidate.namespaceKind === "group"
+      ? "g/"
+      : (candidate.namespaceKind === "user" ? "u/" : "f/")) +
+      candidate.namespaceName +
       "/" +
       (extraDir.length > 0 ? extraDir + "/" : "") +
       fileNameParts.at(-3);
@@ -167,7 +242,7 @@ async function push(opts: GlobalOptions, dir?: string) {
       try {
         contentPath = await findContentFile(candidate.path);
       } catch (e) {
-        console.log(colors.red(e));
+        console.log(colors.red(e.toString()));
         continue;
       }
       await pushScript(

cli/resource-type.ts

@@ -1,10 +1,7 @@
 // deno-lint-ignore-file no-explicit-any
-import { Command } from "https://deno.land/x/cliffy@v0.25.4/command/command.ts";
-import { ResourceService } from "https://deno.land/x/windmill@v1.50.0/mod.ts";
 import { GlobalOptions } from "./types.ts";
-import { colors } from "https://deno.land/x/cliffy@v0.25.4/ansi/colors.ts";
 import { requireLogin, resolveWorkspace } from "./context.ts";
-import { Table } from "https://deno.land/x/cliffy@v0.25.4/table/table.ts";
+import { colors, Command, ResourceService, Table } from "./deps.ts";
 
 type ResourceTypeFile = {
   schema?: any;

cli/resource.ts

@@ -1,22 +1,18 @@
-import { Command } from "https://deno.land/x/cliffy@v0.25.4/command/command.ts";
-import { ResourceService } from "https://deno.land/x/windmill@v1.50.0/mod.ts";
 import { GlobalOptions } from "./types.ts";
-import { colors } from "https://deno.land/x/cliffy@v0.25.4/ansi/colors.ts";
-import { requireLogin, resolveWorkspace } from "./context.ts";
-import { Resource } from "https://deno.land/x/windmill@v1.50.0/windmill-api/index.ts";
-import { Table } from "https://deno.land/x/cliffy@v0.25.4/table/table.ts";
+import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
+import { colors, Command, Resource, ResourceService, Table } from "./deps.ts";
 
 type ResourceFile = {
   value: any;
   description?: string;
   resource_type: string;
-  is_oauth?: boolean;
+  is_oauth?: boolean; // deprecated
 };
 
 export async function pushResource(
   workspace: string,
   filePath: string,
-  remotePath: string
+  remotePath: string,
 ) {
   const data: ResourceFile = JSON.parse(await Deno.readTextFile(filePath));
   if (
@@ -30,26 +26,26 @@ export async function pushResource(
       workspace: workspace,
       path: remotePath,
     });
+
     if (existing.resource_type != data.resource_type) {
       console.log(
         colors.red.underline.bold(
           "Remote resource at " +
             remotePath +
-            " exists & has a different resource type. This cannot be updated. If you wish to do this anyways, consider deleting the remote resource."
-        )
+            " exists & has a different resource type. This cannot be updated. If you wish to do this anyways, consider deleting the remote resource.",
+        ),
       );
       return;
     }
-    if (existing.is_oauth != data.is_oauth) {
+
+    if (typeof data.is_oauth !== "undefined") {
       console.log(
-        colors.red.underline.bold(
-          "Remote resource at " +
-            remotePath +
-            " exists & has a different oauth state. This cannot be updated. If you wish to do this anyways, consider deleting the remote resource."
-        )
+        colors.yellow(
+          "! is_oauth has been removed in newer versions. Ignoring.",
+        ),
       );
-      return;
     }
+
     await ResourceService.updateResource({
       workspace: workspace,
       path: remotePath,
@@ -60,6 +56,14 @@ export async function pushResource(
       },
     });
   } else {
+    if (typeof data.is_oauth !== "undefined") {
+      console.log(
+        colors.yellow(
+          "! is_oauth has been removed in newer versions. Ignoring.",
+        ),
+      );
+    }
+
     console.log(colors.yellow("Creating new resource..."));
     await ResourceService.createResource({
       workspace: workspace,
@@ -68,7 +72,6 @@ export async function pushResource(
         resource_type: data.resource_type,
         value: data.value,
         description: data.description,
-        is_oauth: data.is_oauth,
       },
     });
   }
@@ -79,12 +82,7 @@ async function push(opts: PushOptions, filePath: string, remotePath: string) {
   const workspace = await resolveWorkspace(opts);
   await requireLogin(opts);
 
-  if (!(remotePath.startsWith("g") || remotePath.startsWith("u"))) {
-    console.log(
-      colors.red(
-        "Given remote path looks invalid. Remote paths are typicall of the form <u|g>/<username|group>/..."
-      )
-    );
+  if (!await validatePath(opts, remotePath)) {
     return;
   }
 
@@ -131,7 +129,7 @@ const command = new Command()
   .action(list as any)
   .command(
     "push",
-    "push a local resource spec. This overrides any remote versions."
+    "push a local resource spec. This overrides any remote versions.",
   )
   .arguments("<file_path:string> <remote_path:string>")
   .action(push as any);

cli/script.ts

@@ -1,15 +1,11 @@
 // deno-lint-ignore-file no-explicit-any
-import { Command } from "https://deno.land/x/cliffy@v0.25.4/command/command.ts";
-import { ScriptService } from "https://deno.land/x/windmill@v1.50.0/mod.ts";
 import { GlobalOptions } from "./types.ts";
-import { colors } from "https://deno.land/x/cliffy@v0.25.4/ansi/colors.ts";
-import { resolveWorkspace, requireLogin } from "./context.ts";
+import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
 import {
   JobService,
   Script,
 } from "https://deno.land/x/windmill@v1.50.0/windmill-api/index.ts";
-import { Table } from "https://deno.land/x/cliffy@v0.25.4/table/table.ts";
-import { readAll } from "https://deno.land/std@0.165.0/streams/mod.ts";
+import { colors, Command, readAll, ScriptService, Table } from "./deps.ts";
 
 type ScriptFile = {
   parent_hash?: string;
@@ -26,15 +22,10 @@ async function push(
   opts: PushOptions,
   filePath: string,
   remotePath: string,
-  contentPath?: string
+  contentPath?: string,
 ) {
   const workspace = await resolveWorkspace(opts);
-  if (!(remotePath.startsWith("g") || remotePath.startsWith("u"))) {
-    console.log(
-      colors.red(
-        "Given remote path looks invalid. Remote paths are typicall of the form <u|g>/<username|group>/..."
-      )
-    );
+  if (!await validatePath(opts, remotePath)) {
     return;
   }
 
@@ -58,9 +49,9 @@ async function push(
 
 export async function findContentFile(filePath: string) {
   const candidates = [
-    filePath.replace(".json", ".ts"),
-    filePath.replace(".json", ".py"),
-    filePath.replace(".json", ".go"),
+    filePath.replace(".script.json", ".ts"),
+    filePath.replace(".script.json", ".py"),
+    filePath.replace(".script.json", ".go"),
   ];
   const validCandidates = (
     await Promise.all(
@@ -71,7 +62,7 @@ export async function findContentFile(filePath: string) {
           .then((e) => {
             return { path: x, file: e };
           });
-      })
+      }),
     )
   )
     .filter((x) => x.file)
@@ -79,7 +70,7 @@ export async function findContentFile(filePath: string) {
   if (validCandidates.length > 1) {
     throw new Error(
       "No content path given and more then one candidate found: " +
-        validCandidates.join(", ")
+        validCandidates.join(", "),
     );
   }
   if (validCandidates.length < 1) {
@@ -92,7 +83,7 @@ export async function pushScript(
   filePath: string,
   contentPath: string,
   workspace: string,
-  remotePath: string
+  remotePath: string,
 ) {
   const data: ScriptFile = JSON.parse(await Deno.readTextFile(filePath));
   const content = await Deno.readTextFile(contentPath);
@@ -170,7 +161,7 @@ async function list(opts: GlobalOptions & { showArchived?: boolean }) {
         x.language,
         x.created_at,
         x.created_by,
-      ])
+      ]),
     )
     .render();
 }
@@ -221,7 +212,7 @@ async function run(
     input: string[];
     silent: boolean;
   },
-  path: string
+  path: string,
 ) {
   const workspace = await resolveWorkspace(opts);
   await requireLogin(opts);
@@ -239,13 +230,12 @@ async function run(
 
   while (true) {
     try {
-      const result =
-        (
-          await JobService.getCompletedJob({
-            workspace: workspace.workspaceId,
-            id,
-          })
-        ).result ?? {};
+      const result = (
+        await JobService.getCompletedJob({
+          workspace: workspace.workspaceId,
+          id,
+        })
+      ).result ?? {};
       console.log(result);
 
       break;
@@ -311,7 +301,7 @@ export async function track_job(workspace: string, id: string) {
     if (running && updates.running === false) {
       running = false;
       console.log(
-        colors.yellow("Job suspended. Waiting for it to continue...")
+        colors.yellow("Job suspended. Waiting for it to continue..."),
       );
     }
   }
@@ -352,7 +342,7 @@ const command = new Command()
   .action(list as any)
   .command(
     "push",
-    "push a local script spec. This overrides any remote versions."
+    "push a local script spec. This overrides any remote versions.",
   )
   .arguments("<file_path:string> <remote_path:string> [content_path:string]")
   .action(push as any)
@@ -363,11 +353,11 @@ const command = new Command()
   .arguments("<path:string>")
   .option(
     "-i --input [inputs...:string]",
-    "Inputs specified as JSON objects or simply as <name>=<value>. Supports file inputs using @<filename> and stdin using @- these also need to be formatted as JSON. Later inputs override earlier ones."
+    "Inputs specified as JSON objects or simply as <name>=<value>. Supports file inputs using @<filename> and stdin using @- these also need to be formatted as JSON. Later inputs override earlier ones.",
   )
   .option(
     "-s --silent",
-    "Do not ouput anything other then the final output. Useful for scripting."
+    "Do not ouput anything other then the final output. Useful for scripting.",
   )
   .action(run as any);
 

cli/store.ts

@@ -1,5 +1,4 @@
-import dir from "https://deno.land/x/dir@1.5.1/mod.ts";
-import * as fs from "https://deno.land/std@0.161.0/fs/mod.ts";
+import { dir, ensureDir } from "./deps.ts";
 
 function hash_string(str: string): number {
   let hash = 0,
@@ -16,13 +15,13 @@ function hash_string(str: string): number {
 
 export async function getRootStore(): Promise<string> {
   const store = dir("config") + "/windmill/";
-  await fs.ensureDir(store);
+  await ensureDir(store);
   return store;
 }
 
 export async function getStore(baseUrl: string): Promise<string> {
   const baseHash = Math.abs(hash_string(baseUrl)).toString(16);
   const baseStore = (await getRootStore()) + baseHash + "/";
-  await fs.ensureDir(baseStore);
+  await ensureDir(baseStore);
   return baseStore;
 }

cli/tmp/f/my_folder/folder.meta.json

@@ -0,0 +1,10 @@
+{
+  "workspace_id": "admins",
+  "name": "my_folder",
+  "display_name": "my_folder",
+  "owners": [],
+  "extra_perms": {
+    "u/test": true,
+    "u/admin@windmill.dev": false
+  }
+}

cli/tmp/u/admin/hub_sync.script.json

@@ -0,0 +1,12 @@
+{
+  "summary": "Syncronize Hub Resource types with starter workspace",
+  "description": "Basic administrative script to sync latest resource types from hub. Recommended to run at least once. On a schedule by default.",
+  "schema": {
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "properties": {},
+    "required": [],
+    "type": "object"
+  },
+  "is_template": false,
+  "lock": []
+}

cli/tmp/u/admin/hub_sync.ts

@@ -0,0 +1,13 @@
+import wmill from "https://deno.land/x/wmill@v1.55.0/main.ts";
+
+export async function main() {
+  await run(
+    "workspace", "add", "__automation", "starter", Deno.env.get("WM_BASE_URL") + "/", "--token", Deno.env.get("WM_TOKEN"));
+
+  await run("hub", "pull");
+}
+
+async function run(...cmd: string[]) {
+  console.log("Running \"" + cmd.join(' ') + "\"");
+  await wmill.parse(cmd);
+}

cli/user.ts

@@ -1,12 +1,14 @@
 // deno-lint-ignore-file no-explicit-any
-import { Command } from "https://deno.land/x/cliffy@v0.25.4/command/command.ts";
-import { Table } from "https://deno.land/x/cliffy@v0.25.4/table/table.ts";
-import { UserService } from "https://deno.land/x/windmill@v1.50.0/mod.ts";
-import { GlobalUserInfo } from "https://deno.land/x/windmill@v1.50.0/windmill-api/index.ts";
-import { passwordGenerator } from "https://deno.land/x/password_generator@latest/mod.ts"; // TODO: I think the version is called latest, but it's still pinned.
 import { requireLogin } from "./context.ts";
 import { GlobalOptions } from "./types.ts";
-import { colors } from "https://deno.land/x/cliffy@v0.25.4/ansi/colors.ts";
+import {
+  colors,
+  Command,
+  GlobalUserInfo,
+  passwordGenerator,
+  Table,
+  UserService,
+} from "./deps.ts";
 
 async function list(opts: GlobalOptions) {
   await requireLogin(opts);

cli/variable.ts

@@ -1,10 +1,7 @@
 // deno-lint-ignore-file no-explicit-any
-import { colors } from "https://deno.land/x/cliffy@v0.25.4/ansi/colors.ts";
-import { Command } from "https://deno.land/x/cliffy@v0.25.4/command/command.ts";
-import { Table } from "https://deno.land/x/cliffy@v0.25.4/table/table.ts";
-import { VariableService } from "https://deno.land/x/windmill@v1.50.0/mod.ts";
-import { requireLogin, resolveWorkspace } from "./context.ts";
+import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
 import { GlobalOptions } from "./types.ts";
+import { colors, Command, Table, VariableService } from "./deps.ts";
 
 async function list(opts: GlobalOptions) {
   const workspace = await resolveWorkspace(opts);
@@ -41,12 +38,7 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
   const workspace = await resolveWorkspace(opts);
   await requireLogin(opts);
 
-  if (!(remotePath.startsWith("g") || remotePath.startsWith("u"))) {
-    console.log(
-      colors.red(
-        "Given remote path looks invalid. Remote paths are typicall of the form <u|g>/<username|group>/...",
-      ),
-    );
+  if (!await validatePath(opts, remotePath)) {
     return;
   }
 

cli/workspace.ts

@@ -1,12 +1,8 @@
 // deno-lint-ignore-file no-explicit-any
-import { Command } from "https://deno.land/x/cliffy@v0.25.4/command/command.ts";
-import { Table } from "https://deno.land/x/cliffy@v0.25.4/table/table.ts";
 import { GlobalOptions } from "./types.ts";
-import { DelimiterStream } from "https://deno.land/std@0.165.0/streams/mod.ts";
-import { colors } from "https://deno.land/x/cliffy@v0.25.4/ansi/colors.ts";
 import { getRootStore } from "./store.ts";
-import { Input } from "https://deno.land/x/cliffy@v0.25.4/prompt/input.ts";
 import { loginInteractive, tryGetLoginInfo } from "./login.ts";
+import { colors, Command, DelimiterStream, Input, Table } from "./deps.ts";
 
 export type Workspace = {
   remote: string;