From adf991cd2d175e645046a33a3fe1da2a709b276c Mon Sep 17 00:00:00 2001 From: Kai Jellinghaus Date: Fri, 23 Dec 2022 03:06:12 +0100 Subject: [PATCH] feat!(cli): Folders support & Less Tarball nesting (#1040) * Basic folder support * Proper Folder Support + deps.ts * Upgrade Versions * Add folder meta to tarball * Remove tarball folders * Minor fixes * Fix typo * Remove extra_perms check * Use new endpoint * Use new untar location * Fix shrinking files --- backend/windmill-api/src/workspaces.rs | 44 +++--- cli/context.ts | 90 ++++++++++-- cli/deps.ts | 32 +++++ cli/examples/f/my_folder/folder.meta.json | 6 + cli/flow.ts | 23 ++- cli/folder.ts | 90 ++++++++++++ cli/hub.ts | 2 +- cli/login.ts | 5 +- cli/main.ts | 20 ++- cli/pull.ts | 31 ++-- cli/push.ts | 163 ++++++++++++++++------ cli/resource-type.ts | 5 +- cli/resource.ts | 48 +++---- cli/script.ts | 54 +++---- cli/store.ts | 7 +- cli/tmp/f/my_folder/folder.meta.json | 10 ++ cli/tmp/u/admin/hub_sync.script.json | 12 ++ cli/tmp/u/admin/hub_sync.ts | 13 ++ cli/user.ts | 14 +- cli/variable.ts | 14 +- cli/workspace.ts | 6 +- 21 files changed, 489 insertions(+), 200 deletions(-) create mode 100644 cli/deps.ts create mode 100644 cli/examples/f/my_folder/folder.meta.json create mode 100644 cli/folder.ts create mode 100644 cli/tmp/f/my_folder/folder.meta.json create mode 100644 cli/tmp/u/admin/hub_sync.script.json create mode 100644 cli/tmp/u/admin/hub_sync.ts diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 680dc477c3..65c704f3c5 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -8,6 +8,7 @@ use crate::{ db::{UserDB, DB}, + folders::Folder, resources::{Resource, ResourceType}, users::{Authed, WorkspaceInvite}, utils::require_super_admin, @@ -700,6 +701,22 @@ async fn tarball_workspace( let file = File::create(&file_path).await?; let mut a = tokio_tar::Builder::new(file); + { + let folders = sqlx::query_as::<_, Folder>("SELECT * FROM folder WHERE workspace_id = $1") + .bind(&w_id) + .fetch_all(&db) + .await?; + + for folder in folders { + write_to_archive( + serde_json::to_string_pretty(&folder).unwrap(), + format!("f/{}/folder.meta.json", folder.name), + &mut a, + ) + .await?; + } + } + { let scripts = sqlx::query_as::<_, Script>( "SELECT * FROM script as o WHERE workspace_id = $1 AND archived = false @@ -717,12 +734,7 @@ async fn tarball_workspace( ScriptLang::Go => "go", ScriptLang::Bash => "sh", }; - write_to_archive( - script.content, - format!("scripts/{}.{}", script.path, ext), - &mut a, - ) - .await?; + write_to_archive(script.content, format!("{}.{}", script.path, ext), &mut a).await?; let lock = script .lock @@ -738,12 +750,7 @@ async fn tarball_workspace( lock, }; let metadata_str = serde_json::to_string_pretty(&metadata).unwrap(); - write_to_archive( - metadata_str, - format!("scripts/{}.script.json", script.path), - &mut a, - ) - .await?; + write_to_archive(metadata_str, format!("{}.script.json", script.path), &mut a).await?; } } @@ -760,7 +767,7 @@ async fn tarball_workspace( let resource_str = serde_json::to_string_pretty(&resource).unwrap(); write_to_archive( resource_str, - format!("resources/{}.resource.json", resource.path), + format!("{}.resource.json", resource.path), &mut a, ) .await?; @@ -780,7 +787,7 @@ async fn tarball_workspace( let resource_str = serde_json::to_string_pretty(&resource_type).unwrap(); write_to_archive( resource_str, - format!("resource_types/{}.resource-type.json", resource_type.name), + format!("{}.resource-type.json", resource_type.name), &mut a, ) .await?; @@ -797,7 +804,7 @@ async fn tarball_workspace( for flow in flows { let flow_str = serde_json::to_string_pretty(&flow).unwrap(); - write_to_archive(flow_str, format!("flows/{}.flow.json", flow.path), &mut a).await?; + write_to_archive(flow_str, format!("{}.flow.json", flow.path), &mut a).await?; } } @@ -811,12 +818,7 @@ async fn tarball_workspace( for var in variables { let flow_str = serde_json::to_string_pretty(&var).unwrap(); - write_to_archive( - flow_str, - format!("variables/{}.variable.json", var.path), - &mut a, - ) - .await?; + write_to_archive(flow_str, format!("{}.variable.json", var.path), &mut a).await?; } } a.into_inner().await?; diff --git a/cli/context.ts b/cli/context.ts index f65149a1ad..1b524814e0 100644 --- a/cli/context.ts +++ b/cli/context.ts @@ -1,6 +1,5 @@ // deno-lint-ignore-file no-explicit-any -import { colors } from "https://deno.land/x/cliffy@v0.25.4/ansi/colors.ts"; -import { setClient } from "https://deno.land/x/windmill@v1.50.0/mod.ts"; +import { colors, setClient } from "./deps.ts"; import { tryGetLoginInfo } from "./login.ts"; import { GlobalOptions } from "./types.ts"; import { @@ -16,29 +15,47 @@ export type Context = { token: string; }; -export async function resolveWorkspace( +async function tryResolveWorkspace( opts: GlobalOptions, -): Promise { +): Promise< + { isError: false; value: Workspace } | { isError: true; error: string } +> { const cache = (opts as any).__secret_workspace; if (cache) return cache; if (opts.workspace) { const e = await getWorkspaceByName(opts.workspace); if (!e) { - console.log(colors.red.underline("Given workspace does not exist.")); - return Deno.exit(-1); + return { + isError: true, + error: colors.red.underline("Given workspace does not exist."), + }; } (opts as any).__secret_workspace = e; - return e; + return { isError: false, value: e }; } const defaultWorkspace = await getActiveWorkspace(opts); if (!defaultWorkspace) { - console.log(colors.red.underline("No workspace given and no default set.")); - return Deno.exit(-3); + return { + isError: true, + error: colors.red.underline("No workspace given and no default set."), + }; } - return defaultWorkspace; + return { isError: false, value: defaultWorkspace }; +} + +export async function resolveWorkspace( + opts: GlobalOptions, +): Promise { + const res = await tryResolveWorkspace(opts); + if (res.isError) { + console.log(res.error); + return Deno.exit(-1); + } else { + return res.value; + } } export async function requireLogin(opts: GlobalOptions) { @@ -51,3 +68,56 @@ export async function requireLogin(opts: GlobalOptions) { setClient(token, workspace.remote.substring(0, workspace.remote.length - 1)); } + +export async function tryResolveVersion( + opts: GlobalOptions, +): Promise { + if ((opts as any).__cache_version) { + return (opts as any).__cache_version; + } + + const workspaceRes = await tryResolveWorkspace(opts); + if (workspaceRes.isError) return undefined; + + const response = await fetch( + new URL(new URL(workspaceRes.value.remote).origin + "/api/version"), + ); + const version = await response.text(); + try { + return Number.parseInt( + version.split("-", 1)[0].replaceAll(".", "").replace("v", ""), + ); + } catch { + return undefined; + } +} + +export async function validatePath( + opts: GlobalOptions, + path: string, +): Promise { + const backendVersion = await tryResolveVersion(opts); + if (path.startsWith("f")) { + if (!backendVersion || backendVersion >= 1550) { + return true; + } + console.log( + `Attempting to use folders, but the current remote does not have support. Remote version is ${backendVersion} but folders are supported from 1560.`, + ); + return false; + } + + if ( + !(path.startsWith("g") || + path.startsWith("u")) + ) { + console.log( + colors.red( + "Given remote path looks invalid. Remote paths are typically of the form //...", + ), + ); + return false; + } + + return true; +} diff --git a/cli/deps.ts b/cli/deps.ts new file mode 100644 index 0000000000..12a4658b9b --- /dev/null +++ b/cli/deps.ts @@ -0,0 +1,32 @@ +// windmill +export { setClient } from "https://deno.land/x/windmill@v1.56.0/mod.ts"; +export * from "https://deno.land/x/windmill@v1.56.0/windmill-api/index.t"; + +// cliffy +export { Command } from "https://deno.land/x/cliffy@v0.25.6/command/command.ts"; +export { Table } from "https://deno.land/x/cliffy@v0.25.6/table/table.ts"; +export { colors } from "https://deno.land/x/cliffy@v0.25.6/ansi/colors.ts"; +export { Secret } from "https://deno.land/x/cliffy@v0.25.6/prompt/secret.ts"; +export { Select } from "https://deno.land/x/cliffy@v0.25.6/prompt/select.ts"; +export { Confirm } from "https://deno.land/x/cliffy@v0.25.6/prompt/confirm.ts"; +export { Input } from "https://deno.land/x/cliffy@v0.25.6/prompt/input.ts"; +export { + DenoLandProvider, + UpgradeCommand, +} from "https://deno.land/x/cliffy@v0.25.6/command/upgrade/mod.ts"; + +// std +export { Untar } from "https://deno.land/std@0.170.0/archive/untar.ts"; +export * as path from "https://deno.land/std@0.170.0/path/mod.ts"; +export { ensureDir } from "https://deno.land/std@0.170.0/fs/ensure_dir.ts"; +export { + copy, + readAll, + readerFromStreamReader, +} from "https://deno.land/std@0.170.0/streams/mod.ts"; +export { DelimiterStream } from "https://deno.land/std@0.170.0/streams/mod.ts"; + +// other +export { getAvailablePort } from "https://deno.land/x/port@1.0.0/mod.ts"; +export { default as dir } from "https://deno.land/x/dir@1.5.1/mod.ts"; +export { passwordGenerator } from "https://deno.land/x/password_generator@latest/mod.ts"; // TODO: I think the version is called latest, but it's still pinned. diff --git a/cli/examples/f/my_folder/folder.meta.json b/cli/examples/f/my_folder/folder.meta.json new file mode 100644 index 0000000000..14ad698648 --- /dev/null +++ b/cli/examples/f/my_folder/folder.meta.json @@ -0,0 +1,6 @@ +{ + "owners": [ + "u/kai", + "u/test" + ] +} diff --git a/cli/flow.ts b/cli/flow.ts index c39848385a..1d2171fa08 100644 --- a/cli/flow.ts +++ b/cli/flow.ts @@ -1,28 +1,21 @@ // deno-lint-ignore-file no-explicit-any -import { Command } from "https://deno.land/x/cliffy@v0.25.4/command/command.ts"; -import { - FlowService, - JobService, -} from "https://deno.land/x/windmill@v1.50.0/mod.ts"; import { GlobalOptions } from "./types.ts"; import { + colors, + Command, Flow, + FlowService, + JobService, OpenFlow, -} from "https://deno.land/x/windmill@v1.50.0/windmill-api/index.ts"; -import { colors } from "https://deno.land/x/cliffy@v0.25.4/ansi/colors.ts"; -import { requireLogin, resolveWorkspace } from "./context.ts"; -import { Table } from "https://deno.land/x/cliffy@v0.25.4/table/table.ts"; + Table, +} from "./deps.ts"; +import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; import { resolve, track_job } from "./script.ts"; type Options = GlobalOptions; async function push(opts: Options, filePath: string, remotePath: string) { - if (!(remotePath.startsWith("g") || remotePath.startsWith("u"))) { - console.log( - colors.red( - "Given remote path looks invalid. Remote paths are typicall of the form //...", - ), - ); + if (!await validatePath(opts, remotePath)) { return; } const workspace = await resolveWorkspace(opts); diff --git a/cli/folder.ts b/cli/folder.ts new file mode 100644 index 0000000000..03e857276d --- /dev/null +++ b/cli/folder.ts @@ -0,0 +1,90 @@ +import { colors, Command, Folder, FolderService } from "./deps.ts"; +import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; +import { GlobalOptions } from "./types.ts"; + +async function push(opts: GlobalOptions, filePath: string, remotePath: string) { + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + if (!await validatePath(opts, remotePath)) { + return; + } + + const fstat = await Deno.stat(filePath); + if (!fstat.isFile) { + throw new Error("file path must refer to a file."); + } + + console.log(colors.bold.yellow("Pushing resource...")); + + await pushFolder(workspace.workspaceId, filePath, remotePath); + console.log(colors.bold.underline.green("Resource successfully pushed")); +} + +type FolderFile = { + owners: Array | undefined; + extra_perms: Record | undefined; +}; + +export async function pushFolder( + workspace: string, + filePath: string, + remotePath: string, +) { + if (remotePath.startsWith("/")) { + remotePath = remotePath.substring(1); + } + if (remotePath.startsWith("f/")) { + remotePath = remotePath.substring(2); + } + const data: FolderFile = JSON.parse(await Deno.readTextFile(filePath)); + let optFolder: Folder | undefined; + try { + optFolder = await FolderService.getFolder({ workspace, name: remotePath }); + } catch { + optFolder = undefined; + } + + if (optFolder) { + // for (const [k, v] of Object.entries(optFolder.extra_perms)) { + // if (!data.extra_perms || data.extra_perms[k] !== v) { + // console.log(colors.red.underline.bold(`Extra Perms missmatch on ${k}`)); + // return; + // } + // } + + console.log(colors.yellow("Updating existing folder...")); + await FolderService.updateFolder({ + workspace, + name: remotePath, + requestBody: { + extra_perms: data.extra_perms, + owners: data.owners, + }, + }); + } else { + console.log(colors.yellow("Creating new folder...")); + await FolderService.createFolder({ + workspace, + requestBody: { + name: remotePath, + extra_perms: data.extra_perms, + owners: data.owners, + }, + }); + + // HACK: Workaround backend automatically adding current user to folder. + await pushFolder(workspace, filePath, remotePath); + } +} + +const command = new Command() + .description("resource related commands") + .command( + "push", + "push a local folder spec. This overrides any remote versions.", + ) + .arguments(" ") + .action(push as any); + +export default command; diff --git a/cli/hub.ts b/cli/hub.ts index 406640fe2a..65379b91be 100644 --- a/cli/hub.ts +++ b/cli/hub.ts @@ -1,4 +1,4 @@ -import { Command } from "https://deno.land/x/cliffy@v0.25.4/command/command.ts"; +import { Command } from "./deps.ts"; import { requireLogin, resolveWorkspace } from "./context.ts"; import { pushResourceTypeDef } from "./resource-type.ts"; import { GlobalOptions } from "./types.ts"; diff --git a/cli/login.ts b/cli/login.ts index 7de52b843c..2e6227cb69 100644 --- a/cli/login.ts +++ b/cli/login.ts @@ -1,8 +1,5 @@ -import { Select } from "https://deno.land/x/cliffy@v0.25.4/prompt/select.ts"; import { GlobalOptions } from "./types.ts"; -import { colors } from "https://deno.land/x/cliffy@v0.25.4/ansi/colors.ts"; -import { getAvailablePort } from "https://deno.land/x/port@1.0.0/mod.ts"; -import { Secret } from "https://deno.land/x/cliffy@v0.25.4/prompt/secret.ts"; +import { colors, getAvailablePort, Secret, Select } from "./deps.ts"; export async function loginInteractive(remote: string) { let token: string | undefined; diff --git a/cli/main.ts b/cli/main.ts index ec9d5fb2ba..d9d49578e8 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -1,8 +1,4 @@ -import { Command } from "https://deno.land/x/cliffy@v0.25.4/command/mod.ts"; -import { - DenoLandProvider, - UpgradeCommand, -} from "https://deno.land/x/cliffy@v0.25.4/command/upgrade/mod.ts"; +import { Command, DenoLandProvider, UpgradeCommand } from "./deps.ts"; import flow from "./flow.ts"; import script from "./script.ts"; import workspace from "./workspace.ts"; @@ -12,6 +8,9 @@ import variable from "./variable.ts"; import push from "./push.ts"; import pull from "./pull.ts"; import hub from "./hub.ts"; +// import folder from "./folder.ts"; +import { tryResolveVersion } from "./context.ts"; +import { GlobalOptions } from "./types.ts"; const VERSION = "v1.55.0"; @@ -36,6 +35,17 @@ const command = new Command() .command("push", push) .command("pull", pull) .command("hub", hub) + // .command("folder", folder) + .command("version", "Show version information") + .action(async (opts) => { + console.log("CLI build against " + VERSION); + const backendVersion = await tryResolveVersion(opts as GlobalOptions); + if (backendVersion) { + console.log("Backend Version: " + backendVersion); + } else { + console.log("Cannot resolve Backend Version"); + } + }) .command( "upgrade", new UpgradeCommand({ diff --git a/cli/pull.ts b/cli/pull.ts index 935aa25320..4a5264979c 100644 --- a/cli/pull.ts +++ b/cli/pull.ts @@ -1,16 +1,16 @@ // deno-lint-ignore-file no-explicit-any -import { Untar } from "https://deno.land/std@0.162.0/archive/tar.ts"; -import { Command } from "https://deno.land/x/cliffy@v0.25.4/command/command.ts"; -import { - copy, - readerFromStreamReader, -} from "https://deno.land/std@0.162.0/streams/mod.ts"; import { resolveWorkspace } from "./context.ts"; import { GlobalOptions } from "./types.ts"; -import * as path from "https://deno.land/std@0.162.0/path/mod.ts"; -import { colors } from "https://deno.land/x/cliffy@v0.25.4/ansi/colors.ts"; -import { ensureDir } from "https://deno.land/std@0.162.0/fs/ensure_dir.ts"; -import { Confirm } from "https://deno.land/x/cliffy@v0.25.4/prompt/confirm.ts"; +import { + colors, + Command, + Confirm, + copy, + ensureDir, + path, + readerFromStreamReader, + Untar, +} from "./deps.ts"; async function pull(opts: GlobalOptions & { override: boolean }, dir: string) { const workspace = await resolveWorkspace(opts); @@ -29,8 +29,12 @@ async function pull(opts: GlobalOptions & { override: boolean }, dir: string) { ); if (!tarResponse.ok) { - console.log(colors.red("Failed to request tarball from API")); - console.log(colors.red(await tarResponse.text())); + console.log( + colors.red( + "Failed to request tarball from API " + tarResponse.statusText, + ), + ); + console.log(await tarResponse.text()); return; } @@ -71,7 +75,8 @@ async function pull(opts: GlobalOptions & { override: boolean }, dir: string) { } } const file = await Deno.open(filePath, { write: true, create: true }); - await copy(entry, file); + const len = await copy(entry, file); + await file.truncate(len); file.close(); } console.log(colors.green("Done. Wrote all files to disk.")); diff --git a/cli/push.ts b/cli/push.ts index 371bc95593..8f86ca7106 100644 --- a/cli/push.ts +++ b/cli/push.ts @@ -1,7 +1,5 @@ // deno-lint-ignore-file no-explicit-any -import { colors } from "https://deno.land/x/cliffy@v0.25.4/ansi/colors.ts"; -import { Command } from "https://deno.land/x/cliffy@v0.25.4/command/command.ts"; -import * as path from "https://deno.land/std@0.162.0/path/mod.ts"; +import { colors, Command, path } from "./deps.ts"; import { requireLogin, resolveWorkspace } from "./context.ts"; import { pushFlow } from "./flow.ts"; import { pushResource } from "./resource.ts"; @@ -9,34 +7,73 @@ import { findContentFile, pushScript } from "./script.ts"; import { GlobalOptions } from "./types.ts"; import { pushVariable } from "./variable.ts"; import { pushResourceType } from "./resource-type.ts"; +import { pushFolder } from "./folder.ts"; type Candidate = { path: string; - group: boolean; - groupOrUsername: string; + namespaceKind: "user" | "group" | "folder"; + namespaceName: string; }; -async function findCandidateFiles(dir: string): Promise { - const candidates: Candidate[] = []; - if (dir.startsWith(".")) return []; + +type ResourceTypeCandidate = { + path: string; +}; + +type FolderCandidate = { + path: string; + namespaceName: string; +}; + +async function findCandidateFiles( + dir: string, +): Promise< + { + normal: Candidate[]; + resourceTypes: ResourceTypeCandidate[]; + folders: FolderCandidate[]; + } +> { + dir = path.resolve(dir); + if (path.dirname(dir).startsWith(".")) { + return { normal: [], resourceTypes: [], folders: [] }; + } + const normalCandidates: Candidate[] = []; + const resourceTypeCandidates: ResourceTypeCandidate[] = []; + const folderCandidates: FolderCandidate[] = []; for await (const e of Deno.readDir(dir)) { if (e.isDirectory) { - if (e.name == "u" || e.name == "g") { + if (e.name == "u" || e.name == "g" || e.name == "f") { // TODO: Check version for f const newDir = dir + (dir.endsWith("/") ? "" : "/") + e.name; for await (const e2 of Deno.readDir(newDir)) { if (e2.isDirectory) { - if (e2.name.startsWith(".")) return []; - const groupOrUserName = e2.name; + if (e2.name.startsWith(".")) continue; + const namespaceName = e2.name; const stack: string[] = []; - stack.push(newDir + "/" + groupOrUserName + "/"); + { + const path = newDir + "/" + namespaceName + "/"; + stack.push(path); + try { + await Deno.stat(path + "folder.meta.json"); + folderCandidates.push({ + namespaceName, + path: path + "folder.meta.json", + }); + } catch {} + } while (stack.length > 0) { const dir2 = stack.pop()!; for await (const e3 of Deno.readDir(dir2)) { if (e3.isFile) { - candidates.push({ + if (e3.name === "folder.meta.json") continue; + normalCandidates.push({ path: dir2 + e3.name, - group: e.name == "g", - groupOrUsername: groupOrUserName, + namespaceKind: e.name == "g" + ? "group" + : e.name == "u" + ? "user" + : "folder", + namespaceName: namespaceName, }); } else { stack.push(dir2 + e3.name + "/"); @@ -51,21 +88,27 @@ async function findCandidateFiles(dir: string): Promise { "Including organizational folder " + e.name + " in push!", ), ); - candidates.push(...(await findCandidateFiles(path.join(dir, e.name)))); + const { normal, resourceTypes, folders } = await findCandidateFiles( + path.join(dir, e.name), + ); + normalCandidates.push(...normal); + resourceTypeCandidates.push(...resourceTypes); + folderCandidates.push(...folders); } } else { // handle root files if (e.name.endsWith(".resource-type.json")) { - candidates.push({ - group: false, - groupOrUsername: "", + resourceTypeCandidates.push({ path: dir + (dir.endsWith("/") ? "" : "/") + e.name, }); - console.log(candidates); } } } - return candidates; + return { + normal: normalCandidates, + folders: folderCandidates, + resourceTypes: resourceTypeCandidates, + }; } async function push(opts: GlobalOptions, dir?: string) { @@ -74,16 +117,55 @@ async function push(opts: GlobalOptions, dir?: string) { await requireLogin(opts); console.log(colors.blue("Searching Directory...")); - const candidates: Candidate[] = await findCandidateFiles(dir); - console.log(colors.blue("Found " + candidates.length + " candidates")); - for (const candidate of candidates) { + const { normal, resourceTypes, folders } = await findCandidateFiles(dir); + console.log( + colors.blue( + "Found " + (normal.length + resourceTypes.length + folders.length) + + " candidates", + ), + ); + for (const resourceType of resourceTypes) { + const fileName = resourceType.path.substring( + resourceType.path.lastIndexOf("/") + 1, + ); + const fileNameParts = fileName.split("."); + // invalid file names, like my.cool.script.script.json. Not valid. + if (fileNameParts.length != 3) { + console.log( + colors.yellow("invalid file name found at " + resourceType.path), + ); + continue; + } + + // filter out non-json files. Note that we filter out script contents above, so this is really an error. + if (fileNameParts.at(-1) != "json") { + console.log(colors.yellow("non-JSON file found at " + resourceType.path)); + continue; + } + + console.log("pushing resource type " + fileNameParts.at(-3)!); + await pushResourceType( + workspace.workspaceId, + resourceType.path, + fileNameParts.at(-3)!, + ); + } + for (const folder of folders) { + await pushFolder( + workspace.workspaceId, + folder.path, + "f/" + folder.namespaceName, + ); + } + for (const candidate of normal) { // full file name. No leading /. includes .type.json const fileName = candidate.path.substring( candidate.path.lastIndexOf("/") + 1, ); // figure out just the path after ...../u|g/username|group/ (in extra dir) const dirParts = candidate.path.split("/").filter((x) => x.length > 0); - const gIndex = dirParts.findIndex((x) => x == "u" || x == "g"); + // TODO: check version for folder + const gIndex = dirParts.findIndex((x) => x == "u" || x == "g" || x == "f"); const extraDir = dirParts.slice(gIndex + 2, -1).join("/"); // file name parts has .json (hopefully) at -1, type at -2, and the actual name at -3. Dots in names are not allowed. @@ -116,22 +198,13 @@ async function push(opts: GlobalOptions, dir?: string) { // get the type & filter it for valid ones. const type = fileNameParts.at(-2); if (type == "resource-type") { - if (candidate.group == false && candidate.groupOrUsername == "") { - console.log("pushing resource type " + fileNameParts.at(-3)!); - await pushResourceType( - workspace.workspaceId, - candidate.path, - fileNameParts.at(-3)!, - ); - } else { - console.log( - colors.yellow( - "Found resource type file at " + - candidate.path + - " this appears to be inside a path folder. Resource types are not addressed by path. Place them at the root or inside only an organizational folder. Ignoring this file!", - ), - ); - } + console.log( + colors.yellow( + "Found resource type file at " + + candidate.path + + " this appears to be inside a path folder. Resource types are not addressed by path. Place them at the root or inside only an organizational folder. Ignoring this file!", + ), + ); continue; } @@ -150,8 +223,10 @@ async function push(opts: GlobalOptions, dir?: string) { } // create the remotePath for the API - const remotePath = (candidate.group ? "g/" : "u/") + - candidate.groupOrUsername + + const remotePath = (candidate.namespaceKind === "group" + ? "g/" + : (candidate.namespaceKind === "user" ? "u/" : "f/")) + + candidate.namespaceName + "/" + (extraDir.length > 0 ? extraDir + "/" : "") + fileNameParts.at(-3); @@ -167,7 +242,7 @@ async function push(opts: GlobalOptions, dir?: string) { try { contentPath = await findContentFile(candidate.path); } catch (e) { - console.log(colors.red(e)); + console.log(colors.red(e.toString())); continue; } await pushScript( diff --git a/cli/resource-type.ts b/cli/resource-type.ts index c11799849c..e2a3ee52ca 100644 --- a/cli/resource-type.ts +++ b/cli/resource-type.ts @@ -1,10 +1,7 @@ // deno-lint-ignore-file no-explicit-any -import { Command } from "https://deno.land/x/cliffy@v0.25.4/command/command.ts"; -import { ResourceService } from "https://deno.land/x/windmill@v1.50.0/mod.ts"; import { GlobalOptions } from "./types.ts"; -import { colors } from "https://deno.land/x/cliffy@v0.25.4/ansi/colors.ts"; import { requireLogin, resolveWorkspace } from "./context.ts"; -import { Table } from "https://deno.land/x/cliffy@v0.25.4/table/table.ts"; +import { colors, Command, ResourceService, Table } from "./deps.ts"; type ResourceTypeFile = { schema?: any; diff --git a/cli/resource.ts b/cli/resource.ts index ea76458d59..9e02e3070d 100644 --- a/cli/resource.ts +++ b/cli/resource.ts @@ -1,22 +1,18 @@ -import { Command } from "https://deno.land/x/cliffy@v0.25.4/command/command.ts"; -import { ResourceService } from "https://deno.land/x/windmill@v1.50.0/mod.ts"; import { GlobalOptions } from "./types.ts"; -import { colors } from "https://deno.land/x/cliffy@v0.25.4/ansi/colors.ts"; -import { requireLogin, resolveWorkspace } from "./context.ts"; -import { Resource } from "https://deno.land/x/windmill@v1.50.0/windmill-api/index.ts"; -import { Table } from "https://deno.land/x/cliffy@v0.25.4/table/table.ts"; +import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; +import { colors, Command, Resource, ResourceService, Table } from "./deps.ts"; type ResourceFile = { value: any; description?: string; resource_type: string; - is_oauth?: boolean; + is_oauth?: boolean; // deprecated }; export async function pushResource( workspace: string, filePath: string, - remotePath: string + remotePath: string, ) { const data: ResourceFile = JSON.parse(await Deno.readTextFile(filePath)); if ( @@ -30,26 +26,26 @@ export async function pushResource( workspace: workspace, path: remotePath, }); + if (existing.resource_type != data.resource_type) { console.log( colors.red.underline.bold( "Remote resource at " + remotePath + - " exists & has a different resource type. This cannot be updated. If you wish to do this anyways, consider deleting the remote resource." - ) + " exists & has a different resource type. This cannot be updated. If you wish to do this anyways, consider deleting the remote resource.", + ), ); return; } - if (existing.is_oauth != data.is_oauth) { + + if (typeof data.is_oauth !== "undefined") { console.log( - colors.red.underline.bold( - "Remote resource at " + - remotePath + - " exists & has a different oauth state. This cannot be updated. If you wish to do this anyways, consider deleting the remote resource." - ) + colors.yellow( + "! is_oauth has been removed in newer versions. Ignoring.", + ), ); - return; } + await ResourceService.updateResource({ workspace: workspace, path: remotePath, @@ -60,6 +56,14 @@ export async function pushResource( }, }); } else { + if (typeof data.is_oauth !== "undefined") { + console.log( + colors.yellow( + "! is_oauth has been removed in newer versions. Ignoring.", + ), + ); + } + console.log(colors.yellow("Creating new resource...")); await ResourceService.createResource({ workspace: workspace, @@ -68,7 +72,6 @@ export async function pushResource( resource_type: data.resource_type, value: data.value, description: data.description, - is_oauth: data.is_oauth, }, }); } @@ -79,12 +82,7 @@ async function push(opts: PushOptions, filePath: string, remotePath: string) { const workspace = await resolveWorkspace(opts); await requireLogin(opts); - if (!(remotePath.startsWith("g") || remotePath.startsWith("u"))) { - console.log( - colors.red( - "Given remote path looks invalid. Remote paths are typicall of the form //..." - ) - ); + if (!await validatePath(opts, remotePath)) { return; } @@ -131,7 +129,7 @@ const command = new Command() .action(list as any) .command( "push", - "push a local resource spec. This overrides any remote versions." + "push a local resource spec. This overrides any remote versions.", ) .arguments(" ") .action(push as any); diff --git a/cli/script.ts b/cli/script.ts index eb70548304..749768f5a8 100644 --- a/cli/script.ts +++ b/cli/script.ts @@ -1,15 +1,11 @@ // deno-lint-ignore-file no-explicit-any -import { Command } from "https://deno.land/x/cliffy@v0.25.4/command/command.ts"; -import { ScriptService } from "https://deno.land/x/windmill@v1.50.0/mod.ts"; import { GlobalOptions } from "./types.ts"; -import { colors } from "https://deno.land/x/cliffy@v0.25.4/ansi/colors.ts"; -import { resolveWorkspace, requireLogin } from "./context.ts"; +import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; import { JobService, Script, } from "https://deno.land/x/windmill@v1.50.0/windmill-api/index.ts"; -import { Table } from "https://deno.land/x/cliffy@v0.25.4/table/table.ts"; -import { readAll } from "https://deno.land/std@0.165.0/streams/mod.ts"; +import { colors, Command, readAll, ScriptService, Table } from "./deps.ts"; type ScriptFile = { parent_hash?: string; @@ -26,15 +22,10 @@ async function push( opts: PushOptions, filePath: string, remotePath: string, - contentPath?: string + contentPath?: string, ) { const workspace = await resolveWorkspace(opts); - if (!(remotePath.startsWith("g") || remotePath.startsWith("u"))) { - console.log( - colors.red( - "Given remote path looks invalid. Remote paths are typicall of the form //..." - ) - ); + if (!await validatePath(opts, remotePath)) { return; } @@ -58,9 +49,9 @@ async function push( export async function findContentFile(filePath: string) { const candidates = [ - filePath.replace(".json", ".ts"), - filePath.replace(".json", ".py"), - filePath.replace(".json", ".go"), + filePath.replace(".script.json", ".ts"), + filePath.replace(".script.json", ".py"), + filePath.replace(".script.json", ".go"), ]; const validCandidates = ( await Promise.all( @@ -71,7 +62,7 @@ export async function findContentFile(filePath: string) { .then((e) => { return { path: x, file: e }; }); - }) + }), ) ) .filter((x) => x.file) @@ -79,7 +70,7 @@ export async function findContentFile(filePath: string) { if (validCandidates.length > 1) { throw new Error( "No content path given and more then one candidate found: " + - validCandidates.join(", ") + validCandidates.join(", "), ); } if (validCandidates.length < 1) { @@ -92,7 +83,7 @@ export async function pushScript( filePath: string, contentPath: string, workspace: string, - remotePath: string + remotePath: string, ) { const data: ScriptFile = JSON.parse(await Deno.readTextFile(filePath)); const content = await Deno.readTextFile(contentPath); @@ -170,7 +161,7 @@ async function list(opts: GlobalOptions & { showArchived?: boolean }) { x.language, x.created_at, x.created_by, - ]) + ]), ) .render(); } @@ -221,7 +212,7 @@ async function run( input: string[]; silent: boolean; }, - path: string + path: string, ) { const workspace = await resolveWorkspace(opts); await requireLogin(opts); @@ -239,13 +230,12 @@ async function run( while (true) { try { - const result = - ( - await JobService.getCompletedJob({ - workspace: workspace.workspaceId, - id, - }) - ).result ?? {}; + const result = ( + await JobService.getCompletedJob({ + workspace: workspace.workspaceId, + id, + }) + ).result ?? {}; console.log(result); break; @@ -311,7 +301,7 @@ export async function track_job(workspace: string, id: string) { if (running && updates.running === false) { running = false; console.log( - colors.yellow("Job suspended. Waiting for it to continue...") + colors.yellow("Job suspended. Waiting for it to continue..."), ); } } @@ -352,7 +342,7 @@ const command = new Command() .action(list as any) .command( "push", - "push a local script spec. This overrides any remote versions." + "push a local script spec. This overrides any remote versions.", ) .arguments(" [content_path:string]") .action(push as any) @@ -363,11 +353,11 @@ const command = new Command() .arguments("") .option( "-i --input [inputs...:string]", - "Inputs specified as JSON objects or simply as =. Supports file inputs using @ and stdin using @- these also need to be formatted as JSON. Later inputs override earlier ones." + "Inputs specified as JSON objects or simply as =. Supports file inputs using @ and stdin using @- these also need to be formatted as JSON. Later inputs override earlier ones.", ) .option( "-s --silent", - "Do not ouput anything other then the final output. Useful for scripting." + "Do not ouput anything other then the final output. Useful for scripting.", ) .action(run as any); diff --git a/cli/store.ts b/cli/store.ts index 0c1875682f..2fa6529da9 100644 --- a/cli/store.ts +++ b/cli/store.ts @@ -1,5 +1,4 @@ -import dir from "https://deno.land/x/dir@1.5.1/mod.ts"; -import * as fs from "https://deno.land/std@0.161.0/fs/mod.ts"; +import { dir, ensureDir } from "./deps.ts"; function hash_string(str: string): number { let hash = 0, @@ -16,13 +15,13 @@ function hash_string(str: string): number { export async function getRootStore(): Promise { const store = dir("config") + "/windmill/"; - await fs.ensureDir(store); + await ensureDir(store); return store; } export async function getStore(baseUrl: string): Promise { const baseHash = Math.abs(hash_string(baseUrl)).toString(16); const baseStore = (await getRootStore()) + baseHash + "/"; - await fs.ensureDir(baseStore); + await ensureDir(baseStore); return baseStore; } diff --git a/cli/tmp/f/my_folder/folder.meta.json b/cli/tmp/f/my_folder/folder.meta.json new file mode 100644 index 0000000000..42d14ea3d0 --- /dev/null +++ b/cli/tmp/f/my_folder/folder.meta.json @@ -0,0 +1,10 @@ +{ + "workspace_id": "admins", + "name": "my_folder", + "display_name": "my_folder", + "owners": [], + "extra_perms": { + "u/test": true, + "u/admin@windmill.dev": false + } +} \ No newline at end of file diff --git a/cli/tmp/u/admin/hub_sync.script.json b/cli/tmp/u/admin/hub_sync.script.json new file mode 100644 index 0000000000..e9e1636e1e --- /dev/null +++ b/cli/tmp/u/admin/hub_sync.script.json @@ -0,0 +1,12 @@ +{ + "summary": "Syncronize Hub Resource types with starter workspace", + "description": "Basic administrative script to sync latest resource types from hub. Recommended to run at least once. On a schedule by default.", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": {}, + "required": [], + "type": "object" + }, + "is_template": false, + "lock": [] +} \ No newline at end of file diff --git a/cli/tmp/u/admin/hub_sync.ts b/cli/tmp/u/admin/hub_sync.ts new file mode 100644 index 0000000000..0a369ae9e5 --- /dev/null +++ b/cli/tmp/u/admin/hub_sync.ts @@ -0,0 +1,13 @@ +import wmill from "https://deno.land/x/wmill@v1.55.0/main.ts"; + +export async function main() { + await run( + "workspace", "add", "__automation", "starter", Deno.env.get("WM_BASE_URL") + "/", "--token", Deno.env.get("WM_TOKEN")); + + await run("hub", "pull"); +} + +async function run(...cmd: string[]) { + console.log("Running \"" + cmd.join(' ') + "\""); + await wmill.parse(cmd); +} \ No newline at end of file diff --git a/cli/user.ts b/cli/user.ts index 0b0bfc09db..cccb60334b 100644 --- a/cli/user.ts +++ b/cli/user.ts @@ -1,12 +1,14 @@ // deno-lint-ignore-file no-explicit-any -import { Command } from "https://deno.land/x/cliffy@v0.25.4/command/command.ts"; -import { Table } from "https://deno.land/x/cliffy@v0.25.4/table/table.ts"; -import { UserService } from "https://deno.land/x/windmill@v1.50.0/mod.ts"; -import { GlobalUserInfo } from "https://deno.land/x/windmill@v1.50.0/windmill-api/index.ts"; -import { passwordGenerator } from "https://deno.land/x/password_generator@latest/mod.ts"; // TODO: I think the version is called latest, but it's still pinned. import { requireLogin } from "./context.ts"; import { GlobalOptions } from "./types.ts"; -import { colors } from "https://deno.land/x/cliffy@v0.25.4/ansi/colors.ts"; +import { + colors, + Command, + GlobalUserInfo, + passwordGenerator, + Table, + UserService, +} from "./deps.ts"; async function list(opts: GlobalOptions) { await requireLogin(opts); diff --git a/cli/variable.ts b/cli/variable.ts index 3b9651660f..a90d9a8b38 100644 --- a/cli/variable.ts +++ b/cli/variable.ts @@ -1,10 +1,7 @@ // deno-lint-ignore-file no-explicit-any -import { colors } from "https://deno.land/x/cliffy@v0.25.4/ansi/colors.ts"; -import { Command } from "https://deno.land/x/cliffy@v0.25.4/command/command.ts"; -import { Table } from "https://deno.land/x/cliffy@v0.25.4/table/table.ts"; -import { VariableService } from "https://deno.land/x/windmill@v1.50.0/mod.ts"; -import { requireLogin, resolveWorkspace } from "./context.ts"; +import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; import { GlobalOptions } from "./types.ts"; +import { colors, Command, Table, VariableService } from "./deps.ts"; async function list(opts: GlobalOptions) { const workspace = await resolveWorkspace(opts); @@ -41,12 +38,7 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) { const workspace = await resolveWorkspace(opts); await requireLogin(opts); - if (!(remotePath.startsWith("g") || remotePath.startsWith("u"))) { - console.log( - colors.red( - "Given remote path looks invalid. Remote paths are typicall of the form //...", - ), - ); + if (!await validatePath(opts, remotePath)) { return; } diff --git a/cli/workspace.ts b/cli/workspace.ts index 044dad8f43..0e2e7e9bbf 100644 --- a/cli/workspace.ts +++ b/cli/workspace.ts @@ -1,12 +1,8 @@ // deno-lint-ignore-file no-explicit-any -import { Command } from "https://deno.land/x/cliffy@v0.25.4/command/command.ts"; -import { Table } from "https://deno.land/x/cliffy@v0.25.4/table/table.ts"; import { GlobalOptions } from "./types.ts"; -import { DelimiterStream } from "https://deno.land/std@0.165.0/streams/mod.ts"; -import { colors } from "https://deno.land/x/cliffy@v0.25.4/ansi/colors.ts"; import { getRootStore } from "./store.ts"; -import { Input } from "https://deno.land/x/cliffy@v0.25.4/prompt/input.ts"; import { loginInteractive, tryGetLoginInfo } from "./login.ts"; +import { colors, Command, DelimiterStream, Input, Table } from "./deps.ts"; export type Workspace = { remote: string;