diff --git a/cli/context.ts b/cli/context.ts index 1b524814e0..1b5574a3d6 100644 --- a/cli/context.ts +++ b/cli/context.ts @@ -1,10 +1,12 @@ // deno-lint-ignore-file no-explicit-any -import { colors, setClient } from "./deps.ts"; -import { tryGetLoginInfo } from "./login.ts"; +import { colors, setClient, UserService } from "./deps.ts"; +import { loginInteractive, tryGetLoginInfo } from "./login.ts"; import { GlobalOptions } from "./types.ts"; import { + addWorkspace, getActiveWorkspace, getWorkspaceByName, + removeWorkspace, Workspace, } from "./workspace.ts"; @@ -67,6 +69,27 @@ export async function requireLogin(opts: GlobalOptions) { } setClient(token, workspace.remote.substring(0, workspace.remote.length - 1)); + + try { + await UserService.globalWhoami(); + } catch { + console.log( + "! Could not reach API given existing credentials. Attempting to reauth...", + ); + const newToken = await loginInteractive(workspace.remote); + if (!newToken) { + throw new Error("Could not reauth"); + } + removeWorkspace(workspace.name); + workspace.token = newToken; + addWorkspace(workspace); + + setClient( + token, + workspace.remote.substring(0, workspace.remote.length - 1), + ); + await UserService.globalWhoami(); + } } export async function tryResolveVersion( diff --git a/cli/decoverto.ts b/cli/decoverto.ts new file mode 100644 index 0000000000..98f1a157c2 --- /dev/null +++ b/cli/decoverto.ts @@ -0,0 +1,8 @@ +// globally shared decoverto instance +import { Decoverto } from "npm:decoverto"; +const decoverto = new Decoverto(); + +// TODO: Properly type FlowModule + +export { Any, array, map, MapShape, model, property } from "npm:decoverto"; +export { decoverto }; diff --git a/cli/deps.ts b/cli/deps.ts index 9da34f87c9..6c2458183c 100644 --- a/cli/deps.ts +++ b/cli/deps.ts @@ -1,32 +1,40 @@ // windmill -export { setClient } from "https://deno.land/x/windmill@v1.56.0/mod.ts"; -export * from "https://deno.land/x/windmill@v1.56.0/windmill-api/index.ts"; +export { setClient } from "https://deno.land/x/windmill@v1.61.1/mod.ts"; +export * from "https://deno.land/x/windmill@v1.61.1/windmill-api/index.ts"; // cliffy -export { Command } from "https://deno.land/x/cliffy@v0.25.6/command/command.ts"; -export { Table } from "https://deno.land/x/cliffy@v0.25.6/table/table.ts"; -export { colors } from "https://deno.land/x/cliffy@v0.25.6/ansi/colors.ts"; -export { Secret } from "https://deno.land/x/cliffy@v0.25.6/prompt/secret.ts"; -export { Select } from "https://deno.land/x/cliffy@v0.25.6/prompt/select.ts"; -export { Confirm } from "https://deno.land/x/cliffy@v0.25.6/prompt/confirm.ts"; -export { Input } from "https://deno.land/x/cliffy@v0.25.6/prompt/input.ts"; +export { Command } from "https://deno.land/x/cliffy@v0.25.7/command/command.ts"; +export { Table } from "https://deno.land/x/cliffy@v0.25.7/table/table.ts"; +export { colors } from "https://deno.land/x/cliffy@v0.25.7/ansi/colors.ts"; +export { Secret } from "https://deno.land/x/cliffy@v0.25.7/prompt/secret.ts"; +export { Select } from "https://deno.land/x/cliffy@v0.25.7/prompt/select.ts"; +export { Confirm } from "https://deno.land/x/cliffy@v0.25.7/prompt/confirm.ts"; +export { Input } from "https://deno.land/x/cliffy@v0.25.7/prompt/input.ts"; export { DenoLandProvider, UpgradeCommand, -} from "https://deno.land/x/cliffy@v0.25.6/command/upgrade/mod.ts"; +} from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/mod.ts"; // std -export { Untar } from "https://deno.land/std@0.170.0/archive/untar.ts"; -export * as path from "https://deno.land/std@0.170.0/path/mod.ts"; -export { ensureDir } from "https://deno.land/std@0.170.0/fs/ensure_dir.ts"; +export { Untar } from "https://deno.land/std@0.176.0/archive/untar.ts"; +export * as path from "https://deno.land/std@0.176.0/path/mod.ts"; +export { ensureDir } from "https://deno.land/std@0.176.0/fs/ensure_dir.ts"; export { copy, readAll, readerFromStreamReader, -} from "https://deno.land/std@0.170.0/streams/mod.ts"; -export { DelimiterStream } from "https://deno.land/std@0.170.0/streams/mod.ts"; +} from "https://deno.land/std@0.176.0/streams/mod.ts"; +export { DelimiterStream } from "https://deno.land/std@0.176.0/streams/mod.ts"; +export { iterateReader } from "https://deno.land/std@0.176.0/streams/iterate_reader.ts"; // other export { getAvailablePort } from "https://deno.land/x/port@1.0.0/mod.ts"; export { default as dir } from "https://deno.land/x/dir@1.5.1/mod.ts"; export { passwordGenerator } from "https://deno.land/x/password_generator@latest/mod.ts"; // TODO: I think the version is called latest, but it's still pinned. +export { nanoid } from "https://deno.land/x/nanoid@v3.0.0/mod.ts"; +export * as cbor from "https://deno.land/x/cbor@v1.4.1/index.js"; +export { default as Murmurhash3 } from "https://deno.land/x/murmurhash@v1.0.0/mod.ts"; +export { + default as microdiff, +} from "https://deno.land/x/microdiff@v1.3.1/index.ts"; +export { default as objectHash } from "https://deno.land/x/object_hash@2.0.3.1/mod.ts"; diff --git a/cli/examples/u/admin/fib/fib.script.ts b/cli/examples/u/admin/fib/fib.ts similarity index 100% rename from cli/examples/u/admin/fib/fib.script.ts rename to cli/examples/u/admin/fib/fib.ts diff --git a/cli/examples/u/admin/go_test/go_test.script.go b/cli/examples/u/admin/go_test/go_test.go similarity index 100% rename from cli/examples/u/admin/go_test/go_test.script.go rename to cli/examples/u/admin/go_test/go_test.go diff --git a/cli/flow.ts b/cli/flow.ts index 1d2171fa08..a8e2af2e24 100644 --- a/cli/flow.ts +++ b/cli/flow.ts @@ -1,16 +1,145 @@ // deno-lint-ignore-file no-explicit-any -import { GlobalOptions } from "./types.ts"; +import { + Difference, + GlobalOptions, + PushDiffs, + Resource, + setValueByPath, +} from "./types.ts"; import { colors, Command, Flow, + FlowModule, FlowService, JobService, - OpenFlow, + microdiff, + OpenFlowWPath, Table, } from "./deps.ts"; import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; import { resolve, track_job } from "./script.ts"; +import { Any, array, decoverto, model, property } from "./decoverto.ts"; + +@model() +export class FlowValueFilePart { + @property(array(Any)) + modules: Array; + @property(Any) + failure_module?: FlowModule; + @property(() => Boolean) + same_worker?: boolean; + + constructor(modules: Array) { + this.modules = modules; + } +} + +// this is effectively "OpenFlow" but a copy as it is accepted by the CLI +@model() +export class FlowFile implements Resource, PushDiffs { + @property(() => String) + summary: string; + @property(() => String) + description?: string; + @property(() => FlowValueFilePart) + value: FlowValueFilePart; + @property(Any) + schema?: any; + + constructor(summary: string, value: FlowValueFilePart) { + this.summary = summary; + this.value = value; + } + async pushDiffs( + workspace: string, + remotePath: string, + diffs: Difference[], + ): Promise { + if ( + await FlowService.existsFlowByPath({ + workspace: workspace, + path: remotePath, + }) + ) { + console.log( + colors.bold.yellow( + `Applying ${diffs.length} diffs to existing flow...`, + ), + ); + + // TODO: Make these optional in backend (not path ofc) + const changeset: OpenFlowWPath = { + path: remotePath, + summary: this.summary, + value: this.value, + description: this.description, // This is OpenAPIed as optional, but isn't + schema: this.schema, // Same + }; + for (const diff of diffs) { + if ( + diff.type !== "REMOVE" && + ( + diff.path[0] !== "value" && ( + diff.path.length !== 1 || + !["summary", "description", "schema"].includes( + diff.path[0] as string, + ) + ) + ) + ) { + throw new Error("Invalid flow diff with path " + diff.path); + } + if (diff.type === "CREATE" || diff.type === "CHANGE") { + setValueByPath(changeset, diff.path, diff.value); + } else if (diff.type === "REMOVE") { + setValueByPath(changeset, diff.path, null); + } + } + const hasChanges = Object.values(changeset).some((v) => + v !== null && typeof v !== "undefined" + ); + if (!hasChanges) { + console.log(colors.yellow("! Skipping empty changeset")); + return; + } + + await FlowService.updateFlow({ + workspace: workspace, + path: remotePath, + requestBody: changeset, + }); + } else { + console.log(colors.bold.yellow("Creating new flow...")); + await FlowService.createFlow({ + workspace: workspace, + requestBody: { + path: remotePath, + summary: this.summary, + value: this.value, + schema: this.schema, + description: this.description, + }, + }); + } + } + async push(workspace: string, remotePath: string): Promise { + let remote: Flow | undefined; + try { + remote = await FlowService.getFlowByPath({ + workspace, + path: remotePath, + }); + } catch { + remote = undefined; + } + await this.pushDiffs( + workspace, + remotePath, + microdiff(remote ?? {}, this, { cyclesFix: false }), + ); + } +} type Options = GlobalOptions; @@ -30,38 +159,10 @@ export async function pushFlow( workspace: string, remotePath: string, ) { - const data: OpenFlow = JSON.parse(await Deno.readTextFile(filePath)); - if ( - await FlowService.existsFlowByPath({ - workspace: workspace, - path: remotePath, - }) - ) { - console.log(colors.bold.yellow("Updating existing flow...")); - await FlowService.updateFlow({ - workspace: workspace, - path: remotePath, - requestBody: { - path: remotePath, - summary: data.summary, - value: data.value, - schema: data.schema, - description: data.description, - }, - }); - } else { - console.log(colors.bold.yellow("Creating new flow...")); - await FlowService.createFlow({ - workspace: workspace, - requestBody: { - path: remotePath, - summary: data.summary, - value: data.value, - schema: data.schema, - description: data.description, - }, - }); - } + const data = decoverto.type(FlowFile).rawToInstance( + await Deno.readTextFile(filePath), + ); + await data.push(workspace, remotePath); } async function list(opts: GlobalOptions & { showArchived?: boolean }) { diff --git a/cli/folder.ts b/cli/folder.ts index 03e857276d..102942ac9b 100644 --- a/cli/folder.ts +++ b/cli/folder.ts @@ -1,6 +1,122 @@ -import { colors, Command, Folder, FolderService } from "./deps.ts"; +import { colors, Command, Folder, FolderService, microdiff } from "./deps.ts"; import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; -import { GlobalOptions } from "./types.ts"; +import { + Difference, + GlobalOptions, + PushDiffs, + Resource, + setValueByPath, +} from "./types.ts"; +import { + array, + decoverto, + map, + MapShape, + model, + property, +} from "./decoverto.ts"; + +@model() +export class FolderFile implements Resource, PushDiffs { + @property(array(() => String)) + owners: Array | undefined; + @property(map(() => String, () => Boolean, { shape: MapShape.Object })) + extra_perms: Map | undefined; + + async push(workspace: string, remotePath: string): Promise { + if (remotePath.startsWith("/")) { + remotePath = remotePath.substring(1); + } + if (remotePath.startsWith("f/")) { + remotePath = remotePath.substring(2); + } + + let existing: Folder | undefined; + try { + existing = await FolderService.getFolder({ workspace, name: remotePath }); + } catch { + existing = undefined; + } + await this.pushDiffs( + workspace, + remotePath, + microdiff(existing ?? {}, this, { cyclesFix: false }), + ); + } + + async pushDiffs( + workspace: string, + remotePath: string, + diffs: Difference[], + ): Promise { + if (remotePath.startsWith("/")) { + remotePath = remotePath.substring(1); + } + if (remotePath.startsWith("f/")) { + remotePath = remotePath.substring(2); + } + + // TODO: Support this in backend + let exists: boolean; + try { + exists = !!await FolderService.getFolder({ workspace, name: remotePath }); + } catch { + exists = false; + } + if (exists) { + console.log( + colors.bold.yellow( + `Applying ${diffs.length} diffs to existing folder...`, + ), + ); + + const changeset: { + owners?: string[] | undefined; + extra_perms?: any; + } = {}; + for (const diff of diffs) { + if ( + diff.type !== "REMOVE" && + ( + diff.path.length !== 1 || + !["owners", "extra_perms"].includes(diff.path[0] as string) + ) + ) { + throw new Error("Invalid folder diff with path " + diff.path); + } + if (diff.type === "CREATE" || diff.type === "CHANGE") { + setValueByPath(changeset, diff.path, diff.value); + } else if (diff.type === "REMOVE") { + setValueByPath(changeset, diff.path, null); + } + } + + const hasChanges = Object.values(changeset).some((v) => + v !== null && typeof v !== "undefined" + ); + if (!hasChanges) { + console.log(colors.yellow("! Skipping empty changeset")); + return; + } + + await FolderService.updateFolder({ + workspace: workspace, + name: remotePath, + requestBody: changeset, + }); + } else { + console.log(colors.bold.yellow("Creating new folder...")); + await FolderService.createFolder({ + workspace: workspace, + requestBody: { + name: remotePath, + extra_perms: this.extra_perms, + owners: this.owners, + }, + }); + } + } +} async function push(opts: GlobalOptions, filePath: string, remotePath: string) { const workspace = await resolveWorkspace(opts); @@ -21,61 +137,15 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) { console.log(colors.bold.underline.green("Resource successfully pushed")); } -type FolderFile = { - owners: Array | undefined; - extra_perms: Record | undefined; -}; - export async function pushFolder( workspace: string, filePath: string, remotePath: string, ) { - if (remotePath.startsWith("/")) { - remotePath = remotePath.substring(1); - } - if (remotePath.startsWith("f/")) { - remotePath = remotePath.substring(2); - } - const data: FolderFile = JSON.parse(await Deno.readTextFile(filePath)); - let optFolder: Folder | undefined; - try { - optFolder = await FolderService.getFolder({ workspace, name: remotePath }); - } catch { - optFolder = undefined; - } - - if (optFolder) { - // for (const [k, v] of Object.entries(optFolder.extra_perms)) { - // if (!data.extra_perms || data.extra_perms[k] !== v) { - // console.log(colors.red.underline.bold(`Extra Perms missmatch on ${k}`)); - // return; - // } - // } - - console.log(colors.yellow("Updating existing folder...")); - await FolderService.updateFolder({ - workspace, - name: remotePath, - requestBody: { - extra_perms: data.extra_perms, - owners: data.owners, - }, - }); - } else { - console.log(colors.yellow("Creating new folder...")); - await FolderService.createFolder({ - workspace, - requestBody: { - name: remotePath, - extra_perms: data.extra_perms, - owners: data.owners, - }, - }); - - // HACK: Workaround backend automatically adding current user to folder. - await pushFolder(workspace, filePath, remotePath); - } + const data = decoverto.type(FolderFile).rawToInstance( + await Deno.readTextFile(filePath), + ); + data.push(workspace, remotePath); } const command = new Command() diff --git a/cli/hub.ts b/cli/hub.ts index 41b0e6d2c0..526df43bbc 100644 --- a/cli/hub.ts +++ b/cli/hub.ts @@ -1,6 +1,6 @@ import { Command } from "./deps.ts"; import { requireLogin, resolveWorkspace } from "./context.ts"; -import { pushResourceTypeDef } from "./resource-type.ts"; +import { ResourceTypeFile } from "./resource-type.ts"; import { GlobalOptions } from "./types.ts"; async function pull(opts: GlobalOptions) { @@ -56,14 +56,10 @@ async function pull(opts: GlobalOptions) { const x of list ) { console.log("syncing " + x.name); - await pushResourceTypeDef( - workspace.workspaceId, - x.name, - { - description: x.description, - schema: JSON.parse(x.schema), - }, - ); + const f = new ResourceTypeFile(); + f.description = x.description; + f.schema = JSON.parse(x.schema); + await f.push(workspace.workspaceId, x.name); } } diff --git a/cli/main.ts b/cli/main.ts index ec7d76f6fc..55ee5d6d9f 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -8,13 +8,14 @@ import variable from "./variable.ts"; import push from "./push.ts"; import pull from "./pull.ts"; import hub from "./hub.ts"; -// import folder from "./folder.ts"; +import folder from "./folder.ts"; +import sync from "./sync.ts"; import { tryResolveVersion } from "./context.ts"; import { GlobalOptions } from "./types.ts"; const VERSION = "v1.61.1"; -const command = new Command() +let command: any = new Command() .name("wmill") .description("A simple CLI tool for windmill.") .globalOption( @@ -32,10 +33,9 @@ const command = new Command() .command("resource", resource) .command("user", user) .command("variable", variable) - .command("push", push) - .command("pull", pull) .command("hub", hub) - // .command("folder", folder) + .command("folder", folder) + .command("sync", sync) .command("version", "Show version information") .action(async (opts) => { console.log("CLI build against " + VERSION); @@ -61,6 +61,12 @@ const command = new Command() }), ); +if (Number.parseInt(VERSION.replace("v", "").replace(".", "")) > 1700) { + command = command + .command("push", push) + .command("pull", pull); +} + try { await command.parse(Deno.args); } catch (e) { diff --git a/cli/pull.ts b/cli/pull.ts index 4a5264979c..6d5d5f13da 100644 --- a/cli/pull.ts +++ b/cli/pull.ts @@ -1,20 +1,11 @@ // deno-lint-ignore-file no-explicit-any -import { resolveWorkspace } from "./context.ts"; import { GlobalOptions } from "./types.ts"; -import { - colors, - Command, - Confirm, - copy, - ensureDir, - path, - readerFromStreamReader, - Untar, -} from "./deps.ts"; - -async function pull(opts: GlobalOptions & { override: boolean }, dir: string) { - const workspace = await resolveWorkspace(opts); +import { colors, Command, readerFromStreamReader, Untar } from "./deps.ts"; +import { Workspace } from "./workspace.ts"; +export async function downloadTar( + workspace: Workspace, +): Promise { const requestHeaders: HeadersInit = new Headers(); requestHeaders.set("Authorization", "Bearer " + workspace.token); requestHeaders.set("Content-Type", "application/octet-stream"); @@ -35,51 +26,29 @@ async function pull(opts: GlobalOptions & { override: boolean }, dir: string) { ), ); console.log(await tarResponse.text()); - return; + return undefined; } const streamReader = tarResponse.body?.getReader(); if (!streamReader) { console.log(colors.red("Failed to read tar request body")); - return; + return undefined; } console.log(colors.yellow("Streaming tarball to disk...")); const denoReader = readerFromStreamReader(streamReader); const untar = new Untar(denoReader); - for await (const entry of untar) { - console.log(entry.fileName); - const filePath = path.resolve(dir, entry.fileName); - if (entry.type === "directory") { - await ensureDir(filePath); - continue; - } - await ensureDir(path.dirname(filePath)); - if (!opts.override) { - let exists = false; - try { - const _stat = await Deno.stat(filePath); - exists = true; - } catch { - exists = false; - } - if (exists) { - if ( - !(await Confirm.prompt( - "Conflict at " + - filePath + - " do you want to override the local version?", - )) - ) { - continue; - } - } - } - const file = await Deno.open(filePath, { write: true, create: true }); - const len = await copy(entry, file); - await file.truncate(len); - file.close(); - } - console.log(colors.green("Done. Wrote all files to disk.")); + return untar; +} + +async function stub( + _opts: GlobalOptions & { override: boolean }, + _dir: string, +) { + console.log( + colors.red.underline( + 'Pull is deprecated. Use "sync pull --raw" instead. See for more information.', + ), + ); } const command = new Command() @@ -87,6 +56,6 @@ const command = new Command() "Pull all definitions in the current workspace from the API and write them to disk.", ) .arguments("") - .action(pull as any); + .action(stub as any); export default command; diff --git a/cli/push.ts b/cli/push.ts index 8f86ca7106..7667f2fbc1 100644 --- a/cli/push.ts +++ b/cli/push.ts @@ -1,266 +1,21 @@ // deno-lint-ignore-file no-explicit-any -import { colors, Command, path } from "./deps.ts"; -import { requireLogin, resolveWorkspace } from "./context.ts"; -import { pushFlow } from "./flow.ts"; -import { pushResource } from "./resource.ts"; -import { findContentFile, pushScript } from "./script.ts"; +import { colors, Command } from "./deps.ts"; import { GlobalOptions } from "./types.ts"; -import { pushVariable } from "./variable.ts"; -import { pushResourceType } from "./resource-type.ts"; -import { pushFolder } from "./folder.ts"; -type Candidate = { - path: string; - namespaceKind: "user" | "group" | "folder"; - namespaceName: string; -}; - -type ResourceTypeCandidate = { - path: string; -}; - -type FolderCandidate = { - path: string; - namespaceName: string; -}; - -async function findCandidateFiles( - dir: string, -): Promise< - { - normal: Candidate[]; - resourceTypes: ResourceTypeCandidate[]; - folders: FolderCandidate[]; - } -> { - dir = path.resolve(dir); - if (path.dirname(dir).startsWith(".")) { - return { normal: [], resourceTypes: [], folders: [] }; - } - const normalCandidates: Candidate[] = []; - const resourceTypeCandidates: ResourceTypeCandidate[] = []; - const folderCandidates: FolderCandidate[] = []; - for await (const e of Deno.readDir(dir)) { - if (e.isDirectory) { - if (e.name == "u" || e.name == "g" || e.name == "f") { // TODO: Check version for f - const newDir = dir + (dir.endsWith("/") ? "" : "/") + e.name; - for await (const e2 of Deno.readDir(newDir)) { - if (e2.isDirectory) { - if (e2.name.startsWith(".")) continue; - const namespaceName = e2.name; - const stack: string[] = []; - { - const path = newDir + "/" + namespaceName + "/"; - stack.push(path); - try { - await Deno.stat(path + "folder.meta.json"); - folderCandidates.push({ - namespaceName, - path: path + "folder.meta.json", - }); - } catch {} - } - - while (stack.length > 0) { - const dir2 = stack.pop()!; - for await (const e3 of Deno.readDir(dir2)) { - if (e3.isFile) { - if (e3.name === "folder.meta.json") continue; - normalCandidates.push({ - path: dir2 + e3.name, - namespaceKind: e.name == "g" - ? "group" - : e.name == "u" - ? "user" - : "folder", - namespaceName: namespaceName, - }); - } else { - stack.push(dir2 + e3.name + "/"); - } - } - } - } - } - } else { - console.log( - colors.yellow( - "Including organizational folder " + e.name + " in push!", - ), - ); - const { normal, resourceTypes, folders } = await findCandidateFiles( - path.join(dir, e.name), - ); - normalCandidates.push(...normal); - resourceTypeCandidates.push(...resourceTypes); - folderCandidates.push(...folders); - } - } else { - // handle root files - if (e.name.endsWith(".resource-type.json")) { - resourceTypeCandidates.push({ - path: dir + (dir.endsWith("/") ? "" : "/") + e.name, - }); - } - } - } - return { - normal: normalCandidates, - folders: folderCandidates, - resourceTypes: resourceTypeCandidates, - }; -} - -async function push(opts: GlobalOptions, dir?: string) { - dir = dir ?? Deno.cwd(); - const workspace = await resolveWorkspace(opts); - await requireLogin(opts); - - console.log(colors.blue("Searching Directory...")); - const { normal, resourceTypes, folders } = await findCandidateFiles(dir); +async function stub( + _opts: GlobalOptions, + _dir?: string, +) { console.log( - colors.blue( - "Found " + (normal.length + resourceTypes.length + folders.length) + - " candidates", + colors.red.underline( + 'Push is deprecated. Use "sync push --raw" instead. See for more information.', ), ); - for (const resourceType of resourceTypes) { - const fileName = resourceType.path.substring( - resourceType.path.lastIndexOf("/") + 1, - ); - const fileNameParts = fileName.split("."); - // invalid file names, like my.cool.script.script.json. Not valid. - if (fileNameParts.length != 3) { - console.log( - colors.yellow("invalid file name found at " + resourceType.path), - ); - continue; - } - - // filter out non-json files. Note that we filter out script contents above, so this is really an error. - if (fileNameParts.at(-1) != "json") { - console.log(colors.yellow("non-JSON file found at " + resourceType.path)); - continue; - } - - console.log("pushing resource type " + fileNameParts.at(-3)!); - await pushResourceType( - workspace.workspaceId, - resourceType.path, - fileNameParts.at(-3)!, - ); - } - for (const folder of folders) { - await pushFolder( - workspace.workspaceId, - folder.path, - "f/" + folder.namespaceName, - ); - } - for (const candidate of normal) { - // full file name. No leading /. includes .type.json - const fileName = candidate.path.substring( - candidate.path.lastIndexOf("/") + 1, - ); - // figure out just the path after ...../u|g/username|group/ (in extra dir) - const dirParts = candidate.path.split("/").filter((x) => x.length > 0); - // TODO: check version for folder - const gIndex = dirParts.findIndex((x) => x == "u" || x == "g" || x == "f"); - const extraDir = dirParts.slice(gIndex + 2, -1).join("/"); - - // file name parts has .json (hopefully) at -1, type at -2, and the actual name at -3. Dots in names are not allowed. - const fileNameParts = fileName.split("."); - - // filter out script content files - if ( - fileNameParts.at(-1) == "ts" || - fileNameParts.at(-1) == "py" || - fileNameParts.at(-1) == "go" - ) { - // probably part of a script. Silent ignore. - continue; - } - - // invalid file names, like my.cool.script.script.json. Not valid. - if (fileNameParts.length != 3) { - console.log( - colors.yellow("invalid file name found at " + candidate.path), - ); - continue; - } - - // filter out non-json files. Note that we filter out script contents above, so this is really an error. - if (fileNameParts.at(-1) != "json") { - console.log(colors.yellow("non-JSON file found at " + candidate.path)); - continue; - } - - // get the type & filter it for valid ones. - const type = fileNameParts.at(-2); - if (type == "resource-type") { - console.log( - colors.yellow( - "Found resource type file at " + - candidate.path + - " this appears to be inside a path folder. Resource types are not addressed by path. Place them at the root or inside only an organizational folder. Ignoring this file!", - ), - ); - continue; - } - - if ( - type != "flow" && - type != "resource" && - type != "script" && - type != "variable" - ) { - console.log( - colors.yellow( - "file with invalid type " + type + " found at " + candidate.path, - ), - ); - continue; - } - - // create the remotePath for the API - const remotePath = (candidate.namespaceKind === "group" - ? "g/" - : (candidate.namespaceKind === "user" ? "u/" : "f/")) + - candidate.namespaceName + - "/" + - (extraDir.length > 0 ? extraDir + "/" : "") + - fileNameParts.at(-3); - - console.log("pushing " + type + " to " + remotePath); - - if (type == "flow") { - await pushFlow(candidate.path, workspace.workspaceId, remotePath); - } else if (type == "resource") { - await pushResource(workspace.workspaceId, candidate.path, remotePath); - } else if (type == "script") { - let contentPath: string; - try { - contentPath = await findContentFile(candidate.path); - } catch (e) { - console.log(colors.red(e.toString())); - continue; - } - await pushScript( - candidate.path, - contentPath, - workspace.workspaceId, - remotePath, - ); - } else if (type == "variable") { - await pushVariable(workspace.workspaceId, candidate.path, remotePath); - } - } - console.log(colors.underline.bold.green("Successfully Pushed all files.")); } const command = new Command() .description("Push all files from a folder") .arguments("[dir:string]") - .action(push as any); + .action(stub as any); export default command; diff --git a/cli/resource-type.ts b/cli/resource-type.ts index e2a3ee52ca..d4483039c3 100644 --- a/cli/resource-type.ts +++ b/cli/resource-type.ts @@ -1,65 +1,129 @@ // deno-lint-ignore-file no-explicit-any -import { GlobalOptions } from "./types.ts"; +import { + Difference, + GlobalOptions, + PushDiffs, + Resource as ResourceI, + setValueByPath, +} from "./types.ts"; import { requireLogin, resolveWorkspace } from "./context.ts"; -import { colors, Command, ResourceService, Table } from "./deps.ts"; +import { + colors, + Command, + EditResourceType, + microdiff, + ResourceService, + ResourceType, + Table, +} from "./deps.ts"; +import { Any, decoverto, model, property } from "./decoverto.ts"; -type ResourceTypeFile = { +@model() +export class ResourceTypeFile implements ResourceI, PushDiffs { + @property(Any) schema?: any; + @property(() => String) description?: string; -}; + + async push(workspace: string, remotePath: string): Promise { + let existing: ResourceType | undefined; + try { + existing = await ResourceService.getResourceType({ + workspace, + path: remotePath, + }); + } catch { + existing = undefined; + } + this.pushDiffs( + workspace, + remotePath, + microdiff(existing ?? {}, this, { cyclesFix: false }), + ); + } + + async pushDiffs( + workspace: string, + remotePath: string, + diffs: Difference[], + ): Promise { + if ( + await ResourceService.existsResourceType({ + workspace: workspace, + path: remotePath, + }) + ) { + if ( + (await ResourceService.listResourceType({ workspace })).findIndex((x) => + x.name === remotePath + ) === -1 + ) { + console.log( + "Resource type " + remotePath + + " is already taken for the current workspace, but cannot be updated. Is this a conflict with starter?", + ); + return; + } + console.log( + colors.yellow( + `Applying ${diffs.length} diffs to existing resource type...`, + ), + ); + const changeset: EditResourceType = {}; + for (const diff of diffs) { + if ( + diff.type !== "REMOVE" && + ( + diff.path.length !== 1 || + !["schema", "description"].includes(diff.path[0] as string) + ) + ) { + throw new Error("Invalid resource type diff with path " + diff.path); + } + if (diff.type === "CREATE" || diff.type === "CHANGE") { + setValueByPath(changeset, diff.path, diff.value); + } else if (diff.type === "REMOVE") { + setValueByPath(changeset, diff.path, null); + } + } + + const hasChanges = Object.values(changeset).some((v) => + v !== null && typeof v !== "undefined" + ); + if (!hasChanges) { + console.log(colors.yellow("! Skipping empty changeset")); + return; + } + + await ResourceService.updateResourceType({ + workspace: workspace, + path: remotePath, + requestBody: changeset, + }); + } else { + console.log(colors.yellow("Creating new resource type...")); + await ResourceService.createResourceType({ + workspace: workspace, + requestBody: { + name: remotePath, + description: this.description, + schema: this.schema, + workspace_id: workspace, + }, + }); + } + } +} export async function pushResourceType( workspace: string, filePath: string, name: string, ) { - const data: ResourceTypeFile = JSON.parse(await Deno.readTextFile(filePath)); - await pushResourceTypeDef(workspace, name, data); -} - -export async function pushResourceTypeDef( - workspace: string, - name: string, - data: ResourceTypeFile, -) { - if ( - await ResourceService.existsResourceType({ - workspace: workspace, - path: name, - }) - ) { - console.log(colors.yellow("Updating existing resource type...")); - if ( - (await ResourceService.listResourceType({ workspace })).findIndex((x) => - x.name === name - ) === -1 - ) { - console.log( - "Resource type " + name + - " is already taken for the current workspace, but cannot be updated. Is this a conflict with starter?", - ); - return; - } - await ResourceService.updateResourceType({ - workspace: workspace, - path: name, - requestBody: { - description: data.description, - schema: data.schema, - }, - }); - } else { - console.log(colors.yellow("Creating new resource type...")); - await ResourceService.createResourceType({ - workspace: workspace, - requestBody: { - name: name, - description: data.description, - schema: data.schema, - workspace_id: workspace, - }, - }); - } + const data: ResourceTypeFile = decoverto.type(ResourceTypeFile).rawToInstance( + await Deno.readTextFile(filePath), + ); + await data.push(workspace, name); } type PushOptions = GlobalOptions; diff --git a/cli/resource.ts b/cli/resource.ts index 9e02e3070d..b6379c049b 100644 --- a/cli/resource.ts +++ b/cli/resource.ts @@ -1,80 +1,142 @@ -import { GlobalOptions } from "./types.ts"; +import { + Difference, + GlobalOptions, + PushDiffs, + Resource as Resource2, + setValueByPath, +} from "./types.ts"; import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; -import { colors, Command, Resource, ResourceService, Table } from "./deps.ts"; +import { + colors, + Command, + EditResource, + microdiff, + Resource, + ResourceService, + Table, +} from "./deps.ts"; +import { Any, decoverto, model, property } from "./decoverto.ts"; -type ResourceFile = { - value: any; +@model() +export class ResourceFile implements Resource2, PushDiffs { + @property(Any) + value?: any; + @property(() => String) description?: string; + @property(() => String) resource_type: string; + @property(() => Boolean) is_oauth?: boolean; // deprecated -}; + + constructor(resource_type: string) { + this.resource_type = resource_type; + } + async pushDiffs( + workspace: string, + remotePath: string, + diffs: Difference[], + ): Promise { + if ( + await ResourceService.existsResource({ + workspace: workspace, + path: remotePath, + }) + ) { + console.log( + colors.yellow(`Applying ${diffs.length} diffs to existing resource...`), + ); + + const changeset: EditResource = { + path: remotePath, // TODO: Remove this in backend + }; + for (const diff of diffs) { + if (diff.path[0] === "is_oauth") { + console.log( + colors.yellow( + "! is_oauth has been removed in newer versions. Ignoring.", + ), + ); + continue; + } + if ( + diff.type !== "REMOVE" && + ( + diff.path[0] !== "value" && ( + diff.path.length !== 1 || + diff.path[0] !== "description" + ) + ) + ) { + throw new Error("Invalid folder diff with path " + diff.path); + } + if (diff.type === "CREATE" || diff.type === "CHANGE") { + setValueByPath(changeset, diff.path, diff.value); + } else if (diff.type === "REMOVE") { + setValueByPath(changeset, diff.path, null); + } + } + + const hasChanges = Object.values(changeset).some((v) => + v !== null && typeof v !== "undefined" + ); + if (!hasChanges) { + console.log(colors.yellow("! Skipping empty changeset")); + return; + } + + await ResourceService.updateResource({ + workspace: workspace, + path: remotePath, + requestBody: changeset, + }); + } else { + if (typeof this.is_oauth !== "undefined") { + console.log( + colors.yellow( + "! is_oauth has been removed in newer versions. Ignoring.", + ), + ); + } + + console.log(colors.yellow("Creating new resource...")); + await ResourceService.createResource({ + workspace: workspace, + requestBody: { + path: remotePath, + resource_type: this.resource_type, + value: this.value, + description: this.description, + }, + }); + } + } + async push(workspace: string, remotePath: string): Promise { + let existing: Resource | undefined; + try { + existing = await ResourceService.getResource({ + workspace, + path: remotePath, + }); + } catch { + existing = undefined; + } + await this.pushDiffs( + workspace, + remotePath, + microdiff(existing ?? {}, this, { cyclesFix: false }), + ); + } +} export async function pushResource( workspace: string, filePath: string, remotePath: string, ) { - const data: ResourceFile = JSON.parse(await Deno.readTextFile(filePath)); - if ( - await ResourceService.existsResource({ - workspace: workspace, - path: remotePath, - }) - ) { - console.log(colors.yellow("Updating existing resource...")); - const existing = await ResourceService.getResource({ - workspace: workspace, - path: remotePath, - }); - - if (existing.resource_type != data.resource_type) { - console.log( - colors.red.underline.bold( - "Remote resource at " + - remotePath + - " exists & has a different resource type. This cannot be updated. If you wish to do this anyways, consider deleting the remote resource.", - ), - ); - return; - } - - if (typeof data.is_oauth !== "undefined") { - console.log( - colors.yellow( - "! is_oauth has been removed in newer versions. Ignoring.", - ), - ); - } - - await ResourceService.updateResource({ - workspace: workspace, - path: remotePath, - requestBody: { - path: remotePath, - value: data.value, - description: data.description, - }, - }); - } else { - if (typeof data.is_oauth !== "undefined") { - console.log( - colors.yellow( - "! is_oauth has been removed in newer versions. Ignoring.", - ), - ); - } - - console.log(colors.yellow("Creating new resource...")); - await ResourceService.createResource({ - workspace: workspace, - requestBody: { - path: remotePath, - resource_type: data.resource_type, - value: data.value, - description: data.description, - }, - }); - } + const data = decoverto.type(ResourceFile).rawToInstance( + await Deno.readTextFile(filePath), + ); + await data.push(workspace, remotePath); } type PushOptions = GlobalOptions; diff --git a/cli/script.ts b/cli/script.ts index 749768f5a8..90ad126b9d 100644 --- a/cli/script.ts +++ b/cli/script.ts @@ -2,20 +2,52 @@ import { GlobalOptions } from "./types.ts"; import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; import { + colors, + Command, JobService, + readAll, Script, -} from "https://deno.land/x/windmill@v1.50.0/windmill-api/index.ts"; -import { colors, Command, readAll, ScriptService, Table } from "./deps.ts"; + ScriptService, + Table, +} from "./deps.ts"; +import { Any, array, decoverto, model, property } from "./decoverto.ts"; -type ScriptFile = { +@model() +export class ScriptFile { + @property(() => String) parent_hash?: string; + @property(() => String) summary: string; + @property(() => String) description: string; + @property(Any) schema?: any; + @property(() => Boolean) is_template?: boolean; + @property(array(() => String)) lock?: Array; + @property({ + toInstance: (data) => { + if (data == null) return data; + + if ( + data === "script" || data === "failure" || data === "trigger" || + data === "command" || data === "approvial" + ) { + return data; + } + + throw new Error("Invalid kind " + data); + }, + toPlain: (data) => data, + }) kind?: "script" | "failure" | "trigger" | "command" | "approval"; -}; + + constructor(summary: string, description: string) { + this.summary = summary; + this.description = description; + } +} type PushOptions = GlobalOptions; async function push( @@ -48,10 +80,12 @@ async function push( } export async function findContentFile(filePath: string) { + console.log("Searching " + filePath); const candidates = [ filePath.replace(".script.json", ".ts"), filePath.replace(".script.json", ".py"), filePath.replace(".script.json", ".go"), + filePath.replace(".script.json", ".sh"), ]; const validCandidates = ( await Promise.all( @@ -79,23 +113,35 @@ export async function findContentFile(filePath: string) { return validCandidates[0]; } +export function inferContentTypeFromFilePath( + contentPath: string, +): "python3" | "deno" | "go" | "bash" { + let language = contentPath.substring(contentPath.lastIndexOf(".")); + if (language == ".ts") language = "deno"; + if (language == ".py") language = "python3"; + if (language == ".sh") language = "bash"; + if (language == ".go") language = "go"; + if ( + language != "python3" && language != "deno" && language != "go" && + language != "bash" + ) { + throw new Error("Invalid language: " + language); + } + return language; +} + export async function pushScript( filePath: string, contentPath: string, workspace: string, remotePath: string, ) { - const data: ScriptFile = JSON.parse(await Deno.readTextFile(filePath)); + const data = decoverto.type(ScriptFile).rawToInstance( + await Deno.readTextFile(filePath), + ); const content = await Deno.readTextFile(contentPath); - let language = contentPath.substring(contentPath.lastIndexOf(".")); - if (language == ".ts") language = "deno"; - if (language == ".py") language = "python3"; - if (language == ".go") language = "go"; - if (language != "python3" && language != "deno" && language != "go") { - throw new Error("Invalid language: " + language); - } - + const language = inferContentTypeFromFilePath(contentPath); let parent_hash = data.parent_hash; if (!parent_hash) { try { diff --git a/cli/sync.ts b/cli/sync.ts new file mode 100644 index 0000000000..2c12ee4d4b --- /dev/null +++ b/cli/sync.ts @@ -0,0 +1,969 @@ +import { requireLogin, resolveWorkspace } from "./context.ts"; +import { getWorkspaceStream, Workspace } from "./workspace.ts"; +import { decoverto, map, MapShape, model, property } from "./decoverto.ts"; +import { + cbor, + colors, + Command, + Confirm, + copy, + ensureDir, + iterateReader, + microdiff, + nanoid, + objectHash, + path, + ScriptService, +} from "./deps.ts"; +import { + Difference, + getTypeStrFromPath, + GlobalOptions, + inferTypeFromPath, + setValueByPath, +} from "./types.ts"; +import { downloadTar } from "./pull.ts"; +import { FolderFile } from "./folder.ts"; +import { ResourceTypeFile } from "./resource-type.ts"; +import { + findContentFile, + inferContentTypeFromFilePath, + pushScript, + ScriptFile, +} from "./script.ts"; +import { ResourceFile } from "./resource.ts"; +import { FlowFile } from "./flow.ts"; +import { VariableFile } from "./variable.ts"; + +type TrackedId = string; +const TrackedId = String; + +const CONTENT_ENCODER: cbor.Encoder = new cbor.Encoder({ pack: true }); + +export class Tracked { + #id: TrackedId; + #parent: State; + + path: string; + + #content_cache?: string; + + constructor(id: TrackedId, parent: State, path: string) { + this.#id = id; + this.#parent = parent; + this.path = path; + } + + async getContent(): Promise { + if (this.#content_cache) { + return this.#content_cache; + } + + if (!this.#parent.stateRoot) { + throw new Error("Parent uninitialized"); + } + + const f = this.#parent.contentFile(this.#id); + if (!f) { + return undefined; + } + + const data = await Deno.readFile( + path.join(this.#parent.stateRoot, ".wmill", f), + ); + const content = CONTENT_ENCODER.decode(data); + this.#content_cache = content; + return content; + } + + getHash(): string { + return this.#parent.hashes.get(this.#id)!; + } + + getId(): TrackedId { + return this.#id; + } +} + +@model() +export class State { + // TODO: Handle script contents, as they are separate files & need to be tracked separately. + // I think the way this makes most sense is actually reating them as a separate kind of file everywhere + // And then move to using a diff-based push system everywhere. This should greatly simplify code & enable this easily + + @property(map(() => TrackedId, () => String, { shape: MapShape.Object })) + hashes: Map; + + @property(map(() => TrackedId, () => String, { shape: MapShape.Object })) + contentFiles: Map; + + @property(map(() => String, () => TrackedId, { shape: MapShape.Object })) + tracked: Map; + + @property(() => String) + workspaceId: string; + + @property(() => String) + remoteUrl: string; + + stateRoot?: string; + + constructor( + hashes: Map, + tracked: Map, + contentFiles: Map, + workspaceId: string, + remoteUrl: string, + ) { + this.hashes = hashes; + this.tracked = tracked; + this.contentFiles = contentFiles; + this.workspaceId = workspaceId; + this.remoteUrl = remoteUrl; + } + + add(path: string) { + if (this.tracked.get(path)) { + throw new Error("Cannot newly track already tracked paths"); + } else { + this.tracked.set(path, nanoid()); + } + } + + public forget(path: string) { + const id = this.tracked.get(path); + if (id) { + this.tracked.delete(path); + this.hashes.delete(id); + } + } + + public contentFile(id: TrackedId): string | undefined { + return this.contentFiles.get(id); + } + + public get(path: string): Tracked { + const id = this.tracked.get(path); + if (!id) { + throw new Error("Could not resolve path " + path); + } + return new Tracked(id, this, path); + } + + public async save(): Promise { + if (!this.stateRoot) { + throw new Error("Uninitialized state root"); + } + const encoder = new cbor.Encoder({}); + const plain = decoverto.type(State).instanceToPlain(this); + const result: Uint8Array = encoder.encode(plain, {}); + await Deno.writeFile(path.join(this.stateRoot, ".wmill", "main"), result, { + create: true, + }); + } + + public static async loadState(dir: string): Promise { + const source = await Deno.readFile(path.join(dir, ".wmill", "main")); + const encoder = new cbor.Encoder({}); + const raw = encoder.decode(source); + const state = decoverto.type(State).plainToInstance( + raw, + ); + state.stateRoot = dir; + return Object.freeze(state); + } +} + +async function getState(opts: GlobalOptions) { + const existingState = await State.loadState(Deno.cwd()); + + const workspaceStream = await getWorkspaceStream(); + const reader = workspaceStream.getReader(); + while (true) { + const res = await reader.read(); + if (res.value) { + if ( + new URL(res.value.remote) == new URL(existingState.remoteUrl) && + res.value.workspaceId == existingState.workspaceId + ) { + opts.workspace = res.value.name; + (opts as any).__secret_workspace = res.value; + break; + } + } + if (res.done) { + break; + } + } + return existingState; +} + +async function updateStateFromRemote( + workspace: Workspace, + state: State, + callback: (filename: string) => PromiseLike | boolean, +) { + const untar = await downloadTar(workspace); + if (!untar) throw new Error("Failed to pull Tar"); + + const decoder = new TextDecoder(); + for await (const entry of untar) { + const id = state.tracked.get(entry.fileName); + if (id) { + let val = ""; + for await (const e of iterateReader(entry)) { + const tmp = decoder.decode(e); + val += tmp; + } + + if (entry.fileName.endsWith(".json")) { + const parsed = JSON.parse(val); + const typed = inferTypeFromPath(entry.fileName, parsed); + + const oldHash = state.hashes.get(id); + const newHash = objectHash(typed); + + if (!oldHash || oldHash !== newHash) { + if (!await callback(entry.fileName)) { + return; // notice that we are not saving + } + state.hashes.set(id, newHash); + + const encoded = CONTENT_ENCODER.encode(typed); + + const fileName = nanoid(); + await Deno.writeFile( + path.join(state.stateRoot!, ".wmill", fileName), + encoded, + { create: true }, + ); + state.contentFiles.set(id, fileName); + } + } else { + const fileName = nanoid(); + await Deno.writeTextFile( + path.join(state.stateRoot!, ".wmill", fileName), + val, + { create: true }, + ); + state.contentFiles.set(id, fileName); + } + } + } + + await state.save(); +} + +async function pull( + opts: GlobalOptions & { raw: boolean; rawOverride: boolean }, +) { + if (opts.raw) { + const opts2 = opts as any; + opts2.override = opts2.rawOverride; + opts2.raw = undefined; + opts2.rawOverride = undefined; + await pullRaw(opts2, Deno.cwd()); + return; + } + + const state = await getState(opts); + + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + console.log("Pulling remote changes"); + await updateStateFromRemote(workspace, state, (_) => true); + + const diffs = diffState(state); + + await copyNonJsonFiles(state); + + console.log(`Applying changes to files`); + for await (const diff of diffs) { + await applyDiff( + diff.diff, + path.join(state.stateRoot!, diff.localPath), + ); + } + console.log(colors.green.underline("Done! All changes applied.")); + + async function applyDiff(diffs: Difference[], file: string) { + ensureDir(path.dirname(file)); + let json; + try { + json = JSON.parse(await Deno.readTextFile(file)); + } catch { + json = {}; + } + // TODO: Delegate the below to the object itself + // This would work by infering the type of `JSON` (which includes then statically typing it using decoverto) and then + // delegating the applying of the diffs to the object via an interface + for (const diff of diffs) { + if (diff.type === "CREATE") { + setValueByPath(json, diff.path, diff.value); + } else if (diff.type === "REMOVE") { + setValueByPath(json, diff.path, undefined); + } else if (diff.type === "CHANGE") { + setValueByPath(json, diff.path, diff.value); + } + } + + // TODO: Use decoverto instanceToPlain below. + await Deno.writeTextFile(file, JSON.stringify(json, undefined, " "), { + create: true, + }); + } + + async function copyNonJsonFiles(state: State): Promise { + for (const t of state.tracked.keys()) { + if (t.endsWith(".json")) { + continue; + } + const entry = state.get(t); + + const target = await Deno.open(entry.path, { create: true, write: true }); + const source = await Deno.open( + path.join( + state.stateRoot!, + ".wmill", + state.contentFile(entry.getId())!, + ), + { + read: true, + write: false, + }, + ); + await source.readable.pipeTo(target.writable); + } + } +} + +async function* diffState(state: State): AsyncGenerator { + for (const t of state.tracked.keys()) { + if (!t.endsWith(".json")) { + continue; + } + const entry = state.get(t); + let fileText; + try { + fileText = await Deno.readTextFile( + path.join(state.stateRoot!, entry.path), + ); + } catch { + fileText = "{}"; + } + const old = JSON.parse(fileText); + const fileHash = objectHash(old); + + if (fileHash !== entry.getHash()) { + const stateContent = await entry.getContent() as any; + + const diff = microdiff( + old, + stateContent ?? {}, + { cyclesFix: false }, + ); + + yield new StateDiff(entry.getId(), entry.path, diff); + } + } +} + +function prettyDiff(diffs: Difference[]) { + for (const diff of diffs) { + let pathString = ""; + for (const pathSegment of diff.path) { + if (typeof pathSegment === "string") { + pathString += "."; + pathString += pathSegment; + } else { + pathString += "["; + pathString += pathSegment; + pathString += "]"; + } + } + if (diff.type === "REMOVE" || diff.type === "CHANGE") { + console.log(colors.red("- " + pathString + " = " + diff.oldValue)); + } + if (diff.type === "CREATE" || diff.type === "CHANGE") { + console.log(colors.green("+ " + pathString + " = " + diff.value)); + } + } +} + +async function push(opts: GlobalOptions & { raw: boolean }) { + if (opts.raw) { + const opts2 = opts as any; + opts2.raw = undefined; + await pushRaw(opts2, undefined); + return; + } + + const state = await getState(opts); + + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + let error = false; + await updateStateFromRemote(workspace, state, async (filename) => { + const e = state.get(filename); + if (!e) { + throw new Error("!? State change on untracked file ?!"); + } + + let fileJSON; + try { + fileJSON = JSON.parse( + await Deno.readTextFile(path.join(state.stateRoot!, e.path)), + ); + } catch { + fileJSON = {}; + } + const file = inferTypeFromPath(e.path, fileJSON); + const eContent = (inferTypeFromPath(e.path, await e.getContent())) ?? {}; + + const fileHash = objectHash(file); + const eHash = objectHash(eContent); + + if (fileHash !== eHash) { + console.log( + colors.red("!! Local and Remote change present. Local diff:"), + ); + prettyDiff(microdiff(eContent as any, file, { cyclesFix: false })); + console.log( + colors.red( + "Consider comitting or otherwise saving your work and pulling to load any remote changes", + ), + ); + error = true; + return false; + } + return true; + }); + if (error) { + return; + } + + for (const p of state.tracked.keys()) { + if (!p.endsWith(".json")) continue; + const entry = state.get(p); + let fileJSON; + try { + fileJSON = JSON.parse( + await Deno.readTextFile(path.join(state.stateRoot!, entry.path)), + ); + } catch { + fileJSON = {}; + } + const file = inferTypeFromPath(entry.path, fileJSON); + const eContent = + (inferTypeFromPath(entry.path, await entry.getContent())) ?? {}; + + const fileHash = objectHash(file); + const eHash = objectHash(eContent); + + if (fileHash !== eHash) { + const remotePath = entry.path.split(".")[0]; + const type = getTypeStrFromPath(entry.path); + if (type === "script") { + // Diffing makes no sense for scripts - instead fetch parent hash & check hash again. + // If hash is still missmatched - create new script as child. + const typed = decoverto.type(ScriptFile).plainToInstance(file); + const contentPath = await findContentFile(entry.path); + const language = inferContentTypeFromFilePath(contentPath); + const content = await Deno.readTextFile(contentPath); + try { + const remote = await ScriptService.getScriptByPath({ + workspace: workspace.workspaceId, + path: remotePath, + }); + if (objectHash(remote) !== fileHash) { + await ScriptService.createScript({ + workspace: workspace.workspaceId, + requestBody: { + content, + description: typed.description, + language, + path: remotePath, + summary: typed.summary, + is_template: typed.is_template, + kind: typed.kind, + lock: undefined, + parent_hash: remote.hash, + schema: typed.schema, + }, + }); + } + } catch { + // no parent hash + await ScriptService.createScript({ + workspace: workspace.workspaceId, + requestBody: { + content, + description: typed.description, + language, + path: remotePath, + summary: typed.summary, + is_template: typed.is_template, + kind: typed.kind, + lock: undefined, + parent_hash: undefined, + schema: typed.schema, + }, + }); + } + } else { + const diff = microdiff(eContent as any, file, { cyclesFix: false }); + await applyDiff( + workspace.workspaceId, + remotePath, + file, + diff, + ); + } + } + } + + let anyRemoteChanges = false; + await updateStateFromRemote(workspace, state, (_) => { + anyRemoteChanges = true; + return true; + }); + if (anyRemoteChanges) { + console.log("New remote changes - consider pulling"); + } + + function applyDiff( + workspace: string, + remotePath: string, + file: + | ScriptFile + | VariableFile + | FlowFile + | ResourceFile + | ResourceTypeFile + | FolderFile, + diffs: Difference[], + ) { + if (file instanceof ScriptFile) { + throw new Error( + "This code path should be unreachable - we should never generate diffs for scripts", + ); + } else if (file instanceof FolderFile) { + const parts = remotePath.split("/"); + if (parts[0] === "f") { + remotePath = parts[1]; + } else { + remotePath = parts[0]; + } + } + return file.pushDiffs(workspace, remotePath, diffs); + } +} + +class StateDiff { + trackedId: TrackedId; + localPath: string; + diff: Difference[]; + + constructor( + trackedId: TrackedId, + localPath: string, + diff: Difference[], + ) { + this.trackedId = trackedId; + this.localPath = localPath; + this.diff = diff; + } +} + +async function add(opts: GlobalOptions, path: string) { + const state = await getState(opts); + + // TODO: Automatically check whether this path exists either locally or on the remote + state.add(path); + + if (path.endsWith(".script.json")) { + try { + const f = await findContentFile(path); + state.add(f); + } catch { + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + try { + const old = await ScriptService.getScriptByPath({ + workspace: workspace.workspaceId, + path: path.split(".")[0], + }); + if (old.language === "python3") { + state.add(path.replace(".script.json", ".py")); + } else if (old.language === "bash") { + state.add(path.replace(".script.json", ".sh")); + } else if (old.language === "deno") { + state.add(path.replace(".script.json", ".ts")); + } else if (old.language === "go") { + state.add(path.replace(".script.json", ".go")); + } else { + throw new Error("Remote returned invalid language?! " + old.language); + } + } catch { + throw new Error( + "Could not infer script language from local or remote. Exiting.", + ); + } + } + } + + await state.save(); +} + +async function init(opts: GlobalOptions) { + const root = Deno.cwd(); + try { + await Deno.mkdir(path.join(root, ".wmill")); + } catch { + console.log( + colors.red( + "! Looks like this folder is already initialized or we are missing permissions to do so. Exiting.", + ), + ); + return; + } + + const workspace = await resolveWorkspace(opts); + + const newState = new State( + new Map(), + new Map(), + new Map(), + workspace.workspaceId, + workspace.remote, + ); + newState.stateRoot = root; + + await newState.save(); +} + +async function pullRaw( + opts: GlobalOptions & { override: boolean }, + dir: string, +) { + const workspace = await resolveWorkspace(opts); + + const untar = await downloadTar(workspace); + if (!untar) return; + + for await (const entry of untar) { + console.log(entry.fileName); + const filePath = path.resolve(dir, entry.fileName); + if (entry.type === "directory") { + await ensureDir(filePath); + continue; + } + await ensureDir(path.dirname(filePath)); + if (!opts.override) { + let exists = false; + try { + const _stat = await Deno.stat(filePath); + exists = true; + } catch { + exists = false; + } + if (exists) { + if ( + !(await Confirm.prompt( + "Conflict at " + + filePath + + " do you want to override the local version?", + )) + ) { + continue; + } + } + } + const file = await Deno.open(filePath, { write: true, create: true }); + const len = await copy(entry, file); + await file.truncate(len); + file.close(); + } + console.log(colors.green("Done. Wrote all files to disk.")); +} + +type PushRawCandidate = { + path: string; + namespaceKind: "user" | "group" | "folder"; + namespaceName: string; +}; + +type PushRawResourceTypeCandidate = { + path: string; +}; + +type PushRawFolderCandidate = { + path: string; + namespaceName: string; +}; + +async function pushRawFindCandidateFiles( + dir: string, +): Promise< + { + normal: PushRawCandidate[]; + resourceTypes: PushRawResourceTypeCandidate[]; + folders: PushRawFolderCandidate[]; + } +> { + dir = path.resolve(dir); + if (path.dirname(dir).startsWith(".")) { + return { normal: [], resourceTypes: [], folders: [] }; + } + const normalCandidates: PushRawCandidate[] = []; + const resourceTypeCandidates: PushRawResourceTypeCandidate[] = []; + const folderCandidates: PushRawFolderCandidate[] = []; + for await (const e of Deno.readDir(dir)) { + if (e.isDirectory) { + if (e.name == "u" || e.name == "g" || e.name == "f") { // TODO: Check version for f + const newDir = dir + (dir.endsWith("/") ? "" : "/") + e.name; + for await (const e2 of Deno.readDir(newDir)) { + if (e2.isDirectory) { + if (e2.name.startsWith(".")) continue; + const namespaceName = e2.name; + const stack: string[] = []; + { + const path = newDir + "/" + namespaceName + "/"; + stack.push(path); + try { + await Deno.stat(path + "folder.meta.json"); + folderCandidates.push({ + namespaceName, + path: path + "folder.meta.json", + }); + } catch {} + } + + while (stack.length > 0) { + const dir2 = stack.pop()!; + for await (const e3 of Deno.readDir(dir2)) { + if (e3.isFile) { + if (e3.name === "folder.meta.json") continue; + normalCandidates.push({ + path: dir2 + e3.name, + namespaceKind: e.name == "g" + ? "group" + : e.name == "u" + ? "user" + : "folder", + namespaceName: namespaceName, + }); + } else { + stack.push(dir2 + e3.name + "/"); + } + } + } + } + } + } else { + console.log( + colors.yellow( + "Including organizational folder " + e.name + " in push!", + ), + ); + const { normal, resourceTypes, folders } = + await pushRawFindCandidateFiles( + path.join(dir, e.name), + ); + normalCandidates.push(...normal); + resourceTypeCandidates.push(...resourceTypes); + folderCandidates.push(...folders); + } + } else { + // handle root files + if (e.name.endsWith(".resource-type.json")) { + resourceTypeCandidates.push({ + path: dir + (dir.endsWith("/") ? "" : "/") + e.name, + }); + } + } + } + return { + normal: normalCandidates, + folders: folderCandidates, + resourceTypes: resourceTypeCandidates, + }; +} + +async function pushRaw(opts: GlobalOptions, dir?: string) { + dir = dir ?? Deno.cwd(); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + console.log(colors.blue("Searching Directory...")); + const { normal, resourceTypes, folders } = await pushRawFindCandidateFiles( + dir, + ); + console.log( + colors.blue( + "Found " + (normal.length + resourceTypes.length + folders.length) + + " candidates", + ), + ); + for (const resourceType of resourceTypes) { + const fileName = resourceType.path.substring( + resourceType.path.lastIndexOf("/") + 1, + ); + const fileNameParts = fileName.split("."); + // invalid file names, like my.cool.script.script.json. Not valid. + if (fileNameParts.length != 3) { + console.log( + colors.yellow("invalid file name found at " + resourceType.path), + ); + continue; + } + + // filter out non-json files. Note that we filter out script contents above, so this is really an error. + if (fileNameParts.at(-1) != "json") { + console.log(colors.yellow("non-JSON file found at " + resourceType.path)); + continue; + } + + console.log("pushing resource type " + fileNameParts.at(-3)!); + await decoverto.type(ResourceTypeFile).rawToInstance( + await Deno.readTextFile(resourceType.path), + ).push(workspace.workspaceId, fileNameParts.at(-3)!); + } + for (const folder of folders) { + await decoverto.type(FolderFile).plainToInstance( + JSON.parse(await Deno.readTextFile(folder.path)), + ).push( + workspace.workspaceId, + "f/" + folder.namespaceName, + ); + } + for (const candidate of normal) { + // full file name. No leading /. includes .type.json + const fileName = candidate.path.substring( + candidate.path.lastIndexOf("/") + 1, + ); + // figure out just the path after ...../u|g/username|group/ (in extra dir) + const dirParts = candidate.path.split("/").filter((x) => x.length > 0); + // TODO: check version for folder + const gIndex = dirParts.findIndex((x) => x == "u" || x == "g" || x == "f"); + const extraDir = dirParts.slice(gIndex + 2, -1).join("/"); + + // file name parts has .json (hopefully) at -1, type at -2, and the actual name at -3. Dots in names are not allowed. + const fileNameParts = fileName.split("."); + + // filter out script content files + if ( + fileNameParts.at(-1) == "ts" || + fileNameParts.at(-1) == "py" || + fileNameParts.at(-1) == "go" + ) { + // probably part of a script. Silent ignore. + continue; + } + + // invalid file names, like my.cool.script.script.json. Not valid. + if (fileNameParts.length != 3) { + console.log( + colors.yellow("invalid file name found at " + candidate.path), + ); + continue; + } + + // filter out non-json files. Note that we filter out script contents above, so this is really an error. + if (fileNameParts.at(-1) != "json") { + console.log(colors.yellow("non-JSON file found at " + candidate.path)); + continue; + } + + // get the type & filter it for valid ones. + const type = fileNameParts.at(-2); + if (type == "resource-type") { + console.log( + colors.yellow( + "Found resource type file at " + + candidate.path + + " this appears to be inside a path folder. Resource types are not addressed by path. Place them at the root or inside only an organizational folder. Ignoring this file!", + ), + ); + continue; + } + + if ( + type != "flow" && + type != "resource" && + type != "script" && + type != "variable" + ) { + console.log( + colors.yellow( + "file with invalid type " + type + " found at " + candidate.path, + ), + ); + continue; + } + + // create the remotePath for the API + const remotePath = (candidate.namespaceKind === "group" + ? "g/" + : (candidate.namespaceKind === "user" ? "u/" : "f/")) + + candidate.namespaceName + + "/" + + (extraDir.length > 0 ? extraDir + "/" : "") + + fileNameParts.at(-3); + + console.log("pushing " + type + " to " + remotePath); + + const typed = inferTypeFromPath( + candidate.path, + JSON.parse(await Deno.readTextFile(candidate.path)), + ); + if (typed instanceof ResourceTypeFile || typed instanceof FolderFile) { + throw new Error( + "Resource Types and Folders should be filtered out at this point!", + ); + } else if (typed instanceof ScriptFile) { + let contentPath: string; + try { + contentPath = await findContentFile(candidate.path); + } catch (e) { + console.log(colors.red(e.toString())); + continue; + } + await pushScript( + candidate.path, + contentPath, + workspace.workspaceId, + remotePath, + ); + } else { + typed.push(workspace.workspaceId, remotePath); + } + } + console.log(colors.underline.bold.green("Successfully Pushed all files.")); +} + +const command = new Command() + .command("init") + .description( + "Initialize this folder as sync root for the currently selected workspace & remote.", + ) + .action(init as any) + .command("add") + .description("Add a local file for tracking") + .arguments("") + .action(add as any) + .command("pull") + .description("Pull any remote changes and apply them locally") + .option("--raw", "Pull without using state.") + .option("--raw-override", "Always override local files with remote.", { + depends: ["raw"], + }) + .action(pull as any) + .command("push") + .description("Push any local changes and apply them remotely") + .option("--raw", "Push without using state.") + .action(push as any); + +export default command; diff --git a/cli/tmp/f/my_folder/folder.meta.json b/cli/tmp/f/my_folder/folder.meta.json deleted file mode 100644 index 42d14ea3d0..0000000000 --- a/cli/tmp/f/my_folder/folder.meta.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "workspace_id": "admins", - "name": "my_folder", - "display_name": "my_folder", - "owners": [], - "extra_perms": { - "u/test": true, - "u/admin@windmill.dev": false - } -} \ No newline at end of file diff --git a/cli/tmp/u/admin/hub_sync.script.json b/cli/tmp/u/admin/hub_sync.script.json deleted file mode 100644 index e9e1636e1e..0000000000 --- a/cli/tmp/u/admin/hub_sync.script.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "summary": "Syncronize Hub Resource types with starter workspace", - "description": "Basic administrative script to sync latest resource types from hub. Recommended to run at least once. On a schedule by default.", - "schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "properties": {}, - "required": [], - "type": "object" - }, - "is_template": false, - "lock": [] -} \ No newline at end of file diff --git a/cli/tmp/u/admin/hub_sync.ts b/cli/tmp/u/admin/hub_sync.ts deleted file mode 100644 index 0a369ae9e5..0000000000 --- a/cli/tmp/u/admin/hub_sync.ts +++ /dev/null @@ -1,13 +0,0 @@ -import wmill from "https://deno.land/x/wmill@v1.55.0/main.ts"; - -export async function main() { - await run( - "workspace", "add", "__automation", "starter", Deno.env.get("WM_BASE_URL") + "/", "--token", Deno.env.get("WM_TOKEN")); - - await run("hub", "pull"); -} - -async function run(...cmd: string[]) { - console.log("Running \"" + cmd.join(' ') + "\""); - await wmill.parse(cmd); -} \ No newline at end of file diff --git a/cli/types.ts b/cli/types.ts index 8dd0ec4c7c..b02502c5f6 100644 --- a/cli/types.ts +++ b/cli/types.ts @@ -1,4 +1,137 @@ +import { decoverto } from "./decoverto.ts"; +import { FlowFile } from "./flow.ts"; +import { ResourceTypeFile } from "./resource-type.ts"; +import { ResourceFile } from "./resource.ts"; +import { ScriptFile } from "./script.ts"; +import { VariableFile } from "./variable.ts"; +import { path } from "./deps.ts"; +import { FolderFile } from "./folder.ts"; + +// TODO: Remove this & replace with a "pull" that lets the object either pull the remote version or return undefined. +// Then combine those with diffing, which then gives the new push impl +export interface Resource { + push(workspace: string, remotePath: string): Promise; +} + +export interface PushDiffs { + pushDiffs( + workspace: string, + remotePath: string, + diffs: Difference[], + ): Promise; +} + +export interface DifferenceCreate { + type: "CREATE"; + path: (string | number)[]; + value: any; +} + +export interface DifferenceRemove { + type: "REMOVE"; + path: (string | number)[]; + oldValue: any; +} + +export interface DifferenceChange { + type: "CHANGE"; + path: (string | number)[]; + value: any; + oldValue: any; +} + +export type Difference = DifferenceCreate | DifferenceRemove | DifferenceChange; + +export function setValueByPath( + obj: any, + path: (string | number)[], + value: any, +) { + let i; + let lastObj = undefined; + for (i = 0; i < path.length - 1; i++) { + if (!obj) { + let oldNewObj; + if (typeof path[i] === "number") { + oldNewObj = []; + } else { + oldNewObj = {}; + } + lastObj[path[i - 1]] = oldNewObj; + obj = oldNewObj; + } + lastObj = obj; + obj = obj[path[i]]; + } + if (!obj) { + let oldNewObj; + if (typeof path[i] === "number") { + oldNewObj = []; + } else { + oldNewObj = {}; + } + lastObj[path[i - 1]] = oldNewObj; + obj = oldNewObj; + } + obj[path[i]] = value; +} + export type GlobalOptions = { workspace: string | undefined; token: string | undefined; }; + +export function inferTypeFromPath( + p: string, + obj: any, +): + | ScriptFile + | VariableFile + | FlowFile + | ResourceFile + | ResourceTypeFile + | FolderFile { + const typeEnding = getTypeStrFromPath(p); + + if (typeEnding === "folder") { + return decoverto.type(FolderFile).plainToInstance(obj); + } else if (typeEnding === "script") { + return decoverto.type(ScriptFile).plainToInstance(obj); + } else if (typeEnding === "variable") { + return decoverto.type(VariableFile).plainToInstance(obj); + } else if (typeEnding === "flow") { + return decoverto.type(FlowFile).plainToInstance(obj); + } else if (typeEnding === "resource") { + return decoverto.type(ResourceFile).plainToInstance(obj); + } else if (typeEnding === "resource-type") { + return decoverto.type(ResourceTypeFile).plainToInstance(obj); + } else { + throw new Error("infer type unreachable"); + } +} + +export function getTypeStrFromPath( + p: string, +): "script" | "variable" | "flow" | "resource" | "resource-type" | "folder" { + const parsed = path.parse(p); + if (parsed.ext !== ".json") { + throw new Error( + "Cannot infer type of non-json file " + JSON.stringify(parsed), + ); + } + + if (parsed.name === "folder.meta") { + return "folder"; + } + + const typeEnding = parsed.name.split(".").at(-1); + if ( + typeEnding === "script" || typeEnding === "variable" || + typeEnding === "flow" || typeEnding === "resource" || + typeEnding === "resource-type" + ) { + return typeEnding; + } else { + throw new Error("Could not infer type of path " + JSON.stringify(parsed)); + } +} diff --git a/cli/variable.ts b/cli/variable.ts index a90d9a8b38..0f07fa69ee 100644 --- a/cli/variable.ts +++ b/cli/variable.ts @@ -1,7 +1,22 @@ // deno-lint-ignore-file no-explicit-any import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; -import { GlobalOptions } from "./types.ts"; -import { colors, Command, Table, VariableService } from "./deps.ts"; +import { + Difference, + GlobalOptions, + PushDiffs, + Resource, + setValueByPath, +} from "./types.ts"; +import { + colors, + Command, + EditVariable, + ListableVariable, + microdiff, + Table, + VariableService, +} from "./deps.ts"; +import { decoverto, model, property } from "./decoverto.ts"; async function list(opts: GlobalOptions) { const workspace = await resolveWorkspace(opts); @@ -26,13 +41,100 @@ async function list(opts: GlobalOptions) { .render(); } -type VariableFile = { +@model() +export class VariableFile implements Resource, PushDiffs { + @property(() => String) value: string; + @property(() => Boolean) is_secret: boolean; + @property(() => String) description: string; + @property(() => Number) account?: number; + @property(() => Boolean) is_oauth?: boolean; -}; + + constructor(value: string, is_secret: boolean, description: string) { + this.value = value; + this.is_secret = is_secret; + this.description = description; + } + async pushDiffs( + workspace: string, + remotePath: string, + diffs: Difference[], + ): Promise { + if (await VariableService.existsVariable({ workspace, path: remotePath })) { + console.log( + colors.bold.yellow( + `Applying ${diffs.length} diffs to existing variable...`, + ), + ); + const changeset: EditVariable = {}; + for (const diff of diffs) { + if ( + diff.type !== "REMOVE" && + ( + diff.path.length !== 1 || + !["path", "value", "is_secret", "description"].includes( + diff.path[0] as string, + ) + ) + ) { + throw new Error("Invalid variable diff with path " + diff.path); + } + if (diff.type === "CREATE" || diff.type === "CHANGE") { + setValueByPath(changeset, diff.path, diff.value); + } else if (diff.type === "REMOVE") { + setValueByPath(changeset, diff.path, null); + } + } + + const hasChanges = Object.values(changeset).some((v) => + v !== null && typeof v !== "undefined" + ); + if (!hasChanges) { + console.log(colors.yellow("! Skipping empty changeset")); + return; + } + + await VariableService.updateVariable({ + workspace, + path: remotePath, + requestBody: changeset, + }); + } else { + console.log(colors.yellow("Creating new variable...")); + await VariableService.createVariable({ + workspace, + requestBody: { + path: remotePath, + description: this.description, + is_secret: this.is_secret, + value: this.value, + account: this.account, + is_oauth: this.is_oauth, + }, + }); + } + } + async push(workspace: string, remotePath: string): Promise { + let existing: ListableVariable | undefined; + try { + existing = await VariableService.getVariable({ + workspace: workspace, + path: remotePath, + }); + } catch { + existing = undefined; + } + await this.pushDiffs( + workspace, + remotePath, + microdiff(existing ?? {}, this, { cyclesFix: false }), + ); + } +} async function push(opts: GlobalOptions, filePath: string, remotePath: string) { const workspace = await resolveWorkspace(opts); @@ -58,72 +160,10 @@ export async function pushVariable( filePath: string, remotePath: string, ) { - const data: VariableFile = JSON.parse(await Deno.readTextFile(filePath)); - if (await VariableService.existsVariable({ workspace, path: remotePath })) { - const existing = await VariableService.getVariable({ - workspace: workspace, - path: remotePath, - }); - if (existing.is_oauth != data.is_oauth) { - console.log( - colors.red.underline.bold( - "Remote variable at " + - remotePath + - " exists & has a different oauth state. This cannot be updated. If you wish to do this anyways, consider deleting the remote resource.", - ), - ); - return; - } - - if (existing.account != data.account) { - console.log( - colors.red.underline.bold( - "Remote variable at " + - remotePath + - " exists & has a different account state. This cannot be updated. If you wish to do this anyways, consider deleting the remote resource.", - ), - ); - return; - } - - if (existing.is_secret && !data.is_secret) { - console.log( - colors.red.underline.bold( - "Remote variable at " + - remotePath + - " exists & is secret. Variables cannot be updated to be no longer secret. If you wish to do this anyways, consider deleting the remote resource.", - ), - ); - return; - } - - const actual_secret = data.is_secret ? true : undefined; - - console.log(colors.yellow("Updating existing variable...")); - await VariableService.updateVariable({ - workspace, - path: remotePath, - requestBody: { - description: data.description, - is_secret: actual_secret, - path: remotePath, - value: data.value, - }, - }); - } else { - console.log(colors.yellow("Creating new variable...")); - await VariableService.createVariable({ - workspace, - requestBody: { - path: remotePath, - description: data.description, - is_secret: data.is_secret, - value: data.value, - account: data.account, - is_oauth: data.is_oauth, - }, - }); - } + const data = decoverto.type(VariableFile).rawToInstance( + await Deno.readTextFile(filePath), + ); + await data.push(workspace, remotePath); } const command = new Command() diff --git a/cli/workspace.ts b/cli/workspace.ts index 73cf168a97..6281f74ef7 100644 --- a/cli/workspace.ts +++ b/cli/workspace.ts @@ -11,14 +11,31 @@ import { Table, WorkspaceService, } from "./deps.ts"; -import { requireLogin } from "./context.ts"; +import { decoverto, model, property } from "./decoverto.ts"; -export type Workspace = { +@model() +export class Workspace { + @property(() => String) remote: string; + @property(() => String) workspaceId: string; + @property(() => String) name: string; + @property(() => String) token: string; -}; + + constructor( + remote: string, + workspaceId: string, + name: string, + token: string, + ) { + this.remote = remote; + this.workspaceId = workspaceId; + this.name = name; + this.token = token; + } +} function makeWorkspaceStream( readable: ReadableStream, @@ -33,7 +50,9 @@ function makeWorkspaceStream( if (line.length <= 2) { return; } - controller.enqueue(JSON.parse(line) as Workspace); + const workspace = decoverto.type(Workspace).rawToInstance(line); + workspace.remote = new URL(workspace.remote).toString(); // add trailing slash in all cases! + controller.enqueue(workspace); } catch { /* ignore */ } @@ -42,14 +61,17 @@ function makeWorkspaceStream( ); } -async function allWorkspaces(): Promise { - try { - const file = await Deno.open((await getRootStore()) + "remotes.ndjson", { - write: false, - read: true, - }); - const workspaceStream = makeWorkspaceStream(file.readable); +export async function getWorkspaceStream() { + const file = await Deno.open((await getRootStore()) + "remotes.ndjson", { + write: false, + read: true, + }); + return makeWorkspaceStream(file.readable); +} +export async function allWorkspaces(): Promise { + try { + const workspaceStream = await getWorkspaceStream(); const workspaces: Workspace[] = []; for await (const workspace of workspaceStream) { workspaces.push(workspace); @@ -87,8 +109,7 @@ export async function getActiveWorkspace( export async function getWorkspaceByName( workspaceName: string, ): Promise { - const file = await Deno.open((await getRootStore()) + "remotes.ndjson"); - const workspaceStream = makeWorkspaceStream(file.readable); + const workspaceStream = await getWorkspaceStream(); for await (const workspace of workspaceStream) { if (workspace.name === workspaceName) { return workspace; @@ -185,6 +206,7 @@ export async function add( remote = new URL(await Input.prompt("Enter the Remote URL")).toString(); } } + remote = new URL(remote).toString(); // add trailing slash in all cases! let token = await tryGetLoginInfo(opts); while (!token) { @@ -192,7 +214,10 @@ export async function add( } if (opts.create) { - setClient(token, remote.endsWith('/') ? remote.substring(0, remote.length - 1) : remote); + setClient( + token, + remote.endsWith("/") ? remote.substring(0, remote.length - 1) : remote, + ); if ( !await WorkspaceService.existsWorkspace({ @@ -224,6 +249,7 @@ export async function add( } export async function addWorkspace(workspace: Workspace) { + workspace.remote = new URL(workspace.remote).toString(); // add trailing slash in all cases! const file = await Deno.open((await getRootStore()) + "remotes.ndjson", { append: true, write: true, @@ -234,7 +260,7 @@ export async function addWorkspace(workspace: Workspace) { file.close(); } -async function remove(_opts: GlobalOptions, name: string) { +export async function removeWorkspace(name: string) { const orgWorkspaces = await allWorkspaces(); await Deno.writeTextFile( (await getRootStore()) + "remotes.ndjson", @@ -243,6 +269,10 @@ async function remove(_opts: GlobalOptions, name: string) { .map((x) => JSON.stringify(x)) .join("\n"), ); +} + +async function remove(_opts: GlobalOptions, name: string) { + await removeWorkspace(name); console.log(colors.green.underline("Succesfully removed workspace!")); }