From 7de518c8d8b8ad10a8b51609f7fa6742438eb3f5 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 5 May 2023 02:51:01 +0200 Subject: [PATCH] fix(cli): refactor cli (#1531) * all * cliUpgrade * refactor entire cli --- cli/apps.ts | 157 ++++++++++++------------------------- cli/decoverto.ts | 8 -- cli/flow.ts | 156 +++++++++++-------------------------- cli/folder.ts | 182 ++++++++++++------------------------------- cli/hub.ts | 46 ++++++----- cli/resource-type.ts | 148 ++++++++++++----------------------- cli/resource.ts | 172 +++++++++++++--------------------------- cli/script.ts | 72 +++++------------ cli/sync.ts | 123 +++++++---------------------- cli/types.ts | 172 +++++++++++++++++++++------------------- cli/variable.ts | 155 +++++++++++------------------------- cli/workspace.ts | 22 +----- 12 files changed, 464 insertions(+), 949 deletions(-) delete mode 100644 cli/decoverto.ts diff --git a/cli/apps.ts b/cli/apps.ts index 6d37d535a4..02e92a69c1 100644 --- a/cli/apps.ts +++ b/cli/apps.ts @@ -1,120 +1,55 @@ +// deno-lint-ignore-file no-explicit-any import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; -import { Any, decoverto, model, property } from "./decoverto.ts"; import { AppService, AppWithLastVersion, colors, Command, ListableApp, - microdiff, Policy, Table, } from "./deps.ts"; import { - Difference, GlobalOptions, - PushDiffs, - Resource, - setValueByPath, + isSuperset, + parseFromFile, + removeType, } from "./types.ts"; -@model() -export class AppFile implements Resource, PushDiffs { - @property(Any) +export interface AppFile { value: any; - @property(() => String) summary: string; - @property(Any) policy: Policy; +} - constructor(value: string, summary: string, policy: Policy) { - this.value = value; - this.summary = summary; - this.policy = policy; - } - async pushDiffs( - workspace: string, - remotePath: string, - diffs: Difference[] - ): Promise { - let app: AppWithLastVersion | undefined = undefined; - try { - app = await AppService.getAppByPath({ workspace, path: remotePath }); - } catch (e) {} - - if (app) { - console.log( - colors.bold.yellow( - `Applying ${diffs.length} diffs to existing app... ${remotePath}` - ) - ); - const changeset: { - summary?: string | undefined; - value?: any; - policy?: Policy | undefined; - } = {}; - for (const diff of diffs) { - if ( - diff.type !== "REMOVE" && - diff.path[0] !== "value" && - diff.path[0] !== "policy" && - (diff.path.length !== 1 || - !["summary"].includes(diff.path[0] as string)) - ) { - throw new Error("Invalid app diff with path " + diff.path); - } - if (diff.type === "CREATE" || diff.type === "CHANGE") { - setValueByPath(changeset, diff.path, diff.value); - } else if (diff.type === "REMOVE") { - setValueByPath(changeset, diff.path, null); - } - } - - if ( - (!changeset?.policy || - JSON.stringify(changeset?.policy) == JSON.stringify(app.policy)) && - (!changeset?.value || - JSON.stringify(changeset?.value) == JSON.stringify(app.value)) && - (!changeset?.summary || changeset.summary == app.summary) - ) { - console.log( - colors.yellow(`No changes to push for app ${remotePath}, skipping`) - ); - return; - } - - const hasChanges = Object.values(changeset).some( - (v) => v !== null && typeof v !== "undefined" - ); - if (!hasChanges) { - return; - } - - await AppService.updateApp({ - workspace, - path: remotePath, - requestBody: changeset, - }); - } else { - console.log(colors.yellow.bold("Creating new app...")); - - await AppService.createApp({ - workspace, - requestBody: { - path: remotePath, - policy: this.policy, - summary: this.summary, - value: this.value, - }, - }); +export async function pushApp( + workspace: string, + remotePath: string, + app: AppFile | AppWithLastVersion | undefined, + newApp: AppFile +): Promise { + remotePath = removeType(remotePath, "app"); + if (app) { + if (isSuperset(newApp, app)) { + return; } - } - async push(workspace: string, remotePath: string): Promise { - await this.pushDiffs( + await AppService.updateApp({ workspace, - remotePath, - microdiff({}, this, { cyclesFix: false }) - ); + path: remotePath, + requestBody: { + ...newApp, + }, + }); + } else { + console.log(colors.yellow.bold("Creating new app...")); + + await AppService.createApp({ + workspace, + requestBody: { + path: remotePath, + ...newApp, + }, + }); } } @@ -154,19 +89,23 @@ async function push(opts: GlobalOptions, filePath: string) { const workspace = await resolveWorkspace(opts); await requireLogin(opts); - await pushApp(filePath, workspace.workspaceId, remotePath); - console.log(colors.bold.underline.green("App pushed")); -} + let app: AppWithLastVersion | undefined = undefined; + try { + app = await AppService.getAppByPath({ + workspace: workspace.workspaceId, + path: remotePath, + }); + } catch { + // app doesn't exist + } -export async function pushApp( - filePath: string, - workspace: string, - remotePath: string -) { - const data = decoverto - .type(AppFile) - .rawToInstance(await Deno.readTextFile(filePath)); - await data.push(workspace, remotePath); + await pushApp( + workspace.workspaceId, + remotePath, + app, + parseFromFile(filePath) + ); + console.log(colors.bold.underline.green("App pushed")); } const command = new Command() diff --git a/cli/decoverto.ts b/cli/decoverto.ts deleted file mode 100644 index 98f1a157c2..0000000000 --- a/cli/decoverto.ts +++ /dev/null @@ -1,8 +0,0 @@ -// globally shared decoverto instance -import { Decoverto } from "npm:decoverto"; -const decoverto = new Decoverto(); - -// TODO: Properly type FlowModule - -export { Any, array, map, MapShape, model, property } from "npm:decoverto"; -export { decoverto }; diff --git a/cli/flow.ts b/cli/flow.ts index 10d2c70f0c..1b802a518f 100644 --- a/cli/flow.ts +++ b/cli/flow.ts @@ -1,10 +1,9 @@ // deno-lint-ignore-file no-explicit-any import { - Difference, GlobalOptions, - PushDiffs, - Resource, - setValueByPath, + isSuperset, + parseFromFile, + removeType, } from "./types.ts"; import { colors, @@ -12,110 +11,47 @@ import { Flow, FlowService, JobService, - microdiff, - OpenFlowWPath, Table, } from "./deps.ts"; import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; import { resolve, track_job } from "./script.ts"; -import { Any, decoverto, model, property } from "./decoverto.ts"; -// this is effectively "OpenFlow" but a copy as it is accepted by the CLI -@model() -export class FlowFile implements Resource, PushDiffs { - @property(() => String) +export interface FlowFile { summary: string; - @property(() => String) description?: string; - @property(Any) value: any; - @property(Any) schema?: any; +} - constructor(value: any, summary?: string) { - this.summary = summary ?? ""; - this.value = value; - } - async pushDiffs( - workspace: string, - remotePath: string, - diffs: Difference[] - ): Promise { - if ( - await FlowService.existsFlowByPath({ - workspace: workspace, - path: remotePath, - }) - ) { - console.log( - colors.bold.yellow( - `Applying ${diffs.length} diffs to existing flow... ${remotePath}` - ) - ); +export async function pushFlow( + workspace: string, + remotePath: string, + flow: Flow | FlowFile | undefined, + localFlow: FlowFile +): Promise { + remotePath = removeType(remotePath, "flow"); - // TODO: Make these optional in backend (not path ofc) - const changeset: OpenFlowWPath = { - path: remotePath, - summary: this.summary, - value: this.value, - description: this.description, // This is OpenAPIed as optional, but isn't - schema: this.schema, // Same - }; - const base_changeset = { ...changeset }; - for (const diff of diffs) { - if ( - diff.type !== "REMOVE" && - diff.path[0] !== "value" && - (diff.path.length !== 1 || - !["summary", "description", "schema"].includes( - diff.path[0] as string - )) - ) { - throw new Error("Invalid flow diff with path " + diff.path); - } - if (diff.type === "CREATE" || diff.type === "CHANGE") { - setValueByPath(changeset, diff.path, diff.value); - } else if (diff.type === "REMOVE") { - setValueByPath(changeset, diff.path, null); - } - } - const hasChanges = Object.values(changeset).some( - (v) => v !== null && typeof v !== "undefined" - ); - if (!hasChanges) { - return; - } - - const update = { - ...changeset, - ...base_changeset, - }; - - await FlowService.updateFlow({ - workspace: workspace, - path: remotePath, - requestBody: update, - }); - } else { - console.log(colors.bold.yellow("Creating new flow...")); - await FlowService.createFlow({ - workspace: workspace, - requestBody: { - path: remotePath, - summary: this.summary, - value: this.value, - schema: this.schema, - description: this.description, - }, - }); + if (flow) { + if (isSuperset(localFlow, flow)) { + return; } - } - async push(workspace: string, remotePath: string): Promise { - await this.pushDiffs( - workspace, - remotePath, - microdiff({}, this, { cyclesFix: false }) - ); + await FlowService.updateFlow({ + workspace: workspace, + path: remotePath, + requestBody: { + path: remotePath, + ...localFlow, + }, + }); + } else { + console.log(colors.bold.yellow("Creating new flow...")); + await FlowService.createFlow({ + workspace: workspace, + requestBody: { + path: remotePath, + ...localFlow, + }, + }); } } @@ -127,22 +63,24 @@ async function push(opts: Options, filePath: string, remotePath: string) { } const workspace = await resolveWorkspace(opts); await requireLogin(opts); - - await pushFlow(filePath, workspace.workspaceId, remotePath); + let flow: Flow | undefined = undefined; + try { + flow = await FlowService.getFlowByPath({ + workspace: workspace.workspaceId, + path: remotePath, + }); + } catch { + // flow doesn't exist + } + await pushFlow( + workspace.workspaceId, + remotePath, + flow, + parseFromFile(filePath) + ); console.log(colors.bold.underline.green("Flow pushed")); } -export async function pushFlow( - filePath: string, - workspace: string, - remotePath: string -) { - const data = decoverto - .type(FlowFile) - .rawToInstance(await Deno.readTextFile(filePath)); - await data.push(workspace, remotePath); -} - async function list(opts: GlobalOptions & { showArchived?: boolean }) { const workspace = await resolveWorkspace(opts); await requireLogin(opts); diff --git a/cli/folder.ts b/cli/folder.ts index 4849b2820e..f400d2aeb1 100644 --- a/cli/folder.ts +++ b/cli/folder.ts @@ -1,135 +1,52 @@ -import { colors, Command, FolderService, microdiff } from "./deps.ts"; +// deno-lint-ignore-file no-explicit-any +import { colors, Command, Folder, FolderService } from "./deps.ts"; import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; -import { - Difference, - GlobalOptions, - PushDiffs, - Resource, - setValueByPath, -} from "./types.ts"; -import { - array, - decoverto, - map, - MapShape, - model, - property, -} from "./decoverto.ts"; +import { GlobalOptions, isSuperset, parseFromFile } from "./types.ts"; -@model() -export class FolderFile implements Resource, PushDiffs { - @property(array(() => String)) +export interface FolderFile { owners: Array | undefined; - @property( - map( - () => String, - () => Boolean, - { shape: MapShape.Object } - ) - ) extra_perms: Map | undefined; - @property(() => String) display_name: string | undefined; +} - async push(workspace: string, remotePath: string): Promise { - if (remotePath.startsWith("/")) { - remotePath = remotePath.substring(1); - } - if (remotePath.startsWith("f/")) { - remotePath = remotePath.substring(2); - } - - await this.pushDiffs( - workspace, - remotePath, - microdiff({}, this, { cyclesFix: false }) - ); +export async function pushFolder( + workspace: string, + name: string, + folder: Folder | FolderFile | undefined, + localFolder: FolderFile +): Promise { + if (name.startsWith("/")) { + name = name.substring(1); + } + if (name.startsWith("f/")) { + name = name.substring(2); } - async pushDiffs( - workspace: string, - remotePath: string, - diffs: Difference[] - ): Promise { - if (remotePath.startsWith("/")) { - remotePath = remotePath.substring(1); + if (folder) { + if (isSuperset(localFolder, folder)) { + return; } - if (remotePath.startsWith("f/")) { - remotePath = remotePath.substring(2); - } - - // TODO: Support this in backend - let exists: boolean; try { - exists = !!(await FolderService.getFolder({ - workspace, - name: remotePath, - })); - } catch { - exists = false; - } - if (exists) { - console.log( - colors.bold.yellow( - `Applying ${diffs.length} diffs to existing folder... ${remotePath}` - ) - ); - - const changeset: { - owners?: string[] | undefined; - extra_perms?: any; - display_name?: string | undefined; - } = {}; - for (const diff of diffs) { - if ( - diff.type !== "REMOVE" && - (diff.path.length !== 1 || - !["owners", "extra_perms", "display_name"].includes( - diff.path[0] as string - )) - ) { - console.log(diff.path); - throw new Error("Invalid folder diff with path " + diff.path); - } - if (diff.type === "CREATE" || diff.type === "CHANGE") { - setValueByPath(changeset, diff.path, diff.value); - } else if (diff.type === "REMOVE") { - setValueByPath(changeset, diff.path, null); - } - } - - const hasChanges = Object.values(changeset).some( - (v) => v !== null && typeof v !== "undefined" - ); - if (!hasChanges) { - return; - } - try { - await FolderService.updateFolder({ - workspace: workspace, - name: remotePath, - requestBody: { - ...changeset, - extra_perms: changeset.extra_perms - ? Object.fromEntries(this.extra_perms?.entries() ?? []) - : undefined, - }, - }); - } catch (e) { - console.error(colors.red.bold(e.body)); - throw e; - } - } else { - console.log(colors.bold.yellow("Creating new folder: " + remotePath)); - await FolderService.createFolder({ + await FolderService.updateFolder({ workspace: workspace, + name: name, requestBody: { - name: remotePath, - extra_perms: Object.fromEntries(this.extra_perms?.entries() ?? []), - owners: this.owners, + ...localFolder, }, }); + } catch (e) { + console.error(colors.red.bold(e.body)); + throw e; } + } else { + console.log(colors.bold.yellow("Creating new folder: " + name)); + await FolderService.createFolder({ + workspace: workspace, + requestBody: { + name: name, + ...localFolder, + }, + }); } } @@ -146,21 +63,24 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) { throw new Error("file path must refer to a file."); } - console.log(colors.bold.yellow("Pushing resource...")); + console.log(colors.bold.yellow("Pushing folder...")); + let folder: Folder | undefined = undefined; + try { + folder = await FolderService.getFolder({ + workspace: workspace.workspaceId, + name: remotePath, + }); + } catch { + // folder doesn't exist + } - await pushFolder(workspace.workspaceId, filePath, remotePath); - console.log(colors.bold.underline.green("Resource pushed")); -} - -export async function pushFolder( - workspace: string, - filePath: string, - remotePath: string -) { - const data = decoverto - .type(FolderFile) - .rawToInstance(await Deno.readTextFile(filePath)); - data.push(workspace, remotePath); + await pushFolder( + workspace.workspaceId, + remotePath, + folder, + parseFromFile(filePath) + ); + console.log(colors.bold.underline.green("Folder pushed")); } const command = new Command() diff --git a/cli/hub.ts b/cli/hub.ts index bb28cc701f..e97a27298c 100644 --- a/cli/hub.ts +++ b/cli/hub.ts @@ -1,6 +1,7 @@ -import { Command } from "./deps.ts"; +// deno-lint-ignore-file no-explicit-any +import { Command, ResourceService } from "./deps.ts"; import { requireLogin, resolveWorkspace } from "./context.ts"; -import { ResourceTypeFile } from "./resource-type.ts"; +import { pushResourceType } from "./resource-type.ts"; import { GlobalOptions } from "./types.ts"; async function pull(opts: GlobalOptions) { @@ -8,7 +9,7 @@ async function pull(opts: GlobalOptions) { if (workspace.workspaceId !== "admins") { console.log( - "Should only sync to admins workspace, but current is not admins.", + "Should only sync to admins workspace, but current is not admins." ); return; } @@ -24,15 +25,12 @@ async function pull(opts: GlobalOptions) { created_by: string; created_at: Date; comments: never[]; - }[] = await fetch( - "https://hub.windmill.dev/resource_types/list", - { - headers: { - "Accept": "application/json", - "X-email": userInfo.email, - }, + }[] = await fetch("https://hub.windmill.dev/resource_types/list", { + headers: { + Accept: "application/json", + "X-email": userInfo.email, }, - ) + }) .then((r) => r.json()) .then((list: { id: number; name: string }[]) => list.map((x) => @@ -40,9 +38,9 @@ async function pull(opts: GlobalOptions) { "https://hub.windmill.dev/resource_types/" + x.id + "/" + x.name, { headers: { - "Accept": "application/json", + Accept: "application/json", }, - }, + } ) ) ) @@ -58,14 +56,22 @@ async function pull(opts: GlobalOptions) { .then((x) => Promise.all(x)) .then((x) => x.filter((x) => x).map((x) => x.resource_type)); - for ( - const x of list - ) { + const resourceTypes = await ResourceService.listResourceType({ + workspace: workspace.workspaceId, + }); + + for (const x of list) { + if (resourceTypes.find((y) => y.name === x.name)) { + console.log("skipping " + x.name); + continue; + } console.log("syncing " + x.name); - const f = new ResourceTypeFile(); - f.description = x.description; - f.schema = JSON.parse(x.schema); - await f.push(workspace.workspaceId, x.name); + await pushResourceType( + workspace.workspaceId, + x.name + ".resource-type.json", + undefined, + x + ); } } diff --git a/cli/resource-type.ts b/cli/resource-type.ts index 031328cbd3..08fb69378e 100644 --- a/cli/resource-type.ts +++ b/cli/resource-type.ts @@ -1,118 +1,53 @@ // deno-lint-ignore-file no-explicit-any import { - Difference, GlobalOptions, - PushDiffs, - Resource as ResourceI, - setValueByPath, + isSuperset, + parseFromFile, + removeType, } from "./types.ts"; import { requireLogin, resolveWorkspace } from "./context.ts"; import { colors, Command, - EditResourceType, - microdiff, ResourceService, + ResourceType, Table, } from "./deps.ts"; -import { Any, decoverto, model, property } from "./decoverto.ts"; -@model() -export class ResourceTypeFile implements ResourceI, PushDiffs { - @property(Any) +export interface ResourceTypeFile { schema?: any; - @property(() => String) description?: string; - - async push(workspace: string, remotePath: string): Promise { - await this.pushDiffs( - workspace, - remotePath, - microdiff({}, this, { cyclesFix: false }), - ); - } - - async pushDiffs( - workspace: string, - remotePath: string, - diffs: Difference[], - ): Promise { - if ( - await ResourceService.existsResourceType({ - workspace: workspace, - path: remotePath, - }) - ) { - if ( - (await ResourceService.listResourceType({ workspace })).findIndex((x) => - x.name === remotePath - ) === -1 - ) { - console.log( - "Resource type " + remotePath + - " is already taken for the current workspace, but cannot be updated. Is this a conflict with starter?", - ); - return; - } - console.log( - colors.yellow.bold( - `Applying ${diffs.length} diffs to existing resource type... ${remotePath}`, - ), - ); - const changeset: EditResourceType = {}; - for (const diff of diffs) { - if ( - diff.type !== "REMOVE" && - ( - diff.path.length !== 1 || - !["schema", "description"].includes(diff.path[0] as string) - ) - ) { - throw new Error("Invalid resource type diff with path " + diff.path); - } - if (diff.type === "CREATE" || diff.type === "CHANGE") { - setValueByPath(changeset, diff.path, diff.value); - } else if (diff.type === "REMOVE") { - setValueByPath(changeset, diff.path, null); - } - } - - const hasChanges = Object.values(changeset).some((v) => - v !== null && typeof v !== "undefined" - ); - if (!hasChanges) { - return; - } - - await ResourceService.updateResourceType({ - workspace: workspace, - path: remotePath, - requestBody: changeset, - }); - } else { - console.log(colors.yellow.bold("Creating new resource type...")); - await ResourceService.createResourceType({ - workspace: workspace, - requestBody: { - name: remotePath, - description: this.description, - schema: this.schema, - workspace_id: workspace, - }, - }); - } - } } export async function pushResourceType( workspace: string, - filePath: string, - name: string, -) { - const data: ResourceTypeFile = decoverto.type(ResourceTypeFile).rawToInstance( - await Deno.readTextFile(filePath), - ); - await data.push(workspace, name); + remotePath: string, + resource: ResourceTypeFile | ResourceType | undefined, + localResource: ResourceTypeFile +): Promise { + remotePath = removeType(remotePath, "resource-type"); + if (resource) { + if (isSuperset(localResource, resource)) { + return; + } + + await ResourceService.updateResourceType({ + workspace: workspace, + path: remotePath, + requestBody: { + ...localResource, + }, + }); + } else { + console.log(colors.yellow.bold("Creating new resource type...")); + await ResourceService.createResourceType({ + workspace: workspace, + requestBody: { + name: remotePath, + ...localResource, + }, + }); + } } type PushOptions = GlobalOptions; @@ -126,7 +61,22 @@ async function push(opts: PushOptions, filePath: string, name: string) { console.log(colors.bold.yellow("Pushing resource...")); - await pushResourceType(workspace.workspaceId, filePath, name); + let resourceType: ResourceType | undefined = undefined; + try { + resourceType = await ResourceService.getResourceType({ + workspace: workspace.workspaceId, + path: name, + }); + } catch { + // resource type doesn't exist + } + + await pushResourceType( + workspace.workspaceId, + name, + resourceType, + parseFromFile(filePath) + ); console.log(colors.bold.underline.green("Resource pushed")); } @@ -150,7 +100,7 @@ const command = new Command() .action(list as any) .command( "push", - "push a local resource spec. This overrides any remote versions.", + "push a local resource spec. This overrides any remote versions." ) .arguments(" ") .action(push as any); diff --git a/cli/resource.ts b/cli/resource.ts index 2c65c7cdbf..8ec2a66ee1 100644 --- a/cli/resource.ts +++ b/cli/resource.ts @@ -1,130 +1,56 @@ +// deno-lint-ignore-file no-explicit-any import { - Difference, GlobalOptions, - PushDiffs, - Resource as Resource2, - setValueByPath, + isSuperset, + parseFromFile, + removeType, } from "./types.ts"; import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; -import { - colors, - Command, - EditResource, - microdiff, - Resource, - ResourceService, - Table, -} from "./deps.ts"; -import { Any, decoverto, model, property } from "./decoverto.ts"; +import { colors, Command, Resource, ResourceService, Table } from "./deps.ts"; -@model() -export class ResourceFile implements Resource2, PushDiffs { - @property(Any) - value?: any; - @property(() => String) +export interface ResourceFile { + value: any; description?: string; - @property(() => String) resource_type: string; - @property(() => Boolean) is_oauth?: boolean; // deprecated - - constructor(resource_type: string) { - this.resource_type = resource_type; - } - async pushDiffs( - workspace: string, - remotePath: string, - diffs: Difference[] - ): Promise { - if ( - await ResourceService.existsResource({ - workspace: workspace, - path: remotePath, - }) - ) { - console.log( - colors.yellow.bold( - `Applying ${diffs.length} diffs to existing resource... ${remotePath}` - ) - ); - - const changeset: EditResource = { - path: remotePath, // TODO: Remove this in backend - }; - for (const diff of diffs) { - if (diff.path[0] === "is_oauth") { - //is_oauth is not updatable - continue; - } - if ( - diff.type !== "REMOVE" && - diff.path[0] !== "value" && - (diff.path.length !== 1 || diff.path[0] !== "description") && - diff.path[0] !== "resource_type" - ) { - console.log( - colors.red("Invalid variable diff with path " + diff.path) - ); - throw new Error("Invalid folder diff with path " + diff.path); - } - if (diff.type === "CREATE" || diff.type === "CHANGE") { - setValueByPath(changeset, diff.path, diff.value); - } else if (diff.type === "REMOVE") { - setValueByPath(changeset, diff.path, null); - } - } - - const hasChanges = Object.values(changeset).some( - (v) => v !== null && typeof v !== "undefined" - ); - if (!hasChanges) { - return; - } - - await ResourceService.updateResource({ - workspace: workspace, - path: remotePath, - requestBody: changeset, - }); - } else { - if (typeof this.is_oauth !== "undefined") { - console.log( - colors.yellow( - "! is_oauth has been removed in newer versions. Ignoring." - ) - ); - } - - console.log(colors.yellow.bold("Creating new resource...")); - await ResourceService.createResource({ - workspace: workspace, - requestBody: { - path: remotePath, - resource_type: this.resource_type, - value: this.value, - description: this.description, - }, - }); - } - } - async push(workspace: string, remotePath: string): Promise { - await this.pushDiffs( - workspace, - remotePath, - microdiff({}, this, { cyclesFix: false }) - ); - } } export async function pushResource( workspace: string, - filePath: string, - remotePath: string -) { - const data = decoverto - .type(ResourceFile) - .rawToInstance(await Deno.readTextFile(filePath)); - await data.push(workspace, remotePath); + remotePath: string, + resource: ResourceFile | Resource | undefined, + localResource: ResourceFile +): Promise { + remotePath = removeType(remotePath, "resource"); + + if (resource) { + if (isSuperset(localResource, resource)) { + return; + } + + await ResourceService.updateResource({ + workspace: workspace, + path: remotePath, + requestBody: { ...localResource }, + }); + } else { + if (localResource.is_oauth) { + console.log( + colors.yellow( + "! is_oauth has been removed in newer versions. Ignoring." + ) + ); + } + + console.log(colors.yellow.bold("Creating new resource...")); + await ResourceService.createResource({ + workspace: workspace, + requestBody: { + path: remotePath, + ...localResource, + }, + }); + } } type PushOptions = GlobalOptions; @@ -142,8 +68,22 @@ async function push(opts: PushOptions, filePath: string, remotePath: string) { } console.log(colors.bold.yellow("Pushing resource...")); + let resource: Resource | undefined = undefined; + try { + resource = await ResourceService.getResource({ + workspace: workspace.workspaceId, + path: remotePath, + }); + } catch { + // flow doesn't exist + } - await pushResource(workspace.workspaceId, filePath, remotePath); + await pushResource( + workspace.workspaceId, + remotePath, + resource, + parseFromFile(filePath) + ); console.log(colors.bold.underline.green(`Resource ${remotePath} pushed`)); } diff --git a/cli/script.ts b/cli/script.ts index d3b7f7ba74..0540a90b96 100644 --- a/cli/script.ts +++ b/cli/script.ts @@ -1,5 +1,5 @@ // deno-lint-ignore-file no-explicit-any -import { GlobalOptions } from "./types.ts"; +import { GlobalOptions, parseFromFile } from "./types.ts"; import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; import { colors, @@ -10,49 +10,17 @@ import { ScriptService, Table, } from "./deps.ts"; -import { Any, array, decoverto, model, property } from "./decoverto.ts"; import { writeAllSync } from "https://deno.land/std@0.176.0/streams/mod.ts"; import { parse as yamlParse } from "https://deno.land/std@0.184.0/yaml/mod.ts"; -@model() -export class ScriptFile { - @property(() => String) +export interface ScriptFile { parent_hash?: string; - @property(() => String) summary: string; - @property(() => String) description: string; - @property(Any) schema?: any; - @property(() => Boolean) is_template?: boolean; - @property(array(() => String)) lock?: Array; - @property({ - toInstance: (data) => { - if (data == null) return data; - - if ( - data === "script" || - data === "failure" || - data === "trigger" || - data === "command" || - data === "approvial" - ) { - return data; - } - - throw new Error("Invalid kind " + data); - }, - toPlain: (data) => data, - }) - @property(() => String) kind?: "script" | "failure" | "trigger" | "command" | "approval"; - - constructor(summary: string, description: string) { - this.summary = summary; - this.description = description; - } } type PushOptions = GlobalOptions; @@ -123,14 +91,11 @@ export async function handleFile( try { await Deno.stat(metaPath); typed = JSON.parse(await Deno.readTextFile(metaPath)); - typed = decoverto.type(ScriptFile).plainToInstance(typed); } catch { const metaPath = remotePath + ".script.yaml"; - let typed = undefined; try { await Deno.stat(metaPath); typed = yamlParse(await Deno.readTextFile(metaPath)); - typed = decoverto.type(ScriptFile).plainToInstance(typed); } catch { // no meta file } @@ -168,7 +133,7 @@ export async function handleFile( return true; } } - + console.log(typed); await ScriptService.createScript({ workspace, requestBody: { @@ -215,16 +180,19 @@ export async function handleFile( } export async function findContentFile(filePath: string) { - const candidates = [ - filePath.replace(".script.json", ".ts"), - filePath.replace(".script.json", ".py"), - filePath.replace(".script.json", ".go"), - filePath.replace(".script.json", ".sh"), - filePath.replace(".script.yaml", ".ts"), - filePath.replace(".script.yaml", ".py"), - filePath.replace(".script.yaml", ".go"), - filePath.replace(".script.yaml", ".sh"), - ]; + const candidates = filePath.endsWith("script.json") + ? [ + filePath.replace(".script.json", ".ts"), + filePath.replace(".script.json", ".py"), + filePath.replace(".script.json", ".go"), + filePath.replace(".script.json", ".sh"), + ] + : [ + filePath.replace(".script.yaml", ".ts"), + filePath.replace(".script.yaml", ".py"), + filePath.replace(".script.yaml", ".go"), + filePath.replace(".script.yaml", ".sh"), + ]; const validCandidates = ( await Promise.all( candidates.map((x) => { @@ -241,7 +209,7 @@ export async function findContentFile(filePath: string) { .map((x) => x.path); if (validCandidates.length > 1) { throw new Error( - "No content path given and more then one candidate found: " + + "No content path given and more than one candidate found: " + validCandidates.join(", ") ); } @@ -276,10 +244,8 @@ export async function pushScript( workspace: string, remotePath: string ) { - const data = filePath - ? decoverto - .type(ScriptFile) - .rawToInstance(await Deno.readTextFile(filePath)) + const data: ScriptFile | undefined = filePath + ? parseFromFile(filePath) : undefined; const content = await Deno.readTextFile(contentPath); diff --git a/cli/sync.ts b/cli/sync.ts index 8fdfbc5748..6b9bd8b293 100644 --- a/cli/sync.ts +++ b/cli/sync.ts @@ -6,7 +6,6 @@ import { ensureDir, gitignore_parser, JSZip, - microdiff, path, ScriptService, FolderService, @@ -16,21 +15,19 @@ import { FlowService, } from "./deps.ts"; import { - Difference, getTypeStrFromPath, GlobalOptions, - inferTypeFromPath, + parseFromPath, + pushObj, + showConflict, + showDiff, } from "./types.ts"; import { downloadZip } from "./pull.ts"; -import { FolderFile } from "./folder.ts"; -import { ResourceTypeFile } from "./resource-type.ts"; -import { handleScriptMetadata, ScriptFile } from "./script.ts"; -import { ResourceFile } from "./resource.ts"; -import { FlowFile } from "./flow.ts"; -import { VariableFile } from "./variable.ts"; + +import { handleScriptMetadata } from "./script.ts"; + import { handleFile } from "./script.ts"; import { equal } from "https://deno.land/x/equal@v1.5.0/mod.ts"; -import * as Diff from "npm:diff"; import { stringify as yamlStringify, parse as yamlParse, @@ -199,6 +196,7 @@ async function elementsToMap( } return map; } + async function compareDynFSElement( els1: DynFSElement, els2: DynFSElement | undefined, @@ -306,7 +304,9 @@ async function pull( : await FSFSElement(path.join(Deno.cwd(), opts.raw ? "" : ".wmill")); const changes = await compareDynFSElement(remote, local, await ignoreF()); - console.log(`remote -> local: ${changes.length} changes to apply`); + console.log( + `remote (${workspace.name}) -> local: ${changes.length} changes to apply` + ); if (changes.length > 0) { prettyChanges(changes); if ( @@ -424,27 +424,6 @@ async function pull( ) ); } - - function showConflict(path: string, local: string, remote: string) { - console.log(colors.yellow(`- ${path}`)); - - let finalString = ""; - for (const part of Diff.diffLines(local, remote)) { - if (part.removed) { - // print red if removed without newline - finalString += `\x1b[31m${part.value}\x1b[0m`; - } else if (part.added) { - // print green if added - finalString += `\x1b[32m${part.value}\x1b[0m`; - } else { - // print white if unchanged - finalString += `\x1b[37m${part.value}\x1b[0m`; - } - } - console.log(finalString); - console.log("\x1b[31mlocal\x1b[31m - \x1b[32mremote\x1b[32m"); - console.log(); - } } function prettyChanges(changes: Change[]) { @@ -461,6 +440,7 @@ function prettyChanges(changes: Change[]) { console.log( colors.yellow(`~ ${getTypeStrFromPath(change.path)} ` + change.path) ); + showDiff(change.before, change.after); } } } @@ -529,7 +509,9 @@ async function push( const local = await FSFSElement(path.join(Deno.cwd(), "")); const changes = await compareDynFSElement(local, remote, await ignoreF()); - console.log(`remote <- local: ${changes.length} changes to apply`); + console.log( + `remote (${workspace.name}) <- local: ${changes.length} changes to apply` + ); if (changes.length > 0) { prettyChanges(changes); if ( @@ -583,20 +565,17 @@ async function push( `Editing ${getTypeStrFromPath(change.path)} ${change.path}` ); } - const obj = inferTypeFromPath(change.path, JSON.parse(change.after)); + const oldObj = parseFromPath(change.path, change.before); + const newObj = parseFromPath(change.path, change.after); - const diff = microdiff( - inferTypeFromPath(change.path, JSON.parse(change.before)), - obj, - { cyclesFix: false } - ); - await applyDiff( + pushObj( workspace.workspaceId, - change.path.split(".")[0], - obj, - diff, - opts.plainSecrets + change.path, + oldObj, + newObj, + opts.plainSecrets ?? false ); + if (!opts.raw && stateExists) { await Deno.writeTextFile(stateTarget, change.after); } @@ -622,20 +601,15 @@ async function push( `Adding ${getTypeStrFromPath(change.path)} ${change.path}` ); } - const obj = inferTypeFromPath( - change.path, - change.path.endsWith(".yaml") - ? yamlParse(change.content) - : JSON.parse(change.content) - ); - const diff = microdiff({}, obj, { cyclesFix: false }); - await applyDiff( + const obj = parseFromPath(change.path, change.content); + pushObj( workspace.workspaceId, - change.path.split(".")[0], + change.path, + undefined, obj, - diff, - opts.plainSecrets + opts.plainSecrets ?? false ); + if (!opts.raw && stateExists) { await Deno.writeTextFile(stateTarget, change.content); } @@ -708,47 +682,10 @@ async function push( } console.log( colors.green.underline( - `Done! All ${changes.length} changes pushed to the remote workspace.` + `Done! All ${changes.length} changes pushed to the remote workspace ${workspace.workspaceId} named ${workspace.name}.` ) ); } - - async function applyDiff( - workspace: string, - remotePath: string, - file: - | ScriptFile - | VariableFile - | FlowFile - | ResourceFile - | ResourceTypeFile - | FolderFile, - diffs: Difference[], - plainSecrets?: boolean - ) { - if (file instanceof ScriptFile) { - throw new Error( - "This code path should be unreachable - we should never generate diffs for scripts" - ); - } else if (file instanceof FolderFile) { - const parts = remotePath.split("/"); - if (parts[0] === "f") { - remotePath = parts[1]; - } else { - remotePath = parts[0]; - } - } - if (diffs.length === 0) { - console.log("No diffs to apply to " + remotePath); - return; - } - try { - await file.pushDiffs(workspace, remotePath, diffs, plainSecrets); - } catch (e) { - console.error("Failing to apply diffs to " + remotePath); - console.error(JSON.stringify(e)); - } - } } const command = new Command() diff --git a/cli/types.ts b/cli/types.ts index eda52c92b2..6c34c2eaf8 100644 --- a/cli/types.ts +++ b/cli/types.ts @@ -1,31 +1,19 @@ -import { decoverto } from "./decoverto.ts"; -import { FlowFile } from "./flow.ts"; -import { ResourceTypeFile } from "./resource-type.ts"; -import { ResourceFile } from "./resource.ts"; -import { ScriptFile } from "./script.ts"; -import { VariableFile } from "./variable.ts"; -import { path } from "./deps.ts"; -import { FolderFile } from "./folder.ts"; -import { AppFile } from "./apps.ts"; +// deno-lint-ignore-file no-explicit-any -// TODO: Remove this & replace with a "pull" that lets the object either pull the remote version or return undefined. -// Then combine those with diffing, which then gives the new push impl -export interface Resource { - push( - workspace: string, - remotePath: string, - plainSecrets?: boolean - ): Promise; -} - -export interface PushDiffs { - pushDiffs( - workspace: string, - remotePath: string, - diffs: Difference[], - plainSecrets?: boolean - ): Promise; -} +import { colors, path } from "./deps.ts"; +import { pushApp } from "./apps.ts"; +import { + parse as yamlParse, + stringify as yamlStringify, +} from "https://deno.land/std@0.184.0/yaml/mod.ts"; +import { equal } from "https://deno.land/x/equal@v1.5.0/equal.ts"; +import { pushFolder } from "./folder.ts"; +import { pushScript } from "./script.ts"; +import { pushFlow } from "./flow.ts"; +import { pushResource } from "./resource.ts"; +import { pushResourceType } from "./resource-type.ts"; +import { pushVariable } from "./variable.ts"; +import * as Diff from "npm:diff"; export interface DifferenceCreate { type: "CREATE"; @@ -48,77 +36,91 @@ export interface DifferenceChange { export type Difference = DifferenceCreate | DifferenceRemove | DifferenceChange; -export function setValueByPath( - obj: any, - path: (string | number)[], - value: any -) { - let i; - let lastObj = undefined; - for (i = 0; i < path.length - 1; i++) { - if (!obj) { - let oldNewObj; - if (typeof path[i] === "number") { - oldNewObj = []; - } else { - oldNewObj = {}; - } - lastObj[path[i - 1]] = oldNewObj; - obj = oldNewObj; - } - lastObj = obj; - obj = obj[path[i]]; - } - if (!obj) { - let oldNewObj; - if (typeof path[i] === "number") { - oldNewObj = []; - } else { - oldNewObj = {}; - } - lastObj[path[i - 1]] = oldNewObj; - obj = oldNewObj; - } - obj[path[i]] = value; -} - export type GlobalOptions = { workspace: string | undefined; token: string | undefined; }; -export function inferTypeFromPath( +export function isSuperset( + subset: Record, + superset: Record +): boolean { + return Object.keys(subset).every((key) => { + const eq = equal(subset[key], superset[key]); + if (!eq) { + console.log( + `Found diff for ${key}:`, + showDiff(yamlStringify(subset[key]), yamlStringify(superset[key])) + ); + } + return eq; + }); +} + +export function showDiff(local: string, remote: string) { + let finalString = ""; + for (const part of Diff.diffLines(local, remote)) { + if (part.removed) { + // print red if removed without newline + finalString += `\x1b[31m${part.value}\x1b[0m`; + } else if (part.added) { + // print green if added + finalString += `\x1b[32m${part.value}\x1b[0m`; + } else { + // print white if unchanged + finalString += `\x1b[37m${part.value}\x1b[0m`; + } + } + console.log(finalString); +} + +export function showConflict(path: string, local: string, remote: string) { + console.log(colors.yellow(`- ${path}`)); + showDiff(local, remote); + console.log("\x1b[31mlocal\x1b[31m - \x1b[32mremote\x1b[32m"); + console.log(); +} + +export function pushObj( + workspace: string, p: string, - obj: any -): - | ScriptFile - | VariableFile - | FlowFile - | ResourceFile - | ResourceTypeFile - | FolderFile - | AppFile { + befObj: any, + newObj: any, + plainSecrets: boolean +) { const typeEnding = getTypeStrFromPath(p); - if (typeEnding === "folder") { - return decoverto.type(FolderFile).plainToInstance(obj); + if (typeEnding === "app") { + pushApp(workspace, p, befObj, newObj); + } else if (typeEnding === "folder") { + pushFolder(workspace, p, befObj, newObj); } else if (typeEnding === "script") { - return decoverto.type(ScriptFile).plainToInstance(obj); + pushScript(workspace, p, befObj, newObj); } else if (typeEnding === "variable") { - return decoverto.type(VariableFile).plainToInstance(obj); + pushVariable(workspace, p, befObj, newObj, plainSecrets); } else if (typeEnding === "flow") { - return decoverto.type(FlowFile).plainToInstance(obj); + pushFlow(workspace, p, befObj, newObj); } else if (typeEnding === "resource") { - return decoverto.type(ResourceFile).plainToInstance(obj); + pushResource(workspace, p, befObj, newObj); } else if (typeEnding === "resource-type") { - return decoverto.type(ResourceTypeFile).plainToInstance(obj); - } else if (typeEnding === "app") { - return decoverto.type(AppFile).plainToInstance(obj); + pushResourceType(workspace, p, befObj, newObj); } else { throw new Error("infer type unreachable"); } } +export function parseFromPath(p: string, content: string): any { + return p.endsWith(".yaml") ? yamlParse(content) : JSON.parse(content); +} +export function parseFromFile(p: string): any { + if (p.endsWith(".json")) { + return JSON.parse(Deno.readTextFileSync(p)); + } else if (p.endsWith(".yaml") || p.endsWith(".yml")) { + return yamlParse(Deno.readTextFileSync(p)); + } else { + throw new Error("Could not read file " + p); + } +} export function getTypeStrFromPath( p: string ): @@ -157,3 +159,13 @@ export function getTypeStrFromPath( throw new Error("Could not infer type of path " + JSON.stringify(parsed)); } } + +export function removeType(str: string, type: string) { + if ( + !str.endsWith("." + type + ".yaml") && + !str.endsWith("." + type + ".json") + ) { + throw new Error(str + " does not end with ." + type + ".(yaml|json)"); + } + return str.slice(0, str.length - type.length - 6); +} diff --git a/cli/variable.ts b/cli/variable.ts index 22ac3a08f0..aa1d0cb6f3 100644 --- a/cli/variable.ts +++ b/cli/variable.ts @@ -1,21 +1,13 @@ // deno-lint-ignore-file no-explicit-any import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; -import { - Difference, - GlobalOptions, - PushDiffs, - Resource, - setValueByPath, -} from "./types.ts"; +import { GlobalOptions, isSuperset, parseFromFile } from "./types.ts"; import { colors, Command, - EditVariable, - microdiff, + ListableVariable, Table, VariableService, } from "./deps.ts"; -import { decoverto, model, property } from "./decoverto.ts"; async function list(opts: GlobalOptions) { const workspace = await resolveWorkspace(opts); @@ -40,102 +32,44 @@ async function list(opts: GlobalOptions) { .render(); } -@model() -export class VariableFile implements Resource, PushDiffs { - @property(() => String) +export interface VariableFile { value: string; - @property(() => Boolean) is_secret: boolean; - @property(() => String) description: string; - @property(() => Number) account?: number; - @property(() => Boolean) is_oauth?: boolean; +} - constructor(value: string, is_secret: boolean, description: string) { - this.value = value; - this.is_secret = is_secret; - this.description = description; - } - async pushDiffs( - workspace: string, - remotePath: string, - diffs: Difference[], - plainSecrets?: boolean - ): Promise { - if (await VariableService.existsVariable({ workspace, path: remotePath })) { - console.log( - colors.bold.yellow( - `Applying ${diffs.length} diffs to existing variable... ${remotePath}` - ) - ); - const changeset: EditVariable = {}; - for (const diff of diffs) { - if ( - diff.type !== "REMOVE" && - (diff.path.length !== 1 || - ![ - "path", - "value", - "is_secret", - "description", - "account", - "is_oauth", - ].includes(diff.path[0] as string)) - ) { - console.log( - colors.red("Invalid variable diff with path " + diff.path) - ); - throw new Error("Invalid variable diff with path " + diff.path); - } - if (diff.type === "CREATE" || diff.type === "CHANGE") { - setValueByPath(changeset, diff.path, diff.value); - } else if (diff.type === "REMOVE") { - setValueByPath(changeset, diff.path, null); - } - } - - const hasChanges = Object.values(changeset).some( - (v) => v !== null && typeof v !== "undefined" - ); - if (!hasChanges) { - return; - } - - await VariableService.updateVariable({ - workspace, - path: remotePath, - alreadyEncrypted: !plainSecrets, - requestBody: changeset, - }); - } else { - console.log(colors.yellow.bold("Creating new variable...")); - await VariableService.createVariable({ - workspace, - alreadyEncrypted: !plainSecrets, - requestBody: { - path: remotePath, - description: this.description, - is_secret: this.is_secret, - value: this.value, - account: this.account, - is_oauth: this.is_oauth, - }, - }); +export async function pushVariable( + workspace: string, + remotePath: string, + variable: VariableFile | ListableVariable | undefined, + localVariable: VariableFile, + plainSecrets: boolean +): Promise { + if (variable) { + if (isSuperset(localVariable, variable)) { + return; } - } - async push( - workspace: string, - remotePath: string, - plainSecrets?: boolean - ): Promise { - await this.pushDiffs( + + await VariableService.updateVariable({ workspace, - remotePath, - microdiff({}, this, { cyclesFix: false }), - plainSecrets - ); + path: remotePath, + alreadyEncrypted: !plainSecrets, + requestBody: { + ...localVariable, + }, + }); + } else { + console.log(colors.yellow.bold("Creating new variable...")); + await VariableService.createVariable({ + workspace, + alreadyEncrypted: !plainSecrets, + requestBody: { + path: remotePath, + ...localVariable, + }, + }); } } @@ -158,27 +92,26 @@ async function push( console.log(colors.bold.yellow("Pushing variable...")); + let variable: ListableVariable | undefined = undefined; + try { + variable = await VariableService.getVariable({ + workspace: workspace.workspaceId, + path: remotePath, + }); + } catch { + // resource type doesn't exist + } + await pushVariable( workspace.workspaceId, - filePath, remotePath, + variable, + parseFromFile(filePath), opts.plainSecrets ); console.log(colors.bold.underline.green(`Variable ${remotePath} pushed`)); } -export async function pushVariable( - workspace: string, - filePath: string, - remotePath: string, - plainSecrets: boolean -) { - const data = decoverto - .type(VariableFile) - .rawToInstance(await Deno.readTextFile(filePath)); - await data.push(workspace, remotePath, plainSecrets); -} - const command = new Command() .description("variable related commands") .action(list as any) diff --git a/cli/workspace.ts b/cli/workspace.ts index 4e60ed4656..c286118ba5 100644 --- a/cli/workspace.ts +++ b/cli/workspace.ts @@ -12,31 +12,13 @@ import { UserService, WorkspaceService, } from "./deps.ts"; -import { decoverto, model, property } from "./decoverto.ts"; import { requireLogin } from "./context.ts"; -@model() -export class Workspace { - @property(() => String) +export interface Workspace { remote: string; - @property(() => String) workspaceId: string; - @property(() => String) name: string; - @property(() => String) token: string; - - constructor( - remote: string, - workspaceId: string, - name: string, - token: string - ) { - this.remote = remote; - this.workspaceId = workspaceId; - this.name = name; - this.token = token; - } } function makeWorkspaceStream( @@ -52,7 +34,7 @@ function makeWorkspaceStream( if (line.length <= 2) { return; } - const workspace = decoverto.type(Workspace).rawToInstance(line); + const workspace = JSON.parse(line) as Workspace; workspace.remote = new URL(workspace.remote).toString(); // add trailing slash in all cases! controller.enqueue(workspace); } catch {