import { requireLogin, resolveWorkspace } from "./context.ts"; import { colors, Command, Confirm, ensureDir, gitignore_parser, JSZip, microdiff, path, ScriptService, FolderService, ResourceService, VariableService, AppService, FlowService, } from "./deps.ts"; import { Difference, getTypeStrFromPath, GlobalOptions, inferTypeFromPath, setValueByPath, } from "./types.ts"; import { downloadZip } from "./pull.ts"; import { FolderFile } from "./folder.ts"; import { ResourceTypeFile } from "./resource-type.ts"; import { handleScriptMetadata, ScriptFile } from "./script.ts"; import { ResourceFile } from "./resource.ts"; import { FlowFile } from "./flow.ts"; import { VariableFile } from "./variable.ts"; import { handleFile } from "./script.ts"; import { equal } from "https://deno.land/x/equal@v1.5.0/mod.ts"; import * as Diff from "npm:diff"; type DynFSElement = { isDirectory: boolean; path: string; getContentBytes(): Promise; getContentText(): Promise; getChildren(): AsyncIterable; }; async function FSFSElement(p: string): Promise { function _internal_element(localP: string, isDir: boolean): DynFSElement { return { isDirectory: isDir, path: localP.substring(p.length + 1), async *getChildren(): AsyncIterable { for await (const e of Deno.readDir(localP)) { yield _internal_element(path.join(localP, e.name), e.isDirectory); } }, async getContentBytes(): Promise { return await Deno.readFile(localP); }, async getContentText(): Promise { return await Deno.readTextFile(localP); }, }; } return _internal_element(p, (await Deno.stat(p)).isDirectory); } function ZipFSElement(zip: JSZip): DynFSElement { function _internal_file(p: string, f: JSZip.JSZipObject): DynFSElement { return { isDirectory: false, path: p, // deno-lint-ignore require-yield async *getChildren(): AsyncIterable { throw new Error("Cannot get children of file"); }, async getContentBytes(): Promise { return await f.async("uint8array"); }, async getContentText(): Promise { return await f.async("text"); }, }; } function _internal_folder(p: string, zip: JSZip): DynFSElement { return { isDirectory: true, path: p, async *getChildren(): AsyncIterable { for (const filename in zip.files) { const file = zip.files[filename]; const totalPath = path.join(p, filename); if (file.dir) { const e = zip.folder(file.name)!; yield _internal_folder(totalPath, e); } else { yield _internal_file(totalPath, file); } } }, async getContentBytes(): Promise { throw new Error("Cannot get content of folder"); }, async getContentText(): Promise { throw new Error("Cannot get content of folder"); }, }; } return _internal_folder("./", zip); } async function* readDirRecursiveWithIgnore( ignore: (path: string, isDirectory: boolean) => boolean, root: DynFSElement ): AsyncGenerator<{ path: string; ignored: boolean; isDirectory: boolean; getContentBytes(): Promise; getContentText(): Promise; }> { const stack: { path: string; isDirectory: boolean; ignored: boolean; c(): AsyncIterable; getContentBytes(): Promise; getContentText(): Promise; }[] = [ { path: root.path, ignored: ignore(root.path, root.isDirectory), isDirectory: root.isDirectory, c: root.getChildren, getContentBytes(): Promise { throw undefined; }, getContentText(): Promise { throw undefined; }, }, ]; while (stack.length > 0) { const e = stack.pop()!; yield e; if (!e.isDirectory) continue; for await (const e2 of e.c()) { stack.push({ path: e2.path, ignored: e.ignored || ignore(e2.path, e2.isDirectory), isDirectory: e2.isDirectory, getContentBytes: e2.getContentBytes, getContentText: e2.getContentText, c: e2.getChildren, }); } } } type Added = { name: "added"; path: string; content: string }; type Deleted = { name: "deleted"; path: string }; type Edit = { name: "edited"; path: string; before: string; after: string }; type Change = Added | Deleted | Edit; async function elementsToMap( els: DynFSElement, ignore: (path: string, isDirectory: boolean) => boolean ): Promise<{ [key: string]: string }> { const map: { [key: string]: string } = {}; for await (const entry of readDirRecursiveWithIgnore(ignore, els)) { if (entry.isDirectory || entry.ignored) continue; const content = await entry.getContentText(); map[entry.path] = content; } return map; } async function compareDynFSElement( els1: DynFSElement, els2: DynFSElement | undefined, ignore: (path: string, isDirectory: boolean) => boolean ): Promise { const [m1, m2] = els2 ? await Promise.all([ elementsToMap(els1, ignore), elementsToMap(els2, ignore), ]) : [await elementsToMap(els1, ignore), {}]; const changes: Change[] = []; for (const [k, v] of Object.entries(m1)) { if (m2[k] === undefined) { changes.push({ name: "added", path: k, content: v }); } else if ( m2[k] != v && (!k.endsWith(".json") || !equal(JSON.parse(v), JSON.parse(m2[k]))) ) { // await Deno.writeTextFile("/tmp/k", m2[k]) // await Deno.writeTextFile("/tmp/v", v) // console.log(k) // if (k.includes("flow")) // Deno.exit(1) changes.push({ name: "edited", path: k, after: v, before: m2[k] }); } } for (const [k] of Object.entries(m2)) { if (m1[k] === undefined) { changes.push({ name: "deleted", path: k }); } } return changes; } const isNotWmillFile = (p: string, isDirectory: boolean) => { if (p.endsWith("/")) { return false; } if (isDirectory) { return !p.startsWith("u/") && !p.startsWith("f/") && !p.startsWith("g/"); } try { const typ = getTypeStrFromPath(p); if (typ == "resource-type") { return p.includes("/"); } else { return !p.startsWith("u/") && !p.startsWith("f/") && !p.startsWith("g/"); } } catch { return true; } }; const isWhitelisted = (p: string) => { return p == "./" || p == "" || p == "u" || p == "f" || p == "g"; }; async function ignoreF() { try { const ignore: { accepts(file: string): boolean; denies(file: string): boolean; } = gitignore_parser.compile(await Deno.readTextFile(".wmillignore")); return (p: string, isDirectory: boolean) => { return ( !isWhitelisted(p) && (isNotWmillFile(p, isDirectory) || ignore.denies(p)) ); }; } catch (e) { return (p: string, isDirectory: boolean) => !isWhitelisted(p) && isNotWmillFile(p, isDirectory); } } async function pull( opts: GlobalOptions & { raw: boolean; yes: boolean; failConflicts: boolean; plainSecrets?: boolean; } ) { if (!opts.raw) { await ensureDir(path.join(Deno.cwd(), ".wmill")); } const workspace = await resolveWorkspace(opts); await requireLogin(opts); console.log( colors.gray( "Computing the files to update locally to match remote (taking .wmillignore into account)" ) ); const remote = ZipFSElement( (await downloadZip(workspace, opts.plainSecrets))! ); const local = opts.raw ? undefined : await FSFSElement(path.join(Deno.cwd(), opts.raw ? "" : ".wmill")); const changes = await compareDynFSElement(remote, local, await ignoreF()); console.log(`remote -> local: ${changes.length} changes to apply`); if (changes.length > 0) { prettyChanges(changes); if ( !opts.yes && !opts.raw && !(await Confirm.prompt({ message: `Do you want to apply these ${changes.length} changes?`, default: true, })) ) { return; } const conflicts = []; console.log(colors.gray(`Applying changes to files ...`)); for await (const change of changes) { const target = path.join(Deno.cwd(), change.path); const stateTarget = path.join(Deno.cwd(), ".wmill", change.path); if (change.name === "edited") { try { const currentLocal = await Deno.readTextFile(target); if (currentLocal !== change.before && currentLocal !== change.after) { console.log( colors.red( `Conflict detected on ${change.path}\nBoth local and remote have been modified.` ) ); if (opts.failConflicts) { conflicts.push({ local: currentLocal, change, path: change.path, }); continue; } else if (opts.yes) { console.log( colors.red( `Override local version with remote since --yes was passed and no --fail-conflicts.` ) ); } else { showConflict(change.path, currentLocal, change.after); if ( await Confirm.prompt( "Preserve local (push to change remote and avoid seeing this again)?" ) ) { continue; } } } } catch {} if (!change.path.endsWith(".json")) { console.log(`Editing script content of ${change.path}`); } else { console.log( `Editing ${getTypeStrFromPath(change.path)} ${change.path}` ); } await Deno.writeTextFile(target, change.after); if (!opts.raw) { await ensureDir(path.dirname(stateTarget)); await Deno.copyFile(target, stateTarget); } } else if (change.name === "added") { await ensureDir(path.dirname(target)); if (!opts.raw) { await ensureDir(path.dirname(stateTarget)); console.log( `Adding ${getTypeStrFromPath(change.path)} ${change.path}` ); } await Deno.writeTextFile(target, change.content); if (!opts.raw) { await Deno.copyFile(target, stateTarget); } } else if (change.name === "deleted") { try { console.log( `Deleting ${getTypeStrFromPath(change.path)} ${change.path}` ); await Deno.remove(target); if (!opts.raw) { await Deno.remove(stateTarget); } } catch (e) { if (!opts.raw) { await Deno.remove(stateTarget); } } } } if (opts.failConflicts) { if (conflicts.length > 0) { console.error(colors.red(`Conflicts were found`)); console.log("Conflicts:"); for (const conflict of conflicts) { showConflict(conflict.path, conflict.local, conflict.change.after); } console.log( colors.red(`Please resolve theses conflicts manually by either: - reverting the content back to its remote (\`wmill pull\` and refuse to preserve local when prompted) - pushing the changes with \`wmill push --skip-pull\` to override wmill with all your local changes `) ); Deno.exit(1); } } console.log( colors.green.underline( `Done! All ${changes.length} changes applied locally.` ) ); } function showConflict(path: string, local: string, remote: string) { console.log(colors.yellow(`- ${path}`)); let finalString = ""; for (const part of Diff.diffLines(local, remote)) { if (part.removed) { // print red if removed without newline finalString += `\x1b[31m${part.value}\x1b[0m`; } else if (part.added) { // print green if added finalString += `\x1b[32m${part.value}\x1b[0m`; } else { // print white if unchanged finalString += `\x1b[37m${part.value}\x1b[0m`; } } console.log(finalString); console.log("\x1b[31mlocal\x1b[31m - \x1b[32mremote\x1b[32m"); console.log(); } } function prettyChanges(changes: Change[]) { for (const change of changes) { if (change.name === "added") { console.log( colors.green(`+ ${getTypeStrFromPath(change.path)} ` + change.path) ); } else if (change.name === "deleted") { console.log( colors.red(`- ${getTypeStrFromPath(change.path)} ` + change.path) ); } else if (change.name === "edited") { console.log( colors.yellow(`~ ${getTypeStrFromPath(change.path)} ` + change.path) ); } } } // function prettyDiff(diffs: Difference[]) { // for (const diff of diffs) { // let pathString = ""; // for (const pathSegment of diff.path) { // if (typeof pathSegment === "string") { // pathString += "."; // pathString += pathSegment; // } else { // pathString += "["; // pathString += pathSegment; // pathString += "]"; // } // } // if (diff.type === "REMOVE" || diff.type === "CHANGE") { // console.log(colors.red("- " + pathString + " = " + diff.oldValue)); // } // if (diff.type === "CREATE" || diff.type === "CHANGE") { // console.log(colors.green("+ " + pathString + " = " + diff.value)); // } // } // } function removeSuffix(str: string, suffix: string) { return str.slice(0, str.length - suffix.length); } async function push( opts: GlobalOptions & { raw: boolean; yes: boolean; skipPull: boolean; failConflicts: boolean; plainSecrets?: boolean; } ) { if (!opts.raw) { if (!opts.skipPull) { console.log( colors.gray("You need to be up-to-date before pushing, pulling first.") ); await pull(opts); console.log(colors.green("Pull done, now pushing.")); console.log(); } } const workspace = await resolveWorkspace(opts); await requireLogin(opts); console.log( colors.gray( "Computing the files to update on the remote to match local (taking .wmillignore into account)" ) ); const remote = opts.raw ? undefined : ZipFSElement((await downloadZip(workspace, opts.plainSecrets))!); const local = await FSFSElement(path.join(Deno.cwd(), "")); const changes = await compareDynFSElement(local, remote, await ignoreF()); console.log(`remote <- local: ${changes.length} changes to apply`); if (changes.length > 0) { prettyChanges(changes); if ( !opts.yes && !(await Confirm.prompt({ message: `Do you want to apply these ${changes.length} changes?`, default: true, })) ) { return; } console.log(colors.gray(`Applying changes to files ...`)); const alreadySynced: string[] = []; for await (const change of changes) { const stateTarget = path.join(Deno.cwd(), ".wmill", change.path); let stateExists = true; try { await Deno.stat(stateTarget); } catch { stateExists = false; } if (change.name === "edited") { if ( await handleScriptMetadata( change.path, workspace.workspaceId, alreadySynced ) ) { if (!opts.raw && stateExists) { await Deno.writeTextFile(stateTarget, change.after); } continue; } else if ( await handleFile( change.path, change.after, workspace.workspaceId, alreadySynced ) ) { if (!opts.raw && stateExists) { await Deno.writeTextFile(stateTarget, change.after); } continue; } if (!opts.raw) { await ensureDir(path.dirname(stateTarget)); console.log( `Editing ${getTypeStrFromPath(change.path)} ${change.path}` ); } const obj = inferTypeFromPath(change.path, JSON.parse(change.after)); const diff = microdiff( inferTypeFromPath(change.path, JSON.parse(change.before)), obj, { cyclesFix: false } ); await applyDiff( workspace.workspaceId, change.path.split(".")[0], obj, diff, opts.plainSecrets ); if (!opts.raw && stateExists) { await Deno.writeTextFile(stateTarget, change.after); } } else if (change.name === "added") { if (change.path.endsWith(".script.json")) { continue; } else if ( await handleFile( change.path, change.content, workspace.workspaceId, alreadySynced ) ) { continue; } if (!opts.raw && stateExists) { await ensureDir(path.dirname(stateTarget)); console.log( `Adding ${getTypeStrFromPath(change.path)} ${change.path}` ); } const obj = inferTypeFromPath(change.path, JSON.parse(change.content)); const diff = microdiff({}, obj, { cyclesFix: false }); await applyDiff( workspace.workspaceId, change.path.split(".")[0], obj, diff, opts.plainSecrets ); if (!opts.raw && stateExists) { await Deno.writeTextFile(stateTarget, change.content); } } else if (change.name === "deleted") { if (!change.path.includes(".json")) { continue; } console.log( `Deleting ${getTypeStrFromPath(change.path)} ${change.path}` ); const typ = getTypeStrFromPath(change.path); const workspaceId = workspace.workspaceId; switch (typ) { case "script": { const script = await ScriptService.getScriptByPath({ workspace: workspaceId, path: removeSuffix(change.path, ".script.json"), }); await ScriptService.deleteScriptByHash({ workspace: workspaceId, hash: script.hash, }); break; } case "folder": await FolderService.deleteFolder({ workspace: workspaceId, name: change.path.split("/")[1], }); break; case "resource": await ResourceService.deleteResource({ workspace: workspaceId, path: removeSuffix(change.path, ".resource.json"), }); break; case "resource-type": await ResourceService.deleteResourceType({ workspace: workspaceId, path: removeSuffix(change.path, ".resource-type.json"), }); break; case "flow": await FlowService.deleteFlowByPath({ workspace: workspaceId, path: removeSuffix(change.path, ".flow.json"), }); break; case "app": await AppService.deleteApp({ workspace: workspaceId, path: removeSuffix(change.path, ".app.json"), }); break; case "variable": await VariableService.deleteVariable({ workspace: workspaceId, path: removeSuffix(change.path, ".variable.json"), }); break; default: break; } try { await Deno.remove(stateTarget); } catch {} } } console.log( colors.green.underline( `Done! All ${changes.length} changes pushed to the remote workspace.` ) ); } async function applyDiff( workspace: string, remotePath: string, file: | ScriptFile | VariableFile | FlowFile | ResourceFile | ResourceTypeFile | FolderFile, diffs: Difference[], plainSecrets?: boolean ) { if (file instanceof ScriptFile) { throw new Error( "This code path should be unreachable - we should never generate diffs for scripts" ); } else if (file instanceof FolderFile) { const parts = remotePath.split("/"); if (parts[0] === "f") { remotePath = parts[1]; } else { remotePath = parts[0]; } } if (diffs.length === 0) { console.log("No diffs to apply to " + remotePath); return; } try { await file.pushDiffs(workspace, remotePath, diffs, plainSecrets); } catch (e) { console.error("Failing to apply diffs to " + remotePath); console.error(JSON.stringify(e)); } } } const command = new Command() .command("pull") .description( "Pull any remote changes and apply them locally. Use --raw for usage without local state tracking." ) .option( "--fail-conflicts", "Error on conflicts (both remote and local have changes on the same item)" ) .option("--yes", "Pull without needing confirmation") .option("--raw", "Pull without using state, just overwrite.") .option("--plain-secrets", "Pull secrets as plain text") .action(pull as any) .command("push") .description( "Push any local changes and apply them remotely. Use --raw for usage without local state tracking." ) .option( "--fail-conflicts", "Error on conflicts (both remote and local have changes on the same item)" ) .option("--skip-pull", "Push without pulling first (you have pulled prior)") .option("--yes", "Push without needing confirmation") .option("--raw", "Push without using state, just overwrite.") .option("--plain-secrets", "Push secrets as plain text") .action(push as any); export default command;