From 89aec0d7c7d52b982dbd4e680269d50248d5d868 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 28 Apr 2024 14:55:33 +0200 Subject: [PATCH] feat(cli): split inline sscripts for apps like for flows --- backend/windmill-worker/src/worker.rs | 8 ++ cli/apps.ts | 83 +++++++++++----- cli/flow.ts | 16 +-- cli/script.ts | 3 +- cli/sync.ts | 135 +++++++++++++++++++------- cli/types.ts | 6 +- 6 files changed, 182 insertions(+), 69 deletions(-) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 78c1dd7d1a..daeb05d9c5 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -3938,6 +3938,14 @@ async fn lock_modules_app( .unwrap_or_default() .to_string(); let mut logs = "".to_string(); + if v.get("lock") + .is_some_and(|x| !x.as_str().unwrap().trim().is_empty()) + { + logs.push_str( + "Found already locked inline script. Skipping lock...\n", + ); + return Ok(Value::Object(m.clone())); + } logs.push_str("Found lockable inline script. Generating lock...\n"); let new_lock = capture_dependency_job( &job.id, diff --git a/cli/apps.ts b/cli/apps.ts index 97faf39c1e..2aff8784c2 100644 --- a/cli/apps.ts +++ b/cli/apps.ts @@ -5,15 +5,13 @@ import { colors, Command, ListableApp, + log, Policy, + SEP, Table, + yamlParse, } from "./deps.ts"; -import { - GlobalOptions, - isSuperset, - parseFromFile, - removeType, -} from "./types.ts"; +import { GlobalOptions, isSuperset } from "./types.ts"; export interface AppFile { value: any; @@ -21,14 +19,20 @@ export interface AppFile { policy: Policy; } +const alreadySynced: string[] = []; + export async function pushApp( workspace: string, - filePath: string, - app: AppFile | undefined, - newApp: AppFile, + remotePath: string, + localPath: string, message?: string ): Promise { - const remotePath = removeType(filePath, "app"); + if (alreadySynced.includes(localPath)) { + return; + } + alreadySynced.push(localPath); + + let app: any = undefined; // deleting old app if it exists in raw mode try { app = await AppService.getAppByPath({ @@ -39,28 +43,61 @@ export async function pushApp( //ignore } - if (app) { - if (isSuperset(newApp, app)) { + if (!localPath.endsWith(SEP)) { + localPath += SEP; + } + const localAppRaw = await Deno.readTextFile(localPath + "app.yaml"); + const localApp = yamlParse(localAppRaw) as AppFile; + + function replaceInlineScripts(rec: any) { + if (!rec) { return; } + if (typeof rec == "object") { + return Object.entries(rec).flatMap(([k, v]) => { + if (k == "inlineScript" && typeof v == "object") { + const o: Record = v as any; + if (o["content"] && o["content"].startsWith("!inline")) { + const basePath = localPath + o["content"].split(" ")[1]; + o["content"] = Deno.readTextFileSync(basePath); + } + if (o["lock"] && o["lock"].startsWith("!inline")) { + const basePath = localPath + o["lock"].split(" ")[1]; + o["lock"] = Deno.readTextFileSync(basePath); + } + } else { + replaceInlineScripts(v); + } + }); + } + return []; + } + + replaceInlineScripts(localApp.value); + + if (app) { + if (isSuperset(localApp, app)) { + log.info(colors.green(`App ${remotePath} is up to date`)); + return; + } + log.info(colors.bold.yellow(`Updating app ${remotePath}...`)); await AppService.updateApp({ workspace, path: remotePath.replaceAll("\\", "/"), requestBody: { deployment_message: message, - ...newApp, + ...localApp, }, }); } else { - console.log(colors.yellow.bold("Creating new app...")); + log.info(colors.yellow.bold("Creating new app...")); - console.log(message); await AppService.createApp({ workspace, requestBody: { path: remotePath.replaceAll("\\", "/"), deployment_message: message, - ...newApp, + ...localApp, }, }); } @@ -94,28 +131,22 @@ async function list(opts: GlobalOptions) { .render(); } -async function push(opts: GlobalOptions, filePath: string) { - const remotePath = filePath.split(".")[0]; +async function push(opts: GlobalOptions, filePath: string, remotePath: string) { if (!validatePath(remotePath)) { return; } const workspace = await resolveWorkspace(opts); await requireLogin(opts); - await pushApp( - workspace.workspaceId, - filePath, - undefined, - parseFromFile(filePath) - ); - console.log(colors.bold.underline.green("App pushed")); + await pushApp(workspace.workspaceId, remotePath, filePath); + log.info(colors.bold.underline.green("Flow pushed")); } const command = new Command() .description("app related commands") .action(list as any) .command("push", "push a local app ") - .arguments("") + .arguments(" ") .action(push as any); export default command; diff --git a/cli/flow.ts b/cli/flow.ts index 16da96e702..731026ba7a 100644 --- a/cli/flow.ts +++ b/cli/flow.ts @@ -27,13 +27,13 @@ const alreadySynced: string[] = []; export async function pushFlow( workspace: string, remotePath: string, - localFlowPath: string, + localPath: string, message?: string ): Promise { - if (alreadySynced.includes(localFlowPath)) { + if (alreadySynced.includes(localPath)) { return; } - alreadySynced.push(localFlowPath); + alreadySynced.push(localPath); let flow: Flow | undefined = undefined; try { flow = await FlowService.getFlowByPath({ @@ -44,17 +44,17 @@ export async function pushFlow( // flow doesn't exist } - if (!localFlowPath.endsWith(SEP)) { - localFlowPath += SEP; + if (!localPath.endsWith(SEP)) { + localPath += SEP; } - const localFlowRaw = await Deno.readTextFile(localFlowPath + "flow.yaml"); + const localFlowRaw = await Deno.readTextFile(localPath + "flow.yaml"); const localFlow = yamlParse(localFlowRaw) as FlowFile; function replaceInlineScripts(modules: FlowModule[]) { modules.forEach((m) => { if (m.value.type == "rawscript") { const path = m.value.content.split(" ")[1]; - m.value.content = Deno.readTextFileSync(localFlowPath + path); + m.value.content = Deno.readTextFileSync(localPath + path); const lock = m.value.lock; if ( @@ -64,7 +64,7 @@ export async function pushFlow( ) { const path = lock.split(" ")[1]; try { - m.value.lock = Deno.readTextFileSync(localFlowPath + path); + m.value.lock = Deno.readTextFileSync(localPath + path); } catch { log.error(`Lock file ${path} not found`); } diff --git a/cli/script.ts b/cli/script.ts index c5593a5d3c..0efeeb3697 100644 --- a/cli/script.ts +++ b/cli/script.ts @@ -640,7 +640,8 @@ async function generateMetadata( return ( (!isD && !exts.some((ext) => p.endsWith(ext))) || ignore(p, isD) || - p.includes(".flow/") + p.includes(".flow/") || + p.includes(".app/") ); }, false, diff --git a/cli/sync.ts b/cli/sync.ts index 065bcdf973..32e87cead8 100644 --- a/cli/sync.ts +++ b/cli/sync.ts @@ -112,16 +112,15 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement { p: string, f: JSZip.JSZipObject ): Promise { - const isFlow = p.endsWith("flow.json"); - function transformPath() { - if (isFlow) { - return p.replace("flow.json", "flow"); - } else { - return useYaml && p.endsWith(".json") - ? p.replaceAll(".json", ".yaml") - : p; - } - } + const kind: "flow" | "app" | "script" | "other" = p.endsWith("flow.json") + ? "flow" + : p.endsWith("app.json") + ? "app" + : p.endsWith("script.json") + ? "script" + : "other"; + + const isJson = p.endsWith(".json"); interface InlineScript { path: string; @@ -166,12 +165,15 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement { else if (language == "graphql") ext = "gql"; else if (language == "bun") ext = "bun.ts"; else if (language == "nativets") ext = "native.ts"; - else ext = "noext"; + else if (language == "frontend") ext = "frontend.js"; + else ext = "no_ext"; return [`${name}.inline_script.`, ext]; } - function extractInlineScripts(modules: FlowModule[]): InlineScript[] { + function extractInlineScriptsForFlows( + modules: FlowModule[] + ): InlineScript[] { return modules.flatMap((m) => { if (m.value.type == "rawscript") { const [basePath, ext] = assignPath(m.summary, m.value.language); @@ -187,17 +189,19 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement { } return r; } else if (m.value.type == "forloopflow") { - return extractInlineScripts(m.value.modules); + return extractInlineScriptsForFlows(m.value.modules); } else if (m.value.type == "branchall") { return m.value.branches.flatMap((b) => - extractInlineScripts(b.modules) + extractInlineScriptsForFlows(b.modules) ); } else if (m.value.type == "whileloopflow") { - return extractInlineScripts(m.value.modules); + return extractInlineScriptsForFlows(m.value.modules); } else if (m.value.type == "branchone") { return [ - ...m.value.branches.flatMap((b) => extractInlineScripts(b.modules)), - ...extractInlineScripts(m.value.default), + ...m.value.branches.flatMap((b) => + extractInlineScriptsForFlows(b.modules) + ), + ...extractInlineScriptsForFlows(m.value.default), ]; } else { return []; @@ -205,15 +209,63 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement { }); } + function extractInlineScriptsForApps(rec: any): InlineScript[] { + if (!rec) { + return []; + } + if (typeof rec == "object") { + return Object.entries(rec).flatMap(([k, v]) => { + if (k == "inlineScript" && typeof v == "object") { + const o: Record = v as any; + const name = rec["name"]; + const [basePath, ext] = assignPath(name, o["language"]); + const r = []; + if (o["content"]) { + const content = o["content"]; + o["content"] = "!inline " + basePath + ext; + r.push({ + path: basePath + ext, + content: content, + }); + } + if (o["lock"]) { + const lock = o["lock"]; + o["lock"] = "!inline " + basePath + "lock"; + r.push({ + path: basePath + "lock", + content: lock, + }); + } + return r; + } else { + return extractInlineScriptsForApps(v); + } + }); + } + return []; + } + + function transformPath() { + if (kind == "flow") { + return p.replace("flow.json", "flow"); + } else if (kind == "app") { + return p.replace("app.json", "app"); + } else { + return useYaml && isJson ? p.replaceAll(".json", ".yaml") : p; + } + } + const finalPath = transformPath(); const r = [ { - isDirectory: isFlow, + isDirectory: kind == "flow" || kind == "app", path: finalPath, async *getChildren(): AsyncIterable { - if (isFlow) { + if (kind == "flow") { const flow: OpenFlow = JSON.parse(await f.async("text")); - const inlineScripts = extractInlineScripts(flow.value.modules); + const inlineScripts = extractInlineScriptsForFlows( + flow.value.modules + ); for (const s of inlineScripts) { yield { isDirectory: false, @@ -235,20 +287,37 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement { return yamlStringify(flow, yamlOptions); }, }; + } else if (kind == "app") { + const app = JSON.parse(await f.async("text")); + const inlineScripts = extractInlineScriptsForApps(app?.["value"]); + for (const s of inlineScripts) { + yield { + isDirectory: false, + path: path.join(finalPath, s.path), + async *getChildren() {}, + // deno-lint-ignore require-await + async getContentText() { + return s.content; + }, + }; + } + + yield { + isDirectory: false, + path: path.join(finalPath, "app.yaml"), + async *getChildren() {}, + // deno-lint-ignore require-await + async getContentText() { + return yamlStringify(app, yamlOptions); + }, + }; } }, - // async getContentBytes(): Promise { - // return await f.async("uint8array"); - // }, + async getContentText(): Promise { const content = await f.async("text"); - // if (p.endsWith(".script.json")) { - // if (parsed["lock"]) { - // parsed["lock"] = "!inline " + p + ".lock"; - // yield {} - // } - // } - if (p.endsWith(".script.json")) { + + if (kind == "script") { const parsed = JSON.parse(content); if (parsed["lock"]) { parsed["lock"] = "!inline " + removeSuffix(p, ".json") + ".lock"; @@ -258,13 +327,13 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement { : JSON.stringify(parsed, null, 2); } - return useYaml && p.endsWith(".json") + return useYaml && isJson ? yamlStringify(JSON.parse(content), yamlOptions) : content; }, }, ]; - if (p.endsWith("script.json")) { + if (kind == "script") { const content = await f.async("text"); const parsed = JSON.parse(content); const lock = parsed["lock"]; @@ -1066,7 +1135,7 @@ async function push(opts: GlobalOptions & SyncOptions) { case "app": await AppService.deleteApp({ workspace: workspaceId, - path: removeSuffix(change.path, ".app.json"), + path: removeSuffix(change.path, ".app/app.json"), }); break; case "schedule": diff --git a/cli/types.ts b/cli/types.ts index 7d5af19ceb..ef547668c8 100644 --- a/cli/types.ts +++ b/cli/types.ts @@ -111,7 +111,8 @@ export async function pushObj( const typeEnding = getTypeStrFromPath(p); if (typeEnding === "app") { - await pushApp(workspace, p, befObj, newObj, message); + const appName = p.split(".app" + path.sep)[0]; + await pushApp(workspace, appName, appName + ".app", message); } else if (typeEnding === "folder") { await pushFolder(workspace, p, befObj, newObj); } else if (typeEnding === "variable") { @@ -171,6 +172,9 @@ export function getTypeStrFromPath( if (p.includes(".flow" + path.sep)) { return "flow"; } + if (p.includes(".app" + path.sep)) { + return "app"; + } const parsed = path.parse(p); if ( parsed.ext == ".go" ||