starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-18 16:02:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(cli): split inline sscripts for apps like for flows

Browse Source
This commit is contained in:
Ruben Fiszel
2024-04-28 14:55:33 +02:00
parent ba6c705936
commit 89aec0d7c7
6 changed files with 182 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/windmill-worker/src/worker.rs
+8
View File
@@ -3938,6 +3938,14 @@ async fn lock_modules_app(
.unwrap_or_default()
.to_string();
let mut logs = "".to_string();
if v.get("lock")
.is_some_and(|x| !x.as_str().unwrap().trim().is_empty())
{
logs.push_str(
"Found already locked inline script. Skipping lock...\n",
);
return Ok(Value::Object(m.clone()));
}
logs.push_str("Found lockable inline script. Generating lock...\n");
let new_lock = capture_dependency_job(
&job.id,
cli/apps.ts
+57 -26
View File
@@ -5,15 +5,13 @@ import {
colors,
Command,
ListableApp,
log,
Policy,
SEP,
Table,
yamlParse,
} from "./deps.ts";
import {
GlobalOptions,
isSuperset,
parseFromFile,
removeType,
} from "./types.ts";
import { GlobalOptions, isSuperset } from "./types.ts";
export interface AppFile {
value: any;
@@ -21,14 +19,20 @@ export interface AppFile {
policy: Policy;
}
const alreadySynced: string[] = [];
export async function pushApp(
workspace: string,
filePath: string,
app: AppFile | undefined,
newApp: AppFile,
remotePath: string,
localPath: string,
message?: string
): Promise<void> {
const remotePath = removeType(filePath, "app");
if (alreadySynced.includes(localPath)) {
return;
}
alreadySynced.push(localPath);
let app: any = undefined;
// deleting old app if it exists in raw mode
try {
app = await AppService.getAppByPath({
@@ -39,28 +43,61 @@ export async function pushApp(
//ignore
}
if (app) {
if (isSuperset(newApp, app)) {
if (!localPath.endsWith(SEP)) {
localPath += SEP;
}
const localAppRaw = await Deno.readTextFile(localPath + "app.yaml");
const localApp = yamlParse(localAppRaw) as AppFile;
function replaceInlineScripts(rec: any) {
if (!rec) {
return;
}
if (typeof rec == "object") {
return Object.entries(rec).flatMap(([k, v]) => {
if (k == "inlineScript" && typeof v == "object") {
const o: Record<string, any> = v as any;
if (o["content"] && o["content"].startsWith("!inline")) {
const basePath = localPath + o["content"].split(" ")[1];
o["content"] = Deno.readTextFileSync(basePath);
}
if (o["lock"] && o["lock"].startsWith("!inline")) {
const basePath = localPath + o["lock"].split(" ")[1];
o["lock"] = Deno.readTextFileSync(basePath);
}
} else {
replaceInlineScripts(v);
}
});
}
return [];
}
replaceInlineScripts(localApp.value);
if (app) {
if (isSuperset(localApp, app)) {
log.info(colors.green(`App ${remotePath} is up to date`));
return;
}
log.info(colors.bold.yellow(`Updating app ${remotePath}...`));
await AppService.updateApp({
workspace,
path: remotePath.replaceAll("\\", "/"),
requestBody: {
deployment_message: message,
...newApp,
...localApp,
},
});
} else {
console.log(colors.yellow.bold("Creating new app..."));
log.info(colors.yellow.bold("Creating new app..."));
console.log(message);
await AppService.createApp({
workspace,
requestBody: {
path: remotePath.replaceAll("\\", "/"),
deployment_message: message,
...newApp,
...localApp,
},
});
}
@@ -94,28 +131,22 @@ async function list(opts: GlobalOptions) {
.render();
}
async function push(opts: GlobalOptions, filePath: string) {
const remotePath = filePath.split(".")[0];
async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
if (!validatePath(remotePath)) {
return;
}
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
await pushApp(
workspace.workspaceId,
filePath,
undefined,
parseFromFile(filePath)
);
console.log(colors.bold.underline.green("App pushed"));
await pushApp(workspace.workspaceId, remotePath, filePath);
log.info(colors.bold.underline.green("Flow pushed"));
}
const command = new Command()
.description("app related commands")
.action(list as any)
.command("push", "push a local app ")
.arguments("<file_path:file>")
.arguments("<file_path:string> <remote_path:string>")
.action(push as any);
export default command;
cli/flow.ts
+8 -8
View File
@@ -27,13 +27,13 @@ const alreadySynced: string[] = [];
export async function pushFlow(
workspace: string,
remotePath: string,
localFlowPath: string,
localPath: string,
message?: string
): Promise<void> {
if (alreadySynced.includes(localFlowPath)) {
if (alreadySynced.includes(localPath)) {
return;
}
alreadySynced.push(localFlowPath);
alreadySynced.push(localPath);
let flow: Flow | undefined = undefined;
try {
flow = await FlowService.getFlowByPath({
@@ -44,17 +44,17 @@ export async function pushFlow(
// flow doesn't exist
}
if (!localFlowPath.endsWith(SEP)) {
localFlowPath += SEP;
if (!localPath.endsWith(SEP)) {
localPath += SEP;
}
const localFlowRaw = await Deno.readTextFile(localFlowPath + "flow.yaml");
const localFlowRaw = await Deno.readTextFile(localPath + "flow.yaml");
const localFlow = yamlParse(localFlowRaw) as FlowFile;
function replaceInlineScripts(modules: FlowModule[]) {
modules.forEach((m) => {
if (m.value.type == "rawscript") {
const path = m.value.content.split(" ")[1];
m.value.content = Deno.readTextFileSync(localFlowPath + path);
m.value.content = Deno.readTextFileSync(localPath + path);
const lock = m.value.lock;
if (
@@ -64,7 +64,7 @@ export async function pushFlow(
) {
const path = lock.split(" ")[1];
try {
m.value.lock = Deno.readTextFileSync(localFlowPath + path);
m.value.lock = Deno.readTextFileSync(localPath + path);
} catch {
log.error(`Lock file ${path} not found`);
}
cli/script.ts
+2 -1
View File
@@ -640,7 +640,8 @@ async function generateMetadata(
return (
(!isD && !exts.some((ext) => p.endsWith(ext))) ||
ignore(p, isD) ||
p.includes(".flow/")
p.includes(".flow/") ||
p.includes(".app/")
);
},
false,
cli/sync.ts
+102 -33
View File
@@ -112,16 +112,15 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
p: string,
f: JSZip.JSZipObject
): Promise<DynFSElement[]> {
const isFlow = p.endsWith("flow.json");
function transformPath() {
if (isFlow) {
return p.replace("flow.json", "flow");
} else {
return useYaml && p.endsWith(".json")
? p.replaceAll(".json", ".yaml")
: p;
}
}
const kind: "flow" | "app" | "script" | "other" = p.endsWith("flow.json")
? "flow"
: p.endsWith("app.json")
? "app"
: p.endsWith("script.json")
? "script"
: "other";
const isJson = p.endsWith(".json");
interface InlineScript {
path: string;
@@ -166,12 +165,15 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
else if (language == "graphql") ext = "gql";
else if (language == "bun") ext = "bun.ts";
else if (language == "nativets") ext = "native.ts";
else ext = "noext";
else if (language == "frontend") ext = "frontend.js";
else ext = "no_ext";
return [`${name}.inline_script.`, ext];
}
function extractInlineScripts(modules: FlowModule[]): InlineScript[] {
function extractInlineScriptsForFlows(
modules: FlowModule[]
): InlineScript[] {
return modules.flatMap((m) => {
if (m.value.type == "rawscript") {
const [basePath, ext] = assignPath(m.summary, m.value.language);
@@ -187,17 +189,19 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
}
return r;
} else if (m.value.type == "forloopflow") {
return extractInlineScripts(m.value.modules);
return extractInlineScriptsForFlows(m.value.modules);
} else if (m.value.type == "branchall") {
return m.value.branches.flatMap((b) =>
extractInlineScripts(b.modules)
extractInlineScriptsForFlows(b.modules)
);
} else if (m.value.type == "whileloopflow") {
return extractInlineScripts(m.value.modules);
return extractInlineScriptsForFlows(m.value.modules);
} else if (m.value.type == "branchone") {
return [
...m.value.branches.flatMap((b) => extractInlineScripts(b.modules)),
...extractInlineScripts(m.value.default),
...m.value.branches.flatMap((b) =>
extractInlineScriptsForFlows(b.modules)
),
...extractInlineScriptsForFlows(m.value.default),
];
} else {
return [];
@@ -205,15 +209,63 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
});
}
function extractInlineScriptsForApps(rec: any): InlineScript[] {
if (!rec) {
return [];
}
if (typeof rec == "object") {
return Object.entries(rec).flatMap(([k, v]) => {
if (k == "inlineScript" && typeof v == "object") {
const o: Record<string, any> = v as any;
const name = rec["name"];
const [basePath, ext] = assignPath(name, o["language"]);
const r = [];
if (o["content"]) {
const content = o["content"];
o["content"] = "!inline " + basePath + ext;
r.push({
path: basePath + ext,
content: content,
});
}
if (o["lock"]) {
const lock = o["lock"];
o["lock"] = "!inline " + basePath + "lock";
r.push({
path: basePath + "lock",
content: lock,
});
}
return r;
} else {
return extractInlineScriptsForApps(v);
}
});
}
return [];
}
function transformPath() {
if (kind == "flow") {
return p.replace("flow.json", "flow");
} else if (kind == "app") {
return p.replace("app.json", "app");
} else {
return useYaml && isJson ? p.replaceAll(".json", ".yaml") : p;
}
}
const finalPath = transformPath();
const r = [
{
isDirectory: isFlow,
isDirectory: kind == "flow" || kind == "app",
path: finalPath,
async *getChildren(): AsyncIterable<DynFSElement> {
if (isFlow) {
if (kind == "flow") {
const flow: OpenFlow = JSON.parse(await f.async("text"));
const inlineScripts = extractInlineScripts(flow.value.modules);
const inlineScripts = extractInlineScriptsForFlows(
flow.value.modules
);
for (const s of inlineScripts) {
yield {
isDirectory: false,
@@ -235,20 +287,37 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
return yamlStringify(flow, yamlOptions);
},
};
} else if (kind == "app") {
const app = JSON.parse(await f.async("text"));
const inlineScripts = extractInlineScriptsForApps(app?.["value"]);
for (const s of inlineScripts) {
yield {
isDirectory: false,
path: path.join(finalPath, s.path),
async *getChildren() {},
// deno-lint-ignore require-await
async getContentText() {
return s.content;
},
};
}
yield {
isDirectory: false,
path: path.join(finalPath, "app.yaml"),
async *getChildren() {},
// deno-lint-ignore require-await
async getContentText() {
return yamlStringify(app, yamlOptions);
},
};
}
},
// async getContentBytes(): Promise<Uint8Array> {
// return await f.async("uint8array");
// },
async getContentText(): Promise<string> {
const content = await f.async("text");
// if (p.endsWith(".script.json")) {
// if (parsed["lock"]) {
// parsed["lock"] = "!inline " + p + ".lock";
// yield {}
// }
// }
if (p.endsWith(".script.json")) {
if (kind == "script") {
const parsed = JSON.parse(content);
if (parsed["lock"]) {
parsed["lock"] = "!inline " + removeSuffix(p, ".json") + ".lock";
@@ -258,13 +327,13 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
: JSON.stringify(parsed, null, 2);
}
return useYaml && p.endsWith(".json")
return useYaml && isJson
? yamlStringify(JSON.parse(content), yamlOptions)
: content;
},
},
];
if (p.endsWith("script.json")) {
if (kind == "script") {
const content = await f.async("text");
const parsed = JSON.parse(content);
const lock = parsed["lock"];
@@ -1066,7 +1135,7 @@ async function push(opts: GlobalOptions & SyncOptions) {
case "app":
await AppService.deleteApp({
workspace: workspaceId,
path: removeSuffix(change.path, ".app.json"),
path: removeSuffix(change.path, ".app/app.json"),
});
break;
case "schedule":
cli/types.ts
+5 -1
View File
@@ -111,7 +111,8 @@ export async function pushObj(
const typeEnding = getTypeStrFromPath(p);
if (typeEnding === "app") {
await pushApp(workspace, p, befObj, newObj, message);
const appName = p.split(".app" + path.sep)[0];
await pushApp(workspace, appName, appName + ".app", message);
} else if (typeEnding === "folder") {
await pushFolder(workspace, p, befObj, newObj);
} else if (typeEnding === "variable") {
@@ -171,6 +172,9 @@ export function getTypeStrFromPath(
if (p.includes(".flow" + path.sep)) {
return "flow";
}
if (p.includes(".app" + path.sep)) {
return "app";
}
const parsed = path.parse(p);
if (
parsed.ext == ".go" ||