starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-20 16:02:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(cli): improve handling of schema for script bundles

Browse Source
This commit is contained in:
Ruben Fiszel
2024-05-18 10:00:00 +02:00
parent f0b3527e53
commit 32bf061627
4 changed files with 138 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cli/metadata.ts
+53 -36
View File
@@ -24,8 +24,9 @@ import { SchemaProperty } from "./bootstrap/common.ts";
import { ScriptLanguage } from "./script_common.ts";
import { inferContentTypeFromFilePath } from "./script_common.ts";
import { GlobalDeps } from "./script.ts";
import { yamlOptions } from "./sync.ts";
import { findCodebase, yamlOptions } from "./sync.ts";
import { generateHash } from "./utils.ts";
import { SyncCodebase } from "./codebase.ts";
export async function generateAllMetadata() {}
@@ -77,7 +78,8 @@ export async function generateMetadataInternal(
},
dryRun: boolean,
noStaleMessage: boolean,
globalDeps: GlobalDeps
globalDeps: GlobalDeps,
codebases: SyncCodebase[]
): Promise<string | undefined> {
const remotePath = scriptPath
.substring(0, scriptPath.indexOf("."))
@@ -100,7 +102,8 @@ export async function generateMetadataInternal(
const metadataWithType = await parseMetadataFile(
remotePath,
undefined,
globalDeps
globalDeps,
codebases
);
// read script content
@@ -138,7 +141,9 @@ export async function generateMetadataInternal(
);
}
if (!opts.schemaOnly) {
const c = findCodebase(scriptPath, codebases);
if (!opts.schemaOnly && !c) {
await updateScriptLock(
workspace,
scriptContent,
@@ -163,7 +168,7 @@ export async function generateMetadataInternal(
return `${remotePath} (${language})`;
}
async function updateScriptSchema(
export async function updateScriptSchema(
scriptContent: string,
language: ScriptLanguage,
metadataContent: Record<string, any>,
@@ -355,8 +360,8 @@ function sortObject(obj: any): any {
);
}
function argSigToJsonSchemaType(
typ:
export function argSigToJsonSchemaType(
t:
| string
| { resource: string | null }
| {
@@ -371,55 +376,55 @@ function argSigToJsonSchemaType(
oldS: SchemaProperty
): void {
const newS: SchemaProperty = { type: "" };
if (typ === "int") {
if (t === "int") {
newS.type = "integer";
} else if (typ === "float") {
} else if (t === "float") {
newS.type = "number";
} else if (typ === "bool") {
} else if (t === "bool") {
newS.type = "boolean";
} else if (typ === "email") {
} else if (t === "email") {
newS.type = "string";
newS.format = "email";
} else if (typ === "sql") {
} else if (t === "sql") {
newS.type = "string";
newS.format = "sql";
} else if (typ === "yaml") {
} else if (t === "yaml") {
newS.type = "string";
newS.format = "yaml";
} else if (typ === "bytes") {
} else if (t === "bytes") {
newS.type = "string";
newS.contentEncoding = "base64";
} else if (typ === "datetime") {
} else if (t === "datetime") {
newS.type = "string";
newS.format = "date-time";
} else if (typeof typ !== "string" && `object` in typ) {
} else if (typeof t !== "string" && `object` in t) {
newS.type = "object";
if (typ.object) {
const properties: Record<string, SchemaProperty> = {};
for (const prop of typ.object) {
properties[prop.key] = { type: undefined };
if (t.object) {
const properties: Record<string, any> = {};
for (const prop of t.object) {
properties[prop.key] = {};
argSigToJsonSchemaType(prop.typ, properties[prop.key]);
}
newS.properties = properties;
}
} else if (typeof typ !== "string" && `str` in typ) {
} else if (typeof t !== "string" && `str` in t) {
newS.type = "string";
if (typ.str) {
newS.enum = typ.str;
if (t.str) {
newS.enum = t.str;
}
} else if (typeof typ !== "string" && `resource` in typ) {
} else if (typeof t !== "string" && `resource` in t) {
newS.type = "object";
newS.format = `resource-${typ.resource}`;
} else if (typeof typ !== "string" && `list` in typ) {
newS.format = `resource-${t.resource}`;
} else if (typeof t !== "string" && `list` in t) {
newS.type = "array";
if (typ.list === "int" || typ.list === "float") {
if (t.list === "int" || t.list === "float") {
newS.items = { type: "number" };
} else if (typ.list === "bytes") {
} else if (t.list === "bytes") {
newS.items = { type: "string", contentEncoding: "base64" };
} else if (typ.list == "string") {
} else if (t.list == "string") {
newS.items = { type: "string" };
} else if (typ.list && typeof typ.list == "object" && "str" in typ.list) {
newS.items = { type: "string", enum: typ.list.str };
} else if (t.list && typeof t.list == "object" && "str" in t.list) {
newS.items = { type: "string", enum: t.list.str };
} else {
newS.items = { type: "object" };
}
@@ -430,17 +435,27 @@ function argSigToJsonSchemaType(
if (oldS.type != newS.type) {
for (const prop of Object.getOwnPropertyNames(newS)) {
if (prop != "description") {
// @ts-ignore: fix
delete oldS[prop];
}
}
} else if (oldS.format == "date-time" && newS.format != "date-time") {
delete oldS.format;
} else if (
(oldS.format == "date" || oldS.format === "date-time") &&
newS.format == "string"
) {
newS.format = oldS.format;
} else if (newS.format == "date-time" && oldS.format == "date") {
newS.format = "date";
} else if (oldS.items?.type != newS.items?.type) {
delete oldS.items;
}
Object.assign(oldS, newS);
// if (sameItems && savedItems != undefined && savedItems.enum != undefined) {
// sendUserToast(JSON.stringify(savedItems))
// oldS.items = savedItems
// }
if (oldS.format?.startsWith("resource-") && newS.type != "object") {
oldS.format = undefined;
}
@@ -474,7 +489,8 @@ export async function parseMetadataFile(
schemaOnly?: boolean;
})
| undefined,
globalDeps: GlobalDeps
globalDeps: GlobalDeps,
codebases: SyncCodebase[]
): Promise<{ isJson: boolean; payload: any; path: string }> {
let metadataFilePath = scriptPath + ".script.json";
try {
@@ -522,7 +538,8 @@ export async function parseMetadataFile(
generateMetadataIfMissing,
false,
false,
globalDeps
globalDeps,
codebases
);
scriptInitialMetadata = yamlParse(
await Deno.readTextFile(metadataFilePath)
cli/script.ts
+42 -16
View File
@@ -1,5 +1,5 @@
// deno-lint-ignore-file no-explicit-any
import { GlobalOptions } from "./types.ts";
import { GlobalOptions, showDiff } from "./types.ts";
import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
import {
colors,
@@ -21,7 +21,11 @@ import {
} from "./bootstrap/script_bootstrap.ts";
import { Workspace } from "./workspace.ts";
import { generateMetadataInternal, parseMetadataFile } from "./metadata.ts";
import {
generateMetadataInternal,
parseMetadataFile,
updateScriptSchema,
} from "./metadata.ts";
import {
ScriptLanguage,
inferContentTypeFromFilePath,
@@ -74,7 +78,7 @@ async function push(opts: PushOptions, filePath: string) {
await requireLogin(opts);
const codebases = await listSyncCodebases(opts as SyncOptions);
const globalDeps = await findGlobalDeps(codebases);
const globalDeps = await findGlobalDeps();
await handleFile(
filePath,
workspace,
@@ -169,7 +173,8 @@ export async function handleFile(
schemaOnly: codebase ? true : undefined,
}
: undefined,
globalDeps
globalDeps,
codebases
)
)?.payload;
@@ -187,6 +192,22 @@ export async function handleFile(
}
const content = await Deno.readTextFile(path);
if (codebase) {
const typedBefore = JSON.parse(JSON.stringify(typed.schema));
await updateScriptSchema(content, language, typed, path);
if (typedBefore != typed.schema) {
log.info(`Updated metadata for bundle ${path}`);
showDiff(
yamlStringify(typedBefore, yamlOptions),
yamlStringify(typed.schema, yamlOptions)
);
await Deno.writeTextFile(
remotePath + ".script.yaml",
yamlStringify(typed as Record<string, any>, yamlOptions)
);
}
}
const requestBodyCommon = {
content,
description: typed?.description ?? "",
@@ -662,12 +683,16 @@ async function bootstrap(
yamlOptions
);
Deno.writeTextFile(scriptCodeFileFullPath, scriptInitialCode, {
createNew: true,
});
Deno.writeTextFile(scriptMetadataFileFullPath, scriptInitialMetadataYaml, {
await Deno.writeTextFile(scriptCodeFileFullPath, scriptInitialCode, {
createNew: true,
});
await Deno.writeTextFile(
scriptMetadataFileFullPath,
scriptInitialMetadataYaml,
{
createNew: true,
}
);
}
export type GlobalDeps = {
@@ -675,13 +700,11 @@ export type GlobalDeps = {
reqs: Record<string, string>;
composers: Record<string, string>;
};
export async function findGlobalDeps(
codebases: SyncCodebase[]
): Promise<GlobalDeps> {
export async function findGlobalDeps(): Promise<GlobalDeps> {
const pkgs: { [key: string]: string } = {};
const reqs: { [key: string]: string } = {};
const composers: { [key: string]: string } = {};
const els = await FSFSElement(Deno.cwd(), codebases);
const els = await FSFSElement(Deno.cwd(), []);
for await (const entry of readDirRecursiveWithIgnore((p, isDir) => {
p = "/" + p;
return (
@@ -725,7 +748,7 @@ async function generateMetadata(
opts = await mergeConfigWithConfigFile(opts);
const codebases = await listSyncCodebases(opts);
const globalDeps = await findGlobalDeps(codebases);
const globalDeps = await findGlobalDeps();
if (scriptPath) {
// read script metadata file
await generateMetadataInternal(
@@ -734,7 +757,8 @@ async function generateMetadata(
opts,
false,
false,
globalDeps
globalDeps,
codebases
);
} else {
const ignore = await ignoreF(opts);
@@ -760,7 +784,8 @@ async function generateMetadata(
opts,
true,
true,
globalDeps
globalDeps,
codebases
);
if (candidate) {
hasAny = true;
@@ -788,7 +813,8 @@ async function generateMetadata(
opts,
false,
true,
globalDeps
globalDeps,
codebases
);
}
}
cli/sync.ts
+42 -31
View File
@@ -266,7 +266,7 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
const r = [{ path: path, content: content }];
m.value.content = "!inline " + path;
const lock = m.value.lock;
if (lock) {
if (lock && lock != "") {
const lockPath = basePath + "lock";
m.value.lock = "!inline " + lockPath;
r.push({ path: lockPath, content: lock });
@@ -312,7 +312,7 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
content: content,
});
}
if (o["lock"]) {
if (o["lock"] && o["lock"] != "") {
const lock = o["lock"];
o["lock"] = "!inline " + basePath + "lock";
r.push({
@@ -403,8 +403,14 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
if (kind == "script") {
const parsed = JSON.parse(content);
if (parsed["lock"]) {
if (
parsed["lock"] &&
parsed["lock"] != "" &&
parsed["codebase"] == undefined
) {
parsed["lock"] = "!inline " + removeSuffix(p, ".json") + ".lock";
} else {
parsed["lock"] = undefined;
}
return useYaml
? yamlStringify(parsed, yamlOptions)
@@ -421,7 +427,7 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
const content = await f.async("text");
const parsed = JSON.parse(content);
const lock = parsed["lock"];
if (lock) {
if (lock && lock != "") {
r.push({
isDirectory: false,
path: removeSuffix(finalPath, ".json") + ".lock",
@@ -867,40 +873,45 @@ async function pull(opts: GlobalOptions & SyncOptions) {
const target = path.join(Deno.cwd(), change.path);
const stateTarget = path.join(Deno.cwd(), ".wmill", change.path);
if (change.name === "edited") {
try {
const currentLocal = await Deno.readTextFile(target);
if (currentLocal !== change.before && currentLocal !== change.after) {
log.info(
colors.red(
`Conflict detected on ${change.path}\nBoth local and remote have been modified.`
)
);
if (opts.failConflicts) {
conflicts.push({
local: currentLocal,
change,
path: change.path,
});
continue;
} else if (opts.yes) {
if (opts.stateful) {
try {
const currentLocal = await Deno.readTextFile(target);
if (
currentLocal !== change.before &&
currentLocal !== change.after
) {
log.info(
colors.red(
`Override local version with remote since --yes was passed and no --fail-conflicts.`
`Conflict detected on ${change.path}\nBoth local and remote have been modified.`
)
);
} else {
showConflict(change.path, currentLocal, change.after);
if (
await Confirm.prompt(
"Preserve local (push to change remote and avoid seeing this again)?"
)
) {
if (opts.failConflicts) {
conflicts.push({
local: currentLocal,
change,
path: change.path,
});
continue;
} else if (opts.yes) {
log.info(
colors.red(
`Override local version with remote since --yes was passed and no --fail-conflicts.`
)
);
} else {
showConflict(change.path, currentLocal, change.after);
if (
await Confirm.prompt(
"Preserve local (push to change remote and avoid seeing this again)?"
)
) {
continue;
}
}
}
} catch {
// ignore
}
} catch {
// ignore
}
if (!change.path.endsWith(".json") && !change.path.endsWith(".yaml")) {
log.info(`Editing script content of ${change.path}`);
@@ -1083,7 +1094,7 @@ async function push(opts: GlobalOptions & SyncOptions) {
log.info(colors.gray(`Applying changes to files ...`));
const alreadySynced: string[] = [];
const globalDeps = await findGlobalDeps(codebases);
const globalDeps = await findGlobalDeps();
for await (const change of changes) {
const stateTarget = path.join(Deno.cwd(), ".wmill", change.path);
frontend/src/routes/(root)/(logged)/user/(user)/create_workspace/+page.svelte
+1 -11
View File
@@ -58,7 +58,7 @@
if (auto_invite) {
await WorkspaceService.editAutoInvite({
workspace: id,
requestBody: { operator: operatorOnly, invite_all: !isCloudHosted(), auto_add: autoAdd }
requestBody: { operator: operatorOnly, invite_all: !isCloudHosted(), auto_add: true }
})
}
if (openAiKey != '') {
@@ -148,7 +148,6 @@
let auto_invite = false
let operatorOnly = false
let autoAdd = false
</script>
<CenteredModal title="New Workspace">
@@ -216,15 +215,6 @@
<div class="text-xs mb-1 leading-6 pt-2">
Mode <Tooltip>Whether to invite or add users directly to the workspace.</Tooltip>
</div>
<ToggleButtonGroup
selected={autoAdd ? 'add' : 'invite'}
on:selected={(e) => {
autoAdd = e.detail == 'add'
}}
>
<ToggleButton value="invite" size="xs" label="Auto-invite" />
<ToggleButton value="add" size="xs" label="Auto-add" />
</ToggleButtonGroup>
<div class="text-xs mb-1 leading-6 pt-2"
>Role <Tooltip>Role of the auto-invited users</Tooltip></div