starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-10-07 08:02:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(cli): support windows more thoroughly + fix generate flow locks (#3944)

Browse Source 
* all

* all

* all

* fix lock

* fix lock

* fix lock

* fix lock
This commit is contained in:
Ruben Fiszel
2024-06-21 14:23:33 +02:00
committed by GitHub
parent f1f9fe513b
commit 3f0cd6349d
12 changed files with 182 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/Cargo.lock
Generated
+9 -18
View File
@@ -4396,15 +4396,6 @@ dependencies = [
"either",
]
[[package]]
name = "itertools"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
dependencies = [
"either",
]
[[package]]
name = "itoa"
version = "1.0.11"
@@ -10096,7 +10087,7 @@ dependencies = [
"hmac",
"http 1.1.0",
"hyper 1.3.1",
"itertools 0.13.0",
"itertools 0.10.5",
"jsonwebtoken",
"lazy_static",
"magic-crypt",
@@ -10191,7 +10182,7 @@ dependencies = [
"hmac",
"hyper 1.3.1",
"indexmap 2.2.6",
"itertools 0.13.0",
"itertools 0.10.5",
"lazy_static",
"magic-crypt",
"mail-send",
@@ -10254,7 +10245,7 @@ version = "1.350.1"
dependencies = [
"anyhow",
"gosyn",
"itertools 0.13.0",
"itertools 0.10.5",
"lazy_static",
"regex",
"windmill-parser",
@@ -10277,7 +10268,7 @@ version = "1.350.1"
dependencies = [
"anyhow",
"convert_case 0.6.0",
"itertools 0.13.0",
"itertools 0.10.5",
"lazy_static",
"php-parser-rs",
"regex",
@@ -10290,7 +10281,7 @@ name = "windmill-parser-py"
version = "1.350.1"
dependencies = [
"anyhow",
"itertools 0.13.0",
"itertools 0.10.5",
"rustpython-parser",
"serde_json",
"windmill-parser",
@@ -10302,7 +10293,7 @@ version = "1.350.1"
dependencies = [
"anyhow",
"async-recursion",
"itertools 0.13.0",
"itertools 0.10.5",
"lazy_static",
"phf",
"regex",
@@ -10369,14 +10360,14 @@ dependencies = [
"anyhow",
"async-recursion",
"axum",
"bigdecimal 0.4.5",
"bigdecimal 0.3.1",
"chrono",
"chrono-tz",
"cron",
"futures-core",
"hex",
"hmac",
"itertools 0.13.0",
"itertools 0.10.5",
"lazy_static",
"prometheus",
"regex",
@@ -10433,7 +10424,7 @@ dependencies = [
"gcp_auth",
"git-version",
"hex",
"itertools 0.13.0",
"itertools 0.10.5",
"jsonwebtoken",
"lazy_static",
"mysql_async",
backend/windmill-api/src/jobs.rs
+1 -1
View File
@@ -3767,7 +3767,7 @@ async fn run_flow_dependencies_job(
PushIsolationLevel::IsolatedRoot(db.clone(), rsmq),
&w_id,
JobPayload::RawFlowDependencies { path: req.path, flow_value: req.flow_value },
PushArgs::empty(),
HashMap::from([("skip_flow_update".to_string(), to_raw_value(&true))]).into(),
authed.display_username(),
&authed.email,
username_to_permissioned_as(&authed.username),
backend/windmill-worker/src/worker_lockfiles.rs
+20 -8
View File
@@ -527,15 +527,27 @@ pub async fn handle_flow_dependency_job<R: rsmq_async::RsmqConnection + Send + S
"status": "Flow lock generation was canceled",
})));
}
let skip_flow_update = job
.args
.as_ref()
.map(|x| {
x.get("skip_flow_update")
.map(|v| serde_json::from_str::<bool>(v.get()).ok())
.flatten()
})
.flatten()
.unwrap_or(false);
sqlx::query!(
"UPDATE flow SET value = $1 WHERE path = $2 AND workspace_id = $3",
new_flow_value,
job_path,
job.workspace_id
)
.execute(db)
.await?;
if !skip_flow_update {
sqlx::query!(
"UPDATE flow SET value = $1 WHERE path = $2 AND workspace_id = $3",
new_flow_value,
job_path,
job.workspace_id
)
.execute(db)
.await?;
}
tx.commit().await?;
if let Err(e) = handle_deployment_metadata(
cli/apps.ts
+6 -5
View File
@@ -12,6 +12,7 @@ import {
yamlParse,
} from "./deps.ts";
import { GlobalOptions, isSuperset } from "./types.ts";
import { readInlinePathSync } from "./utils.ts";
export interface AppFile {
value: any;
@@ -37,7 +38,7 @@ export async function pushApp(
try {
app = await AppService.getAppByPath({
workspace,
path: remotePath.replaceAll("\\", "/"),
path: remotePath.replaceAll(SEP, "/"),
});
} catch {
//ignore
@@ -59,11 +60,11 @@ export async function pushApp(
const o: Record<string, any> = v as any;
if (o["content"] && o["content"].startsWith("!inline")) {
const basePath = localPath + o["content"].split(" ")[1];
o["content"] = Deno.readTextFileSync(basePath);
o["content"] = readInlinePathSync(basePath);
}
if (o["lock"] && o["lock"].startsWith("!inline")) {
const basePath = localPath + o["lock"].split(" ")[1];
o["lock"] = Deno.readTextFileSync(basePath);
o["lock"] = readInlinePathSync(basePath);
}
} else {
replaceInlineScripts(v);
@@ -83,7 +84,7 @@ export async function pushApp(
log.info(colors.bold.yellow(`Updating app ${remotePath}...`));
await AppService.updateApp({
workspace,
path: remotePath.replaceAll("\\", "/"),
path: remotePath.replaceAll(SEP, "/"),
requestBody: {
deployment_message: message,
...localApp,
@@ -95,7 +96,7 @@ export async function pushApp(
await AppService.createApp({
workspace,
requestBody: {
path: remotePath.replaceAll("\\", "/"),
path: remotePath.replaceAll(SEP, "/"),
deployment_message: message,
...localApp,
},
cli/bootstrap/common.ts
+30 -21
View File
@@ -1,24 +1,33 @@
export type EnumType = string[] | undefined;
export interface SchemaProperty {
type: string | undefined
description?: string
pattern?: string
default?: any
enum?: string[]
contentEncoding?: 'base64' | 'binary'
format?: string
items?: {
type?: 'string' | 'number' | 'bytes' | 'object'
contentEncoding?: 'base64'
enum?: string[]
}
min?: number
max?: number
currency?: string
currencyLocale?: string
multiselect?: boolean
customErrorMessage?: string
properties?: { [name: string]: SchemaProperty }
required?: string[]
showExpr?: string
type: string | undefined;
description?: string;
pattern?: string;
default?: any;
enum?: EnumType;
contentEncoding?: "base64" | "binary";
format?: string;
items?: {
type?: "string" | "number" | "bytes" | "object";
contentEncoding?: "base64";
enum?: string[];
};
min?: number;
max?: number;
currency?: string;
currencyLocale?: string;
multiselect?: boolean;
customErrorMessage?: string;
properties?: { [name: string]: SchemaProperty };
required?: string[];
showExpr?: string;
password?: boolean;
order?: string[];
nullable?: boolean;
dateFormat?: string;
title?: string;
placeholder?: string;
oneOf?: SchemaProperty[];
originalType?: string;
}
cli/flow.ts
+6 -5
View File
@@ -12,11 +12,12 @@ import {
yamlParse,
} from "./deps.ts";
import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
import { exts, resolve, track_job } from "./script.ts";
import { resolve, track_job } from "./script.ts";
import { defaultFlowDefinition } from "./bootstrap/flow_bootstrap.ts";
import { generateFlowLockInternal } from "./metadata.ts";
import { SyncOptions, mergeConfigWithConfigFile } from "./conf.ts";
import { FSFSElement, elementsToMap, ignoreF } from "./sync.ts";
import { readInlinePathSync } from "./utils.ts";
export interface FlowFile {
summary: string;
@@ -46,7 +47,7 @@ export function replaceInlineScripts(
) {
const path = lock.split(" ")[1];
try {
m.value.lock = Deno.readTextFileSync(localPath + path);
m.value.lock = readInlinePathSync(localPath + path);
} catch {
log.error(`Lock file ${path} not found`);
}
@@ -104,9 +105,9 @@ export async function pushFlow(
log.info(colors.bold.yellow(`Updating flow ${remotePath}...`));
await FlowService.updateFlow({
workspace: workspace,
path: remotePath.replaceAll("\\", "/"),
path: remotePath.replaceAll(SEP, "/"),
requestBody: {
path: remotePath.replaceAll("\\", "/"),
path: remotePath.replaceAll(SEP, "/"),
deployment_message: message,
...localFlow,
},
@@ -116,7 +117,7 @@ export async function pushFlow(
await FlowService.createFlow({
workspace: workspace,
requestBody: {
path: remotePath.replaceAll("\\", "/"),
path: remotePath.replaceAll(SEP, "/"),
deployment_message: message,
...localFlow,
},
cli/metadata.ts
+81 -13
View File
@@ -39,7 +39,7 @@ import {
newPathAssigner,
yamlOptions,
} from "./sync.ts";
import { generateHash } from "./utils.ts";
import { generateHash, readInlinePathSync } from "./utils.ts";
import { SyncCodebase } from "./codebase.ts";
import { FlowFile, replaceInlineScripts } from "./flow.ts";
@@ -100,11 +100,11 @@ export async function generateFlowLockInternal(
workspace: Workspace,
justUpdateMetadataLock?: boolean
): Promise<string | undefined> {
if (folder.endsWith("/")) {
if (folder.endsWith(SEP)) {
folder = folder.substring(0, folder.length - 1);
}
const remote_path = folder
.replaceAll("\\", "/")
.replaceAll(SEP, "/")
.substring(0, folder.length - ".flow".length);
if (!justUpdateMetadataLock) {
log.info(`Generating lock for flow ${folder} at ${remote_path}`);
@@ -185,7 +185,7 @@ export async function generateScriptMetadataInternal(
): Promise<string | undefined> {
const remotePath = scriptPath
.substring(0, scriptPath.indexOf("."))
.replaceAll("\\", "/");
.replaceAll(SEP, "/");
const language = inferContentTypeFromFilePath(scriptPath, opts.defaultTs);
@@ -256,7 +256,8 @@ export async function generateScriptMetadataInternal(
metadataParsedContent.lock = "";
}
} else {
metadataParsedContent.lock = "!inline " + remotePath + ".script.lock";
metadataParsedContent.lock =
"!inline " + remotePath.replaceAll(SEP, "/") + ".script.lock";
}
let metaPath = remotePath + ".script.yaml";
@@ -347,7 +348,7 @@ async function updateScriptLock(
}
const lockPath = remotePath + ".script.lock";
await Deno.writeTextFile(lockPath, lock);
metadataContent.lock = "!inline " + lockPath;
metadataContent.lock = "!inline " + lockPath.replaceAll(SEP, "/");
} catch (e) {
throw new Error(
`Failed to generate lockfile. Status was: ${rawResponse.statusText}, ${responseText}, ${e}`
@@ -513,10 +514,18 @@ export function argSigToJsonSchemaType(
| null;
}
| { str: string[] | null }
| { object: { key: string; typ: any }[] },
| { object: { key: string; typ: any }[] }
| {
oneof: [
{
label: string;
properties: { key: string; typ: any }[];
}
];
},
oldS: SchemaProperty
): void {
const newS: SchemaProperty = { type: "" };
let newS: SchemaProperty = { type: "" };
if (t === "int") {
newS.type = "integer";
} else if (t === "float") {
@@ -535,15 +544,43 @@ export function argSigToJsonSchemaType(
} else if (t === "bytes") {
newS.type = "string";
newS.contentEncoding = "base64";
newS.originalType = "bytes";
} else if (t === "datetime") {
newS.type = "string";
newS.format = "date-time";
} else if (typeof t !== "string" && "oneof" in t) {
newS.type = "object";
if (t.oneof) {
newS.oneOf = t.oneof.map((obj) => {
const oldObjS =
oldS.oneOf?.find((o) => o?.title === obj.label) ?? undefined;
const properties: Record<string, any> = {};
for (const prop of obj.properties) {
if (oldObjS?.properties && prop.key in oldObjS?.properties) {
properties[prop.key] = oldObjS?.properties[prop.key];
} else {
properties[prop.key] = { description: "", type: "" };
}
argSigToJsonSchemaType(prop.typ, properties[prop.key]);
}
return {
type: "object",
title: obj.label,
properties,
order: oldObjS?.order ?? undefined,
};
});
}
} else if (typeof t !== "string" && `object` in t) {
newS.type = "object";
if (t.object) {
const properties: Record<string, any> = {};
for (const prop of t.object) {
properties[prop.key] = {};
if (oldS.properties && prop.key in oldS.properties) {
properties[prop.key] = oldS.properties[prop.key];
} else {
properties[prop.key] = { description: "", type: "" };
}
argSigToJsonSchemaType(prop.typ, properties[prop.key]);
}
newS.properties = properties;
@@ -551,7 +588,11 @@ export function argSigToJsonSchemaType(
} else if (typeof t !== "string" && `str` in t) {
newS.type = "string";
if (t.str) {
newS.originalType = "enum";
newS.enum = t.str;
} else {
newS.originalType = "string";
newS.enum = undefined;
}
} else if (typeof t !== "string" && `resource` in t) {
newS.type = "object";
@@ -573,6 +614,32 @@ export function argSigToJsonSchemaType(
newS.type = "object";
}
const preservedFields = [
"description",
"pattern",
"min",
"max",
"currency",
"currencyLocale",
"multiselect",
"customErrorMessage",
"required",
"showExpr",
"password",
"order",
"dateFormat",
"title",
"placeholder",
];
preservedFields.forEach((field) => {
// @ts-ignore
if (oldS[field] !== undefined) {
// @ts-ignore
newS[field] = oldS[field];
}
});
if (oldS.type != newS.type) {
for (const prop of Object.getOwnPropertyNames(newS)) {
if (prop != "description") {
@@ -602,18 +669,19 @@ export function argSigToJsonSchemaType(
oldS.format = undefined;
}
}
////////////////////////////////////////////////////////////////////////////////////////////
// end of refactoring TODO //
////////////////////////////////////////////////////////////////////////////////////////////
export async function replaceLock(o?: { lock?: string | string[] }) {
export function replaceLock(o?: { lock?: string | string[] }) {
if (Array.isArray(o?.lock)) {
o.lock = o.lock.join("\n");
}
if (o?.lock?.startsWith("!inline ")) {
try {
const lockPath = o?.lock?.split(" ")[1];
o.lock = await Deno.readTextFile(lockPath);
o.lock = readInlinePathSync(lockPath);
} catch (e) {
log.info(
colors.yellow(`Failed to read lockfile, doing as if it was empty: ${e}`)
@@ -647,7 +715,7 @@ export async function parseMetadataFile(
metadataFilePath = scriptPath + ".script.yaml";
await Deno.stat(metadataFilePath);
const payload: any = yamlParse(await Deno.readTextFile(metadataFilePath));
await replaceLock(payload);
replaceLock(payload);
return {
path: metadataFilePath,
@@ -687,7 +755,7 @@ export async function parseMetadataFile(
scriptInitialMetadata = yamlParse(
await Deno.readTextFile(metadataFilePath)
) as ScriptMetadata;
await replaceLock(scriptInitialMetadata);
replaceLock(scriptInitialMetadata);
} catch (e) {
log.info(
colors.yellow(
cli/resource.ts
+4 -3
View File
@@ -12,6 +12,7 @@ import {
log,
Resource,
ResourceService,
SEP,
Table,
} from "./deps.ts";
@@ -32,7 +33,7 @@ export async function pushResource(
try {
resource = await ResourceService.getResource({
workspace: workspace,
path: remotePath.replaceAll("\\", "/"),
path: remotePath.replaceAll(SEP, "/"),
});
} catch {
// flow doesn't exist
@@ -45,7 +46,7 @@ export async function pushResource(
await ResourceService.updateResource({
workspace: workspace,
path: remotePath.replaceAll("\\", "/"),
path: remotePath.replaceAll(SEP, "/"),
requestBody: { ...localResource },
});
} else {
@@ -61,7 +62,7 @@ export async function pushResource(
await ResourceService.createResource({
workspace: workspace,
requestBody: {
path: remotePath.replaceAll("\\", "/"),
path: remotePath.replaceAll(SEP, "/"),
...localResource,
},
});
cli/script.ts
+3 -3
View File
@@ -143,7 +143,7 @@ export async function handleFile(
alreadySynced.push(path);
const remotePath = path
.substring(0, path.indexOf("."))
.replaceAll("\\", "/");
.replaceAll(SEP, "/");
const language = inferContentTypeFromFilePath(path, opts?.defaultTs);
@@ -186,7 +186,7 @@ export async function handleFile(
try {
remote = await ScriptService.getScriptByPath({
workspace: workspaceId,
path: remotePath.replaceAll("\\", "/"),
path: remotePath.replaceAll(SEP, "/"),
});
log.debug(`Script ${remotePath} exists on remote`);
} catch {
@@ -214,7 +214,7 @@ export async function handleFile(
content,
description: typed?.description ?? "",
language: language as NewScript["language"],
path: remotePath.replaceAll("\\", "/"),
path: remotePath.replaceAll(SEP, "/"),
summary: typed?.summary ?? "",
kind: typed?.kind,
lock: typed?.lock,
cli/sync.ts
+9 -6
View File
@@ -214,11 +214,11 @@ export function extractInlineScriptsForFlows(
const path = basePath + ext;
const content = m.value.content;
const r = [{ path: path, content: content }];
m.value.content = "!inline " + path;
m.value.content = "!inline " + path.replaceAll(SEP, "/");
const lock = m.value.lock;
if (lock && lock != "") {
const lockPath = basePath + "lock";
m.value.lock = "!inline " + lockPath;
m.value.lock = "!inline " + lockPath.replaceAll(SEP, "/");
r.push({ path: lockPath, content: lock });
}
return r;
@@ -264,7 +264,7 @@ export function extractInlineScriptsForApps(
const r = [];
if (o["content"]) {
const content = o["content"];
o["content"] = "!inline " + basePath + ext;
o["content"] = "!inline " + basePath.replaceAll(SEP, "/") + ext;
r.push({
path: basePath + ext,
content: content,
@@ -272,7 +272,7 @@ export function extractInlineScriptsForApps(
}
if (o["lock"] && o["lock"] != "") {
const lock = o["lock"];
o["lock"] = "!inline " + basePath + "lock";
o["lock"] = "!inline " + basePath.replaceAll(SEP, "/") + "lock";
r.push({
path: basePath + "lock",
content: lock,
@@ -437,7 +437,10 @@ function ZipFSElement(
parsed["lock"] != "" &&
parsed["codebase"] == undefined
) {
parsed["lock"] = "!inline " + removeSuffix(p, ".json") + ".lock";
parsed["lock"] =
"!inline " +
removeSuffix(p.replaceAll(SEP, "/"), ".json") +
".lock";
} else {
parsed["lock"] = undefined;
}
@@ -1127,7 +1130,7 @@ async function push(opts: GlobalOptions & SyncOptions) {
log.info(
colors.gray(
"Computing the files to update on the remote to match local (taking .wmillignore into account)"
"Computing the files to update on the remote to match local (taking wmill.yaml includes/excludes into account)"
)
);
const remote = ZipFSElement(
cli/utils.ts
+9 -1
View File
@@ -2,7 +2,7 @@
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-nocheck This file is copied from a JS project, so it's not type-safe.
import { log, encodeHex } from "./deps.ts";
import { log, encodeHex, SEP } from "./deps.ts";
export function deepEqual<T>(a: T, b: T): boolean {
if (a === b) return true;
@@ -117,3 +117,11 @@ export async function generateHashFromBuffer(
const hashBuffer = await crypto.subtle.digest("SHA-256", content);
return encodeHex(hashBuffer);
}
// export async function readInlinePath(path: string): Promise<string> {
// return await Deno.readTextFile(path.replaceAll("/", SEP));
// }
export function readInlinePathSync(path: string): string {
return Deno.readTextFileSync(path.replaceAll("/", SEP));
}
cli/variable.ts
+4 -3
View File
@@ -12,6 +12,7 @@ import {
Confirm,
ListableVariable,
log,
SEP,
Table,
VariableService,
} from "./deps.ts";
@@ -60,7 +61,7 @@ export async function pushVariable(
try {
variable = await VariableService.getVariable({
workspace: workspace,
path: remotePath.replaceAll("\\", "/"),
path: remotePath.replaceAll(SEP, "/"),
decryptSecret: plainSecrets,
includeEncrypted: true,
});
@@ -79,7 +80,7 @@ export async function pushVariable(
await VariableService.updateVariable({
workspace,
path: remotePath.replaceAll("\\", "/"),
path: remotePath.replaceAll(SEP, "/"),
alreadyEncrypted: !plainSecrets,
requestBody: {
...localVariable,
@@ -93,7 +94,7 @@ export async function pushVariable(
workspace,
alreadyEncrypted: !plainSecrets,
requestBody: {
path: remotePath.replaceAll("\\", "/"),
path: remotePath.replaceAll(SEP, "/"),
...localVariable,
},
});