diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 185e5c1dc0..264f15659a 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -4396,15 +4396,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.11" @@ -10096,7 +10087,7 @@ dependencies = [ "hmac", "http 1.1.0", "hyper 1.3.1", - "itertools 0.13.0", + "itertools 0.10.5", "jsonwebtoken", "lazy_static", "magic-crypt", @@ -10191,7 +10182,7 @@ dependencies = [ "hmac", "hyper 1.3.1", "indexmap 2.2.6", - "itertools 0.13.0", + "itertools 0.10.5", "lazy_static", "magic-crypt", "mail-send", @@ -10254,7 +10245,7 @@ version = "1.350.1" dependencies = [ "anyhow", "gosyn", - "itertools 0.13.0", + "itertools 0.10.5", "lazy_static", "regex", "windmill-parser", @@ -10277,7 +10268,7 @@ version = "1.350.1" dependencies = [ "anyhow", "convert_case 0.6.0", - "itertools 0.13.0", + "itertools 0.10.5", "lazy_static", "php-parser-rs", "regex", @@ -10290,7 +10281,7 @@ name = "windmill-parser-py" version = "1.350.1" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.10.5", "rustpython-parser", "serde_json", "windmill-parser", @@ -10302,7 +10293,7 @@ version = "1.350.1" dependencies = [ "anyhow", "async-recursion", - "itertools 0.13.0", + "itertools 0.10.5", "lazy_static", "phf", "regex", @@ -10369,14 +10360,14 @@ dependencies = [ "anyhow", "async-recursion", "axum", - "bigdecimal 0.4.5", + "bigdecimal 0.3.1", "chrono", "chrono-tz", "cron", "futures-core", "hex", "hmac", - "itertools 0.13.0", + "itertools 0.10.5", "lazy_static", "prometheus", "regex", @@ -10433,7 +10424,7 @@ dependencies = [ "gcp_auth", "git-version", "hex", - "itertools 0.13.0", + "itertools 0.10.5", "jsonwebtoken", "lazy_static", "mysql_async", diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 61ae464c59..1ac4d1dd56 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -3767,7 +3767,7 @@ async fn run_flow_dependencies_job( PushIsolationLevel::IsolatedRoot(db.clone(), rsmq), &w_id, JobPayload::RawFlowDependencies { path: req.path, flow_value: req.flow_value }, - PushArgs::empty(), + HashMap::from([("skip_flow_update".to_string(), to_raw_value(&true))]).into(), authed.display_username(), &authed.email, username_to_permissioned_as(&authed.username), diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 1f86a7887c..c70ee9b487 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -527,15 +527,27 @@ pub async fn handle_flow_dependency_job(v.get()).ok()) + .flatten() + }) + .flatten() + .unwrap_or(false); - sqlx::query!( - "UPDATE flow SET value = $1 WHERE path = $2 AND workspace_id = $3", - new_flow_value, - job_path, - job.workspace_id - ) - .execute(db) - .await?; + if !skip_flow_update { + sqlx::query!( + "UPDATE flow SET value = $1 WHERE path = $2 AND workspace_id = $3", + new_flow_value, + job_path, + job.workspace_id + ) + .execute(db) + .await?; + } tx.commit().await?; if let Err(e) = handle_deployment_metadata( diff --git a/cli/apps.ts b/cli/apps.ts index 2aff8784c2..38b724b994 100644 --- a/cli/apps.ts +++ b/cli/apps.ts @@ -12,6 +12,7 @@ import { yamlParse, } from "./deps.ts"; import { GlobalOptions, isSuperset } from "./types.ts"; +import { readInlinePathSync } from "./utils.ts"; export interface AppFile { value: any; @@ -37,7 +38,7 @@ export async function pushApp( try { app = await AppService.getAppByPath({ workspace, - path: remotePath.replaceAll("\\", "/"), + path: remotePath.replaceAll(SEP, "/"), }); } catch { //ignore @@ -59,11 +60,11 @@ export async function pushApp( const o: Record = v as any; if (o["content"] && o["content"].startsWith("!inline")) { const basePath = localPath + o["content"].split(" ")[1]; - o["content"] = Deno.readTextFileSync(basePath); + o["content"] = readInlinePathSync(basePath); } if (o["lock"] && o["lock"].startsWith("!inline")) { const basePath = localPath + o["lock"].split(" ")[1]; - o["lock"] = Deno.readTextFileSync(basePath); + o["lock"] = readInlinePathSync(basePath); } } else { replaceInlineScripts(v); @@ -83,7 +84,7 @@ export async function pushApp( log.info(colors.bold.yellow(`Updating app ${remotePath}...`)); await AppService.updateApp({ workspace, - path: remotePath.replaceAll("\\", "/"), + path: remotePath.replaceAll(SEP, "/"), requestBody: { deployment_message: message, ...localApp, @@ -95,7 +96,7 @@ export async function pushApp( await AppService.createApp({ workspace, requestBody: { - path: remotePath.replaceAll("\\", "/"), + path: remotePath.replaceAll(SEP, "/"), deployment_message: message, ...localApp, }, diff --git a/cli/bootstrap/common.ts b/cli/bootstrap/common.ts index 782eb9c8b5..44b2026b41 100644 --- a/cli/bootstrap/common.ts +++ b/cli/bootstrap/common.ts @@ -1,24 +1,33 @@ +export type EnumType = string[] | undefined; export interface SchemaProperty { - type: string | undefined - description?: string - pattern?: string - default?: any - enum?: string[] - contentEncoding?: 'base64' | 'binary' - format?: string - items?: { - type?: 'string' | 'number' | 'bytes' | 'object' - contentEncoding?: 'base64' - enum?: string[] - } - min?: number - max?: number - currency?: string - currencyLocale?: string - multiselect?: boolean - customErrorMessage?: string - properties?: { [name: string]: SchemaProperty } - required?: string[] - showExpr?: string + type: string | undefined; + description?: string; + pattern?: string; + default?: any; + enum?: EnumType; + contentEncoding?: "base64" | "binary"; + format?: string; + items?: { + type?: "string" | "number" | "bytes" | "object"; + contentEncoding?: "base64"; + enum?: string[]; + }; + min?: number; + max?: number; + currency?: string; + currencyLocale?: string; + multiselect?: boolean; + customErrorMessage?: string; + properties?: { [name: string]: SchemaProperty }; + required?: string[]; + showExpr?: string; + password?: boolean; + order?: string[]; + nullable?: boolean; + dateFormat?: string; + title?: string; + placeholder?: string; + oneOf?: SchemaProperty[]; + originalType?: string; } diff --git a/cli/flow.ts b/cli/flow.ts index 0b4c793f33..587654bdeb 100644 --- a/cli/flow.ts +++ b/cli/flow.ts @@ -12,11 +12,12 @@ import { yamlParse, } from "./deps.ts"; import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; -import { exts, resolve, track_job } from "./script.ts"; +import { resolve, track_job } from "./script.ts"; import { defaultFlowDefinition } from "./bootstrap/flow_bootstrap.ts"; import { generateFlowLockInternal } from "./metadata.ts"; import { SyncOptions, mergeConfigWithConfigFile } from "./conf.ts"; import { FSFSElement, elementsToMap, ignoreF } from "./sync.ts"; +import { readInlinePathSync } from "./utils.ts"; export interface FlowFile { summary: string; @@ -46,7 +47,7 @@ export function replaceInlineScripts( ) { const path = lock.split(" ")[1]; try { - m.value.lock = Deno.readTextFileSync(localPath + path); + m.value.lock = readInlinePathSync(localPath + path); } catch { log.error(`Lock file ${path} not found`); } @@ -104,9 +105,9 @@ export async function pushFlow( log.info(colors.bold.yellow(`Updating flow ${remotePath}...`)); await FlowService.updateFlow({ workspace: workspace, - path: remotePath.replaceAll("\\", "/"), + path: remotePath.replaceAll(SEP, "/"), requestBody: { - path: remotePath.replaceAll("\\", "/"), + path: remotePath.replaceAll(SEP, "/"), deployment_message: message, ...localFlow, }, @@ -116,7 +117,7 @@ export async function pushFlow( await FlowService.createFlow({ workspace: workspace, requestBody: { - path: remotePath.replaceAll("\\", "/"), + path: remotePath.replaceAll(SEP, "/"), deployment_message: message, ...localFlow, }, diff --git a/cli/metadata.ts b/cli/metadata.ts index 65574eeac1..ba5b85009e 100644 --- a/cli/metadata.ts +++ b/cli/metadata.ts @@ -39,7 +39,7 @@ import { newPathAssigner, yamlOptions, } from "./sync.ts"; -import { generateHash } from "./utils.ts"; +import { generateHash, readInlinePathSync } from "./utils.ts"; import { SyncCodebase } from "./codebase.ts"; import { FlowFile, replaceInlineScripts } from "./flow.ts"; @@ -100,11 +100,11 @@ export async function generateFlowLockInternal( workspace: Workspace, justUpdateMetadataLock?: boolean ): Promise { - if (folder.endsWith("/")) { + if (folder.endsWith(SEP)) { folder = folder.substring(0, folder.length - 1); } const remote_path = folder - .replaceAll("\\", "/") + .replaceAll(SEP, "/") .substring(0, folder.length - ".flow".length); if (!justUpdateMetadataLock) { log.info(`Generating lock for flow ${folder} at ${remote_path}`); @@ -185,7 +185,7 @@ export async function generateScriptMetadataInternal( ): Promise { const remotePath = scriptPath .substring(0, scriptPath.indexOf(".")) - .replaceAll("\\", "/"); + .replaceAll(SEP, "/"); const language = inferContentTypeFromFilePath(scriptPath, opts.defaultTs); @@ -256,7 +256,8 @@ export async function generateScriptMetadataInternal( metadataParsedContent.lock = ""; } } else { - metadataParsedContent.lock = "!inline " + remotePath + ".script.lock"; + metadataParsedContent.lock = + "!inline " + remotePath.replaceAll(SEP, "/") + ".script.lock"; } let metaPath = remotePath + ".script.yaml"; @@ -347,7 +348,7 @@ async function updateScriptLock( } const lockPath = remotePath + ".script.lock"; await Deno.writeTextFile(lockPath, lock); - metadataContent.lock = "!inline " + lockPath; + metadataContent.lock = "!inline " + lockPath.replaceAll(SEP, "/"); } catch (e) { throw new Error( `Failed to generate lockfile. Status was: ${rawResponse.statusText}, ${responseText}, ${e}` @@ -513,10 +514,18 @@ export function argSigToJsonSchemaType( | null; } | { str: string[] | null } - | { object: { key: string; typ: any }[] }, + | { object: { key: string; typ: any }[] } + | { + oneof: [ + { + label: string; + properties: { key: string; typ: any }[]; + } + ]; + }, oldS: SchemaProperty ): void { - const newS: SchemaProperty = { type: "" }; + let newS: SchemaProperty = { type: "" }; if (t === "int") { newS.type = "integer"; } else if (t === "float") { @@ -535,15 +544,43 @@ export function argSigToJsonSchemaType( } else if (t === "bytes") { newS.type = "string"; newS.contentEncoding = "base64"; + newS.originalType = "bytes"; } else if (t === "datetime") { newS.type = "string"; newS.format = "date-time"; + } else if (typeof t !== "string" && "oneof" in t) { + newS.type = "object"; + if (t.oneof) { + newS.oneOf = t.oneof.map((obj) => { + const oldObjS = + oldS.oneOf?.find((o) => o?.title === obj.label) ?? undefined; + const properties: Record = {}; + for (const prop of obj.properties) { + if (oldObjS?.properties && prop.key in oldObjS?.properties) { + properties[prop.key] = oldObjS?.properties[prop.key]; + } else { + properties[prop.key] = { description: "", type: "" }; + } + argSigToJsonSchemaType(prop.typ, properties[prop.key]); + } + return { + type: "object", + title: obj.label, + properties, + order: oldObjS?.order ?? undefined, + }; + }); + } } else if (typeof t !== "string" && `object` in t) { newS.type = "object"; if (t.object) { const properties: Record = {}; for (const prop of t.object) { - properties[prop.key] = {}; + if (oldS.properties && prop.key in oldS.properties) { + properties[prop.key] = oldS.properties[prop.key]; + } else { + properties[prop.key] = { description: "", type: "" }; + } argSigToJsonSchemaType(prop.typ, properties[prop.key]); } newS.properties = properties; @@ -551,7 +588,11 @@ export function argSigToJsonSchemaType( } else if (typeof t !== "string" && `str` in t) { newS.type = "string"; if (t.str) { + newS.originalType = "enum"; newS.enum = t.str; + } else { + newS.originalType = "string"; + newS.enum = undefined; } } else if (typeof t !== "string" && `resource` in t) { newS.type = "object"; @@ -573,6 +614,32 @@ export function argSigToJsonSchemaType( newS.type = "object"; } + const preservedFields = [ + "description", + "pattern", + "min", + "max", + "currency", + "currencyLocale", + "multiselect", + "customErrorMessage", + "required", + "showExpr", + "password", + "order", + "dateFormat", + "title", + "placeholder", + ]; + + preservedFields.forEach((field) => { + // @ts-ignore + if (oldS[field] !== undefined) { + // @ts-ignore + newS[field] = oldS[field]; + } + }); + if (oldS.type != newS.type) { for (const prop of Object.getOwnPropertyNames(newS)) { if (prop != "description") { @@ -602,18 +669,19 @@ export function argSigToJsonSchemaType( oldS.format = undefined; } } + //////////////////////////////////////////////////////////////////////////////////////////// // end of refactoring TODO // //////////////////////////////////////////////////////////////////////////////////////////// -export async function replaceLock(o?: { lock?: string | string[] }) { +export function replaceLock(o?: { lock?: string | string[] }) { if (Array.isArray(o?.lock)) { o.lock = o.lock.join("\n"); } if (o?.lock?.startsWith("!inline ")) { try { const lockPath = o?.lock?.split(" ")[1]; - o.lock = await Deno.readTextFile(lockPath); + o.lock = readInlinePathSync(lockPath); } catch (e) { log.info( colors.yellow(`Failed to read lockfile, doing as if it was empty: ${e}`) @@ -647,7 +715,7 @@ export async function parseMetadataFile( metadataFilePath = scriptPath + ".script.yaml"; await Deno.stat(metadataFilePath); const payload: any = yamlParse(await Deno.readTextFile(metadataFilePath)); - await replaceLock(payload); + replaceLock(payload); return { path: metadataFilePath, @@ -687,7 +755,7 @@ export async function parseMetadataFile( scriptInitialMetadata = yamlParse( await Deno.readTextFile(metadataFilePath) ) as ScriptMetadata; - await replaceLock(scriptInitialMetadata); + replaceLock(scriptInitialMetadata); } catch (e) { log.info( colors.yellow( diff --git a/cli/resource.ts b/cli/resource.ts index 55b1a211ae..7bf439223b 100644 --- a/cli/resource.ts +++ b/cli/resource.ts @@ -12,6 +12,7 @@ import { log, Resource, ResourceService, + SEP, Table, } from "./deps.ts"; @@ -32,7 +33,7 @@ export async function pushResource( try { resource = await ResourceService.getResource({ workspace: workspace, - path: remotePath.replaceAll("\\", "/"), + path: remotePath.replaceAll(SEP, "/"), }); } catch { // flow doesn't exist @@ -45,7 +46,7 @@ export async function pushResource( await ResourceService.updateResource({ workspace: workspace, - path: remotePath.replaceAll("\\", "/"), + path: remotePath.replaceAll(SEP, "/"), requestBody: { ...localResource }, }); } else { @@ -61,7 +62,7 @@ export async function pushResource( await ResourceService.createResource({ workspace: workspace, requestBody: { - path: remotePath.replaceAll("\\", "/"), + path: remotePath.replaceAll(SEP, "/"), ...localResource, }, }); diff --git a/cli/script.ts b/cli/script.ts index 38a4a34062..02442c7b98 100644 --- a/cli/script.ts +++ b/cli/script.ts @@ -143,7 +143,7 @@ export async function handleFile( alreadySynced.push(path); const remotePath = path .substring(0, path.indexOf(".")) - .replaceAll("\\", "/"); + .replaceAll(SEP, "/"); const language = inferContentTypeFromFilePath(path, opts?.defaultTs); @@ -186,7 +186,7 @@ export async function handleFile( try { remote = await ScriptService.getScriptByPath({ workspace: workspaceId, - path: remotePath.replaceAll("\\", "/"), + path: remotePath.replaceAll(SEP, "/"), }); log.debug(`Script ${remotePath} exists on remote`); } catch { @@ -214,7 +214,7 @@ export async function handleFile( content, description: typed?.description ?? "", language: language as NewScript["language"], - path: remotePath.replaceAll("\\", "/"), + path: remotePath.replaceAll(SEP, "/"), summary: typed?.summary ?? "", kind: typed?.kind, lock: typed?.lock, diff --git a/cli/sync.ts b/cli/sync.ts index 9f07217873..aec38102c9 100644 --- a/cli/sync.ts +++ b/cli/sync.ts @@ -214,11 +214,11 @@ export function extractInlineScriptsForFlows( const path = basePath + ext; const content = m.value.content; const r = [{ path: path, content: content }]; - m.value.content = "!inline " + path; + m.value.content = "!inline " + path.replaceAll(SEP, "/"); const lock = m.value.lock; if (lock && lock != "") { const lockPath = basePath + "lock"; - m.value.lock = "!inline " + lockPath; + m.value.lock = "!inline " + lockPath.replaceAll(SEP, "/"); r.push({ path: lockPath, content: lock }); } return r; @@ -264,7 +264,7 @@ export function extractInlineScriptsForApps( const r = []; if (o["content"]) { const content = o["content"]; - o["content"] = "!inline " + basePath + ext; + o["content"] = "!inline " + basePath.replaceAll(SEP, "/") + ext; r.push({ path: basePath + ext, content: content, @@ -272,7 +272,7 @@ export function extractInlineScriptsForApps( } if (o["lock"] && o["lock"] != "") { const lock = o["lock"]; - o["lock"] = "!inline " + basePath + "lock"; + o["lock"] = "!inline " + basePath.replaceAll(SEP, "/") + "lock"; r.push({ path: basePath + "lock", content: lock, @@ -437,7 +437,10 @@ function ZipFSElement( parsed["lock"] != "" && parsed["codebase"] == undefined ) { - parsed["lock"] = "!inline " + removeSuffix(p, ".json") + ".lock"; + parsed["lock"] = + "!inline " + + removeSuffix(p.replaceAll(SEP, "/"), ".json") + + ".lock"; } else { parsed["lock"] = undefined; } @@ -1127,7 +1130,7 @@ async function push(opts: GlobalOptions & SyncOptions) { log.info( colors.gray( - "Computing the files to update on the remote to match local (taking .wmillignore into account)" + "Computing the files to update on the remote to match local (taking wmill.yaml includes/excludes into account)" ) ); const remote = ZipFSElement( diff --git a/cli/utils.ts b/cli/utils.ts index 185fe50144..209ec7b8b8 100644 --- a/cli/utils.ts +++ b/cli/utils.ts @@ -2,7 +2,7 @@ // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-nocheck This file is copied from a JS project, so it's not type-safe. -import { log, encodeHex } from "./deps.ts"; +import { log, encodeHex, SEP } from "./deps.ts"; export function deepEqual(a: T, b: T): boolean { if (a === b) return true; @@ -117,3 +117,11 @@ export async function generateHashFromBuffer( const hashBuffer = await crypto.subtle.digest("SHA-256", content); return encodeHex(hashBuffer); } + +// export async function readInlinePath(path: string): Promise { +// return await Deno.readTextFile(path.replaceAll("/", SEP)); +// } + +export function readInlinePathSync(path: string): string { + return Deno.readTextFileSync(path.replaceAll("/", SEP)); +} diff --git a/cli/variable.ts b/cli/variable.ts index adda3df217..6d51cb2803 100644 --- a/cli/variable.ts +++ b/cli/variable.ts @@ -12,6 +12,7 @@ import { Confirm, ListableVariable, log, + SEP, Table, VariableService, } from "./deps.ts"; @@ -60,7 +61,7 @@ export async function pushVariable( try { variable = await VariableService.getVariable({ workspace: workspace, - path: remotePath.replaceAll("\\", "/"), + path: remotePath.replaceAll(SEP, "/"), decryptSecret: plainSecrets, includeEncrypted: true, }); @@ -79,7 +80,7 @@ export async function pushVariable( await VariableService.updateVariable({ workspace, - path: remotePath.replaceAll("\\", "/"), + path: remotePath.replaceAll(SEP, "/"), alreadyEncrypted: !plainSecrets, requestBody: { ...localVariable, @@ -93,7 +94,7 @@ export async function pushVariable( workspace, alreadyEncrypted: !plainSecrets, requestBody: { - path: remotePath.replaceAll("\\", "/"), + path: remotePath.replaceAll(SEP, "/"), ...localVariable, }, });