From b70ca6e3edbf0896d83cea405462dc9b745df210 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Tue, 1 Oct 2024 12:28:46 +0200 Subject: [PATCH 01/38] skip step better UI (#4465) --- backend/windmill-common/src/flow_status.rs | 3 +++ backend/windmill-worker/src/worker_flow.rs | 17 +++++++++++++++++ .../lib/components/FlowStatusViewerInner.svelte | 9 +++++++-- frontend/src/lib/components/ModuleStatus.svelte | 3 +++ .../flows/map/FlowModuleSchemaItem.svelte | 13 +++++++++++++ .../src/lib/components/flows/map/MapItem.svelte | 1 + frontend/src/lib/components/graph/model.ts | 1 + .../graph/renderers/nodes/ModuleNode.svelte | 2 +- frontend/src/lib/components/graph/util.ts | 9 +++++++-- openflow.openapi.yaml | 2 ++ 10 files changed, 55 insertions(+), 5 deletions(-) diff --git a/backend/windmill-common/src/flow_status.rs b/backend/windmill-common/src/flow_status.rs index 035cadecd8..a5c509d6ef 100644 --- a/backend/windmill-common/src/flow_status.rs +++ b/backend/windmill-common/src/flow_status.rs @@ -128,6 +128,7 @@ struct UntaggedFlowStatusModule { while_loop: Option, approvers: Option>, failed_retries: Option>, + skipped: Option, } #[derive(Serialize, Debug, Clone)] @@ -179,6 +180,7 @@ pub enum FlowStatusModule { approvers: Vec, #[serde(skip_serializing_if = "Vec::is_empty")] failed_retries: Vec, + skipped: bool, }, Failure { id: String, @@ -255,6 +257,7 @@ impl<'de> Deserialize<'de> for FlowStatusModule { branch_chosen: untagged.branch_chosen, approvers: untagged.approvers.unwrap_or_default(), failed_retries: untagged.failed_retries.unwrap_or_default(), + skipped: untagged.skipped.unwrap_or(false), }), "Failure" => Ok(FlowStatusModule::Failure { id: untagged diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index c00fd81a44..3f92cfc829 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -561,6 +561,7 @@ pub async fn update_flow_status_after_job_completion_internal< branch_chosen: None, approvers: vec![], failed_retries: vec![], + skipped: false, } } else { success = false; @@ -698,6 +699,20 @@ pub async fn update_flow_status_after_job_completion_internal< } } if success || (flow_jobs.is_some() && (skip_loop_failures || skip_branch_failure)) { + let is_skipped = if current_module.as_ref().is_some_and(|m| m.skip_if.is_some()) { + sqlx::query_scalar!( + "SELECT job_kind = 'identity' FROM completed_job WHERE id = $1", + job_id_for_status + ) + .fetch_one(db) + .await + .map_err(|e| { + Error::InternalErr(format!("error during skip check: {e:#}")) + })? + .unwrap_or(false) + } else { + false + }; success = true; ( true, @@ -709,6 +724,7 @@ pub async fn update_flow_status_after_job_completion_internal< branch_chosen, approvers: vec![], failed_retries: old_status.retry.failed_jobs.clone(), + skipped: is_skipped, }), ) } else { @@ -2344,6 +2360,7 @@ async fn push_next_flow_job branch_chosen: None, approvers: vec![], failed_retries: vec![], + skipped: false, })) .bind(flow_job.id) .execute(db) diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index 0232548183..2dcc2a030b 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -437,7 +437,8 @@ flow_jobs: mod.flow_jobs, flow_jobs_success: mod.flow_jobs_success, iteration_total: mod.iterator?.itered?.length, - retries: mod?.failed_retries?.length + retries: mod?.failed_retries?.length, + skipped: mod.skipped // retries: $flowStateStore?.raw_flow }, force @@ -1054,7 +1055,11 @@ Selected subflow {/if}
- + {#if node.duration_ms} diff --git a/frontend/src/lib/components/ModuleStatus.svelte b/frontend/src/lib/components/ModuleStatus.svelte index b5f4e8b230..c01e793454 100644 --- a/frontend/src/lib/components/ModuleStatus.svelte +++ b/frontend/src/lib/components/ModuleStatus.svelte @@ -7,6 +7,7 @@ export let type: FlowStatusModule['type'] export let scheduled_for: Date | undefined + export let skipped: boolean = false {#if type == 'WaitingForEvents'} @@ -28,6 +29,8 @@ Job is waiting for an executor {/if} +{:else if skipped} + Skipped {:else if type == 'Success'} Success {:else if type == 'Failure'} diff --git a/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte b/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte index bbab6a6f7e..a0896add9a 100644 --- a/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte +++ b/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte @@ -13,6 +13,7 @@ PhoneIncoming, Repeat, Square, + SkipForward, Voicemail, X } from 'lucide-svelte' @@ -33,6 +34,7 @@ export let retry: boolean = false export let cache: boolean = false export let earlyStop: boolean = false + export let skip: boolean = false export let suspend: boolean = false export let sleep: boolean = false export let mock: boolean = false @@ -181,6 +183,17 @@ Early stop/break {/if} + {#if skip} + +
+ +
+ Skip +
+ {/if} {#if suspend}
Date: Tue, 1 Oct 2024 12:38:44 +0200 Subject: [PATCH 02/38] chore(main): release 1.403.0 (#4459) * chore(main): release 1.403.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel --- CHANGELOG.md | 12 +++ backend/Cargo.lock | 95 +++++++++---------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 4 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- python-client/wmill_pg/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 16 files changed, 76 insertions(+), 65 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 848d9a6f10..ef129eb0c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## [1.403.0](https://github.com/windmill-labs/windmill/compare/v1.402.3...v1.403.0) (2024-10-01) + + +### Features + +* flow step skipping ([#4461](https://github.com/windmill-labs/windmill/issues/4461)) ([0df169e](https://github.com/windmill-labs/windmill/commit/0df169e3f996ed54b91569b13cce15d7d019a213)) + + +### Bug Fixes + +* skip one migration to avoid using md5 for azure support ([630ae5d](https://github.com/windmill-labs/windmill/commit/630ae5d425cd9957d674befd2df96e2befec52a3)) + ## [1.402.3](https://github.com/windmill-labs/windmill/compare/v1.402.2...v1.402.3) (2024-09-30) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 0dfc4ee458..f38c3be406 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -595,7 +595,7 @@ dependencies = [ "bytes", "http 1.1.0", "rand 0.8.5", - "reqwest 0.12.7", + "reqwest 0.12.8", "serde", "serde-aux", "serde_json", @@ -1609,9 +1609,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.22" +version = "1.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9540e661f81799159abee814118cc139a2004b3a3aa3ea37724a1b66530b90e0" +checksum = "3bbb537bb4a30b90362caddba8f360c0a56bc13d3a5570028e7197204cb54a17" dependencies = [ "jobserver", "libc", @@ -2708,7 +2708,7 @@ dependencies = [ "dlopen2_derive", "once_cell", "rustls-native-certs 0.7.3", - "rustls-pemfile 2.1.3", + "rustls-pemfile 2.2.0", ] [[package]] @@ -2790,7 +2790,7 @@ dependencies = [ "deno_core", "deno_native_certs", "rustls 0.23.13", - "rustls-pemfile 2.1.3", + "rustls-pemfile 2.2.0", "rustls-tokio-stream", "rustls-webpki 0.102.8", "serde", @@ -4167,9 +4167,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.9.4" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" [[package]] name = "httpdate" @@ -5444,7 +5444,7 @@ dependencies = [ "percent-encoding", "quick-xml 0.36.2", "rand 0.8.5", - "reqwest 0.12.7", + "reqwest 0.12.8", "ring 0.17.8", "serde", "serde_json", @@ -6767,9 +6767,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.7" +version = "0.12.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" +checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b" dependencies = [ "async-compression 0.4.12", "base64 0.22.1", @@ -6795,8 +6795,8 @@ dependencies = [ "pin-project-lite", "quinn", "rustls 0.23.13", - "rustls-native-certs 0.7.3", - "rustls-pemfile 2.1.3", + "rustls-native-certs 0.8.0", + "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", "serde_json", @@ -7126,7 +7126,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" dependencies = [ "openssl-probe", - "rustls-pemfile 2.1.3", + "rustls-pemfile 2.2.0", "rustls-pki-types", "schannel", "security-framework", @@ -7139,7 +7139,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a" dependencies = [ "openssl-probe", - "rustls-pemfile 2.1.3", + "rustls-pemfile 2.2.0", "rustls-pki-types", "schannel", "security-framework", @@ -7156,11 +7156,10 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.3" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" dependencies = [ - "base64 0.22.1", "rustls-pki-types", ] @@ -8042,7 +8041,7 @@ dependencies = [ "paste", "percent-encoding", "rustls 0.23.13", - "rustls-pemfile 2.1.3", + "rustls-pemfile 2.2.0", "serde", "serde_json", "sha2 0.10.8", @@ -9663,7 +9662,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea6023f9fe4b69267ccd3ed7d203d931c43c5f82dbaa0f07202bc17193a5f43" dependencies = [ "loki-api", - "reqwest 0.12.7", + "reqwest 0.12.8", "serde", "serde_json", "snap", @@ -9968,9 +9967,9 @@ dependencies = [ [[package]] name = "unicode-properties" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ea75f83c0137a9b98608359a5f1af8144876eb67bcb1ce837368e906a9f524" +checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" [[package]] name = "unicode-segmentation" @@ -10429,7 +10428,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.402.3" +version = "1.403.0" dependencies = [ "anyhow", "axum", @@ -10447,7 +10446,7 @@ dependencies = [ "prometheus", "quote", "rand 0.8.5", - "reqwest 0.12.7", + "reqwest 0.12.8", "rsmq_async", "serde", "serde_json", @@ -10471,7 +10470,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.402.3" +version = "1.403.0" dependencies = [ "anyhow", "argon2", @@ -10515,7 +10514,7 @@ dependencies = [ "quick_cache", "rand 0.8.5", "regex", - "reqwest 0.12.7", + "reqwest 0.12.8", "rsa 0.7.2", "rsmq_async", "rust-embed", @@ -10555,7 +10554,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.402.3" +version = "1.403.0" dependencies = [ "base64 0.21.7", "chrono", @@ -10573,7 +10572,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.402.3" +version = "1.403.0" dependencies = [ "chrono", "serde", @@ -10586,7 +10585,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.402.3" +version = "1.403.0" dependencies = [ "anyhow", "async-stream", @@ -10613,7 +10612,7 @@ dependencies = [ "prometheus", "rand 0.8.5", "regex", - "reqwest 0.12.7", + "reqwest 0.12.8", "serde", "serde_json", "sha2 0.10.8", @@ -10631,7 +10630,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.402.3" +version = "1.403.0" dependencies = [ "regex", "rsmq_async", @@ -10646,7 +10645,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.402.3" +version = "1.403.0" dependencies = [ "anyhow", "bytes", @@ -10667,7 +10666,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.402.3" +version = "1.403.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -10676,7 +10675,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.402.3" +version = "1.403.0" dependencies = [ "anyhow", "lazy_static", @@ -10688,7 +10687,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.402.3" +version = "1.403.0" dependencies = [ "anyhow", "gosyn", @@ -10700,7 +10699,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.402.3" +version = "1.403.0" dependencies = [ "anyhow", "lazy_static", @@ -10712,7 +10711,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.402.3" +version = "1.403.0" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10723,7 +10722,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.402.3" +version = "1.403.0" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10734,7 +10733,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.402.3" +version = "1.403.0" dependencies = [ "anyhow", "async-recursion", @@ -10752,7 +10751,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.402.3" +version = "1.403.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -10769,7 +10768,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.402.3" +version = "1.403.0" dependencies = [ "anyhow", "lazy_static", @@ -10781,7 +10780,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.402.3" +version = "1.403.0" dependencies = [ "anyhow", "lazy_static", @@ -10799,7 +10798,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.402.3" +version = "1.403.0" dependencies = [ "anyhow", "getrandom 0.2.15", @@ -10820,7 +10819,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.402.3" +version = "1.403.0" dependencies = [ "anyhow", "serde_json", @@ -10830,7 +10829,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.402.3" +version = "1.403.0" dependencies = [ "anyhow", "async-recursion", @@ -10846,7 +10845,7 @@ dependencies = [ "lazy_static", "prometheus", "regex", - "reqwest 0.12.7", + "reqwest 0.12.8", "rsmq_async", "serde", "serde_json", @@ -10863,7 +10862,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.402.3" +version = "1.403.0" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -10873,7 +10872,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.402.3" +version = "1.403.0" dependencies = [ "anyhow", "async-recursion", @@ -10913,7 +10912,7 @@ dependencies = [ "prometheus", "rand 0.8.5", "regex", - "reqwest 0.12.7", + "reqwest 0.12.8", "rsmq_async", "rust_decimal", "serde", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 9b7a0123c8..ab1927c81f 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.402.3" +version = "1.403.0" authors.workspace = true edition.workspace = true @@ -27,7 +27,7 @@ members = [ ] [workspace.package] -version = "1.402.3" +version = "1.403.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index c8c30a7db1..5d5c3fd1df 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.402.3 + version: 1.403.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 2d88e6abed..c877f27c70 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.402.3"; +export const VERSION = "v1.403.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/main.ts b/cli/main.ts index c1aa86cf3e..0a3e71b1f9 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -60,7 +60,7 @@ export { // } // }); -export const VERSION = "1.402.3"; +export const VERSION = "1.403.0"; const command = new Command() .name("wmill") diff --git a/frontend/package-lock.json b/frontend/package-lock.json index fbfe82be4e..091f654742 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.402.3", + "version": "1.403.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.402.3", + "version": "1.403.0", "license": "AGPL-3.0", "dependencies": { "@aws-crypto/sha256-js": "^4.0.0", diff --git a/frontend/package.json b/frontend/package.json index b85fc1caba..06d9dbdb9c 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.402.3", + "version": "1.403.0", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index 8487b85565..26b7000d6c 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.402.3" -wmill_pg = ">=1.402.3" +wmill = ">=1.403.0" +wmill_pg = ">=1.403.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index e95266c52f..7407b45e7b 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.402.3 + version: 1.403.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 89445c5e22..8f030752a6 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. -ModuleVersion = '1.402.3' +ModuleVersion = '1.403.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 5746949537..4946f00d1d 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.402.3" +version = "1.403.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index 5e4f589322..9621e6fd35 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.402.3" +version = "1.403.0" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index a4dfd07ee9..6d9056ba38 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.402.3", + "version": "1.403.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index c6fbf1c41c..48d7b0cac0 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.402.3", + "version": "1.403.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index cce8c1101e..7b3e7a790c 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.402.3 +1.403.0 From ae6d99b9f46d8cefa9166072bb6c24e0fb8be8a0 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Tue, 1 Oct 2024 13:34:31 +0200 Subject: [PATCH 03/38] fix sqlx skip step (#4466) --- ...57ae030463c139c39072777e453bfb7e9c0c3.json | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 backend/.sqlx/query-829130d74c107e4e1a86f3e772657ae030463c139c39072777e453bfb7e9c0c3.json diff --git a/backend/.sqlx/query-829130d74c107e4e1a86f3e772657ae030463c139c39072777e453bfb7e9c0c3.json b/backend/.sqlx/query-829130d74c107e4e1a86f3e772657ae030463c139c39072777e453bfb7e9c0c3.json new file mode 100644 index 0000000000..e00aba3aab --- /dev/null +++ b/backend/.sqlx/query-829130d74c107e4e1a86f3e772657ae030463c139c39072777e453bfb7e9c0c3.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT job_kind = 'identity' FROM completed_job WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "829130d74c107e4e1a86f3e772657ae030463c139c39072777e453bfb7e9c0c3" +} From 73ab8e1653d6e0c0c69fa7dcd96583f25d13ef86 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 1 Oct 2024 16:03:58 +0200 Subject: [PATCH 04/38] fix: fix new instance db setup --- .../create_workspace_without_md5.sql | 6 +++--- backend/windmill-api/src/db.rs | 11 +++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/backend/custom_migrations/create_workspace_without_md5.sql b/backend/custom_migrations/create_workspace_without_md5.sql index 4a818d6611..fc140d9e57 100644 --- a/backend/custom_migrations/create_workspace_without_md5.sql +++ b/backend/custom_migrations/create_workspace_without_md5.sql @@ -1,8 +1,8 @@ INSERT INTO workspace(id, name, owner) VALUES - ('admins', 'Admins', 'admin@windmill.dev'); + ('admins', 'Admins', 'admin@windmill.dev') ON CONFLICT DO NOTHING; INSERT INTO workspace_settings (workspace_id) VALUES - ('admins'); + ('admins') ON CONFLICT DO NOTHING; INSERT INTO workspace_key (workspace_id, kind, key) @@ -13,4 +13,4 @@ INSERT INTO workspace_key FROM generate_series(1, 32) -- generates 32 characters ), '' -)); \ No newline at end of file +)) ON CONFLICT DO NOTHING; \ No newline at end of file diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index 41761d53b9..d6e6ed1a1d 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -137,6 +137,17 @@ impl Migrate for CustomMigrator { "../../custom_migrations/create_workspace_without_md5.sql" )) .await?; + let _ = sqlx::query( + r#" + INSERT INTO _sqlx_migrations ( version, description, success, checksum, execution_time ) + VALUES ( $1, $2, TRUE, $3, -1 ) ON CONFLICT DO NOTHING + "#, + ) + .bind(migration.version) + .bind(&*migration.description) + .bind(&*migration.checksum) + .execute(&mut *self.inner) + .await?; return Ok(std::time::Duration::from_secs(0)); } else { let r = self.inner.apply(migration).await; From 0476f98231d922a78157c8df64dc71e93f3c0dc9 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 1 Oct 2024 16:07:57 +0200 Subject: [PATCH 05/38] chore(main): release 1.403.1 (#4467) * chore(main): release 1.403.1 * Apply automatic changes --------- Co-authored-by: rubenfiszel --- CHANGELOG.md | 7 +++ backend/Cargo.lock | 48 +++++++++---------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 4 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- python-client/wmill_pg/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 16 files changed, 48 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef129eb0c6..86b2d85219 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [1.403.1](https://github.com/windmill-labs/windmill/compare/v1.403.0...v1.403.1) (2024-10-01) + + +### Bug Fixes + +* fix new instance db setup ([73ab8e1](https://github.com/windmill-labs/windmill/commit/73ab8e1653d6e0c0c69fa7dcd96583f25d13ef86)) + ## [1.403.0](https://github.com/windmill-labs/windmill/compare/v1.402.3...v1.403.0) (2024-10-01) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index f38c3be406..bbd1c7cdb6 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1609,9 +1609,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.23" +version = "1.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bbb537bb4a30b90362caddba8f360c0a56bc13d3a5570028e7197204cb54a17" +checksum = "812acba72f0a070b003d3697490d2b55b837230ae7c6c6497f05cc2ddbb8d938" dependencies = [ "jobserver", "libc", @@ -10428,7 +10428,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.403.0" +version = "1.403.1" dependencies = [ "anyhow", "axum", @@ -10470,7 +10470,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.403.0" +version = "1.403.1" dependencies = [ "anyhow", "argon2", @@ -10554,7 +10554,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.403.0" +version = "1.403.1" dependencies = [ "base64 0.21.7", "chrono", @@ -10572,7 +10572,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.403.0" +version = "1.403.1" dependencies = [ "chrono", "serde", @@ -10585,7 +10585,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.403.0" +version = "1.403.1" dependencies = [ "anyhow", "async-stream", @@ -10630,7 +10630,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.403.0" +version = "1.403.1" dependencies = [ "regex", "rsmq_async", @@ -10645,7 +10645,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.403.0" +version = "1.403.1" dependencies = [ "anyhow", "bytes", @@ -10666,7 +10666,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.403.0" +version = "1.403.1" dependencies = [ "convert_case 0.6.0", "serde", @@ -10675,7 +10675,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.403.0" +version = "1.403.1" dependencies = [ "anyhow", "lazy_static", @@ -10687,7 +10687,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.403.0" +version = "1.403.1" dependencies = [ "anyhow", "gosyn", @@ -10699,7 +10699,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.403.0" +version = "1.403.1" dependencies = [ "anyhow", "lazy_static", @@ -10711,7 +10711,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.403.0" +version = "1.403.1" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10722,7 +10722,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.403.0" +version = "1.403.1" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10733,7 +10733,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.403.0" +version = "1.403.1" dependencies = [ "anyhow", "async-recursion", @@ -10751,7 +10751,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.403.0" +version = "1.403.1" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -10768,7 +10768,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.403.0" +version = "1.403.1" dependencies = [ "anyhow", "lazy_static", @@ -10780,7 +10780,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.403.0" +version = "1.403.1" dependencies = [ "anyhow", "lazy_static", @@ -10798,7 +10798,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.403.0" +version = "1.403.1" dependencies = [ "anyhow", "getrandom 0.2.15", @@ -10819,7 +10819,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.403.0" +version = "1.403.1" dependencies = [ "anyhow", "serde_json", @@ -10829,7 +10829,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.403.0" +version = "1.403.1" dependencies = [ "anyhow", "async-recursion", @@ -10862,7 +10862,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.403.0" +version = "1.403.1" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -10872,7 +10872,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.403.0" +version = "1.403.1" dependencies = [ "anyhow", "async-recursion", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index ab1927c81f..692edd8f0b 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.403.0" +version = "1.403.1" authors.workspace = true edition.workspace = true @@ -27,7 +27,7 @@ members = [ ] [workspace.package] -version = "1.403.0" +version = "1.403.1" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 5d5c3fd1df..1ca35ac564 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.403.0 + version: 1.403.1 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index c877f27c70..8a0e53bb91 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.403.0"; +export const VERSION = "v1.403.1"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/main.ts b/cli/main.ts index 0a3e71b1f9..7cac847e6c 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -60,7 +60,7 @@ export { // } // }); -export const VERSION = "1.403.0"; +export const VERSION = "1.403.1"; const command = new Command() .name("wmill") diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 091f654742..65aee2cd28 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.403.0", + "version": "1.403.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.403.0", + "version": "1.403.1", "license": "AGPL-3.0", "dependencies": { "@aws-crypto/sha256-js": "^4.0.0", diff --git a/frontend/package.json b/frontend/package.json index 06d9dbdb9c..61554a5db8 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.403.0", + "version": "1.403.1", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index 26b7000d6c..8a901e1fc6 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.403.0" -wmill_pg = ">=1.403.0" +wmill = ">=1.403.1" +wmill_pg = ">=1.403.1" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 7407b45e7b..fb285ccd64 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.403.0 + version: 1.403.1 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 8f030752a6..8037a060db 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. -ModuleVersion = '1.403.0' +ModuleVersion = '1.403.1' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 4946f00d1d..c4300a04af 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.403.0" +version = "1.403.1" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index 9621e6fd35..d1ad5c634b 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.403.0" +version = "1.403.1" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 6d9056ba38..108b7e5afc 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.403.0", + "version": "1.403.1", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 48d7b0cac0..98e7b62919 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.403.0", + "version": "1.403.1", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index 7b3e7a790c..ff5ef8e01f 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.403.0 +1.403.1 From 99911dc21b84a273baf6aca5dfc5ddd12f45ad28 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 1 Oct 2024 16:46:25 +0200 Subject: [PATCH 06/38] refresh superadmin state on workspace list page refresh superadmin state on workspace list page --- .../(root)/(logged)/user/(user)/workspaces/+page.svelte | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frontend/src/routes/(root)/(logged)/user/(user)/workspaces/+page.svelte b/frontend/src/routes/(root)/(logged)/user/(user)/workspaces/+page.svelte index 89b0341747..21ca506c8b 100644 --- a/frontend/src/routes/(root)/(logged)/user/(user)/workspaces/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/user/(user)/workspaces/+page.svelte @@ -24,6 +24,7 @@ import { isCloudHosted } from '$lib/cloud' import { emptyString } from '$lib/utils' import { getUserExt } from '$lib/user' + import { refreshSuperadmin } from '$lib/refreshUser' let invites: WorkspaceInvite[] = [] let list_all_as_super_admin: boolean = false @@ -45,6 +46,7 @@ } async function loadWorkspaces() { + console.log('loading workspaces', $usersWorkspaceStore) if (!$usersWorkspaceStore) { try { usersWorkspaceStore.set(await WorkspaceService.listUserWorkspaces()) @@ -76,7 +78,7 @@ } $: list_all_as_super_admin != undefined && $userWorkspaces && handleListWorkspaces() - $: adminsInstance = workspaces?.find((x) => x.id == 'admins') + $: adminsInstance = workspaces?.find((x) => x.id == 'admins') || $superadmin $: nonAdminWorkspaces = (workspaces ?? []).filter((x) => x.id != 'admins') $: noWorkspaces = $superadmin && nonAdminWorkspaces.length == 0 @@ -97,6 +99,7 @@ getCreateWorkspaceRequireSuperadmin() } + refreshSuperadmin() loadInvites() loadWorkspaces() From 9ac3b6b1d5d64d7467dd80506f8a8d772c4630bd Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 1 Oct 2024 16:53:07 +0200 Subject: [PATCH 07/38] fix(cli): improve schedule path handling on windows --- cli/schedule.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cli/schedule.ts b/cli/schedule.ts index fe2db1af19..25ffdd5b49 100644 --- a/cli/schedule.ts +++ b/cli/schedule.ts @@ -1,5 +1,5 @@ // deno-lint-ignore-file no-explicit-any -import { colors, Command, log, Table } from "./deps.ts"; +import { colors, Command, log, SEP, Table } from "./deps.ts"; import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; import * as wmill from "./gen/services.gen.ts"; @@ -42,8 +42,7 @@ export async function pushSchedule( schedule: Schedule | ScheduleFile | undefined, localSchedule: ScheduleFile ): Promise { - path = removeType(path, "schedule"); - + path = removeType(path, "schedule").replaceAll(SEP, "/"); log.debug(`Processing local schedule ${path}`); // deleting old app if it exists in raw mode From 3134f79ced80aab86912643ab7a60dcf909ab104 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Wed, 2 Oct 2024 12:07:11 +0200 Subject: [PATCH 08/38] fix(frontend): disable runnable field on route editor from detail panel (#4469) --- .../lib/components/triggers/RouteEditorInner.svelte | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/frontend/src/lib/components/triggers/RouteEditorInner.svelte b/frontend/src/lib/components/triggers/RouteEditorInner.svelte index 2fa7998295..19846e5c24 100644 --- a/frontend/src/lib/components/triggers/RouteEditorInner.svelte +++ b/frontend/src/lib/components/triggers/RouteEditorInner.svelte @@ -28,6 +28,7 @@ let script_path = '' let initialScriptPath = '' + let fixedScriptPath = '' let drawerLoading = true export async function openEdit(ePath: string, isFlow: boolean) { @@ -46,7 +47,7 @@ } } - export async function openNew(nis_flow: boolean, initial_script_path?: string) { + export async function openNew(nis_flow: boolean, fixedScriptPath_?: string) { drawerLoading = true try { drawer?.openDrawer() @@ -58,8 +59,9 @@ initialRoutePath = '' route_path = '' http_method = 'post' - initialScriptPath = initial_script_path ?? '' - script_path = initialScriptPath + initialScriptPath = '' + fixedScriptPath = fixedScriptPath_ ?? '' + script_path = fixedScriptPath path = '' initialPath = '' dirtyPath = false @@ -282,8 +284,8 @@

Date: Thu, 3 Oct 2024 11:55:26 +0200 Subject: [PATCH 09/38] fix: fix id editor for app --- frontend/src/lib/components/IdEditorInput.svelte | 1 - 1 file changed, 1 deletion(-) diff --git a/frontend/src/lib/components/IdEditorInput.svelte b/frontend/src/lib/components/IdEditorInput.svelte index dae667c867..1f390d517a 100644 --- a/frontend/src/lib/components/IdEditorInput.svelte +++ b/frontend/src/lib/components/IdEditorInput.svelte @@ -47,7 +47,6 @@
Date: Thu, 3 Oct 2024 13:11:58 +0200 Subject: [PATCH 10/38] deno_core is an optional feature flag (#4473) * full * all * add deno_core as features * all * remove warnings * all --- .github/DockerfileBackendTests | 2 + .github/workflows/backend-test.yml | 4 +- .github/workflows/build-staging-image.yml | 2 +- .github/workflows/docker-image-rpi4.yml | 2 +- .github/workflows/docker-image.yml | 12 +- backend/Cargo.lock | 1 + backend/Cargo.toml | 4 +- backend/src/main.rs | 1 + backend/windmill-worker/Cargo.toml | 37 ++- backend/windmill-worker/build.rs | 16 + backend/windmill-worker/src/bun_executor.rs | 93 +++--- backend/windmill-worker/src/job_logger.rs | 2 +- backend/windmill-worker/src/js_eval.rs | 307 ++++++++++++-------- cli/instance.ts | 39 ++- 14 files changed, 326 insertions(+), 196 deletions(-) diff --git a/.github/DockerfileBackendTests b/.github/DockerfileBackendTests index 9fa9d788ff..03f51de332 100644 --- a/.github/DockerfileBackendTests +++ b/.github/DockerfileBackendTests @@ -43,6 +43,8 @@ RUN wget https://golang.org/dl/go1.21.5.linux-amd64.tar.gz && tar -C /usr/local ENV PATH="${PATH}:/usr/local/go/bin" ENV GO_PATH=/usr/local/go/bin/go +RUN curl -LsSf https://astral.sh/uv/install.sh | sh + ENV TZ=Etc/UTC ENV PYTHON_VERSION 3.11.4 diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index 9963ecd4d1..bedd324f9e 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -44,5 +44,5 @@ jobs: mkdir frontend/build && cd backend && touch windmill-api/openapi-deref.yaml && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill - DISABLE_EMBEDDING=true RUST_LOG=info cargo test --features enterprise - --all -- --nocapture + DISABLE_EMBEDDING=true RUST_LOG=info cargo test --features + enterprise,deno_core --all -- --nocapture diff --git a/.github/workflows/build-staging-image.yml b/.github/workflows/build-staging-image.yml index 744ae83aae..724fde4e59 100644 --- a/.github/workflows/build-staging-image.yml +++ b/.github/workflows/build-staging-image.yml @@ -62,7 +62,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core tags: | ${{ steps.meta-ee-public.outputs.tags }} labels: | diff --git a/.github/workflows/docker-image-rpi4.yml b/.github/workflows/docker-image-rpi4.yml index 9fc20c3a87..71a796f679 100644 --- a/.github/workflows/docker-image-rpi4.yml +++ b/.github/workflows/docker-image-rpi4.yml @@ -67,7 +67,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=embedding,parquet,openidconnect + features=embedding,parquet,openidconnect,deno_core tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev ${{ steps.meta-public.outputs.tags }} diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 8cac56c35b..7089a870c9 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -1,8 +1,10 @@ env: REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.event_name != 'pull_request' && github.repository || + IMAGE_NAME: + ${{ github.event_name != 'pull_request' && github.repository || 'windmill-labs/windmill-test' }} - DEV_SHA: ${{ github.event_name != 'pull_request' && 'dev' || format('pr-{0}', + DEV_SHA: + ${{ github.event_name != 'pull_request' && 'dev' || format('pr-{0}', github.event.number) }} name: Build windmill:main @@ -75,7 +77,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=embedding,parquet,openidconnect,jemalloc + features=embedding,parquet,openidconnect,jemalloc,deno_core tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} ${{ steps.meta-public.outputs.tags }} @@ -136,7 +138,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} ${{ steps.meta-ee-public.outputs.tags }} @@ -198,7 +200,7 @@ jobs: platforms: linux/amd64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core PYTHON_IMAGE=python:3.12.2-slim-bookworm tags: | ${{ steps.meta-ee-public-py312.outputs.tags }} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index bbd1c7cdb6..39f4a3e7d4 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -10919,6 +10919,7 @@ dependencies = [ "serde_json", "sha2 0.10.8", "sqlx", + "swc_ecma_parser 0.144.3", "tar", "tiberius", "tokio", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 692edd8f0b..1537837c3f 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -57,6 +57,7 @@ cloud = ["windmill-queue/cloud", "windmill-worker/cloud"] jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"] tantivy = ["dep:windmill-indexer", "windmill-api/tantivy"] sqlx = ["windmill-worker/sqlx"] +deno_core = ["windmill-worker/deno_core", "dep:deno_core"] [dependencies] anyhow.workspace = true @@ -85,7 +86,7 @@ uuid.workspace = true gethostname.workspace = true serde_json.workspace = true serde.workspace = true -deno_core.workspace = true +deno_core = { workspace = true, optional = true } object_store = { workspace = true, optional = true } pg-embed = {git = "https://github.com/faokunega/pg-embed", optional = true, default-features = false, features = ['rt_tokio']} quote.workspace = true @@ -105,6 +106,7 @@ serde.workspace = true windmill-api-client.workspace = true deno_core = { workspace = true, features = ["include_js_files_for_snapshotting", "unsafe_use_unprotected_platform"] } + [workspace.dependencies] windmill-api = { path = "./windmill-api", default-features = false } windmill-queue = { path = "./windmill-queue" } diff --git a/backend/src/main.rs b/backend/src/main.rs index 0a59e4af07..3d97e5f1f7 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -118,6 +118,7 @@ where } pub fn main() -> anyhow::Result<()> { + #[cfg(feature = "deno_core")] deno_core::JsRuntime::init_platform(None); create_and_run_current_thread_inner(windmill_main()) } diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index c05bc26fc3..e5b47b7a81 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -18,6 +18,7 @@ parquet = ["windmill-common/parquet", "dep:object_store"] flow_testing = [] cloud = [] sqlx = [] +deno_core = ["dep:deno_fetch", "dep:deno_webidl", "dep:deno_web", "dep:deno_net", "dep:deno_console", "dep:deno_url", "dep:deno_core", "dep:deno_ast", "dep:deno_tls"] [dependencies] windmill-queue.workspace = true @@ -59,15 +60,15 @@ once_cell.workspace = true rsmq_async.workspace = true tokio-postgres.workspace = true bit-vec.workspace = true -deno_fetch.workspace = true -deno_webidl.workspace = true -deno_web.workspace = true -deno_net.workspace = true -deno_console.workspace = true -deno_url.workspace = true -deno_core.workspace = true -deno_ast.workspace = true -deno_tls.workspace = true +deno_fetch = { workspace = true, optional = true } +deno_webidl = { workspace = true, optional = true } +deno_web = { workspace = true, optional = true } +deno_net = { workspace = true, optional = true } +deno_console = { workspace = true, optional = true } +deno_url = { workspace = true, optional = true } +deno_core = { workspace = true, optional = true } +deno_ast = { workspace = true, optional = true } +deno_tls = { workspace = true, optional = true } postgres-native-tls.workspace = true native-tls.workspace = true mysql_async.workspace = true @@ -89,13 +90,17 @@ tar.workspace = true object_store = { workspace = true, optional = true} convert_case.workspace = true yaml-rust.workspace = true +swc_ecma_parser.workspace = true + [build-dependencies] -deno_fetch.workspace = true -deno_webidl.workspace = true -deno_web.workspace = true -deno_console.workspace = true -deno_url.workspace = true -deno_core.workspace = true -deno_net.workspace = true +deno_fetch = { workspace = true, optional = true } +deno_webidl = { workspace = true, optional = true } +deno_web = { workspace = true, optional = true } +deno_net = { workspace = true, optional = true } +deno_console = { workspace = true, optional = true } +deno_url = { workspace = true, optional = true } +deno_core = { workspace = true, optional = true } +deno_ast = { workspace = true, optional = true } +deno_tls = { workspace = true, optional = true } zstd.workspace = true diff --git a/backend/windmill-worker/build.rs b/backend/windmill-worker/build.rs index 8fa581fd5c..66bc66959f 100644 --- a/backend/windmill-worker/build.rs +++ b/backend/windmill-worker/build.rs @@ -1,13 +1,22 @@ +#[cfg(feature = "deno_core")] use deno_fetch::FetchPermissions; +#[cfg(feature = "deno_core")] use deno_net::NetPermissions; +#[cfg(feature = "deno_core")] use deno_web::{BlobStore, TimersPermission}; +#[cfg(feature = "deno_core")] use std::env; +#[cfg(feature = "deno_core")] use std::io::Write; +#[cfg(feature = "deno_core")] use std::path::PathBuf; +#[cfg(feature = "deno_core")] use std::sync::Arc; +#[cfg(feature = "deno_core")] pub struct PermissionsContainer; +#[cfg(feature = "deno_core")] impl FetchPermissions for PermissionsContainer { #[inline(always)] fn check_net_url( @@ -28,6 +37,7 @@ impl FetchPermissions for PermissionsContainer { } } +#[cfg(feature = "deno_core")] impl TimersPermission for PermissionsContainer { #[inline(always)] fn allow_hrtime(&mut self) -> bool { @@ -35,6 +45,7 @@ impl TimersPermission for PermissionsContainer { } } +#[cfg(feature = "deno_core")] impl NetPermissions for PermissionsContainer { fn check_read( &mut self, @@ -61,12 +72,14 @@ impl NetPermissions for PermissionsContainer { } } +#[cfg(feature = "deno_core")] deno_core::extension!( fetch, esm_entry_point = "ext:fetch/src/runtime.js", esm = ["src/runtime.js"], ); +#[cfg(feature = "deno_core")] fn main() { println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap()); println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap()); @@ -122,3 +135,6 @@ fn main() { println!("cargo:rerun-if-changed={}", path.display()); } } + +#[cfg(not(feature = "deno_core"))] +fn main() {} diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index e17f140b32..b63ae6c001 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -1,8 +1,14 @@ -use std::{collections::HashMap, fs, io, path::Path, process::Stdio, time::Instant}; +#[cfg(feature = "deno_core")] +use std::time::Instant; +use std::{collections::HashMap, fs, io, path::Path, process::Stdio}; use base64::Engine; use itertools::Itertools; + +#[cfg(not(feature = "deno_core"))] +use serde_json::value::to_raw_value; use serde_json::value::RawValue; + use sha2::Digest; use uuid::Uuid; use windmill_parser_ts::remove_pinned_imports; @@ -1190,51 +1196,58 @@ try {{ } } if annotation.native_mode { - let env_code = format!( + #[cfg(not(feature = "deno_core"))] + return Ok(to_raw_value("").unwrap()); + + #[cfg(feature = "deno_core")] + { + let env_code = format!( "const process = {{ env: {{}} }};\nconst BASE_URL = '{base_internal_url}';\nconst BASE_INTERNAL_URL = '{base_internal_url}';\nprocess.env['BASE_URL'] = BASE_URL;process.env['BASE_INTERNAL_URL'] = BASE_INTERNAL_URL;\n{}", reserved_variables .iter() .map(|(k, v)| format!("process.env['{}'] = '{}';\n", k, v)) .collect::>() .join("\n")); - let js_code = read_file_content(&format!("{job_dir}/main.js")).await?; - let started_at = Instant::now(); - let args = crate::common::build_args_map(job, client, db) - .await? - .map(sqlx::types::Json); - let job_args = if args.is_some() { - args.as_ref() - } else { - job.args.as_ref() - }; - let result = crate::js_eval::eval_fetch_timeout( - env_code, - inner_content.clone(), - js_code, - job_args, - job.id, - job.timeout, - db, - mem_peak, - canceled_by, - worker_name, - &job.workspace_id, - false, - occupancy_metrics, - ) - .await?; - tracing::info!( - "Executed native code in {}ms", - started_at.elapsed().as_millis() - ); - append_logs( - &job.id, - &job.workspace_id, - format!("{}\n{}", init_logs, result.1), - db, - ) - .await; - return Ok(result.0); + let js_code = read_file_content(&format!("{job_dir}/main.js")).await?; + let started_at = Instant::now(); + let args = crate::common::build_args_map(job, client, db) + .await? + .map(sqlx::types::Json); + let job_args = if args.is_some() { + args.as_ref() + } else { + job.args.as_ref() + }; + + let result = crate::js_eval::eval_fetch_timeout( + env_code, + inner_content.clone(), + js_code, + job_args, + job.id, + job.timeout, + db, + mem_peak, + canceled_by, + worker_name, + &job.workspace_id, + false, + occupancy_metrics, + ) + .await?; + tracing::info!( + "Executed native code in {}ms", + started_at.elapsed().as_millis() + ); + append_logs( + &job.id, + &job.workspace_id, + format!("{}\n{}", init_logs, result.1), + db, + ) + .await; + return Ok(result.0); + } } append_logs(&job.id, &job.workspace_id, init_logs, db).await; diff --git a/backend/windmill-worker/src/job_logger.rs b/backend/windmill-worker/src/job_logger.rs index 5193fe79aa..05b9e5812b 100644 --- a/backend/windmill-worker/src/job_logger.rs +++ b/backend/windmill-worker/src/job_logger.rs @@ -1,5 +1,5 @@ -use deno_ast::swc::parser::lexer::util::CharExt; use itertools::Itertools; +use swc_ecma_parser::lexer::util::CharExt; #[cfg(all(feature = "enterprise", feature = "parquet"))] use object_store::path::Path; diff --git a/backend/windmill-worker/src/js_eval.rs b/backend/windmill-worker/src/js_eval.rs index 24f83c86af..23029202e5 100644 --- a/backend/windmill-worker/src/js_eval.rs +++ b/backend/windmill-worker/src/js_eval.rs @@ -6,56 +6,72 @@ * LICENSE-AGPL for a copy of the license. */ +#[cfg(feature = "deno_core")] use std::{ cell::RefCell, - collections::HashMap, env, io::{self, BufReader}, rc::Rc, - sync::Arc, }; +use std::{collections::HashMap, sync::Arc}; + +#[cfg(feature = "deno_core")] use deno_ast::ParseParams; +#[cfg(feature = "deno_core")] use deno_core::{ error::AnyError, op2, serde_v8, url, v8::{self, IsolateHandle}, Extension, JsRuntime, OpState, PollEventLoopOptions, RuntimeOptions, }; +#[cfg(feature = "deno_core")] use deno_fetch::FetchPermissions; +#[cfg(feature = "deno_core")] use deno_net::NetPermissions; +#[cfg(feature = "deno_core")] use deno_tls::{rustls::RootCertStore, rustls_pemfile}; +#[cfg(feature = "deno_core")] use deno_web::{BlobStore, TimersPermission}; +#[cfg(feature = "deno_core")] use itertools::Itertools; use lazy_static::lazy_static; use regex::Regex; use serde_json::value::RawValue; use sqlx::types::Json; + +#[cfg(feature = "deno_core")] use tokio::{ sync::{mpsc, oneshot}, time::timeout, }; use uuid::Uuid; -use windmill_common::{error::Error, flow_status::JobResult, DB}; + +#[cfg(feature = "deno_core")] +use windmill_common::error::Error; + +use windmill_common::{flow_status::JobResult, DB}; use windmill_queue::CanceledBy; -use crate::{ - common::{unsafe_raw, OccupancyMetrics}, - handle_child::run_future_with_polling_update_job_poller, - AuthedClient, -}; +use crate::{common::OccupancyMetrics, AuthedClient}; + +#[cfg(feature = "deno_core")] +use crate::{common::unsafe_raw, handle_child::run_future_with_polling_update_job_poller}; #[derive(Debug, Clone)] pub struct IdContext { pub flow_job: Uuid, + #[allow(dead_code)] pub steps_results: HashMap, pub previous_id: String, } +#[cfg(feature = "deno_core")] pub struct ContainerRootCertStoreProvider { root_cert_store: RootCertStore, } +#[cfg(feature = "deno_core")] impl ContainerRootCertStoreProvider { fn new() -> ContainerRootCertStoreProvider { return ContainerRootCertStoreProvider { @@ -73,14 +89,17 @@ impl ContainerRootCertStoreProvider { } } +#[cfg(feature = "deno_core")] impl deno_tls::RootCertStoreProvider for ContainerRootCertStoreProvider { fn get_or_try_init(&self) -> Result<&RootCertStore, AnyError> { Ok(&self.root_cert_store) } } +#[cfg(feature = "deno_core")] pub struct PermissionsContainer; +#[cfg(feature = "deno_core")] impl FetchPermissions for PermissionsContainer { #[inline(always)] fn check_net_url( @@ -101,6 +120,7 @@ impl FetchPermissions for PermissionsContainer { } } +#[cfg(feature = "deno_core")] impl TimersPermission for PermissionsContainer { #[inline(always)] fn allow_hrtime(&mut self) -> bool { @@ -108,6 +128,7 @@ impl TimersPermission for PermissionsContainer { } } +#[cfg(feature = "deno_core")] impl NetPermissions for PermissionsContainer { fn check_read( &mut self, @@ -134,6 +155,7 @@ impl NetPermissions for PermissionsContainer { } } +#[cfg(feature = "deno_core")] pub struct OptAuthedClient(Option); pub async fn eval_timeout( @@ -142,7 +164,7 @@ pub async fn eval_timeout( flow_input: Option>>>, authed_client: Option<&AuthedClient>, by_id: Option, - ctx: Option>, + #[allow(unused_variables)] ctx: Option>, ) -> anyhow::Result> { let expr = expr.trim().to_string(); @@ -212,121 +234,133 @@ pub async fn eval_timeout( } } - let expr2 = expr.clone(); - let (sender, mut receiver) = oneshot::channel::(); - let has_client = authed_client.is_some(); - let authed_client = authed_client.cloned(); - timeout( - std::time::Duration::from_millis(10000), - tokio::task::spawn_blocking(move || { - let mut ops = vec![op_get_context()]; + #[cfg(not(feature = "deno_core"))] + { + #[allow(unreachable_code)] + return todo!(); + } - if authed_client.is_some() { - ops.extend([ - // An op for summing an array of numbers - // The op-layer automatically deserializes inputs - // and serializes the returned Result & value - op_variable(), - op_resource(), - ]) - } + #[cfg(feature = "deno_core")] + { + let expr2 = expr.clone(); + let (sender, mut receiver) = oneshot::channel::(); + let has_client = authed_client.is_some(); + let authed_client = authed_client.cloned(); + return timeout( + std::time::Duration::from_millis(10000), + tokio::task::spawn_blocking(move || { + let mut ops = vec![op_get_context()]; - if by_id.is_some() && authed_client.is_some() { - ops.push(op_get_result()); - ops.push(op_get_id()); - } - - let ext = Extension { name: "js_eval", ops: ops.into(), ..Default::default() }; - let exts = vec![ext]; - // Use our snapshot to provision our new runtime - let options = RuntimeOptions { - extensions: exts, - // startup_snapshot: Some(Snapshot::Static(buffer)), - ..Default::default() - }; - - let mut context_keys = transform_context - .keys() - .filter(|x| expr.contains(&x.to_string())) - .map(|x| x.clone()) - .collect_vec(); - - if !context_keys.contains(&"previous_result".to_string()) - && (p_ids.is_some() && p_ids.as_ref().unwrap().iter().any(|x| expr.contains(x))) - || expr.contains("error") - { - // tracing::error!("PREVIOUS_RESULT"); - context_keys.push("previous_result".to_string()); - } - let has_flow_input = expr.contains("flow_input"); - if has_flow_input { - context_keys.push("flow_input".to_string()) - } - - let mut js_runtime = JsRuntime::new(options); - { - let op_state = js_runtime.op_state(); - let mut op_state = op_state.borrow_mut(); - let mut client = authed_client.clone(); - if let Some(client) = client.as_mut() { - client.force_client = Some( - reqwest::ClientBuilder::new() - .user_agent("windmill/beta") - .danger_accept_invalid_certs( - std::env::var("ACCEPT_INVALID_CERTS").is_ok(), - ) - .build() - .unwrap(), - ); + if authed_client.is_some() { + ops.extend([ + // An op for summing an array of numbers + // The op-layer automatically deserializes inputs + // and serializes the returned Result & value + op_variable(), + op_resource(), + ]) } - op_state.put(OptAuthedClient(client)); - op_state.put(TransformContext { - flow_input: if has_flow_input { flow_input } else { None }, - envs: transform_context - .into_iter() - .filter(|(a, _)| context_keys.contains(a)) - .collect(), - }) - } - sender - .send(js_runtime.v8_isolate().thread_safe_handle()) - .map_err(|_| Error::ExecutionErr("impossible to send v8 isolate".to_string()))?; + if by_id.is_some() && authed_client.is_some() { + ops.push(op_get_result()); + ops.push(op_get_id()); + } - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build()?; + let ext = Extension { name: "js_eval", ops: ops.into(), ..Default::default() }; + let exts = vec![ext]; + // Use our snapshot to provision our new runtime + let options = RuntimeOptions { + extensions: exts, + // startup_snapshot: Some(Snapshot::Static(buffer)), + ..Default::default() + }; - // pretty frail but this it to make the expr more user friendly and not require the user to write await - let expr = ["variable", "resource"] - .into_iter() - .fold(expr, replace_with_await); + let mut context_keys = transform_context + .keys() + .filter(|x| expr.contains(&x.to_string())) + .map(|x| x.clone()) + .collect_vec(); - let expr = replace_with_await_result(expr); + if !context_keys.contains(&"previous_result".to_string()) + && (p_ids.is_some() && p_ids.as_ref().unwrap().iter().any(|x| expr.contains(x))) + || expr.contains("error") + { + // tracing::error!("PREVIOUS_RESULT"); + context_keys.push("previous_result".to_string()); + } + let has_flow_input = expr.contains("flow_input"); + if has_flow_input { + context_keys.push("flow_input".to_string()) + } - let r = runtime.block_on(eval( - &mut js_runtime, - &expr, - context_keys, - by_id, - has_client, - ctx, - ))?; + let mut js_runtime = JsRuntime::new(options); + { + let op_state = js_runtime.op_state(); + let mut op_state = op_state.borrow_mut(); + let mut client = authed_client.clone(); + if let Some(client) = client.as_mut() { + client.force_client = Some( + reqwest::ClientBuilder::new() + .user_agent("windmill/beta") + .danger_accept_invalid_certs( + std::env::var("ACCEPT_INVALID_CERTS").is_ok(), + ) + .build() + .unwrap(), + ); + } + op_state.put(OptAuthedClient(client)); + op_state.put(TransformContext { + flow_input: if has_flow_input { flow_input } else { None }, + envs: transform_context + .into_iter() + .filter(|(a, _)| context_keys.contains(a)) + .collect(), + }) + } - Ok(r) as anyhow::Result> - }), - ) - .await - .map_err(|_| { - if let Ok(isolate) = receiver.try_recv() { - isolate.terminate_execution(); - }; - Error::ExecutionErr(format!( - "The expression of evaluation `{expr2}` took too long to execute (>10000ms)" - )) - })?? + sender + .send(js_runtime.v8_isolate().thread_safe_handle()) + .map_err(|_| { + Error::ExecutionErr("impossible to send v8 isolate".to_string()) + })?; + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + + // pretty frail but this it to make the expr more user friendly and not require the user to write await + let expr = ["variable", "resource"] + .into_iter() + .fold(expr, replace_with_await); + + let expr = replace_with_await_result(expr); + + let r = runtime.block_on(eval( + &mut js_runtime, + &expr, + context_keys, + by_id, + has_client, + ctx, + ))?; + + Ok(r) as anyhow::Result> + }), + ) + .await + .map_err(|_| { + if let Ok(isolate) = receiver.try_recv() { + isolate.terminate_execution(); + }; + Error::ExecutionErr(format!( + "The expression of evaluation `{expr2}` took too long to execute (>10000ms)" + )) + })??; + } } +#[cfg(feature = "deno_core")] fn replace_with_await(expr: String, fn_name: &str) -> String { let sep = format!("{}(", fn_name); let mut split = expr.split(&sep); @@ -345,10 +379,12 @@ lazy_static! { Regex::new(r"^(https?)://(([^:@\s]+):([^:@\s]+)@)?([^:@\s]+)(:(\d+))?$").unwrap(); } +#[cfg(feature = "deno_core")] fn replace_with_await_result(expr: String) -> String { RE.replace_all(&expr, "(await $r)").to_string() } +#[cfg(feature = "deno_core")] fn add_closing_bracket(s: &str) -> String { let mut s = s.to_string(); let mut level = 1; @@ -368,6 +404,7 @@ fn add_closing_bracket(s: &str) -> String { s } +#[cfg(feature = "deno_core")] async fn eval( context: &mut JsRuntime, expr: &str, @@ -508,6 +545,7 @@ function get_from_env(name) {{ // } // TODO: Can we a) share the api configuration here somehow or b) just implement this natively in deno, via the deno client? +#[cfg(feature = "deno_core")] #[op2(async)] #[string] async fn op_variable( @@ -522,6 +560,7 @@ async fn op_variable( } } +#[cfg(feature = "deno_core")] #[op2(async)] #[string] async fn op_get_result( @@ -540,6 +579,7 @@ async fn op_get_result( } } +#[cfg(feature = "deno_core")] #[op2(async)] #[string] async fn op_get_id( @@ -563,6 +603,7 @@ async fn op_get_id( } } +#[cfg(feature = "deno_core")] #[op2(async)] #[string] async fn op_resource( @@ -580,11 +621,13 @@ async fn op_resource( } } +#[cfg(feature = "deno_core")] pub struct TransformContext { pub envs: HashMap>>, pub flow_input: Option>>>, } +#[cfg(feature = "deno_core")] #[op2] #[string] fn op_get_context(op_state: Rc>, #[string] id: &str) -> String { @@ -605,6 +648,7 @@ fn op_get_context(op_state: Rc>, #[string] id: &str) -> String } } +#[cfg(feature = "deno_core")] pub fn transpile_ts(expr: String) -> anyhow::Result { let parsed = deno_ast::parse_module(ParseParams { specifier: url::Url::parse("file:///eval.ts")?, @@ -621,21 +665,30 @@ pub fn transpile_ts(expr: String) -> anyhow::Result { .text) } +#[cfg(not(feature = "deno_core"))] +pub fn transpile_ts(_expr: String) -> anyhow::Result { + Ok("require deno".to_string()) +} + +#[cfg(feature = "deno_core")] static RUNTIME_SNAPSHOT: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/FETCH_SNAPSHOT.bin")); +#[cfg(feature = "deno_core")] pub struct MainArgs { args: Vec>>, } +#[cfg(feature = "deno_core")] pub struct LogString { pub s: String, } +#[cfg(feature = "deno_core")] pub struct NativeAnnotation { pub useragent: Option, pub proxy: Option<(String, Option<(String, String)>)>, } - +#[cfg(feature = "deno_core")] pub fn get_annotation(inner_content: &str) -> NativeAnnotation { let mut res = NativeAnnotation { useragent: None, proxy: None }; @@ -655,6 +708,7 @@ pub fn get_annotation(inner_content: &str) -> NativeAnnotation { res } +#[cfg(feature = "deno_core")] fn capture_proxy(s: &str) -> Option<(String, Option<(String, String)>)> { RE_PROXY.captures(s).map(|x| { ( @@ -675,7 +729,27 @@ fn capture_proxy(s: &str) -> Option<(String, Option<(String, String)>)> { ) }) } +#[cfg(not(feature = "deno_core"))] +pub async fn eval_fetch_timeout( + _env_code: String, + _ts_expr: String, + _js_expr: String, + _args: Option<&Json>>>, + _job_id: Uuid, + _job_timeout: Option, + _db: &DB, + _mem_peak: &mut i32, + _canceled_by: &mut Option, + _worker_name: &str, + _w_id: &str, + _load_client: bool, + _occupation_metrics: &mut OccupancyMetrics, +) -> anyhow::Result<(Box, String)> { + use serde_json::value::to_raw_value; + Ok((to_raw_value("require deno_core").unwrap(), "".to_string())) +} +#[cfg(feature = "deno_core")] pub async fn eval_fetch_timeout( env_code: String, ts_expr: String, @@ -851,8 +925,10 @@ pub async fn eval_fetch_timeout( Ok((res, format!("{extra_logs}{logs}"))) } +#[cfg(feature = "deno_core")] const WINDMILL_CLIENT: &str = include_str!("./windmill-client.js"); +#[cfg(feature = "deno_core")] async fn eval_fetch( js_runtime: &mut JsRuntime, expr: &str, @@ -898,6 +974,7 @@ import("file:///eval.ts").then((module) => module.main(...args)).then(JSON.strin Ok(unsafe_raw(r.unwrap_or_else(|| "null".to_string()))) } +#[cfg(feature = "deno_core")] #[op2] #[serde] fn op_get_static_args(op_state: Rc>) -> Vec> { @@ -910,6 +987,7 @@ fn op_get_static_args(op_state: Rc>) -> Vec> { .collect_vec() } +#[cfg(feature = "deno_core")] #[op2(fast)] fn op_log(op_state: Rc>, #[string] log: &str) { // tracing::error!("log: |{}|", log); @@ -920,6 +998,7 @@ fn op_log(op_state: Rc>, #[string] log: &str) { .push_str(log); } +#[cfg(feature = "deno_core")] #[cfg(test)] mod tests { diff --git a/cli/instance.ts b/cli/instance.ts index cf2d5f5b02..2354acdec4 100644 --- a/cli/instance.ts +++ b/cli/instance.ts @@ -183,10 +183,13 @@ export type InstanceSyncOptions = { yes?: boolean; }; -export async function pickInstance(opts: InstanceSyncOptions, allowNew: boolean) { +export async function pickInstance( + opts: InstanceSyncOptions, + allowNew: boolean +) { const instances = await allInstances(); if (opts.baseUrl && opts.token) { - log.info("Using instance fully defined by --base-url and --token") + log.info("Using instance fully defined by --base-url and --token"); return { name: "custom", remote: opts.baseUrl, @@ -335,12 +338,14 @@ async function instancePull(opts: GlobalOptions & InstanceSyncOptions) { ); if (localWorkspacesToDelete.length > 0) { - const confirmDelete = await Confirm.prompt({ - message: - "Do you want to delete the local copy of workspaces that don't exist anymore on the instance?\n" + - localWorkspacesToDelete.map((w) => w.workspaceId).join(", "), - default: true, - }); + const confirmDelete = + opts.yes || + (await Confirm.prompt({ + message: + "Do you want to delete the local copy of workspaces that don't exist anymore on the instance?\n" + + localWorkspacesToDelete.map((w) => w.workspaceId).join(", "), + default: true, + })); if (confirmDelete) { for (const workspace of localWorkspacesToDelete) { @@ -485,12 +490,14 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) { (w) => !localWorkspaces.find((l) => l.workspaceId === w.id) ); if (workspacesToDelete.length > 0) { - const confirmDelete = await Confirm.prompt({ - message: - "Do you want to delete the following remote workspaces that don't exist locally?\n" + - workspacesToDelete.map((w) => w.id).join(", "), - default: true, - }); + const confirmDelete = + opts.yes || + (await Confirm.prompt({ + message: + "Do you want to delete the following remote workspaces that don't exist locally?\n" + + workspacesToDelete.map((w) => w.id).join(", "), + default: true, + })); if (confirmDelete) { for (const workspace of workspacesToDelete) { @@ -544,7 +551,9 @@ async function whoami(opts: {}) { log.info(colors.green.underline(`global whoami infos:`)); log.info(JSON.stringify(whoamiInfo, null, 2)); } catch (error) { - log.error(colors.red(`Failed to retrieve whoami information: ${error.message}`)); + log.error( + colors.red(`Failed to retrieve whoami information: ${error.message}`) + ); } } From 48a85e1732e52f76b4e2cb65ff7acd214624501e Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 3 Oct 2024 13:27:37 +0200 Subject: [PATCH 11/38] remove pg-embed --- backend/Cargo.toml | 2 -- backend/src/main.rs | 10 --------- backend/src/pg_embed.rs | 46 ----------------------------------------- 3 files changed, 58 deletions(-) delete mode 100644 backend/src/pg_embed.rs diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 1537837c3f..632f7d3b22 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -47,7 +47,6 @@ stripe = ["windmill-api/stripe"] benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark", "windmill-common/benchmark"] flamegraph = ["windmill-common/flamegraph", "windmill-worker/flamegraph"] loki = ["windmill-common/loki"] -pg_embed = ["dep:pg-embed"] embedding = ["windmill-api/embedding"] parquet = ["windmill-api/parquet", "windmill-common/parquet", "windmill-worker/parquet", "windmill-indexer/parquet", "dep:object_store"] prometheus = ["windmill-common/prometheus", "windmill-api/prometheus", "windmill-worker/prometheus", "windmill-queue/prometheus"] @@ -88,7 +87,6 @@ serde_json.workspace = true serde.workspace = true deno_core = { workspace = true, optional = true } object_store = { workspace = true, optional = true } -pg-embed = {git = "https://github.com/faokunega/pg-embed", optional = true, default-features = false, features = ['rt_tokio']} quote.workspace = true diff --git a/backend/src/main.rs b/backend/src/main.rs index 3d97e5f1f7..0c06702e4c 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -92,9 +92,6 @@ const DEFAULT_SERVER_BIND_ADDR: Ipv4Addr = Ipv4Addr::new(0, 0, 0, 0); mod ee; mod monitor; -#[cfg(feature = "pg_embed")] -mod pg_embed; - #[inline(always)] fn create_and_run_current_thread_inner(future: F) -> R where @@ -342,13 +339,6 @@ async fn windmill_main() -> anyhow::Result<()> { config }); - #[cfg(feature = "pg_embed")] - let _pg = { - let (db_url, pg) = pg_embed::start().await.expect("pg embed"); - tracing::info!("Use embedded pg: {db_url}"); - std::env::set_var("DATABASE_URL", db_url); - pg - }; tracing::info!("Connecting to database..."); let db = windmill_common::connect_db(server_mode, indexer_mode).await?; diff --git a/backend/src/pg_embed.rs b/backend/src/pg_embed.rs deleted file mode 100644 index ad5529dc02..0000000000 --- a/backend/src/pg_embed.rs +++ /dev/null @@ -1,46 +0,0 @@ -use pg_embed::pg_enums::PgAuthMethod; -use pg_embed::pg_fetch::PgFetchSettings; -use pg_embed::postgres::{PgEmbed, PgSettings}; -use std::path::PathBuf; -use std::time::Duration; - -pub async fn start() -> anyhow::Result<(String, PgEmbed)> { - let pg_settings = PgSettings { - database_dir: PathBuf::from("/tmp/db"), - port: 6543, - user: "postgres".to_string(), - password: "password".to_string(), - auth_method: PgAuthMethod::Plain, - persistent: false, - timeout: Some(Duration::from_secs(15)), - migration_dir: None, - }; - - let fetch_settings = PgFetchSettings { - version: pg_embed::pg_fetch::PostgresVersion("15.3.0"), - - ..Default::default() - }; - - tracing::info!( - "Fetch settings: {:?} {:?}", - fetch_settings.operating_system, - fetch_settings.architecture - ); - - let mut pg = PgEmbed::new(pg_settings, fetch_settings).await?; - - pg.setup().await.expect("pg setup"); - - pg.start_db().await.expect("pg start db"); - - //TODO: re-enable this to make it work - // if !pg.database_exists("windmill").await.expect("db exists") { - // pg.create_database("windmill") - // .await - // .expect("pg create database"); - // } - - let uri = pg.full_db_uri("windmill"); - Ok((uri, pg)) -} From 45ccd45e306c66931880a9b8fd48bfe684c774ac Mon Sep 17 00:00:00 2001 From: Guilhem Date: Thu, 3 Oct 2024 14:15:01 +0200 Subject: [PATCH 12/38] feat(frontend): add quick access menu in flow editor (#4415) * (frontend) add quick access menu in flow editor * (frontend) add quick access menu in flow editor * improve UI * make design prettier * add scroll effects * improve loading preview * change no items found * prevent scroll using menu * change user folder button * set default integration icon * reduce column width * ajust font * add defaults script button * add shadow divider * fix scroll * add chevron * Change toogle bar * Add preprocessor menu * add handler * simplify scroll * fix display * fix minor issues * delete useless log * revert node tree changes * merge main * fix z-index issues * iterate * fix: improve allowed domains setting for sso * chore(main): release 1.402.3 (#4458) * chore(main): release 1.402.3 * Apply automatic changes --------- Co-authored-by: rubenfiszel * improve allowed domains change handling * send stats when renewing key if last >24h (#4430) * feat: send stats when renewing key if last >24h * nits * fix: sqlx * nit * renewal reason * stats reason * update ee ref * Update ee-repo-ref.txt --------- Co-authored-by: Ruben Fiszel * fix: skip one migration to avoid using md5 for azure support * all * all * Apply automatic changes * all * done? * nit * nit * nit * nit * nit noAi if prefilter is not all * fix shadow * fix error handler * fix error handler * Polishing default script settings * all * all * full * all * add deno_core as features * all * remove warnings * all * npm check * npm check * new script script * nits * nits item 0 * nits item 0 --------- Co-authored-by: Guilhem Le Mouel Co-authored-by: Ruben Fiszel Co-authored-by: Ruben Fiszel Co-authored-by: rubenfiszel Co-authored-by: HugoCasa Co-authored-by: Guilhem --- ...40930183601_add_preprocessor_kind.down.sql | 1 + ...0240930183601_add_preprocessor_kind.up.sql | 2 + frontend/package.json | 8 + .../src/lib/components/DefaultScripts.svelte | 12 +- .../lib/components/DefaultScriptsInner.svelte | 23 +- frontend/src/lib/components/Dev.svelte | 3 +- .../src/lib/components/FlowBuilder.svelte | 33 +- frontend/src/lib/components/Scrollable.svelte | 57 +++ .../components/ToggleHubWorkspaceQuick.svelte | 15 + .../lib/components/common/menu/Menu.svelte | 3 +- .../lib/components/common/popup/Popup.svelte | 35 +- .../components/common/popup/PopupV2.svelte | 63 +++ .../lib/components/copilot/RegexGen.svelte | 6 +- .../src/lib/components/copilot/StepGen.svelte | 12 +- .../components/copilot/StepGenQuick.svelte | 67 +++ frontend/src/lib/components/copilot/flow.ts | 69 ++- .../lib/components/flows/FlowEditor.svelte | 3 +- .../flows/content/FlowInputsFlowQuick.svelte | 73 +++ .../flows/content/FlowInputsQuick.svelte | 455 ++++++++++++++++++ .../flows/content/GenAiQuick.svelte | 33 ++ .../flows/map/FlowErrorHandlerItem.svelte | 139 ++++-- .../flows/map/FlowModuleSchemaMap.svelte | 76 ++- .../flows/map/InsertModuleButton.svelte | 307 ++++++------ .../flows/map/InsertTriggerButton.svelte | 52 -- .../pickers/FlowScriptPickerQuick.svelte | 56 +++ .../flows/pickers/FlowToplevelNode.svelte | 22 + .../flows/pickers/PickHubScriptQuick.svelte | 175 +++++++ .../flows/pickers/TopLevelNode.svelte | 60 +++ .../pickers/WorkspaceScriptPickerQuick.svelte | 136 ++++++ frontend/src/lib/components/flows/types.ts | 1 + .../lib/components/graph/FlowGraphV2.svelte | 5 +- .../src/lib/components/graph/graphBuilder.ts | 1 + .../graph/renderers/edges/BaseEdge.svelte | 53 +- .../graph/renderers/nodes/InputNode.svelte | 49 +- .../components/home/ListFiltersQuick.svelte | 54 +++ .../lib/components/icons/WindmillIcon2.svelte | 154 ++++++ frontend/src/lib/components/icons/index.ts | 2 + frontend/src/lib/script_helpers.ts | 28 +- frontend/src/routes/flows/dev/+page.svelte | 3 +- 39 files changed, 2000 insertions(+), 346 deletions(-) create mode 100644 backend/migrations/20240930183601_add_preprocessor_kind.down.sql create mode 100644 backend/migrations/20240930183601_add_preprocessor_kind.up.sql create mode 100644 frontend/src/lib/components/Scrollable.svelte create mode 100644 frontend/src/lib/components/ToggleHubWorkspaceQuick.svelte create mode 100644 frontend/src/lib/components/common/popup/PopupV2.svelte create mode 100644 frontend/src/lib/components/copilot/StepGenQuick.svelte create mode 100644 frontend/src/lib/components/flows/content/FlowInputsFlowQuick.svelte create mode 100644 frontend/src/lib/components/flows/content/FlowInputsQuick.svelte create mode 100644 frontend/src/lib/components/flows/content/GenAiQuick.svelte delete mode 100644 frontend/src/lib/components/flows/map/InsertTriggerButton.svelte create mode 100644 frontend/src/lib/components/flows/pickers/FlowScriptPickerQuick.svelte create mode 100644 frontend/src/lib/components/flows/pickers/FlowToplevelNode.svelte create mode 100644 frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte create mode 100644 frontend/src/lib/components/flows/pickers/TopLevelNode.svelte create mode 100644 frontend/src/lib/components/flows/pickers/WorkspaceScriptPickerQuick.svelte create mode 100644 frontend/src/lib/components/home/ListFiltersQuick.svelte create mode 100644 frontend/src/lib/components/icons/WindmillIcon2.svelte diff --git a/backend/migrations/20240930183601_add_preprocessor_kind.down.sql b/backend/migrations/20240930183601_add_preprocessor_kind.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20240930183601_add_preprocessor_kind.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20240930183601_add_preprocessor_kind.up.sql b/backend/migrations/20240930183601_add_preprocessor_kind.up.sql new file mode 100644 index 0000000000..d70dc181aa --- /dev/null +++ b/backend/migrations/20240930183601_add_preprocessor_kind.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +ALTER TYPE SCRIPT_KIND ADD VALUE IF NOT EXISTS 'preprocessor'; \ No newline at end of file diff --git a/frontend/package.json b/frontend/package.json index 61554a5db8..b6ce8cd73c 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -177,6 +177,11 @@ "svelte": "./package/components/icons/WindmillIcon.svelte", "default": "./package/components/icons/WindmillIcon.svelte" }, + "./components/icons/WindmillIcon2.svelte": { + "types": "./package/components/icons/WindmillIcon2.d.ts", + "svelte": "./package/components/icons/WindmillIcon2.svelte", + "default": "./package/components/icons/WindmillIcon2.svelte" + }, "./components/IconedResourceType.svelte": { "types": "./package/components/IconedResourceType.svelte.d.ts", "svelte": "./package/components/IconedResourceType.svelte", @@ -358,6 +363,9 @@ "components/icons/WindmillIcon.svelte": [ "./package/components/icons/WindmillIcon.svelte.d.ts" ], + "components/icons/WindmillIcon2.svelte": [ + "./package/components/icons/WindmillIcon2.svelte.d.ts" + ], "components/scriptEditor/LogPanel.svelte": [ "./package/components/scriptEditor/LogPanel.svelte.d.ts" ], diff --git a/frontend/src/lib/components/DefaultScripts.svelte b/frontend/src/lib/components/DefaultScripts.svelte index 4c6383c3fb..c53c590954 100644 --- a/frontend/src/lib/components/DefaultScripts.svelte +++ b/frontend/src/lib/components/DefaultScripts.svelte @@ -7,10 +7,14 @@ import DefaultScriptsInner from './DefaultScriptsInner.svelte' let drawer: Drawer + export let placement: 'left' | 'right' = 'left' + + export let size: 'xs3' | 'xs2' = 'xs2' + export let noText = false {#if $userStore?.is_admin || $userStore?.is_super_admin} - + @@ -19,8 +23,10 @@ on:click={drawer?.openDrawer} startIcon={{ icon: SettingsIcon }} color="light" - size="xs2" + {size} btnClasses="!text-tertiary" - variant="contained">defaults + {noText ? '' : 'defaults'} + {/if} diff --git a/frontend/src/lib/components/DefaultScriptsInner.svelte b/frontend/src/lib/components/DefaultScriptsInner.svelte index db0dfa2500..62809c18e3 100644 --- a/frontend/src/lib/components/DefaultScriptsInner.svelte +++ b/frontend/src/lib/components/DefaultScriptsInner.svelte @@ -6,6 +6,7 @@ import { defaultScriptLanguages } from '$lib/scripts' import Alert from './common/alert/Alert.svelte' + export let small = false $: langs = computeLangs($defaultScripts) function computeLangs(defaultScripts: WorkspaceDefaultScripts | undefined): Script['language'][] { @@ -31,24 +32,32 @@ } - + This setting is only available to admins and will affect all users in the workspace. -
+
{#each langs as lang, i (lang)}

{lang}

+ class="w-full p-2 rounded {small + ? '' + : 'border border-secondary'} grid grid-cols-3 items-center" + >

{lang}

{#if i > 0} - {/if} {#if i < langs.length - 1} - changePosition(i ?? 0, false)} + class={small ? 'mr-2 text-secondary text-sm' : 'text-lg mr-2'} + title="Move down">↓ {/if}
diff --git a/frontend/src/lib/components/Dev.svelte b/frontend/src/lib/components/Dev.svelte index 343291adb4..aa6a66b1d4 100644 --- a/frontend/src/lib/components/Dev.svelte +++ b/frontend/src/lib/components/Dev.svelte @@ -496,7 +496,8 @@ saveDraft: () => {}, initialPath: '', flowInputsStore: writable({}), - customUi: {} + customUi: {}, + insertButtonOpen: writable(false) }) $: updateFlow($flowStore) diff --git a/frontend/src/lib/components/FlowBuilder.svelte b/frontend/src/lib/components/FlowBuilder.svelte index 5de1ecf251..f33841a4f6 100644 --- a/frontend/src/lib/components/FlowBuilder.svelte +++ b/frontend/src/lib/components/FlowBuilder.svelte @@ -394,6 +394,7 @@ selectedIdStore.set(selectedId) } + let insertButtonOpen = writable(false) setContext('FlowEditorContext', { selectedId: selectedIdStore, schedule: scheduleStore, @@ -408,7 +409,8 @@ saveDraft, initialPath, flowInputsStore: writable({}), - customUi + customUi, + insertButtonOpen }) async function loadSchedule() { @@ -461,20 +463,24 @@ } break case 'ArrowDown': { - let ids = generateIds() - let idx = ids.indexOf($selectedIdStore) - if (idx > -1 && idx < ids.length - 1) { - $selectedIdStore = ids[idx + 1] - event.preventDefault() + if (!$insertButtonOpen) { + let ids = generateIds() + let idx = ids.indexOf($selectedIdStore) + if (idx > -1 && idx < ids.length - 1) { + $selectedIdStore = ids[idx + 1] + event.preventDefault() + } } break } case 'ArrowUp': { - let ids = generateIds() - let idx = ids.indexOf($selectedIdStore) - if (idx > 0 && idx < ids.length) { - $selectedIdStore = ids[idx - 1] - event.preventDefault() + if (!$insertButtonOpen) { + let ids = generateIds() + let idx = ids.indexOf($selectedIdStore) + if (idx > 0 && idx < ids.length) { + $selectedIdStore = ids[idx - 1] + event.preventDefault() + } } break } @@ -561,6 +567,8 @@ kind: string app: string ask_id: number + id: number + version_id: number }[] } catch (err) { if (err.name !== 'CancelError') throw err @@ -890,7 +898,8 @@ const snakeKey = snakeCase(key) if ( schemaProperty && - (!$flowStore.schema || !(snakeKey in ($flowStore.schema.properties as any) ?? {})) // prevent overriding flow inputs + (!$flowStore.schema || + !(snakeKey in ($flowStore?.schema?.properties ?? ({} as any)))) // prevent overriding flow inputs ) { copilotFlowInputs[snakeKey] = schemaProperty if (schema?.required.includes(snakeKey)) { diff --git a/frontend/src/lib/components/Scrollable.svelte b/frontend/src/lib/components/Scrollable.svelte new file mode 100644 index 0000000000..ff8e370e86 --- /dev/null +++ b/frontend/src/lib/components/Scrollable.svelte @@ -0,0 +1,57 @@ + + +
+
+ +
+ {#if !isAtBottom && isScrollable} +
+ {/if} +
diff --git a/frontend/src/lib/components/ToggleHubWorkspaceQuick.svelte b/frontend/src/lib/components/ToggleHubWorkspaceQuick.svelte new file mode 100644 index 0000000000..c48a33a6b1 --- /dev/null +++ b/frontend/src/lib/components/ToggleHubWorkspaceQuick.svelte @@ -0,0 +1,15 @@ + + +
+ + + + + +
diff --git a/frontend/src/lib/components/common/menu/Menu.svelte b/frontend/src/lib/components/common/menu/Menu.svelte index 173eae2208..92ced7e4be 100644 --- a/frontend/src/lib/components/common/menu/Menu.svelte +++ b/frontend/src/lib/components/common/menu/Menu.svelte @@ -54,7 +54,8 @@ 'bottom-start': 'origin-top-left left-0', 'bottom-end': 'origin-top-right right-0', 'top-start': 'origin-bottom-left left-0 bottom-0', - 'top-end': 'origin-bottom-right right-0 bottom-0' + 'top-end': 'origin-bottom-right right-0 bottom-0', + 'top-center': 'origin-top-left -top-full left-1/2 transform -translate-x-1/2 -translate-y-full' } const dispatch = createEventDispatcher() diff --git a/frontend/src/lib/components/common/popup/Popup.svelte b/frontend/src/lib/components/common/popup/Popup.svelte index e2ccc22b74..eb5c15189f 100644 --- a/frontend/src/lib/components/common/popup/Popup.svelte +++ b/frontend/src/lib/components/common/popup/Popup.svelte @@ -9,12 +9,13 @@ } export let containerClasses: string = 'rounded-lg shadow-md border p-4 bg-surface' - + export let floatingClasses: string = '' const [floatingRef, floatingContent] = createFloatingActions(floatingConfig) export let blockOpen = false export let shouldUsePortal: boolean = true export let target: string | HTMLElement | undefined = undefined + export let noTransition = false @@ -24,22 +25,30 @@
-
- - +
+ {#if !noTransition} + + +
+ +
+
+
+ {:else} +
- + {/if}
diff --git a/frontend/src/lib/components/common/popup/PopupV2.svelte b/frontend/src/lib/components/common/popup/PopupV2.svelte new file mode 100644 index 0000000000..a70bc1c2be --- /dev/null +++ b/frontend/src/lib/components/common/popup/PopupV2.svelte @@ -0,0 +1,63 @@ + + +
+ +
+ + + {#if open} +
+
{ + if (acceptClickoutside) { + acceptClickoutside = false + open = false + } + }} + > + +
+
+ {/if} +
diff --git a/frontend/src/lib/components/copilot/RegexGen.svelte b/frontend/src/lib/components/copilot/RegexGen.svelte index 0817167bf5..b2c2199385 100644 --- a/frontend/src/lib/components/copilot/RegexGen.svelte +++ b/frontend/src/lib/components/copilot/RegexGen.svelte @@ -19,7 +19,7 @@ const dispatch = createEventDispatcher() async function onGenerate() { - if (funcDesc.length <= 0) { + if (funcDesc?.length <= 0) { return } savePrompt() @@ -122,7 +122,7 @@ bind:this={input} bind:value={funcDesc} on:keypress={({ key }) => { - if (key === 'Enter' && funcDesc.length > 0) { + if (key === 'Enter' && funcDesc?.length > 0) { close(input || null) onGenerate() } @@ -139,7 +139,7 @@ close(input || null) onGenerate() }} - disabled={funcDesc.length <= 0} + disabled={funcDesc?.length <= 0} iconOnly startIcon={{ icon: Wand2 }} /> diff --git a/frontend/src/lib/components/copilot/StepGen.svelte b/frontend/src/lib/components/copilot/StepGen.svelte index 8b2aa3605b..f2a7617b70 100644 --- a/frontend/src/lib/components/copilot/StepGen.svelte +++ b/frontend/src/lib/components/copilot/StepGen.svelte @@ -110,7 +110,7 @@ f={(x) => (emptyString(x.summary) ? x.path : x.summary + ' (' + x.path + ')')} /> -
+
{ - if (funcDesc.length > 2) { + if (funcDesc?.length > 2) { getHubCompletions(funcDesc) } else { hubCompletions = [] @@ -126,14 +126,14 @@ }} placeholder="Search {trigger ? 'triggers' : 'scripts'} or AI gen" /> - {#if funcDesc.length === 0} + {#if funcDesc?.length === 0} {/if}
- {#if !disableAi && funcDesc.length > 0} + {#if !disableAi && funcDesc?.length > 0}
{/if} - {#if funcDesc.length > 0 && filteredItems.length > 0} + {#if funcDesc?.length > 0 && filteredItems?.length > 0}

Workspace {trigger ? 'Triggers' : 'Scripts'}

    - {#each filteredItems.slice(0, 3) as item (item.path)} + {#each filteredItems?.slice(0, 3) ?? [] as item (item.path)}
  • +
  • + {/each} +
+ {:else} + {#each Array(10).fill(0) as _} + + {/each} + {/if} +
diff --git a/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte new file mode 100644 index 0000000000..894c8c52fd --- /dev/null +++ b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte @@ -0,0 +1,455 @@ + + + +
+ {#if selectedKind != 'preprocessor'} + + {#if ['script', 'trigger', 'approval', 'preprocessor', 'failure'].includes(selectedKind)} + {#if (preFilter === 'all' && owners.length > 0) || preFilter === 'workspace'} + {#if preFilter !== 'workspace'} +
Workspace Folders
+ {/if} + + {#if owners.length > 0} + {#each owners as owner (owner)} +
+ +
+ {/each} + {:else} +
+ No items found. +
+ {/if} + {/if} + + {#if preFilter === 'hub' || preFilter === 'all'} + {#if preFilter == 'all'} +
Integrations
+ {/if} + + {/if} + {:else if selectedKind === 'flow'} + {#if owners.length > 0} + {#each owners as owner (owner)} +
+ +
+ {/each} + {/if} + {/if} +
+ {/if} + + {#if kind == 'script'} + {#each topLevelNodes as [label, kind], i (label)} + { + dispatch('new', { kind }) + }} + {label} + selected={selectedByKeyboard === i} + /> + {/each} + {/if} + + {#if inlineScripts?.length > 0} +
+
New {selectedKind != 'script' ? selectedKind + ' ' : ''}script
+ {#if $userStore?.is_admin || $userStore?.is_super_admin} + {#if !openScriptSettings} + + {/if} + {/if} +
+ {#if openScriptSettings} +
+ +
+ {/if} + {#each inlineScripts as [label, lang], i (lang)} + { + if (lang == 'docker') { + if (isCloudHosted()) { + sendUserToast( + 'You cannot use Docker scripts on the multi-tenant platform. Use a dedicated instance or self-host windmill instead.', + true, + [ + { + label: 'Learn more', + callback: () => { + window.open('https://www.windmill.dev/docs/advanced/docker', '_blank') + } + } + ] + ) + return + } + } + + dispatch('new', { + kind: 'script', + inlineScript: { + language: lang == 'docker' ? 'bash' : lang, + kind: selectedKind, + subkind: + lang == 'docker' + ? 'docker' + : selectedKind == 'preprocessor' + ? 'preprocessor' + : 'flow', + summary + } + }) + }} + /> + {/each} + {/if} + + {#if !disableAi && funcDesc?.length > 0 && kind != 'failure' && kind != 'preprocessor' && (selectedKind == 'script' || selectedKind == 'trigger') && preFilter == 'all'} +
    +
  • { + lang = 'bun' + onGenerate() + }} + /> +
  • +
  • + { + lang = 'python3' + onGenerate() + }} + /> +
  • +
+ {/if} + + {#if (!selected || selected?.kind === 'owner') && (preFilter === 'workspace' || preFilter === 'all')} + {#if !selected && (preFilter !== 'workspace' || funcDesc?.length > 0)} +
Workspace
+ {/if} + + {/if} + + {#if selectedKind != 'preprocessor' && selectedKind != 'flow'} + {#if (!selected || selected?.kind === 'integrations') && (preFilter === 'hub' || preFilter === 'all')} + {#if !selected && preFilter !== 'hub'} +
Hub
+ {/if} + + {/if} + {/if} +
+
+ + diff --git a/frontend/src/lib/components/flows/content/GenAiQuick.svelte b/frontend/src/lib/components/flows/content/GenAiQuick.svelte new file mode 100644 index 0000000000..ef59d9d6d7 --- /dev/null +++ b/frontend/src/lib/components/flows/content/GenAiQuick.svelte @@ -0,0 +1,33 @@ + + + + diff --git a/frontend/src/lib/components/flows/map/FlowErrorHandlerItem.svelte b/frontend/src/lib/components/flows/map/FlowErrorHandlerItem.svelte index 9faf6d862a..fbae0cf433 100644 --- a/frontend/src/lib/components/flows/map/FlowErrorHandlerItem.svelte +++ b/frontend/src/lib/components/flows/map/FlowErrorHandlerItem.svelte @@ -3,37 +3,52 @@ import { getContext } from 'svelte' import { classNames, emptySchema } from '$lib/utils' import type { FlowModuleState } from '../flowState' - import Toggle from '$lib/components/Toggle.svelte' import { NEVER_TESTED_THIS_FAR } from '../models' import type { FlowCopilotContext } from '$lib/components/copilot/flow' import { fade } from 'svelte/transition' - import { Bug } from 'lucide-svelte' + import { Bug, X } from 'lucide-svelte' + import InsertModuleButton from '$lib/components/flows/map/InsertModuleButton.svelte' + import { createInlineScriptModule, pickScript } from '$lib/components/flows/flowStateUtils' + import type { FlowModule, RawScript } from '$lib/gen' + import { twMerge } from 'tailwind-merge' export let small: boolean const { selectedId, flowStateStore, flowStore } = getContext('FlowEditorContext') - function onToggle() { - if ($flowStore?.value?.failure_module) { - $flowStore.value.failure_module = undefined - // By default, we return to settings when disabling the failure module - $selectedId = 'settings-metadata' - } else { - const failureModule: FlowModuleState = { - schema: emptySchema(), - previewResult: NEVER_TESTED_THIS_FAR - } - - $flowStore.value.failure_module = { - id: 'failure', - value: { type: 'identity' } - } - $flowStateStore['failure'] = failureModule - - $selectedId = 'failure' - $flowStore = $flowStore + async function insertNewFailureModule( + inlineScript?: { + language: RawScript['language'] + subkind: 'pgsql' | 'flow' + }, + wsScript?: { path: string; summary: string; hash: string | undefined } + ) { + var module: FlowModule = { + id: 'failure', + value: { type: 'identity' } } + var state: FlowModuleState = { + schema: emptySchema(), + previewResult: NEVER_TESTED_THIS_FAR + } + + if (inlineScript) { + ;[module, state] = await createInlineScriptModule( + inlineScript.language, + 'failure', + inlineScript.subkind, + 'failure' + ) + } else if (wsScript) { + ;[module, state] = await pickScript(wsScript.path, wsScript.summary, module.id, wsScript.hash) + } + + $flowStore.value.failure_module = module + $flowStateStore[module.id] = state + + $selectedId = 'failure' + $flowStore = $flowStore } const { currentStepStore: copilotCurrentStepStore } = @@ -42,45 +57,71 @@ +
{ if ($copilotCurrentStepStore !== undefined) return if ($flowStore?.value?.failure_module) { $selectedId = 'failure' - } else { - onToggle() } }} - class={classNames( - 'z-10', - $copilotCurrentStepStore !== undefined ? 'border-gray-500/75' : 'cursor-pointer', - 'border transition-colors duration-[400ms] ease-linear rounded-sm px-2 py-1 bg-surface text-sm flex justify-between items-center flex-row overflow-x-hidden relative', - $selectedId?.includes('failure') ? 'outline outline-offset-1 outline-2 outline-slate-900 dark:outline-slate-900/0 dark:bg-surface-secondary dark:border-gray-400' : '' - )} - style={small ? 'min-width: 200px' : 'min-width: 275px'} > {#if $copilotCurrentStepStore !== undefined}
{/if} -
- - Error Handler +
+
-
- {#if Boolean($flowStore?.value?.failure_module)} - - {$flowStore.value.failure_module?.summary || - ($flowStore.value.failure_module?.value.type === 'rawscript' - ? `${$flowStore.value.failure_module?.value.language}` - : 'TBD')} - - {/if} -
- + {#if !$flowStore?.value?.failure_module} +
Error Handler
+ {:else} +
+ {$flowStore.value.failure_module?.summary || + ($flowStore.value.failure_module?.value.type === 'rawscript' + ? `${$flowStore.value.failure_module?.value.language}` + : 'TBD')} +
+ {/if} + + {#if !$flowStore?.value?.failure_module} + { + insertNewFailureModule(e.detail.inlineScript) + }} + on:pickScript={(e) => { + insertNewFailureModule(undefined, e.detail) + }} + kind="failure" + /> + {:else} + + {/if}
diff --git a/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte b/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte index bc2af7f82e..c2919e41d7 100644 --- a/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte +++ b/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte @@ -2,15 +2,17 @@ import type { FlowEditorContext } from '../types' import { createEventDispatcher, getContext, tick } from 'svelte' import { + createInlineScriptModule, createBranchAll, createBranches, createLoop, createWhileLoop, deleteFlowStateById, emptyModule, - pickScript + pickScript, + pickFlow } from '$lib/components/flows/flowStateUtils' - import type { FlowModule } from '$lib/gen' + import type { FlowModule, RawScript, Script } from '$lib/gen' import { emptyFlowModuleState, initFlowStepWarnings } from '../utils' import FlowSettingsItem from './FlowSettingsItem.svelte' import FlowConstantsItem from './FlowConstantsItem.svelte' @@ -31,8 +33,6 @@ import { tutorialInProgress } from '$lib/tutorialUtils' import FlowGraphV2 from '$lib/components/graph/FlowGraphV2.svelte' import { replaceId } from '../flowStore' - import { emptySchema } from '$lib/utils' - import { NEVER_TESTED_THIS_FAR } from '../models' export let modules: FlowModule[] | undefined export let sidebarSize: number | undefined = undefined @@ -61,12 +61,22 @@ | 'trigger' | 'approval' | 'end', - wsScript?: { path: string; summary: string; hash: string | undefined } + wsScript?: { path: string; summary: string; hash: string | undefined }, + wsFlow?: { path: string; summary: string }, + inlineScript?: { + language: RawScript['language'] + kind: Script['kind'] + subkind: 'pgsql' | 'flow' + id: string + summary?: string + } ): Promise { push(history, $flowStore) var module = emptyModule($flowStateStore, $flowStore, kind == 'flow') var state = emptyFlowModuleState() - if (wsScript) { + if (wsFlow) { + ;[module, state] = await pickFlow(wsFlow.path, wsFlow.summary, module.id) + } else if (wsScript) { ;[module, state] = await pickScript(wsScript.path, wsScript.summary, module.id, wsScript.hash) } else if (kind == 'forloop') { ;[module, state] = await createLoop( @@ -89,11 +99,49 @@ module.summary = 'Terminate flow' module.stop_after_if = { skip_if_stopped: false, expr: 'true' } } + if (inlineScript) { + const { language, kind, subkind } = inlineScript + ;[module, state] = await createInlineScriptModule( + language, + kind, + subkind, + module.id, + module.summary + ) + } if (!modules) return [module] modules.splice(index, 0, module) return modules } + async function insertNewPreprocessorModule( + inlineScript?: { + language: RawScript['language'] + subkind: 'pgsql' | 'flow' + }, + wsScript?: { path: string; summary: string; hash: string | undefined } + ) { + var module: FlowModule = { + id: 'preprocessor', + value: { type: 'identity' } + } + var state = emptyFlowModuleState() + + if (inlineScript) { + ;[module, state] = await createInlineScriptModule( + inlineScript.language, + 'script', + inlineScript.subkind, + 'preprocessor' + ) + } else if (wsScript) { + ;[module, state] = await pickScript(wsScript.path, wsScript.summary, module.id, wsScript.hash) + } + + $flowStore.value.preprocessor_module = module + $flowStateStore[module.id] = state + } + function removeAtId(modules: FlowModule[], id: string): FlowModule[] { const index = modules.findIndex((mod) => mod.id == id) if (index != -1) { @@ -311,22 +359,16 @@ $moving = undefined } else { if (detail.detail === 'preprocessor') { - const preprocessorModule = { - schema: emptySchema(), - previewResult: NEVER_TESTED_THIS_FAR - } - $flowStore.value.preprocessor_module = { - id: 'preprocessor', - value: { type: 'identity' } - } - $flowStateStore['preprocessor'] = preprocessorModule + insertNewPreprocessorModule(detail.inlineScript, detail.script) $selectedId = 'preprocessor' } else { await insertNewModuleAtIndex( detail.modules, detail.index ?? 0, - detail.detail, - detail.script + detail.kind, + detail.script, + detail.flow, + detail.inlineScript ) $selectedId = detail.modules[detail.index ?? 0].id } diff --git a/frontend/src/lib/components/flows/map/InsertModuleButton.svelte b/frontend/src/lib/components/flows/map/InsertModuleButton.svelte index 40cce9648f..351299ca8d 100644 --- a/frontend/src/lib/components/flows/map/InsertModuleButton.svelte +++ b/frontend/src/lib/components/flows/map/InsertModuleButton.svelte @@ -1,169 +1,200 @@ - - + + + + + + -
- + +
{ + e.stopPropagation() + }} + role="none" + > +
+ + {#if selectedKind != 'preprocessor' && selectedKind != 'flow'} + + {/if} +
- {#if funcDesc.length === 0} -
- - {#if customUi?.triggers != false && trigger} - - {/if} - - - - - - - - - {#if customUi?.flowNode != false} - - {/if} - {#if stop} - - {/if} -
- {/if} + /> + { + close(null) + dispatch('new', { kind: 'whileloop' }) + }} + /> + { + close(null) + dispatch('new', { kind: 'branchone' }) + }} + /> + { + close(null) + dispatch('new', { kind: 'branchall' }) + }} + /> +
+ {/if} + + { + close(null) + }} + on:new + on:pickScript + on:pickFlow + {preFilter} + {small} + /> +
- + diff --git a/frontend/src/lib/components/flows/map/InsertTriggerButton.svelte b/frontend/src/lib/components/flows/map/InsertTriggerButton.svelte deleted file mode 100644 index 35dc331161..0000000000 --- a/frontend/src/lib/components/flows/map/InsertTriggerButton.svelte +++ /dev/null @@ -1,52 +0,0 @@ - - - - - {#if !disableAi} - - {/if} - {#if funcDesc.length === 0} -
- -
- {/if} -
diff --git a/frontend/src/lib/components/flows/pickers/FlowScriptPickerQuick.svelte b/frontend/src/lib/components/flows/pickers/FlowScriptPickerQuick.svelte new file mode 100644 index 0000000000..2f4319e561 --- /dev/null +++ b/frontend/src/lib/components/flows/pickers/FlowScriptPickerQuick.svelte @@ -0,0 +1,56 @@ + + + + + diff --git a/frontend/src/lib/components/flows/pickers/FlowToplevelNode.svelte b/frontend/src/lib/components/flows/pickers/FlowToplevelNode.svelte new file mode 100644 index 0000000000..0248437963 --- /dev/null +++ b/frontend/src/lib/components/flows/pickers/FlowToplevelNode.svelte @@ -0,0 +1,22 @@ + + + + + diff --git a/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte b/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte new file mode 100644 index 0000000000..122e4a3125 --- /dev/null +++ b/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte @@ -0,0 +1,175 @@ + + + +{#if hubNotAvailable} +
+ Hub not available +
+{:else if loading} + {#each Array(15).fill(0) as _} + + {/each} +{:else if items.length > 0 && apps.length > 0} +
    + {#each items as item, index (item.path)} +
  • + +
  • + {/each} +
+ {#if items.length == 40} +
+ There are more items than being displayed. Refine your search. +
+ {/if} +{/if} diff --git a/frontend/src/lib/components/flows/pickers/TopLevelNode.svelte b/frontend/src/lib/components/flows/pickers/TopLevelNode.svelte new file mode 100644 index 0000000000..af70d38508 --- /dev/null +++ b/frontend/src/lib/components/flows/pickers/TopLevelNode.svelte @@ -0,0 +1,60 @@ + + + diff --git a/frontend/src/lib/components/flows/pickers/WorkspaceScriptPickerQuick.svelte b/frontend/src/lib/components/flows/pickers/WorkspaceScriptPickerQuick.svelte new file mode 100644 index 0000000000..04cc468d83 --- /dev/null +++ b/frontend/src/lib/components/flows/pickers/WorkspaceScriptPickerQuick.svelte @@ -0,0 +1,136 @@ + + + (emptyString(x.summary) ? x.path : x.summary + ' (' + x.path + ')')} +/> + + +{#if filteredItems} + {#if filteredItems.length == 0} +
+ {kind == 'flow' ? 'No flows found.' : 'No scripts found.'} +
+ {/if} +
    + {#each filteredWithOwner ?? [] as { path, hash, summary, marked }, index} +
  • + +
  • + {/each} +
+{:else} + {#each Array(10).fill(0) as _} + + {/each} +{/if} diff --git a/frontend/src/lib/components/flows/types.ts b/frontend/src/lib/components/flows/types.ts index ba8ce95c5b..53f81c5a0d 100644 --- a/frontend/src/lib/components/flows/types.ts +++ b/frontend/src/lib/components/flows/types.ts @@ -41,4 +41,5 @@ export type FlowEditorContext = { initialPath: string flowInputsStore: Writable customUi: FlowBuilderWhitelabelCustomUi + insertButtonOpen: Writable } diff --git a/frontend/src/lib/components/graph/FlowGraphV2.svelte b/frontend/src/lib/components/graph/FlowGraphV2.svelte index 318ccaaa74..e4609f6d28 100644 --- a/frontend/src/lib/components/graph/FlowGraphV2.svelte +++ b/frontend/src/lib/components/graph/FlowGraphV2.svelte @@ -48,8 +48,8 @@ export let selectedId: Writable = writable(undefined) export let insertable = false - export let moving: string | undefined = undefined export let scroll = false + export let moving: string | undefined = undefined // Download: display a top level button to open the graph in a new tab export let download = false @@ -266,6 +266,9 @@
{:else} { + window.dispatchEvent(new Event('focus')) + }} {nodes} {edges} {edgeTypes} diff --git a/frontend/src/lib/components/graph/graphBuilder.ts b/frontend/src/lib/components/graph/graphBuilder.ts index eecaffc7e3..3a77c22b2c 100644 --- a/frontend/src/lib/components/graph/graphBuilder.ts +++ b/frontend/src/lib/components/graph/graphBuilder.ts @@ -549,4 +549,5 @@ export default function graphBuilder( error: e } } + } diff --git a/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte b/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte index f27e89b8e0..584fb44b25 100644 --- a/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte +++ b/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte @@ -7,7 +7,6 @@ import type { Writable } from 'svelte/store' import type { GraphEventHandlers } from '../../graphBuilder' import { getStraightLinePath } from '../utils' - import InsertTriggerButton from '$lib/components/flows/map/InsertTriggerButton.svelte' import { twMerge } from 'tailwind-merge' export let sourceX: number @@ -47,33 +46,44 @@ const { useDataflow } = getContext<{ useDataflow: Writable }>('FlowGraphContext') - - let menuOpen = false {#if data?.insertable && !$useDataflow && !data?.moving}
{ - data?.eventHandlers.insert({ modules: data.modules, index: data.index, detail: e.detail }) - }} - on:insert={(e) => { + // console.log('new', e) data?.eventHandlers.insert({ modules: data.modules, index: data.index, - script: e.detail, - detail: 'script' + kind: e.detail.kind, + inlineScript: e.detail.inlineScript + }) + }} + on:pickScript={(e) => { + // console.log('pickScript', e) + data?.eventHandlers.insert({ + modules: data.modules, + index: data.index, + script: e.detail + }) + }} + on:pickFlow={(e) => { + // console.log('pickFlow', e) + data?.eventHandlers.insert({ + modules: data.modules, + index: data.index, + flow: e.detail }) }} - bind:open={menuOpen} />
{#if data.enableTrigger} @@ -81,23 +91,34 @@ class="edgeButtonContainer nodrag nopan" style:transform="translate(100%, 50%) translate({sourceX}px,{sourceY + 2}px)" > - { + // console.log('new', e) data?.eventHandlers.insert({ modules: data.modules, index: data.index, - detail: e.detail + kind: e.detail.kind, + inlineScript: e.detail.inlineScript }) }} - on:insert={(e) => { + on:pickScript={(e) => { + // console.log('pickScript', e) data?.eventHandlers.insert({ modules: data.modules, index: data.index, - script: e.detail, - detail: 'script' + script: e.detail }) }} + on:pickFlow={(e) => { + // console.log('pickFlow', e) + data?.eventHandlers.insert({ + modules: data.modules, + index: data.index, + flow: e.detail + }) + }} + kind="trigger" index={data?.index ?? 0} modules={data?.modules ?? []} /> diff --git a/frontend/src/lib/components/graph/renderers/nodes/InputNode.svelte b/frontend/src/lib/components/graph/renderers/nodes/InputNode.svelte index 0cacdbfc47..7690fae71d 100644 --- a/frontend/src/lib/components/graph/renderers/nodes/InputNode.svelte +++ b/frontend/src/lib/components/graph/renderers/nodes/InputNode.svelte @@ -1,21 +1,24 @@ + +{#if Array.isArray(filters) && filters.length > 0} + {#each filters as filter (filter)} +
+ +
+ {/each} +{/if} diff --git a/frontend/src/lib/components/icons/WindmillIcon2.svelte b/frontend/src/lib/components/icons/WindmillIcon2.svelte new file mode 100644 index 0000000000..568e4cb05f --- /dev/null +++ b/frontend/src/lib/components/icons/WindmillIcon2.svelte @@ -0,0 +1,154 @@ + + + + + + + + + + + + + + diff --git a/frontend/src/lib/components/icons/index.ts b/frontend/src/lib/components/icons/index.ts index 78ec4769d2..8fb92470d8 100644 --- a/frontend/src/lib/components/icons/index.ts +++ b/frontend/src/lib/components/icons/index.ts @@ -18,6 +18,7 @@ import S3Icon from './S3Icon.svelte' import Slack from './Slack.svelte' import TogglIcon from './TogglIcon.svelte' import WindmillIcon from './WindmillIcon.svelte' +import WindmillIcon2 from './WindmillIcon2.svelte' import MailchimpIcon from './MailchimpIcon.svelte' import SendgridIcon from './SendgridIcon.svelte' import SendflakeIcon from './SendflakeIcon.svelte' @@ -211,6 +212,7 @@ export { Slack, TogglIcon, WindmillIcon, + WindmillIcon2, MailchimpIcon, SendgridIcon, LinkedinIcon, diff --git a/frontend/src/lib/script_helpers.ts b/frontend/src/lib/script_helpers.ts index 3dcade90c0..d0e7e26570 100644 --- a/frontend/src/lib/script_helpers.ts +++ b/frontend/src/lib/script_helpers.ts @@ -350,6 +350,27 @@ export async function main() { } ` +export const BUN_INIT_CODE_TRIGGER = `import * as wmill from "windmill-client" + +export async function main() { + + // A common trigger script would follow this pattern: + // 1. Get the last saved state + // const state = await wmill.getState() + // 2. Get the actual state from the external service + // const newState = await (await fetch('https://hacker-news.firebaseio.com/v0/topstories.json')).json() + // 3. Compare the two states and update the internal state + // await wmill.setState(newState) + // 4. Return the new rows + // return range from (state to newState) + + return [1,2,3] + + // In subsequent scripts, you may refer to each row/value returned by the trigger script using + // 'flow_input.iter.value' +} +` + export const GO_INIT_CODE_TRIGGER = `package inner import ( @@ -715,7 +736,9 @@ export function initialCode( } else if (language == 'ansible') { return ANSIBLE_PLAYBOOK_INIT_CODE } else if (language == 'bun' || language == 'bunnative') { - if (language == 'bunnative' || subkind === 'bunnative') { + if (kind == 'trigger') { + return BUN_INIT_CODE_TRIGGER + } else if (language == 'bunnative' || subkind === 'bunnative') { return BUNNATIVE_INIT_CODE } else if (kind === 'approval') { return BUN_INIT_CODE_APPROVAL @@ -723,8 +746,7 @@ export function initialCode( return BUN_FAILURE_MODULE_CODE } else if (subkind === 'preprocessor') { return BUN_PREPROCESSOR_MODULE_CODE - } - if (subkind === 'flow') { + } else if (subkind === 'flow') { return BUN_INIT_CODE_CLEAR } diff --git a/frontend/src/routes/flows/dev/+page.svelte b/frontend/src/routes/flows/dev/+page.svelte index 6cd02642a4..3f12f3f5a7 100644 --- a/frontend/src/routes/flows/dev/+page.svelte +++ b/frontend/src/routes/flows/dev/+page.svelte @@ -97,7 +97,8 @@ saveDraft: () => {}, initialPath: '', flowInputsStore: writable({}), - customUi: {} + customUi: {}, + insertButtonOpen: writable(false) }) type LastEdit = { From f25eb3455f3d447c1f9e3f7a47d2338890ba03f6 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 3 Oct 2024 14:34:33 +0200 Subject: [PATCH 13/38] padding nits --- .../src/lib/components/flows/content/FlowInputsQuick.svelte | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte index 894c8c52fd..a5afaa0dd6 100644 --- a/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte +++ b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte @@ -224,7 +224,7 @@
{/each} +
{:else}
No items found. From b69bbceedb8f7fa36b2326b91b7922bd1c962338 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 3 Oct 2024 14:53:39 +0200 Subject: [PATCH 14/38] chore(main): release 1.404.0 (#4468) * chore(main): release 1.404.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel --- CHANGELOG.md | 14 + backend/Cargo.lock | 261 ++++++------------ backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 4 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- python-client/wmill_pg/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 16 files changed, 120 insertions(+), 189 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86b2d85219..05bc5c98a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## [1.404.0](https://github.com/windmill-labs/windmill/compare/v1.403.1...v1.404.0) (2024-10-03) + + +### Features + +* **frontend:** add quick access menu in flow editor ([#4415](https://github.com/windmill-labs/windmill/issues/4415)) ([45ccd45](https://github.com/windmill-labs/windmill/commit/45ccd45e306c66931880a9b8fd48bfe684c774ac)) + + +### Bug Fixes + +* **cli:** improve schedule path handling on windows ([9ac3b6b](https://github.com/windmill-labs/windmill/commit/9ac3b6b1d5d64d7467dd80506f8a8d772c4630bd)) +* fix id editor for app ([8e58e43](https://github.com/windmill-labs/windmill/commit/8e58e4320a31d71c40a5ed352416a4c2dd3adb26)) +* **frontend:** disable runnable field on route editor from detail panel ([#4469](https://github.com/windmill-labs/windmill/issues/4469)) ([3134f79](https://github.com/windmill-labs/windmill/commit/3134f79ced80aab86912643ab7a60dcf909ab104)) + ## [1.403.1](https://github.com/windmill-labs/windmill/compare/v1.403.0...v1.403.1) (2024-10-01) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 39f4a3e7d4..34aae89874 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -40,22 +40,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" dependencies = [ "cfg-if", - "cipher 0.3.0", + "cipher", "cpufeatures", "opaque-debug", ] -[[package]] -name = "aes" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if", - "cipher 0.4.4", - "cpufeatures", -] - [[package]] name = "ahash" version = "0.7.8" @@ -187,19 +176,6 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" -[[package]] -name = "archiver-rs" -version = "0.5.1" -source = "git+https://github.com/gz/archiver-rs.git?branch=patch-1#a73cef92c2a5b8f48c2a4a9e889952072e03b4b7" -dependencies = [ - "bzip2", - "flate2", - "tar", - "thiserror", - "xz2", - "zip", -] - [[package]] name = "argon2" version = "0.5.3" @@ -209,7 +185,7 @@ dependencies = [ "base64ct", "blake2", "cpufeatures", - "password-hash 0.5.0", + "password-hash", ] [[package]] @@ -383,7 +359,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 2.5.0", + "indexmap 2.6.0", "lexical-core", "num", "serde", @@ -510,11 +486,11 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa" +checksum = "7e614738943d3f68c628ae3dbce7c3daffb196665f82f8c8ea6b65de73c79429" dependencies = [ - "brotli", + "brotli 7.0.0", "bzip2", "flate2", "futures-core", @@ -643,9 +619,9 @@ dependencies = [ [[package]] name = "async-stream" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" dependencies = [ "async-stream-impl", "futures-core", @@ -654,9 +630,9 @@ dependencies = [ [[package]] name = "async-stream-impl" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", @@ -1334,7 +1310,7 @@ dependencies = [ "arrayvec", "cc", "cfg-if", - "constant_time_eq 0.3.1", + "constant_time_eq", ] [[package]] @@ -1363,7 +1339,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2cb03d1bed155d89dce0f845b7899b18a9a163e148fd004e1c28421a783e2d8e" dependencies = [ "block-padding", - "cipher 0.3.0", + "cipher", ] [[package]] @@ -1420,6 +1396,17 @@ dependencies = [ "brotli-decompressor", ] +[[package]] +name = "brotli" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + [[package]] name = "brotli-decompressor" version = "4.0.1" @@ -1712,16 +1699,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common", - "inout", -] - [[package]] name = "clang-sys" version = "1.8.1" @@ -1735,9 +1712,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.18" +version = "4.5.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" +checksum = "7be5744db7978a28d9df86a214130d106a89ce49644cbc4e3f0c22c3fba30615" dependencies = [ "clap_builder", "clap_derive", @@ -1745,9 +1722,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.18" +version = "4.5.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" +checksum = "a5fbc17d3ef8278f55b282b2a2e75ae6f6c7d4bb70ed3d0382375104bfafdb4b" dependencies = [ "anstream", "anstyle", @@ -1898,12 +1875,6 @@ dependencies = [ "unicode-xid", ] -[[package]] -name = "constant_time_eq" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" - [[package]] name = "constant_time_eq" version = "0.3.1" @@ -2259,7 +2230,7 @@ dependencies = [ "arrow-array", "arrow-ipc", "arrow-schema", - "async-compression 0.4.12", + "async-compression 0.4.13", "async-trait", "bytes", "bzip2", @@ -2282,7 +2253,7 @@ dependencies = [ "glob", "half", "hashbrown 0.14.5", - "indexmap 2.5.0", + "indexmap 2.6.0", "itertools 0.12.1", "log", "num_cpus", @@ -2451,7 +2422,7 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr", "hashbrown 0.14.5", - "indexmap 2.5.0", + "indexmap 2.6.0", "itertools 0.12.1", "log", "regex-syntax 0.8.5", @@ -2480,7 +2451,7 @@ dependencies = [ "half", "hashbrown 0.14.5", "hex", - "indexmap 2.5.0", + "indexmap 2.6.0", "itertools 0.12.1", "log", "paste", @@ -2524,7 +2495,7 @@ dependencies = [ "futures", "half", "hashbrown 0.14.5", - "indexmap 2.5.0", + "indexmap 2.6.0", "itertools 0.12.1", "log", "once_cell", @@ -2929,7 +2900,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac41dd49fb554432020d52c875fc290e110113f864c6b1b525cd62c7e7747a5d" dependencies = [ "byteorder", - "cipher 0.3.0", + "cipher", "opaque-debug", ] @@ -3906,7 +3877,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.5.0", + "indexmap 2.6.0", "slab", "tokio", "tokio-util", @@ -3925,7 +3896,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.1.0", - "indexmap 2.5.0", + "indexmap 2.6.0", "slab", "tokio", "tokio-util", @@ -3974,6 +3945,12 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "hashbrown" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" + [[package]] name = "hashlink" version = "0.9.1" @@ -4379,12 +4356,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.14.5", + "hashbrown 0.15.0", "serde", ] @@ -4407,15 +4384,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac" -[[package]] -name = "inout" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" -dependencies = [ - "generic-array", -] - [[package]] name = "instant" version = "0.1.13" @@ -4550,7 +4518,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", ] [[package]] @@ -4848,7 +4816,7 @@ version = "3.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c42f95f9d296f2dcb50665f507ed5a68a171453142663ce44d77a4eb217b053" dependencies = [ - "aes 0.7.5", + "aes", "base64 0.21.7", "block-modes", "crc-any", @@ -5695,7 +5663,7 @@ dependencies = [ "arrow-schema", "arrow-select", "base64 0.22.1", - "brotli", + "brotli 6.0.0", "bytes", "chrono", "flate2", @@ -5725,17 +5693,6 @@ dependencies = [ "regex", ] -[[package]] -name = "password-hash" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" -dependencies = [ - "base64ct", - "rand_core 0.6.4", - "subtle", -] - [[package]] name = "password-hash" version = "0.5.0" @@ -5759,18 +5716,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" -[[package]] -name = "pbkdf2" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" -dependencies = [ - "digest 0.10.7", - "hmac", - "password-hash 0.4.2", - "sha2 0.10.8", -] - [[package]] name = "pem" version = "1.1.1" @@ -5821,25 +5766,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.5.0", -] - -[[package]] -name = "pg-embed" -version = "0.7.2" -source = "git+https://github.com/faokunega/pg-embed#72db5e053f0afac6eee51d3baa2fd5c90803e02d" -dependencies = [ - "archiver-rs", - "async-trait", - "bytes", - "dirs", - "futures", - "lazy_static", - "log", - "reqwest 0.11.27", - "thiserror", - "tokio", - "zip", + "indexmap 2.6.0", ] [[package]] @@ -6771,7 +6698,7 @@ version = "0.12.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b" dependencies = [ - "async-compression 0.4.12", + "async-compression 0.4.13", "base64 0.22.1", "bytes", "encoding_rs", @@ -7524,7 +7451,7 @@ version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", "itoa", "memchr", "ryu", @@ -7620,15 +7547,15 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.9.0" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cecfa94848272156ea67b2b1a53f20fc7bc638c4a46d2f8abde08f05f4b857" +checksum = "9720086b3357bcb44fce40117d769a4d068c70ecfa190850a980a71755f66fcc" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.5.0", + "indexmap 2.6.0", "serde", "serde_derive", "serde_json", @@ -7638,9 +7565,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.9.0" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8fee4991ef4f274617a51ad4af30519438dacb2f56ac773b08a1922ff743350" +checksum = "5f1abbfe725f27678f4663bcacb75a83e829fd464c25d78dd038a3a29e307cec" dependencies = [ "darling 0.20.10", "proc-macro2", @@ -7654,7 +7581,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", "itoa", "ryu", "serde", @@ -8034,7 +7961,7 @@ dependencies = [ "hashbrown 0.14.5", "hashlink", "hex", - "indexmap 2.5.0", + "indexmap 2.6.0", "log", "memchr", "once_cell", @@ -8411,7 +8338,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84b67e115ab136fe0eb03558bb0508ca7782eeb446a96d165508c48617e3fd94" dependencies = [ "anyhow", - "indexmap 2.5.0", + "indexmap 2.6.0", "serde", "serde_json", "swc_cached", @@ -8562,7 +8489,7 @@ checksum = "d37dc505c92af56d0f77cf6f31a6ccd37ac40cad1e01ff77277e0b1c70e8f8ff" dependencies = [ "better_scoped_tls", "bitflags 2.6.0", - "indexmap 2.5.0", + "indexmap 2.6.0", "once_cell", "phf", "rustc-hash 1.1.0", @@ -8631,7 +8558,7 @@ checksum = "446da32cac8299973aaf1d37496562bfd0c1e4f3c3ab5d0af6f07f42e8184102" dependencies = [ "base64 0.21.7", "dashmap", - "indexmap 2.5.0", + "indexmap 2.6.0", "once_cell", "serde", "sha1", @@ -8670,7 +8597,7 @@ version = "0.130.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13e62b199454a576c5fdbd7e1bef8ab88a395427456d8a713d994b7d469833aa" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", "num_cpus", "once_cell", "rustc-hash 1.1.0", @@ -9453,7 +9380,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", "serde", "serde_spanned", "toml_datetime", @@ -9466,7 +9393,7 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", "serde", "serde_spanned", "toml_datetime", @@ -9540,7 +9467,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" dependencies = [ - "async-compression 0.4.12", + "async-compression 0.4.13", "bitflags 2.6.0", "bytes", "futures-core", @@ -9925,9 +9852,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" [[package]] name = "unicode-id" @@ -10428,7 +10355,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.403.1" +version = "1.404.0" dependencies = [ "anyhow", "axum", @@ -10442,7 +10369,6 @@ dependencies = [ "lazy_static", "object_store", "once_cell", - "pg-embed", "prometheus", "quote", "rand 0.8.5", @@ -10470,7 +10396,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.403.1" +version = "1.404.0" dependencies = [ "anyhow", "argon2", @@ -10554,7 +10480,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.403.1" +version = "1.404.0" dependencies = [ "base64 0.21.7", "chrono", @@ -10572,7 +10498,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.403.1" +version = "1.404.0" dependencies = [ "chrono", "serde", @@ -10585,7 +10511,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.403.1" +version = "1.404.0" dependencies = [ "anyhow", "async-stream", @@ -10603,7 +10529,7 @@ dependencies = [ "hex", "hmac", "hyper 1.4.1", - "indexmap 2.5.0", + "indexmap 2.6.0", "itertools 0.13.0", "lazy_static", "magic-crypt", @@ -10630,7 +10556,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.403.1" +version = "1.404.0" dependencies = [ "regex", "rsmq_async", @@ -10645,7 +10571,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.403.1" +version = "1.404.0" dependencies = [ "anyhow", "bytes", @@ -10666,7 +10592,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.403.1" +version = "1.404.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -10675,7 +10601,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.403.1" +version = "1.404.0" dependencies = [ "anyhow", "lazy_static", @@ -10687,7 +10613,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.403.1" +version = "1.404.0" dependencies = [ "anyhow", "gosyn", @@ -10699,7 +10625,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.403.1" +version = "1.404.0" dependencies = [ "anyhow", "lazy_static", @@ -10711,7 +10637,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.403.1" +version = "1.404.0" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10722,7 +10648,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.403.1" +version = "1.404.0" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10733,7 +10659,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.403.1" +version = "1.404.0" dependencies = [ "anyhow", "async-recursion", @@ -10751,7 +10677,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.403.1" +version = "1.404.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -10768,7 +10694,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.403.1" +version = "1.404.0" dependencies = [ "anyhow", "lazy_static", @@ -10780,7 +10706,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.403.1" +version = "1.404.0" dependencies = [ "anyhow", "lazy_static", @@ -10798,7 +10724,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.403.1" +version = "1.404.0" dependencies = [ "anyhow", "getrandom 0.2.15", @@ -10819,7 +10745,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.403.1" +version = "1.404.0" dependencies = [ "anyhow", "serde_json", @@ -10829,7 +10755,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.403.1" +version = "1.404.0" dependencies = [ "anyhow", "async-recursion", @@ -10862,7 +10788,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.403.1" +version = "1.404.0" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -10872,7 +10798,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.403.1" +version = "1.404.0" dependencies = [ "anyhow", "async-recursion", @@ -11296,18 +11222,9 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" dependencies = [ - "aes 0.8.4", "byteorder", - "bzip2", - "constant_time_eq 0.1.5", "crc32fast", "crossbeam-utils", - "flate2", - "hmac", - "pbkdf2", - "sha1", - "time", - "zstd 0.11.2+zstd.1.5.2", ] [[package]] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 632f7d3b22..30d1e63c98 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.403.1" +version = "1.404.0" authors.workspace = true edition.workspace = true @@ -27,7 +27,7 @@ members = [ ] [workspace.package] -version = "1.403.1" +version = "1.404.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 1ca35ac564..24025318a7 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.403.1 + version: 1.404.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 8a0e53bb91..d645983b64 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.403.1"; +export const VERSION = "v1.404.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/main.ts b/cli/main.ts index 7cac847e6c..550e6fe076 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -60,7 +60,7 @@ export { // } // }); -export const VERSION = "1.403.1"; +export const VERSION = "1.404.0"; const command = new Command() .name("wmill") diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 65aee2cd28..bda1ed9443 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.403.1", + "version": "1.404.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.403.1", + "version": "1.404.0", "license": "AGPL-3.0", "dependencies": { "@aws-crypto/sha256-js": "^4.0.0", diff --git a/frontend/package.json b/frontend/package.json index b6ce8cd73c..f7117231b7 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.403.1", + "version": "1.404.0", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index 8a901e1fc6..767b97d966 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.403.1" -wmill_pg = ">=1.403.1" +wmill = ">=1.404.0" +wmill_pg = ">=1.404.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index fb285ccd64..5b5cf59ec7 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.403.1 + version: 1.404.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 8037a060db..d74129549c 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. -ModuleVersion = '1.403.1' +ModuleVersion = '1.404.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index c4300a04af..021029a756 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.403.1" +version = "1.404.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index d1ad5c634b..54c21dfe83 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.403.1" +version = "1.404.0" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 108b7e5afc..4d61c8e117 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.403.1", + "version": "1.404.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 98e7b62919..da34f2c55a 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.403.1", + "version": "1.404.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index ff5ef8e01f..b50eeb55a4 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.403.1 +1.404.0 From 92f61f07ed6d354407d26843e3a270b95bae90bc Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 3 Oct 2024 15:09:58 +0200 Subject: [PATCH 15/38] fix: flow picker of flows --- .../flows/content/FlowInputsQuick.svelte | 10 +++++- .../pickers/WorkspaceScriptPickerQuick.svelte | 32 +++++++++++-------- .../components/home/ListFiltersQuick.svelte | 4 +++ 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte index a5afaa0dd6..0483b1e9de 100644 --- a/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte +++ b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte @@ -256,7 +256,14 @@ {#if preFilter == 'all'}
Integrations
{/if} - + { + selectedByKeyboard = 0 + }} + filters={integrations} + bind:selectedFilter={selected} + resourceType + /> {/if} {:else if selectedKind === 'flow'} {#if owners.length > 0} @@ -415,6 +422,7 @@ kind={selectedKind} selected={selectedByKeyboard - inlineScripts?.length - aiLength - topLevelNodes.length} on:pickScript + on:pickFlow /> {/if} diff --git a/frontend/src/lib/components/flows/pickers/WorkspaceScriptPickerQuick.svelte b/frontend/src/lib/components/flows/pickers/WorkspaceScriptPickerQuick.svelte index 04cc468d83..bd91c76015 100644 --- a/frontend/src/lib/components/flows/pickers/WorkspaceScriptPickerQuick.svelte +++ b/frontend/src/lib/components/flows/pickers/WorkspaceScriptPickerQuick.svelte @@ -64,24 +64,28 @@ const dispatch = createEventDispatcher() let lockHash = false - function onKeyDown(e: KeyboardEvent) { - if ( - selected != undefined && - filteredItems && - selected >= 0 && - selected < filteredItems.length && - e.key === 'Enter' - ) { - e.preventDefault() - let item = filteredItems[selected] - dispatch('pickScript', { path: item.path, hash: lockHash ? item.hash : undefined }) - } - } - $: filteredWithOwner = ownerFilter != undefined ? filteredItems?.filter((x) => x.path.startsWith(ownerFilter?.name!)) : filteredItems + + function onKeyDown(e: KeyboardEvent) { + if ( + selected != undefined && + filteredWithOwner && + selected >= 0 && + selected < filteredWithOwner.length && + e.key === 'Enter' + ) { + e.preventDefault() + let item = filteredWithOwner[selected] + if (kind == 'flow') { + dispatch('pickFlow', { path: item.path }) + } else { + dispatch('pickScript', { path: item.path, hash: lockHash ? item.hash : undefined }) + } + } + } {#if Array.isArray(filters) && filters.length > 0} @@ -27,6 +30,7 @@ on:click={() => { selectedFilter = selectedAppFilter == filter ? undefined : { kind: 'integrations', name: filter } + dispatch('selected') }} >
From 642f3876fa3acb060690d4685f4fb8423b40cece Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 3 Oct 2024 15:13:22 +0200 Subject: [PATCH 16/38] nit hub filtering on flow picker --- .../src/lib/components/flows/content/FlowInputsQuick.svelte | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte index 0483b1e9de..15817edd25 100644 --- a/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte +++ b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte @@ -258,6 +258,7 @@ {/if} { + filteredWorkspaceItems = [] selectedByKeyboard = 0 }} filters={integrations} @@ -431,6 +432,7 @@ {#if !selected && preFilter !== 'hub'}
Hub
{/if} + Date: Thu, 3 Oct 2024 15:13:39 +0200 Subject: [PATCH 17/38] chore(main): release 1.404.1 (#4474) * chore(main): release 1.404.1 * Apply automatic changes --------- Co-authored-by: rubenfiszel --- CHANGELOG.md | 7 +++ backend/Cargo.lock | 44 +++++++++---------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 4 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- python-client/wmill_pg/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 16 files changed, 46 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05bc5c98a2..28c1c60fce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [1.404.1](https://github.com/windmill-labs/windmill/compare/v1.404.0...v1.404.1) (2024-10-03) + + +### Bug Fixes + +* flow picker of flows ([92f61f0](https://github.com/windmill-labs/windmill/commit/92f61f07ed6d354407d26843e3a270b95bae90bc)) + ## [1.404.0](https://github.com/windmill-labs/windmill/compare/v1.403.1...v1.404.0) (2024-10-03) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 34aae89874..278fc815af 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -10355,7 +10355,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.404.0" +version = "1.404.1" dependencies = [ "anyhow", "axum", @@ -10396,7 +10396,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.404.0" +version = "1.404.1" dependencies = [ "anyhow", "argon2", @@ -10480,7 +10480,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.404.0" +version = "1.404.1" dependencies = [ "base64 0.21.7", "chrono", @@ -10498,7 +10498,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.404.0" +version = "1.404.1" dependencies = [ "chrono", "serde", @@ -10511,7 +10511,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.404.0" +version = "1.404.1" dependencies = [ "anyhow", "async-stream", @@ -10556,7 +10556,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.404.0" +version = "1.404.1" dependencies = [ "regex", "rsmq_async", @@ -10571,7 +10571,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.404.0" +version = "1.404.1" dependencies = [ "anyhow", "bytes", @@ -10592,7 +10592,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.404.0" +version = "1.404.1" dependencies = [ "convert_case 0.6.0", "serde", @@ -10601,7 +10601,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.404.0" +version = "1.404.1" dependencies = [ "anyhow", "lazy_static", @@ -10613,7 +10613,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.404.0" +version = "1.404.1" dependencies = [ "anyhow", "gosyn", @@ -10625,7 +10625,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.404.0" +version = "1.404.1" dependencies = [ "anyhow", "lazy_static", @@ -10637,7 +10637,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.404.0" +version = "1.404.1" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10648,7 +10648,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.404.0" +version = "1.404.1" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10659,7 +10659,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.404.0" +version = "1.404.1" dependencies = [ "anyhow", "async-recursion", @@ -10677,7 +10677,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.404.0" +version = "1.404.1" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -10694,7 +10694,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.404.0" +version = "1.404.1" dependencies = [ "anyhow", "lazy_static", @@ -10706,7 +10706,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.404.0" +version = "1.404.1" dependencies = [ "anyhow", "lazy_static", @@ -10724,7 +10724,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.404.0" +version = "1.404.1" dependencies = [ "anyhow", "getrandom 0.2.15", @@ -10745,7 +10745,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.404.0" +version = "1.404.1" dependencies = [ "anyhow", "serde_json", @@ -10755,7 +10755,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.404.0" +version = "1.404.1" dependencies = [ "anyhow", "async-recursion", @@ -10788,7 +10788,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.404.0" +version = "1.404.1" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -10798,7 +10798,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.404.0" +version = "1.404.1" dependencies = [ "anyhow", "async-recursion", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 30d1e63c98..891e46bce9 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.404.0" +version = "1.404.1" authors.workspace = true edition.workspace = true @@ -27,7 +27,7 @@ members = [ ] [workspace.package] -version = "1.404.0" +version = "1.404.1" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 24025318a7..7f28f437eb 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.404.0 + version: 1.404.1 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index d645983b64..9ebf2db1ec 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.404.0"; +export const VERSION = "v1.404.1"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/main.ts b/cli/main.ts index 550e6fe076..789325f243 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -60,7 +60,7 @@ export { // } // }); -export const VERSION = "1.404.0"; +export const VERSION = "1.404.1"; const command = new Command() .name("wmill") diff --git a/frontend/package-lock.json b/frontend/package-lock.json index bda1ed9443..b73a5af0a3 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.404.0", + "version": "1.404.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.404.0", + "version": "1.404.1", "license": "AGPL-3.0", "dependencies": { "@aws-crypto/sha256-js": "^4.0.0", diff --git a/frontend/package.json b/frontend/package.json index f7117231b7..a737397b7d 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.404.0", + "version": "1.404.1", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index 767b97d966..8dbd6a860b 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.404.0" -wmill_pg = ">=1.404.0" +wmill = ">=1.404.1" +wmill_pg = ">=1.404.1" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 5b5cf59ec7..400e1874b1 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.404.0 + version: 1.404.1 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index d74129549c..70f4ec9758 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. -ModuleVersion = '1.404.0' +ModuleVersion = '1.404.1' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 021029a756..f53ed97585 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.404.0" +version = "1.404.1" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index 54c21dfe83..140bfdbfa5 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.404.0" +version = "1.404.1" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 4d61c8e117..0060cf72b9 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.404.0", + "version": "1.404.1", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index da34f2c55a..e505e62523 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.404.0", + "version": "1.404.1", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index b50eeb55a4..7308051af2 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.404.0 +1.404.1 From f5c472727465dd95f5378bc08ee9bbb983f4d259 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Thu, 3 Oct 2024 09:59:12 -0400 Subject: [PATCH 18/38] feat(worker): support workers to run natively on windows (#4446) * minimal code change to get windmill worker on windows for bun and python + rustfmt * adding support for powershell * compiling error on unix * rust linting comments * comments hugo: PSModulePath * comments ruben, refactor to simplify * adding build workflow * editing workflow * editing workflow * editing workflow * editing workflow * editing workflow * skip migration env, ee fixes * improvements powershell * testing windows runner * testing windows runner * testing windows runner * testing windows runner * testing windows runner * install postgres on runner * install postgres on runner * install postgres on runner * install postgres on runner * install postgres on runner * install postgres on runner * install postgres on runner * install postgres on runner * killing process tree in windows * sqlx_offline * install openssl for github windows runner * used pre-installed openssl * used pre-installed openssl * build ee * build ee * build ee * build ee * adding commented out steps for artifact publishing * build on tag matchinv v* pattern * ren instead of mv on Windows Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com> * fix merging issue * gate imports for windows * fixing default cargo home path... * fixing default cargo home path... * comments ruben * make pwsh default modules loading more robust on unix (#4448) Co-authored-by: Ruben Fiszel --------- Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com> Co-authored-by: HugoCasa Co-authored-by: Ruben Fiszel --- .github/workflows/build_windows_worker.yml | 56 +++++++++ .../parsers/windmill-parser-yaml/src/lib.rs | 10 +- backend/src/main.rs | 12 +- backend/windmill-api/src/jobs.rs | 17 +-- backend/windmill-api/src/resources.rs | 5 +- backend/windmill-common/src/worker.rs | 3 + backend/windmill-indexer/src/indexer_ee.rs | 2 +- .../windmill-worker/src/ansible_executor.rs | 9 ++ backend/windmill-worker/src/bash_executor.rs | 113 ++++++++++++++++-- backend/windmill-worker/src/bun_executor.rs | 57 ++++++++- backend/windmill-worker/src/go_executor.rs | 7 +- backend/windmill-worker/src/handle_child.rs | 54 ++++++++- .../windmill-worker/src/python_executor.rs | 57 +++++++-- backend/windmill-worker/src/rust_executor.rs | 51 +++++++- backend/windmill-worker/src/worker.rs | 7 +- 15 files changed, 404 insertions(+), 56 deletions(-) create mode 100644 .github/workflows/build_windows_worker.yml diff --git a/.github/workflows/build_windows_worker.yml b/.github/workflows/build_windows_worker.yml new file mode 100644 index 0000000000..83e0855a50 --- /dev/null +++ b/.github/workflows/build_windows_worker.yml @@ -0,0 +1,56 @@ +name: Build and Publish Windows Worker + +on: + push: + tags: + - "v*" + +env: + CARGO_INCREMENTAL: 0 + SQLX_OFFLINE: true + DISABLE_EMBEDDING: true + RUST_LOG: info + +jobs: + cargo_build_windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + + - name: Read EE repo commit hash + shell: pwsh + run: | + $ee_repo_ref = Get-Content .\backend\ee-repo-ref.txt + echo "ee_repo_ref=$ee_repo_ref" | Out-File -FilePath $env:GITHUB_ENV -Append + + - name: Checkout windmill-ee-private repository + uses: actions/checkout@v4 + with: + repository: windmill-labs/windmill-ee-private + path: ./windmill-ee-private + ref: ${{ env.ee_repo_ref }} + token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} + fetch-depth: 0 + + - name: Substitute EE code + shell: bash + run: | + ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private + + - name: Cargo build windows + timeout-minutes: 60 + run: | + $env:OPENSSL_DIR = "${Env:ProgramFiles}\OpenSSL" + mkdir frontend/build && cd backend + New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force + cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy + + - name: Rename binary with corresponding architecture + run: | + ren ./backend/target/release/windmill.exe ./backend/target/release/windmill-ee.exe + + - name: Attach binary to release + uses: softprops/action-gh-release@v2 + with: + files: | + ./backend/target/release/windmill-ee.exe diff --git a/backend/parsers/windmill-parser-yaml/src/lib.rs b/backend/parsers/windmill-parser-yaml/src/lib.rs index 151d6eb562..f008f2f34a 100644 --- a/backend/parsers/windmill-parser-yaml/src/lib.rs +++ b/backend/parsers/windmill-parser-yaml/src/lib.rs @@ -399,15 +399,11 @@ fn parse_ansible_options(opts: &Vec) -> AnsiblePlaybookOptions { if c > 0 && c <= 6 { ret.verbosity = Some("v".repeat(c.min(6))); } - } } - _ => () - + _ => (), } } - - } } @@ -422,10 +418,10 @@ fn count_consecutive_vs(s: &str) -> usize { if c == 'v' { current_count += 1; if current_count == 6 { - return 6; // Stop early if we reach 6 + return 6; // Stop early if we reach 6 } } else { - current_count = 0; // Reset count if the character is not 'v' + current_count = 0; // Reset count if the character is not 'v' } max_count = max_count.max(current_count); } diff --git a/backend/src/main.rs b/backend/src/main.rs index 0c06702e4c..d7b2d776d6 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -364,8 +364,16 @@ async fn windmill_main() -> anyhow::Result<()> { let is_agent = mode == Mode::Agent; if !is_agent { - // migration code to avoid break - windmill_api::migrate_db(&db).await?; + let skip_migration = std::env::var("SKIP_MIGRATION") + .map(|val| val == "true") + .unwrap_or(false); + + if !skip_migration { + // migration code to avoid break + windmill_api::migrate_db(&db).await?; + } else { + tracing::info!("SKIP_MIGRATION set, skipping db migration...") + } } let (killpill_tx, mut killpill_rx) = tokio::sync::broadcast::channel::<()>(2); diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 156be41b26..a431e4a2f5 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -11,7 +11,6 @@ use axum::http::HeaderValue; use quick_cache::sync::Cache; use serde_json::value::RawValue; use sqlx::Pool; -use windmill_common::error::JsonResult; use std::collections::HashMap; #[cfg(feature = "prometheus")] use std::sync::atomic::Ordering; @@ -19,6 +18,7 @@ use tokio::io::AsyncReadExt; #[cfg(feature = "prometheus")] use tokio::time::Instant; use tower::ServiceBuilder; +use windmill_common::error::JsonResult; use windmill_common::flow_status::{JobResult, RestartedFrom}; use windmill_common::jobs::{ format_completed_job_result, format_result, CompletedJobWithFormattedResult, FormattedResult, @@ -293,11 +293,8 @@ pub fn workspace_unauthed_service() -> Router { pub fn global_root_service() -> Router { Router::new() - .route("/db_clock", get(get_db_clock)) - .route( - "/completed/count_by_tag", - get(count_by_tag), - ) + .route("/db_clock", get(get_db_clock)) + .route("/completed/count_by_tag", get(count_by_tag)) } #[derive(Deserialize)] @@ -4683,8 +4680,8 @@ async fn get_job_update( .fetch_optional(&db) .await?; - let progress: Option = if get_progress == Some(true){ - sqlx::query_scalar!( + let progress: Option = if get_progress == Some(true) { + sqlx::query_scalar!( "SELECT scalar_int FROM job_stats WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", &w_id, job_id, @@ -5115,8 +5112,6 @@ async fn get_completed_job_result( Ok(Json(result).into_response()) } - - #[derive(Deserialize)] struct CountByTagQuery { horizon_secs: Option, @@ -5130,7 +5125,7 @@ struct TagCount { } async fn count_by_tag( - ApiAuthed { email, ..}: ApiAuthed, + ApiAuthed { email, .. }: ApiAuthed, Extension(db): Extension, Query(query): Query, ) -> JsonResult> { diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index 1fb5b12259..bb881d5369 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -58,7 +58,10 @@ pub fn workspaced_service() -> Router { .route("/type/exists/:name", get(exists_resource_type)) .route("/type/update/:name", post(update_resource_type)) .route("/type/delete/:name", delete(delete_resource_type)) - .route("/file_resource_type_to_file_ext_map", get(file_resource_ext_to_resource_type)) + .route( + "/file_resource_type_to_file_ext_map", + get(file_resource_ext_to_resource_type), + ) .route("/type/create", post(create_resource_type)) } diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 9464eccdcc..385fac6163 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -449,10 +449,13 @@ pub async fn save_cache( fn write_binary_file(main_path: &str, byts: &mut bytes::Bytes) -> error::Result<()> { use std::fs::{File, Permissions}; use std::io::Write; + + #[cfg(unix)] use std::os::unix::fs::PermissionsExt; let mut file = File::create(main_path)?; file.write_all(byts)?; + #[cfg(unix)] file.set_permissions(Permissions::from_mode(0o755))?; file.flush()?; Ok(()) diff --git a/backend/windmill-indexer/src/indexer_ee.rs b/backend/windmill-indexer/src/indexer_ee.rs index 79bcbcff77..da92ff0ef8 100644 --- a/backend/windmill-indexer/src/indexer_ee.rs +++ b/backend/windmill-indexer/src/indexer_ee.rs @@ -1,6 +1,6 @@ +use anyhow::anyhow; use sqlx::{Pool, Postgres}; use windmill_common::error::Error; -use anyhow::anyhow; #[derive(Clone)] pub struct IndexReader; diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 6639189f61..7df394a77c 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -1,3 +1,4 @@ +#[cfg(unix)] use std::{ collections::HashMap, os::unix::fs::PermissionsExt, @@ -5,6 +6,13 @@ use std::{ process::Stdio, }; +#[cfg(windows)] +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + process::Stdio, +}; + use anyhow::anyhow; use itertools::Itertools; use serde_json::value::RawValue; @@ -378,6 +386,7 @@ fi let file = write_file(job_dir, "wrapper.sh", &wrapper)?; + #[cfg(unix)] file.metadata()?.permissions().set_mode(0o777); // let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index e9b135d277..1b1fc65f17 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -32,6 +32,9 @@ use crate::{ POWERSHELL_CACHE_DIR, POWERSHELL_PATH, TZ_ENV, }; +#[cfg(windows)] +use crate::SYSTEM_ROOT; + lazy_static::lazy_static! { pub static ref ANSI_ESCAPE_RE: Regex = Regex::new(r"\x1b\[[0-9;]*m").unwrap(); @@ -226,13 +229,19 @@ pub async fn handle_powershell_job( .collect::>() }; + #[cfg(windows)] + let split_char = '\\'; + + #[cfg(unix)] + let split_char = '/'; + let installed_modules = fs::read_dir(POWERSHELL_CACHE_DIR)? .filter_map(|x| { x.ok().map(|x| { x.path() .display() .to_string() - .split('/') + .split(split_char) .last() .unwrap_or_default() .to_lowercase() @@ -289,14 +298,26 @@ pub async fn handle_powershell_job( append_logs(&job.id, &job.workspace_id, logs2, db).await; // make sure default (only allhostsallusers) modules are loaded, disable autoload (cache can be large to explore especially on cloud) and add /tmp/windmill/cache to PSModulePath + #[cfg(unix)] let profile = format!( "$PSModuleAutoloadingPreference = 'None' $PSModulePathBackup = $env:PSModulePath -$env:PSModulePath = ($Env:PSModulePath -split ':')[-1] +$env:PSModulePath = \"$PSHome/Modules\" Get-Module -ListAvailable | Import-Module $env:PSModulePath = \"{}:$PSModulePathBackup\"", POWERSHELL_CACHE_DIR ); + + #[cfg(windows)] + let profile = format!( + "$PSModuleAutoloadingPreference = 'None' +$PSModulePathBackup = $env:PSModulePath +$env:PSModulePath = \"C:\\Program Files\\PowerShell\\7\\Modules\" +Get-Module -ListAvailable | Import-Module +$env:PSModulePath = \"{};$PSModulePathBackup\"", + POWERSHELL_CACHE_DIR + ); + // make sure param() is first let param_match = windmill_parser_bash::RE_POWERSHELL_PARAM.find(&content); let content: String = if let Some(param_match) = param_match { @@ -312,11 +333,29 @@ $env:PSModulePath = \"{}:$PSModulePathBackup\"", }; write_file(job_dir, "main.ps1", content.as_str())?; + + #[cfg(unix)] write_file( job_dir, "wrapper.sh", &format!("set -o pipefail\nset -e\nmkfifo bp\ncat bp | tail -1 > ./result2.out &\n{} -F ./main.ps1 \"$@\" 2>&1 | tee bp\nwait $!", POWERSHELL_PATH.as_str()), )?; + + #[cfg(windows)] + write_file( + job_dir, + "wrapper.ps1", + &format!( + "param([string[]]$args)\n\ + $ErrorActionPreference = 'Stop'\n\ + $pipe = New-TemporaryFile\n\ + & \"{}\" -File ./main.ps1 @args 2>&1 | Tee-Object -FilePath $pipe\n\ + Get-Content -Path $pipe | Select-Object -Last 1 | Set-Content -Path './result2.out'\n\ + Remove-Item $pipe\n", + POWERSHELL_PATH.as_str() + ), + )?; + let token = client.get_token().await; let mut reserved_variables = get_reserved_variables(job, &token, db).await?; reserved_variables.insert("RUST_LOG".to_string(), "info".to_string()); @@ -355,10 +394,24 @@ $env:PSModulePath = \"{}:$PSModulePathBackup\"", .stderr(Stdio::piped()) .spawn()? } else { - let mut cmd_args = vec!["wrapper.sh"]; - cmd_args.extend(pwsh_args.iter().map(|x| x.as_str())); - Command::new(BIN_BASH.as_str()) - .current_dir(job_dir) + let mut cmd; + let mut cmd_args; + + #[cfg(unix)] + { + cmd_args = vec!["wrapper.sh"]; + cmd_args.extend(pwsh_args.iter().map(|x| x.as_str())); + cmd = Command::new(BIN_BASH.as_str()); + } + + #[cfg(windows)] + { + cmd_args = vec![r".\wrapper.ps1".to_string()]; + cmd_args.extend(pwsh_args.iter().map(|x| x.replace("--", "-"))); + cmd = Command::new(POWERSHELL_PATH.as_str()); + } + + cmd.current_dir(job_dir) .env_clear() .envs(envs) .envs(reserved_variables) @@ -366,11 +419,53 @@ $env:PSModulePath = \"{}:$PSModulePathBackup\"", .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("HOME", HOME_ENV.as_str()) - .args(cmd_args) + .args(&cmd_args) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? + .stderr(Stdio::piped()); + + #[cfg(windows)] + { + cmd.env("SystemRoot", SYSTEM_ROOT.as_str()) + .env( + "LOCALAPPDATA", + std::env::var("LOCALAPPDATA") + .unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())), + ) + .env( + "ProgramData", + std::env::var("ProgramData") + .unwrap_or_else(|_| String::from("C:\\ProgramData")), + ) + .env( + "ProgramFiles", + std::env::var("ProgramFiles") + .unwrap_or_else(|_| String::from("C:\\Program Files")), + ) + .env( + "ProgramFiles(x86)", + std::env::var("ProgramFiles(x86)") + .unwrap_or_else(|_| String::from("C:\\Program Files (x86)")), + ) + .env( + "ProgramW6432", + std::env::var("ProgramW6432") + .unwrap_or_else(|_| String::from("C:\\Program Files")), + ) + .env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), + ) + .env( + "PATHEXT", + std::env::var("PATHEXT").unwrap_or_else(|_| { + String::from(".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC;.CPL") + }), + ); + } + + cmd.spawn()? }; + handle_child( &job.id, db, diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index b63ae6c001..7fc786277d 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -29,6 +29,9 @@ use crate::{ NODE_PATH, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, TZ_ENV, }; +#[cfg(windows)] +use crate::SYSTEM_ROOT; + use tokio::{fs::File, process::Command}; use tokio::io::AsyncReadExt; @@ -118,6 +121,9 @@ pub async fn gen_bun_lockfile( .stdout(Stdio::piped()) .stderr(Stdio::piped()); + #[cfg(windows)] + child_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + let mut child_process = start_child_process(child_cmd, &*BUN_PATH).await?; if let Some(db) = db { @@ -251,6 +257,9 @@ pub async fn install_bun_lockfile( .stdout(Stdio::piped()) .stderr(Stdio::piped()); + #[cfg(windows)] + child_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + let mut npm_logs = if npm_mode { "NPM mode\n".to_string() } else { @@ -459,6 +468,10 @@ pub async fn generate_wrapper_mjs( .args(vec!["run", "node_builder.ts"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + + #[cfg(windows)] + child.env("SystemRoot", SYSTEM_ROOT.as_str()); + let child_process = start_child_process(child, &*BUN_PATH).await?; handle_child( job_id, @@ -504,6 +517,10 @@ pub async fn generate_bun_bundle( .args(vec!["run", "node_builder.ts"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + + #[cfg(windows)] + child.env("SystemRoot", SYSTEM_ROOT.as_str()); + let mut child_process = start_child_process(child, &*BUN_PATH).await?; if let Some(db) = db { handle_child( @@ -546,7 +563,11 @@ pub async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result<()> { if is_tar { extract_tar(fs::read(bun_cache_path)?.into(), job_dir).await?; } else { + #[cfg(unix)] tokio::fs::symlink(&bun_cache_path, dst).await?; + + #[cfg(windows)] + std::os::windows::fs::symlink_dir(&bun_cache_path, &dst)?; } } else if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS .read() @@ -559,7 +580,11 @@ pub async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result<()> { if is_tar { extract_tar(bytes, job_dir).await?; } else { + #[cfg(unix)] tokio::fs::symlink(bun_cache_path, dst).await?; + + #[cfg(windows)] + std::os::windows::fs::symlink_dir(&bun_cache_path, &dst)?; } // extract_tar(bytes, job_dir).await?; @@ -728,6 +753,10 @@ async fn compute_bundle_local_and_remote_path( let hash = windmill_common::utils::calculate_hash(&input_src); let local_path = format!("{BUN_BUNDLE_CACHE_DIR}/{hash}"); + + #[cfg(windows)] + let local_path = local_path.replace("/tmp", r"C:\tmp").replace("/", r"\"); + let remote_path = format!("{BUN_BUNDLE_OBJECT_STORE_PREFIX}{hash}"); (local_path, remote_path) } @@ -821,10 +850,23 @@ pub async fn handle_bun_job( )); } - let mut gbuntar_name = None; + let mut gbuntar_name: Option = None; if has_bundle_cache { - let target = format!("{job_dir}/main.js"); - std::os::unix::fs::symlink(&local_path, &target).map_err(|e| { + let target; + let symlink; + + #[cfg(unix)] + { + target = format!("{job_dir}/main.js"); + symlink = std::os::unix::fs::symlink(&local_path, &target); + } + #[cfg(windows)] + { + target = format!("{job_dir}\\main.js"); + symlink = std::os::windows::fs::symlink_dir(&local_path, &target); + } + + symlink.map_err(|e| { error::Error::ExecutionErr(format!( "could not copy cached binary from {local_path} to {job_dir}/main: {e:?}" )) @@ -1339,6 +1381,10 @@ try {{ .args(vec!["--preserve-symlinks", &script_path]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + + #[cfg(windows)] + bun_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + bun_cmd } else { let script_path = format!("{job_dir}/wrapper.mjs"); @@ -1365,8 +1411,13 @@ try {{ .args(args) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + + #[cfg(windows)] + bun_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + bun_cmd }; + start_child_process( cmd, if annotation.nodejs_mode { diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index b139859005..c171e52156 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -225,7 +225,12 @@ func Run(req Req) (interface{{}}, error){{ } } else { let target = format!("{job_dir}/main"); - std::os::unix::fs::symlink(&bin_path, &target).map_err(|e| { + #[cfg(unix)] + let symlink = std::os::unix::fs::symlink(&bin_path, &target); + #[cfg(windows)] + let symlink = std::os::windows::fs::symlink_dir(&bin_path, &target); + + symlink.map_err(|e| { Error::ExecutionErr(format!( "could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}" )) diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index d588ea8fc4..791b60b9a6 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -6,7 +6,11 @@ use nix::sys::signal::{self, Signal}; use nix::unistd::Pid; use sqlx::{Pool, Postgres}; +#[cfg(windows)] +use std::process::Stdio; use tokio::fs::File; +#[cfg(windows)] +use tokio::process::Command; use windmill_common::error::to_anyhow; use windmill_common::error::{self, Error}; @@ -49,6 +53,33 @@ use crate::{MAX_RESULT_SIZE, MAX_WAIT_FOR_SIGINT, MAX_WAIT_FOR_SIGTERM}; lazy_static::lazy_static! { pub static ref SLOW_LOGS: bool = std::env::var("SLOW_LOGS").ok().is_some_and(|x| x == "1" || x == "true"); } + +// - kill windows process along with all child processes +#[cfg(windows)] +async fn kill_process_tree(pid: Option) -> Result<(), String> { + let pid = match pid { + Some(pid) => pid, + None => return Err("No PID provided to kill.".to_string()), + }; + + let output = Command::new("cmd") + .args(&["/C", "taskkill", "/PID", &pid.to_string(), "/T", "/F"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await + .map_err(|e| format!("Failed to execute taskkill: {}", e))?; + + if output.status.success() { + Ok(()) + } else { + Err(format!( + "Failed to kill process tree. Error: {}", + String::from_utf8_lossy(&output.stderr) + )) + } +} + /// - wait until child exits and return with exit status /// - read lines from stdout and stderr and append them to the "queue"."logs" /// quitting early if output exceedes MAX_LOG_SIZE characters (not bytes) @@ -196,9 +227,26 @@ pub async fn handle_child( } } } - /* send SIGKILL and reap child process */ - let (_, kill) = future::join(set_reason, child.kill()).await; - kill.map(|()| Err(kill_reason)) + #[cfg(windows)] + { + let pid_to_kill = child.id(); + match kill_process_tree(pid_to_kill).await { + Ok(_) => tracing::debug!( + "successfully killed process tree with PID: {:?}", + pid_to_kill + ), + Err(e) => tracing::error!("failed to kill process tree: {:?}", e), + }; + set_reason.await; + return Ok(Err(kill_reason)); + } + + #[cfg(unix)] + { + /* send SIGKILL and reap child process */ + let (_, kill) = future::join(set_reason, child.kill()).await; + kill.map(|()| Err(kill_reason)) + } }; /* a future that reads output from the child and appends to the database */ diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 87c1c829ae..a5717ed12b 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -64,6 +64,9 @@ use crate::{ PIP_INDEX_URL, TZ_ENV, }; +#[cfg(windows)] +use crate::SYSTEM_ROOT; + pub async fn create_dependencies_dir(job_dir: &str) { DirBuilder::new() .recursive(true) @@ -399,6 +402,9 @@ except BaseException as e: let mut reserved_variables = get_reserved_variables(job, &client.token, db).await?; let additional_python_paths_folders = additional_python_paths.iter().join(":"); + #[cfg(windows)] + let additional_python_paths_folders = additional_python_paths_folders.replace(":", ";"); + if !*DISABLE_NSJAIL { let shared_deps = additional_python_paths .into_iter() @@ -475,6 +481,10 @@ mount {{ .args(vec!["-u", "-m", "wrapper"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + + #[cfg(windows)] + python_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + start_child_process(python_cmd, PYTHON_PATH.as_str()).await? }; @@ -1016,7 +1026,12 @@ pub async fn handle_python_reqs( start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await? } else { let fssafe_req = NON_ALPHANUM_CHAR.replace_all(&req, "_").to_string(); + #[cfg(unix)] let req = format!("'{}'", req); + + #[cfg(windows)] + let req = format!("{}", req); + let mut command_args = vec![ PYTHON_PATH.as_str(), "-m", @@ -1072,19 +1087,35 @@ pub async fn handle_python_reqs( tracing::debug!("pip install command: {:?}", command_args); - let mut flock_cmd = Command::new(FLOCK_PATH.as_str()); - flock_cmd - .env_clear() - .envs(envs) - .args([ - "-x", - &format!("{}/pip-{}.lock", LOCK_CACHE_DIR, fssafe_req), - "--command", - &command_args.join(" "), - ]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - start_child_process(flock_cmd, FLOCK_PATH.as_str()).await? + #[cfg(unix)] + { + let mut flock_cmd = Command::new(FLOCK_PATH.as_str()); + flock_cmd + .env_clear() + .envs(envs) + .args([ + "-x", + &format!("{}/pip-{}.lock", LOCK_CACHE_DIR, fssafe_req), + "--command", + &command_args.join(" "), + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + start_child_process(flock_cmd, FLOCK_PATH.as_str()).await? + } + + #[cfg(windows)] + { + let mut pip_cmd = Command::new(PYTHON_PATH.as_str()); + pip_cmd + .env_clear() + .envs(envs) + .env("SystemRoot", SYSTEM_ROOT.as_str()) + .args(&command_args[1..]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + start_child_process(pip_cmd, PYTHON_PATH.as_str()).await? + } }; let child = handle_child( diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index ab6fdd247f..6a0090ab55 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -23,12 +23,28 @@ use crate::{ RUST_CACHE_DIR, TZ_ENV, }; +#[cfg(windows)] +use crate::SYSTEM_ROOT; + const NSJAIL_CONFIG_RUN_RUST_CONTENT: &str = include_str!("../nsjail/run.rust.config.proto"); lazy_static::lazy_static! { - static ref CARGO_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| "/usr/local/cargo".to_string()); - static ref RUSTUP_HOME: String = std::env::var("RUSTUP_HOME").unwrap_or_else(|_| "/usr/local/rustup".to_string()); - static ref CARGO_PATH: String = format!("{}/bin/cargo", std::env::var("CARGO_HOME").unwrap_or("/usr/local/cargo/bin/cargo".to_string())); + static ref HOME_DIR: String = std::env::var("HOME").expect("Could not find the HOME environment variable"); + static ref CARGO_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| { CARGO_HOME_DEFAULT.clone() }); + static ref RUSTUP_HOME: String = std::env::var("RUSTUP_HOME").unwrap_or_else(|_| { RUSTUP_HOME_DEFAULT.clone() }); + static ref CARGO_PATH: String = format!("{}/bin/cargo", CARGO_HOME.as_str()); +} + +#[cfg(windows)] +lazy_static::lazy_static! { + static ref CARGO_HOME_DEFAULT: String = format!("{}\\.cargo", *HOME_DIR); + static ref RUSTUP_HOME_DEFAULT: String = format!("{}\\.rustup", *HOME_DIR); +} + +#[cfg(unix)] +lazy_static::lazy_static! { + static ref CARGO_HOME_DEFAULT: String = "/usr/local/cargo".to_string(); + static ref RUSTUP_HOME_DEFAULT: String = "/usr/local/rustup".to_string(); } const RUST_OBJECT_STORE_PREFIX: &str = "rustbin/"; @@ -126,6 +142,14 @@ pub async fn generate_cargo_lockfile( .args(vec!["generate-lockfile"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + #[cfg(windows)] + { + gen_lockfile_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + gen_lockfile_cmd.env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()), + ); + } let gen_lockfile_process = start_child_process(gen_lockfile_cmd, CARGO_PATH.as_str()).await?; handle_child( job_id, @@ -176,6 +200,16 @@ pub async fn build_rust_crate( .args(vec!["build", "--release"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + + #[cfg(windows)] + { + build_rust_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + build_rust_cmd.env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()), + ); + } + let build_rust_process = start_child_process(build_rust_cmd, CARGO_PATH.as_str()).await?; handle_child( job_id, @@ -279,7 +313,13 @@ pub async fn handle_rust_job( let cache_logs = if cache { let target = format!("{job_dir}/main"); - std::os::unix::fs::symlink(&bin_path, &target).map_err(|e| { + + #[cfg(unix)] + let symlink = std::os::unix::fs::symlink(&bin_path, &target); + #[cfg(windows)] + let symlink = std::os::windows::fs::symlink_dir(&bin_path, &target); + + symlink.map_err(|e| { Error::ExecutionErr(format!( "could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}" )) @@ -360,6 +400,9 @@ pub async fn handle_rust_job( .stdout(Stdio::piped()) .stderr(Stdio::piped()); + #[cfg(windows)] + run_rust.env("SystemRoot", SYSTEM_ROOT.as_str()); + start_child_process(run_rust, compiled_executable_name).await? }; handle_child( diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 6c0fce3d3b..918c563ea3 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -348,7 +348,6 @@ lazy_static::lazy_static! { pub static ref PIP_INDEX_URL: Arc>> = Arc::new(RwLock::new(None)); pub static ref JOB_DEFAULT_TIMEOUT: Arc>> = Arc::new(RwLock::new(None)); - static ref MAX_TIMEOUT: u64 = std::env::var("TIMEOUT") .ok() .and_then(|x| x.parse::().ok()) @@ -389,6 +388,12 @@ lazy_static::lazy_static! { } + +#[cfg(windows)] +lazy_static::lazy_static! { + pub static ref SYSTEM_ROOT: String = std::env::var("SystemRoot").unwrap_or_else(|_| "C:\\Windows".to_string()); +} + //only matter if CLOUD_HOSTED pub const MAX_RESULT_SIZE: usize = 1024 * 1024 * 2; // 2MB From b54c9ee657cc88fabe694cae39dc0d3c1918fcbb Mon Sep 17 00:00:00 2001 From: pyranota <92104930+pyranota@users.noreply.github.com> Date: Thu, 3 Oct 2024 14:00:35 +0000 Subject: [PATCH 19/38] feat: Replace `pip-compile` with `uv` (#4460) * Update `shell.nix` - Replace pip-compile with uv packages - Pin rust version - Add var to trigger windmill print more info in stdout * Replace `pip-compile` with `uv` (dirty + untested) * Fix arguments passed to uv Some of the flags are included by default in UV and can be safely removed: - --resolver=backtracking - --no-emit-index-url Also uv does not support `--pip-args` and suggests to directly pass args to uv. * Remove extra `dbg!` * Replace 'pip-compile' with 'uv' in Dockerfile * Add fallback option to `pip-compile` (Disabled) * Add `uv` to `docker/DockerfileSlim*` * Add `get_annotation_python` and rename `get_annotation` to `get_annotation_ts` * Add option to fallback to pip-compile Put `# no_uv` on top the file for specific python script Or set `USE_PIP_COMPILE` variable to `true` * Put back `pip-tools` into shell.nix * Make sure lockfile resolves again if `#no_uv` used Add #no_uv to the end of requirements (requirements.in) That way if something breaks for customer, then they put #no_uv and new lockfile will be resolved * Put `pip install pip-tools` in original spot * Fix compilation error * Fix EE compilation error error[E0658]: attributes on expressions are experimental --> windmill-worker/src/python_executor.rs:144:5 | 144 | #[cfg(feature = "enterprise")] | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: see issue #15701 for more information * Add `no_cache` annotation Will force recalculation of lockfile And block uv from using cached values * Target uv cache to /tmp/windmill/cache * Prohibit uv from managing python * Add uv to DockerfileBackendTests * Pin uv version to 0.4.18 in Dockerfiles * Dont put `#no_uv` in requirements.in Instead postfix hash for requirements.in with `-no_uv` * Push Warning to logs if fallbacked to pip-compile --------- Co-authored-by: Ruben Fiszel --- .github/DockerfileBackendTests | 3 +- Dockerfile | 3 + backend/src/main.rs | 3 +- backend/windmill-api/src/jobs.rs | 2 +- backend/windmill-api/src/scripts.rs | 4 +- backend/windmill-common/src/worker.rs | 24 +- .../windmill-worker/src/ansible_executor.rs | 6 +- backend/windmill-worker/src/bun_executor.rs | 8 +- .../windmill-worker/src/python_executor.rs | 253 +++++++++++++----- backend/windmill-worker/src/worker.rs | 1 + .../windmill-worker/src/worker_lockfiles.rs | 19 +- docker/DockerfileSlim | 3 + docker/DockerfileSlimEe | 2 + shell.nix | 8 +- 14 files changed, 240 insertions(+), 99 deletions(-) diff --git a/.github/DockerfileBackendTests b/.github/DockerfileBackendTests index 03f51de332..c3990c2b4d 100644 --- a/.github/DockerfileBackendTests +++ b/.github/DockerfileBackendTests @@ -43,7 +43,8 @@ RUN wget https://golang.org/dl/go1.21.5.linux-amd64.tar.gz && tar -C /usr/local ENV PATH="${PATH}:/usr/local/go/bin" ENV GO_PATH=/usr/local/go/bin/go -RUN curl -LsSf https://astral.sh/uv/install.sh | sh +# Install UV +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh ENV TZ=Etc/UTC diff --git a/Dockerfile b/Dockerfile index 3bc5088aed..d5d4b8cc92 100644 --- a/Dockerfile +++ b/Dockerfile @@ -158,6 +158,9 @@ RUN set -eux; \ ENV PATH="${PATH}:/usr/local/go/bin" ENV GO_PATH=/usr/local/go/bin/go +# Install UV +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh + RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - RUN apt-get -y update && apt-get install -y curl nodejs awscli && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/backend/src/main.rs b/backend/src/main.rs index d7b2d776d6..c99e551ddb 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -67,7 +67,7 @@ use windmill_worker::{ get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR, - RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, + RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR, }; use crate::monitor::{ @@ -873,6 +873,7 @@ pub async fn run_workers( }; let lang = if &ns.language == &ScriptLang::Bun || &ns.language == &ScriptLang::Bunnative { - let anns = get_annotation(&ns.content); + let anns = get_annotation_ts(&ns.content); if anns.native_mode { ScriptLang::Bunnative } else { diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 385fac6163..89f2c148e0 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -303,14 +303,14 @@ fn parse_file(path: &str) -> Option { .flatten() } -pub struct Annotations { +pub struct TypeScriptAnnotations { pub npm_mode: bool, pub nodejs_mode: bool, pub native_mode: bool, pub nobundling: bool, } -pub fn get_annotation(inner_content: &str) -> Annotations { +pub fn get_annotation_ts(inner_content: &str) -> TypeScriptAnnotations { let annotations = inner_content .lines() .take_while(|x| x.starts_with("//")) @@ -324,7 +324,25 @@ pub fn get_annotation(inner_content: &str) -> Annotations { let nobundling: bool = annotations.contains(&"nobundling".to_string()) || nodejs_mode || *DISABLE_BUNDLING; - Annotations { npm_mode, nodejs_mode, native_mode, nobundling } + TypeScriptAnnotations { npm_mode, nodejs_mode, native_mode, nobundling } +} + +pub struct PythonAnnotations { + pub no_uv: bool, + pub no_cache: bool, +} + +pub fn get_annotation_python(inner_content: &str) -> PythonAnnotations { + let annotations = inner_content + .lines() + .take_while(|x| x.starts_with("#")) + .map(|x| x.to_string().replace("#", "").trim().to_string()) + .collect_vec(); + + let no_uv: bool = annotations.contains(&"no_uv".to_string()); + let no_cache: bool = annotations.contains(&"no_cache".to_string()); + + PythonAnnotations { no_uv, no_cache } } pub struct SqlAnnotations { diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 7df394a77c..9c2474d95f 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -33,7 +33,7 @@ use crate::{ OccupancyMetrics, }, handle_child::handle_child, - python_executor::{create_dependencies_dir, handle_python_reqs, pip_compile}, + python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile}, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, TZ_ENV, }; @@ -80,7 +80,7 @@ async fn handle_ansible_python_deps( if requirements.is_empty() { "".to_string() } else { - pip_compile( + uv_pip_compile( job_id, &requirements, mem_peak, @@ -90,6 +90,8 @@ async fn handle_ansible_python_deps( worker_name, w_id, &mut Some(occupancy_metrics), + false, + false, ) .await .map_err(|e| { diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 7fc786277d..c45d4c2853 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -47,7 +47,7 @@ use windmill_common::{ get_latest_hash_for_path, jobs::{QueuedJob, PREPROCESSOR_FAKE_ENTRYPOINT}, scripts::ScriptLang, - worker::{exists_in_cache, get_annotation, save_cache, write_file}, + worker::{exists_in_cache, get_annotation_ts, save_cache, write_file}, DB, }; @@ -663,7 +663,7 @@ pub async fn prebundle_bun_script( if exists_in_cache(&local_path, &remote_path).await { return Ok(()); } - let annotation = get_annotation(inner_content); + let annotation = get_annotation_ts(inner_content); if annotation.nobundling { return Ok(()); } @@ -800,7 +800,7 @@ pub async fn handle_bun_job( new_args: &mut Option>>, occupancy_metrics: &mut OccupancyMetrics, ) -> error::Result> { - let mut annotation = windmill_common::worker::get_annotation(inner_content); + let mut annotation = windmill_common::worker::get_annotation_ts(inner_content); let (mut has_bundle_cache, cache_logs, local_path, remote_path) = if requirements_o.is_some() && !annotation.nobundling && codebase.is_none() { @@ -1525,7 +1525,7 @@ pub async fn start_worker( let common_bun_proc_envs: HashMap = get_common_bun_proc_envs(Some(&base_internal_url)).await; - let mut annotation = windmill_common::worker::get_annotation(inner_content); + let mut annotation = windmill_common::worker::get_annotation_ts(inner_content); //TODO: remove this when bun dedicated workers work without issues annotation.nodejs_mode = true; diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index a5717ed12b..baefdfd9bc 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -36,6 +36,9 @@ lazy_static::lazy_static! { static ref PIP_TRUSTED_HOST: Option = std::env::var("PIP_TRUSTED_HOST").ok(); static ref PIP_INDEX_CERT: Option = std::env::var("PIP_INDEX_CERT").ok(); + static ref USE_PIP_COMPILE: bool = std::env::var("USE_PIP_COMPILE") + .ok().map(|flag| flag == "true").unwrap_or(false); + static ref RELATIVE_IMPORT_REGEX: Regex = Regex::new(r#"(import|from)\s(((u|f)\.)|\.)"#).unwrap(); @@ -61,7 +64,7 @@ use crate::{ handle_child::handle_child, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, HTTPS_PROXY, HTTP_PROXY, LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, - PIP_INDEX_URL, TZ_ENV, + PIP_INDEX_URL, TZ_ENV, UV_CACHE_DIR, }; #[cfg(windows)] @@ -96,7 +99,7 @@ pub fn handle_ephemeral_token(x: String) -> String { x } -pub async fn pip_compile( +pub async fn uv_pip_compile( job_id: &Uuid, requirements: &str, mem_peak: &mut i32, @@ -106,6 +109,10 @@ pub async fn pip_compile( worker_name: &str, w_id: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + // Fallback to pip-compile. Will be removed in future + mut no_uv: bool, + // Debug-only flag + no_cache: bool, ) -> error::Result { let mut logs = String::new(); logs.push_str(&format!("\nresolving dependencies...")); @@ -142,83 +149,178 @@ pub async fn pip_compile( #[cfg(feature = "enterprise")] let requirements = replace_pip_secret(db, w_id, &requirements, worker_name, job_id).await?; - let req_hash = format!("py-{}", calculate_hash(&requirements)); - if let Some(cached) = sqlx::query_scalar!( - "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1", - req_hash - ) - .fetch_optional(db) - .await? - { - logs.push_str(&format!("\nfound cached resolution: {req_hash}")); - return Ok(cached); + let mut req_hash = format!("py-{}", calculate_hash(&requirements)); + + if no_uv || *USE_PIP_COMPILE { + logs.push_str(&format!("\nFallback to pip-compile (Deprecated!)")); + // Set no_uv if not setted + no_uv = true; + // Make sure that if we put #no_uv (switch to pip-compile) to python code or used `USE_PIP_COMPILE=true` variable. + // Windmill will recalculate lockfile using pip-compile and dont take potentially broken lockfile (generated by uv) from cache (our db). + // It will recalculate lockfile even if inputs have not been changed. + req_hash.push_str("-no_uv"); + // Will be in format: + // py-000..000-no_uv + } + if !no_cache { + if let Some(cached) = sqlx::query_scalar!( + "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1", + req_hash + ) + .fetch_optional(db) + .await? + { + logs.push_str(&format!("\nfound cached resolution: {req_hash}")); + return Ok(cached); + } } let file = "requirements.in"; write_file(job_dir, file, &requirements)?; - let mut args = vec![ - "-q", - "--no-header", - file, - "--resolver=backtracking", - "--strip-extras", - ]; - let mut pip_args = vec![]; - let pip_extra_index_url = PIP_EXTRA_INDEX_URL - .read() - .await - .clone() - .map(handle_ephemeral_token); - if let Some(url) = pip_extra_index_url.as_ref() { - args.extend(["--extra-index-url", url, "--no-emit-index-url"]); - pip_args.push(format!("--extra-index-url {}", url)); - } - let pip_index_url = PIP_INDEX_URL - .read() - .await - .clone() - .map(handle_ephemeral_token); - if let Some(url) = pip_index_url.as_ref() { - args.extend(["--index-url", url, "--no-emit-index-url"]); - pip_args.push(format!("--index-url {}", url)); - } - if let Some(host) = PIP_TRUSTED_HOST.as_ref() { - args.extend(["--trusted-host", host]); - } - if let Some(cert_path) = PIP_INDEX_CERT.as_ref() { - args.extend(["--cert", cert_path]); - } - let pip_args_str = pip_args.join(" "); - if pip_args.len() > 0 { - args.extend(["--pip-args", &pip_args_str]); - } - tracing::debug!("pip-compile args: {:?}", args); + // Fallback pip-compile. Will be removed in future + if no_uv { + tracing::debug!("Fallback to pip-compile"); + + let mut args = vec![ + "-q", + "--no-header", + file, + "--resolver=backtracking", + "--strip-extras", + ]; + let mut pip_args = vec![]; + let pip_extra_index_url = PIP_EXTRA_INDEX_URL + .read() + .await + .clone() + .map(handle_ephemeral_token); + if let Some(url) = pip_extra_index_url.as_ref() { + args.extend(["--extra-index-url", url, "--no-emit-index-url"]); + pip_args.push(format!("--extra-index-url {}", url)); + } + let pip_index_url = PIP_INDEX_URL + .read() + .await + .clone() + .map(handle_ephemeral_token); + if let Some(url) = pip_index_url.as_ref() { + args.extend(["--index-url", url, "--no-emit-index-url"]); + pip_args.push(format!("--index-url {}", url)); + } + if let Some(host) = PIP_TRUSTED_HOST.as_ref() { + args.extend(["--trusted-host", host]); + } + if let Some(cert_path) = PIP_INDEX_CERT.as_ref() { + args.extend(["--cert", cert_path]); + } + let pip_args_str = pip_args.join(" "); + if pip_args.len() > 0 { + args.extend(["--pip-args", &pip_args_str]); + } + tracing::debug!("pip-compile args: {:?}", args); + + let mut child_cmd = Command::new("pip-compile"); + child_cmd + .current_dir(job_dir) + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let child_process = start_child_process(child_cmd, "pip-compile").await?; + append_logs(&job_id, &w_id, logs, db).await; + handle_child( + job_id, + db, + mem_peak, + canceled_by, + child_process, + false, + worker_name, + &w_id, + "pip-compile", + None, + false, + occupancy_metrics, + ) + .await + .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?; + } else { + let mut args = vec![ + "pip", + "compile", + "-q", + "--no-header", + file, + "--strip-extras", + "-o", + "requirements.txt", + // Prefer main index over extra + // https://docs.astral.sh/uv/pip/compatibility/#packages-that-exist-on-multiple-indexes + // TODO: Use env variable that can be toggled from UI + "--index-strategy", + "unsafe-best-match", + // Target to /tmp/windmill/cache/uv + "--cache-dir", + UV_CACHE_DIR, + // We dont want UV to manage python installations + "--python-preference", + "only-system", + "--no-python-downloads", + ]; + if no_cache { + args.extend(["--no-cache"]); + } + let pip_extra_index_url = PIP_EXTRA_INDEX_URL + .read() + .await + .clone() + .map(handle_ephemeral_token); + if let Some(url) = pip_extra_index_url.as_ref() { + args.extend(["--extra-index-url", url]); + } + let pip_index_url = PIP_INDEX_URL + .read() + .await + .clone() + .map(handle_ephemeral_token); + if let Some(url) = pip_index_url.as_ref() { + args.extend(["--index-url", url]); + } + if let Some(host) = PIP_TRUSTED_HOST.as_ref() { + args.extend(["--trusted-host", host]); + } + if let Some(cert_path) = PIP_INDEX_CERT.as_ref() { + args.extend(["--cert", cert_path]); + } + tracing::debug!("uv args: {:?}", args); + + let mut child_cmd = Command::new("uv"); + child_cmd + .current_dir(job_dir) + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let child_process = start_child_process(child_cmd, "uv").await?; + append_logs(&job_id, &w_id, logs, db).await; + handle_child( + job_id, + db, + mem_peak, + canceled_by, + child_process, + false, + worker_name, + &w_id, + // TODO: Rename to uv-pip-compile? + "uv", + None, + false, + occupancy_metrics, + ) + .await + .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?; + } - let mut child_cmd = Command::new("pip-compile"); - child_cmd - .current_dir(job_dir) - .args(args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let child_process = start_child_process(child_cmd, "pip-compile").await?; - append_logs(&job_id, &w_id, logs, db).await; - handle_child( - job_id, - db, - mem_peak, - canceled_by, - child_process, - false, - worker_name, - &w_id, - "pip-compile", - None, - false, - occupancy_metrics, - ) - .await - .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?; let path_lock = format!("{job_dir}/requirements.txt"); let mut file = File::open(path_lock).await?; let mut req_content = "".to_string(); @@ -784,6 +886,7 @@ async fn handle_python_deps( let requirements = match requirements_o { Some(r) => r, None => { + let annotation = windmill_common::worker::get_annotation_python(inner_content); let mut already_visited = vec![]; let requirements = windmill_parser_py_imports::parse_python_imports( @@ -798,7 +901,7 @@ async fn handle_python_deps( if requirements.is_empty() { "".to_string() } else { - pip_compile( + uv_pip_compile( job_id, &requirements, mem_peak, @@ -808,6 +911,8 @@ async fn handle_python_deps( worker_name, w_id, occupancy_metrics, + annotation.no_uv, + annotation.no_cache, ) .await .map_err(|e| { diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 918c563ea3..83e8adff25 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -237,6 +237,7 @@ pub const ROOT_CACHE_NOMOUNT_DIR: &str = concatcp!(TMP_DIR, "/cache_nomount/"); pub const LOCK_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "lock"); pub const PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "pip"); +pub const UV_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "uv"); pub const TAR_PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/pip"); pub const DENO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "deno"); pub const DENO_CACHE_DIR_DEPS: &str = concatcp!(ROOT_CACHE_DIR, "deno/deps"); diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 6f25ec606b..ed89c535e8 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -12,7 +12,7 @@ use windmill_common::flows::{FlowModule, FlowModuleValue}; use windmill_common::get_latest_deployed_hash_for_path; use windmill_common::jobs::JobPayload; use windmill_common::scripts::ScriptHash; -use windmill_common::worker::{get_annotation, to_raw_value, to_raw_value_owned, write_file}; +use windmill_common::worker::{get_annotation_ts, to_raw_value, to_raw_value_owned, write_file}; use windmill_common::{ error::{self, to_anyhow}, flows::FlowValue, @@ -26,7 +26,7 @@ use windmill_parser_ts::parse_expr_for_imports; use windmill_queue::{append_logs, CanceledBy, PushIsolationLevel}; use crate::common::OccupancyMetrics; -use crate::python_executor::{create_dependencies_dir, handle_python_reqs, pip_compile}; +use crate::python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile}; use crate::rust_executor::{build_rust_crate, compute_rust_hash, generate_cargo_lockfile}; use crate::{ bun_executor::gen_bun_lockfile, @@ -953,7 +953,7 @@ async fn lock_modules<'c>( } if language == ScriptLang::Bun || language == ScriptLang::Bunnative { - let anns = get_annotation(&content); + let anns = get_annotation_ts(&content); if anns.native_mode && language == ScriptLang::Bun { language = ScriptLang::Bunnative; } else if !anns.native_mode && language == ScriptLang::Bunnative { @@ -1003,7 +1003,7 @@ async fn lock_modules<'c>( fn skip_creating_new_lock(language: &ScriptLang, content: &str) -> bool { if language == &ScriptLang::Bun || language == &ScriptLang::Bunnative { - let anns = get_annotation(&content); + let anns = get_annotation_ts(&content); if anns.native_mode && language == &ScriptLang::Bun { return false; } else if !anns.native_mode && language == &ScriptLang::Bunnative { @@ -1077,7 +1077,7 @@ async fn lock_modules_app( match new_lock { Ok(new_lock) => { append_logs(&job.id, &job.workspace_id, logs, db).await; - let anns = get_annotation(&content); + let anns = get_annotation_ts(&content); let nlang = if anns.native_mode && language == ScriptLang::Bun { Some(ScriptLang::Bunnative) } else if !anns.native_mode && language == ScriptLang::Bunnative @@ -1280,7 +1280,7 @@ async fn python_dep( occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> std::result::Result { create_dependencies_dir(job_dir).await; - let req: std::result::Result = pip_compile( + let req: std::result::Result = uv_pip_compile( job_id, &reqs, mem_peak, @@ -1290,6 +1290,8 @@ async fn python_dep( worker_name, w_id, occupancy_metrics, + false, + false, ) .await; // install the dependencies to pre-fill the cache @@ -1434,8 +1436,9 @@ async fn capture_dependency_job( .await } ScriptLang::Bun | ScriptLang::Bunnative => { - let npm_mode = npm_mode - .unwrap_or_else(|| windmill_common::worker::get_annotation(job_raw_code).npm_mode); + let npm_mode = npm_mode.unwrap_or_else(|| { + windmill_common::worker::get_annotation_ts(job_raw_code).npm_mode + }); if !raw_deps { let _ = write_file(job_dir, "main.ts", job_raw_code)?; } diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index b94bdec85e..1b1c1af887 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -17,6 +17,9 @@ ENV TZ=Etc/UTC RUN /usr/local/bin/python3 -m pip install pip-tools +# Install UV +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh + COPY --from=oven/bun:1.1.27 /usr/local/bin/bun /usr/bin/bun # add the docker client to call docker from a worker if enabled diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index b94bdec85e..64364adb37 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -16,6 +16,8 @@ RUN apt-get -y update && apt-get install -y curl nodejs awscli ENV TZ=Etc/UTC RUN /usr/local/bin/python3 -m pip install pip-tools +# Install UV +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh COPY --from=oven/bun:1.1.27 /usr/local/bin/bun /usr/bin/bun diff --git a/shell.nix b/shell.nix index 58085ee675..c3009eb183 100644 --- a/shell.nix +++ b/shell.nix @@ -8,8 +8,8 @@ let "https://github.com/oxalica/rust-overlay/archive/master.tar.gz"); pkgs = import { overlays = [ rust_overlay ]; }; # TODO: Pin version? - rustVersion = "latest"; - # rustVersion = "1.83.0"; + # rustVersion = "latest"; + rustVersion = "2024-09-30"; rust = pkgs.rust-bin.nightly.${rustVersion}.default.override { extensions = [ "rust-src" # for rust-analyzer @@ -27,6 +27,7 @@ in pkgs.mkShell { postgresql watchexec # used in client's dev.nu poetry # for python client + uv python312Packages.pip-tools # pip-compile ]; @@ -54,6 +55,7 @@ in pkgs.mkShell { "-C link-arg=-fuse-ld=${pkgs.mold}/bin/mold -Zshare-generics=y -Z threads=4"; RUSTC_WRAPPER = "${pkgs.sccache}/bin/sccache"; - # Use mold as a linker (for faster compilation) + # Useful for development + RUST_LOG = "debug"; } From 794c4cde3cd47042472dccdf4b60a012014dd26d Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 3 Oct 2024 16:49:20 +0200 Subject: [PATCH 20/38] fix(cli): fix set client of instance when passing token and base url --- cli/instance.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cli/instance.ts b/cli/instance.ts index 2354acdec4..69a69619ea 100644 --- a/cli/instance.ts +++ b/cli/instance.ts @@ -190,6 +190,12 @@ export async function pickInstance( const instances = await allInstances(); if (opts.baseUrl && opts.token) { log.info("Using instance fully defined by --base-url and --token"); + + setClient( + opts.token, + opts.baseUrl.endsWith("/") ? opts.baseUrl.slice(0, -1) : opts.baseUrl + ); + return { name: "custom", remote: opts.baseUrl, From 32e4a745248237dbfd97898860dce645f0ea9b54 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 3 Oct 2024 16:52:39 +0200 Subject: [PATCH 21/38] chore(main): release 1.405.0 (#4475) * chore(main): release 1.405.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel --- CHANGELOG.md | 13 ++++++ backend/Cargo.lock | 44 +++++++++---------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 4 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- python-client/wmill_pg/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 16 files changed, 52 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28c1c60fce..f4e16eb3ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## [1.405.0](https://github.com/windmill-labs/windmill/compare/v1.404.1...v1.405.0) (2024-10-03) + + +### Features + +* Replace `pip-compile` with `uv` ([#4460](https://github.com/windmill-labs/windmill/issues/4460)) ([b54c9ee](https://github.com/windmill-labs/windmill/commit/b54c9ee657cc88fabe694cae39dc0d3c1918fcbb)) +* **worker:** support workers to run natively on windows ([#4446](https://github.com/windmill-labs/windmill/issues/4446)) ([f5c4727](https://github.com/windmill-labs/windmill/commit/f5c472727465dd95f5378bc08ee9bbb983f4d259)) + + +### Bug Fixes + +* **cli:** fix set client of instance when passing token and base url ([794c4cd](https://github.com/windmill-labs/windmill/commit/794c4cde3cd47042472dccdf4b60a012014dd26d)) + ## [1.404.1](https://github.com/windmill-labs/windmill/compare/v1.404.0...v1.404.1) (2024-10-03) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 278fc815af..1b2b218120 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -10355,7 +10355,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.404.1" +version = "1.405.0" dependencies = [ "anyhow", "axum", @@ -10396,7 +10396,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.404.1" +version = "1.405.0" dependencies = [ "anyhow", "argon2", @@ -10480,7 +10480,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.404.1" +version = "1.405.0" dependencies = [ "base64 0.21.7", "chrono", @@ -10498,7 +10498,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.404.1" +version = "1.405.0" dependencies = [ "chrono", "serde", @@ -10511,7 +10511,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.404.1" +version = "1.405.0" dependencies = [ "anyhow", "async-stream", @@ -10556,7 +10556,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.404.1" +version = "1.405.0" dependencies = [ "regex", "rsmq_async", @@ -10571,7 +10571,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.404.1" +version = "1.405.0" dependencies = [ "anyhow", "bytes", @@ -10592,7 +10592,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.404.1" +version = "1.405.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -10601,7 +10601,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.404.1" +version = "1.405.0" dependencies = [ "anyhow", "lazy_static", @@ -10613,7 +10613,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.404.1" +version = "1.405.0" dependencies = [ "anyhow", "gosyn", @@ -10625,7 +10625,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.404.1" +version = "1.405.0" dependencies = [ "anyhow", "lazy_static", @@ -10637,7 +10637,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.404.1" +version = "1.405.0" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10648,7 +10648,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.404.1" +version = "1.405.0" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10659,7 +10659,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.404.1" +version = "1.405.0" dependencies = [ "anyhow", "async-recursion", @@ -10677,7 +10677,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.404.1" +version = "1.405.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -10694,7 +10694,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.404.1" +version = "1.405.0" dependencies = [ "anyhow", "lazy_static", @@ -10706,7 +10706,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.404.1" +version = "1.405.0" dependencies = [ "anyhow", "lazy_static", @@ -10724,7 +10724,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.404.1" +version = "1.405.0" dependencies = [ "anyhow", "getrandom 0.2.15", @@ -10745,7 +10745,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.404.1" +version = "1.405.0" dependencies = [ "anyhow", "serde_json", @@ -10755,7 +10755,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.404.1" +version = "1.405.0" dependencies = [ "anyhow", "async-recursion", @@ -10788,7 +10788,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.404.1" +version = "1.405.0" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -10798,7 +10798,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.404.1" +version = "1.405.0" dependencies = [ "anyhow", "async-recursion", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 891e46bce9..a5445ae3dc 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.404.1" +version = "1.405.0" authors.workspace = true edition.workspace = true @@ -27,7 +27,7 @@ members = [ ] [workspace.package] -version = "1.404.1" +version = "1.405.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 7f28f437eb..bafb0be959 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.404.1 + version: 1.405.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 9ebf2db1ec..eeaa3189a3 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.404.1"; +export const VERSION = "v1.405.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/main.ts b/cli/main.ts index 789325f243..1e3e3c49ac 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -60,7 +60,7 @@ export { // } // }); -export const VERSION = "1.404.1"; +export const VERSION = "1.405.0"; const command = new Command() .name("wmill") diff --git a/frontend/package-lock.json b/frontend/package-lock.json index b73a5af0a3..8b44e00bd6 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.404.1", + "version": "1.405.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.404.1", + "version": "1.405.0", "license": "AGPL-3.0", "dependencies": { "@aws-crypto/sha256-js": "^4.0.0", diff --git a/frontend/package.json b/frontend/package.json index a737397b7d..698fb2798b 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.404.1", + "version": "1.405.0", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index 8dbd6a860b..302cf7fec2 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.404.1" -wmill_pg = ">=1.404.1" +wmill = ">=1.405.0" +wmill_pg = ">=1.405.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 400e1874b1..7139849353 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.404.1 + version: 1.405.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 70f4ec9758..b96f2fc15d 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. -ModuleVersion = '1.404.1' +ModuleVersion = '1.405.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index f53ed97585..8d7317e22e 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.404.1" +version = "1.405.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index 140bfdbfa5..00d5fdd6ad 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.404.1" +version = "1.405.0" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 0060cf72b9..f479d84579 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.404.1", + "version": "1.405.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index e505e62523..bef2616769 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.404.1", + "version": "1.405.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index 7308051af2..45ef41ed49 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.404.1 +1.405.0 From a630acc5fc559306986e05ce87b3c36befdffb5a Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Thu, 3 Oct 2024 12:10:30 -0400 Subject: [PATCH 22/38] fix windows worker gh action build workflow (#4478) --- .github/workflows/build_windows_worker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_windows_worker.yml b/.github/workflows/build_windows_worker.yml index 83e0855a50..c9c4af3d5c 100644 --- a/.github/workflows/build_windows_worker.yml +++ b/.github/workflows/build_windows_worker.yml @@ -47,7 +47,7 @@ jobs: - name: Rename binary with corresponding architecture run: | - ren ./backend/target/release/windmill.exe ./backend/target/release/windmill-ee.exe + Rename-Item -Path ".\backend\target\release\windmill.exe" -NewName "windmill-ee.exe" - name: Attach binary to release uses: softprops/action-gh-release@v2 From c84e6fd05de2bea426cae61fa25db0323b8770f5 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 3 Oct 2024 20:48:26 +0200 Subject: [PATCH 23/38] fix: flow picker of flows + precache hub scripts as bundles --- backend/src/main.rs | 26 ++++++++++++++++--- backend/windmill-worker/src/bun_executor.rs | 8 +++--- backend/windmill-worker/src/lib.rs | 4 ++- .../windmill-worker/src/worker_lockfiles.rs | 2 +- .../pickers/WorkspaceScriptPickerQuick.svelte | 6 ++++- 5 files changed, 36 insertions(+), 10 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index c99e551ddb..6add270221 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -135,6 +135,7 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { })?; create_dir_all(HUB_CACHE_DIR).await?; + create_dir_all(BUN_BUNDLE_CACHE_DIR).await?; for path in paths.values() { tracing::info!("Caching hub script at {path}"); @@ -166,7 +167,7 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { create_dir_all(&job_dir).await?; if let Some(lockfile) = res.lockfile { let _ = windmill_worker::prepare_job_dir(&lockfile, &job_dir).await?; - + let envs = windmill_worker::get_common_bun_proc_envs(None).await; let _ = windmill_worker::install_bun_lockfile( &mut 0, &mut None, @@ -175,11 +176,31 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { None, &job_dir, "cache_init", - windmill_worker::get_common_bun_proc_envs(None).await, + envs.clone(), false, &mut None, ) .await?; + + let _ = windmill_common::worker::write_file(&job_dir, "main.js", &res.content)?; + + if let Err(e) = windmill_worker::prebundle_bun_script( + &res.content, + Some(lockfile), + &path, + &job_id, + "admins", + None, + &job_dir, + "", + "cache_init", + "", + &mut None, + ) + .await + { + panic!("Error prebundling bun script: {e:#}"); + } } else { tracing::warn!("No lockfile found for bun script {path}, skipping..."); } @@ -339,7 +360,6 @@ async fn windmill_main() -> anyhow::Result<()> { config }); - tracing::info!("Connecting to database..."); let db = windmill_common::connect_db(server_mode, indexer_mode).await?; tracing::info!("Database connected"); diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index c45d4c2853..73511faada 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -506,7 +506,7 @@ pub async fn generate_bun_bundle( mem_peak: &mut i32, canceled_by: &mut Option, common_bun_proc_envs: &HashMap, - occupancy_metrics: &mut OccupancyMetrics, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> Result<()> { let mut child = Command::new(&*BUN_PATH); child @@ -535,7 +535,7 @@ pub async fn generate_bun_bundle( "bun build", timeout, false, - &mut Some(occupancy_metrics), + occupancy_metrics, ) .await?; } else { @@ -650,7 +650,7 @@ pub async fn prebundle_bun_script( base_internal_url: &str, worker_name: &str, token: &str, - occupancy_metrics: &mut OccupancyMetrics, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> Result<()> { let (local_path, remote_path) = compute_bundle_local_and_remote_path( inner_content, @@ -1190,7 +1190,7 @@ try {{ mem_peak, canceled_by, &common_bun_proc_envs, - occupancy_metrics, + &mut Some(occupancy_metrics), ) .await?; if !local_path.is_empty() { diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index f1c5bc3926..b790eb3faf 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -34,5 +34,7 @@ pub use worker::*; pub use result_processor::handle_job_error; -pub use bun_executor::{get_common_bun_proc_envs, install_bun_lockfile, prepare_job_dir}; +pub use bun_executor::{ + get_common_bun_proc_envs, install_bun_lockfile, prebundle_bun_script, prepare_job_dir, +}; pub use deno_executor::generate_deno_lock; diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index ed89c535e8..e6458375cd 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -1475,7 +1475,7 @@ async fn capture_dependency_job( base_internal_url, worker_name, &token, - occupancy_metrics, + &mut Some(occupancy_metrics), ) .await?; } diff --git a/frontend/src/lib/components/flows/pickers/WorkspaceScriptPickerQuick.svelte b/frontend/src/lib/components/flows/pickers/WorkspaceScriptPickerQuick.svelte index bd91c76015..0d9e372083 100644 --- a/frontend/src/lib/components/flows/pickers/WorkspaceScriptPickerQuick.svelte +++ b/frontend/src/lib/components/flows/pickers/WorkspaceScriptPickerQuick.svelte @@ -111,7 +111,11 @@ ? 'bg-surface-hover' : ''}" on:click={() => { - dispatch('pickScript', { path, hash: lockHash ? hash : undefined }) + if (kind == 'flow') { + dispatch('pickFlow', { path: path }) + } else { + dispatch('pickScript', { path: path, hash: lockHash ? hash : undefined }) + } }} > {#if kind == 'flow'} From 1dabb1591b295d6d140e457147ed37b224f9fb0c Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 3 Oct 2024 20:52:06 +0200 Subject: [PATCH 24/38] chore(main): release 1.405.1 (#4480) * chore(main): release 1.405.1 * Apply automatic changes --------- Co-authored-by: rubenfiszel --- CHANGELOG.md | 7 +++ backend/Cargo.lock | 48 +++++++++---------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 4 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- python-client/wmill_pg/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 16 files changed, 48 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4e16eb3ce..53a9484577 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [1.405.1](https://github.com/windmill-labs/windmill/compare/v1.405.0...v1.405.1) (2024-10-03) + + +### Bug Fixes + +* flow picker of flows + precache hub scripts as bundles ([c84e6fd](https://github.com/windmill-labs/windmill/commit/c84e6fd05de2bea426cae61fa25db0323b8770f5)) + ## [1.405.0](https://github.com/windmill-labs/windmill/compare/v1.404.1...v1.405.0) (2024-10-03) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 1b2b218120..02f2486760 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -4416,9 +4416,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" +checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" [[package]] name = "is-macro" @@ -10355,7 +10355,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.405.0" +version = "1.405.1" dependencies = [ "anyhow", "axum", @@ -10396,7 +10396,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.405.0" +version = "1.405.1" dependencies = [ "anyhow", "argon2", @@ -10480,7 +10480,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.405.0" +version = "1.405.1" dependencies = [ "base64 0.21.7", "chrono", @@ -10498,7 +10498,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.405.0" +version = "1.405.1" dependencies = [ "chrono", "serde", @@ -10511,7 +10511,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.405.0" +version = "1.405.1" dependencies = [ "anyhow", "async-stream", @@ -10556,7 +10556,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.405.0" +version = "1.405.1" dependencies = [ "regex", "rsmq_async", @@ -10571,7 +10571,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.405.0" +version = "1.405.1" dependencies = [ "anyhow", "bytes", @@ -10592,7 +10592,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.405.0" +version = "1.405.1" dependencies = [ "convert_case 0.6.0", "serde", @@ -10601,7 +10601,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.405.0" +version = "1.405.1" dependencies = [ "anyhow", "lazy_static", @@ -10613,7 +10613,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.405.0" +version = "1.405.1" dependencies = [ "anyhow", "gosyn", @@ -10625,7 +10625,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.405.0" +version = "1.405.1" dependencies = [ "anyhow", "lazy_static", @@ -10637,7 +10637,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.405.0" +version = "1.405.1" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10648,7 +10648,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.405.0" +version = "1.405.1" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10659,7 +10659,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.405.0" +version = "1.405.1" dependencies = [ "anyhow", "async-recursion", @@ -10677,7 +10677,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.405.0" +version = "1.405.1" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -10694,7 +10694,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.405.0" +version = "1.405.1" dependencies = [ "anyhow", "lazy_static", @@ -10706,7 +10706,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.405.0" +version = "1.405.1" dependencies = [ "anyhow", "lazy_static", @@ -10724,7 +10724,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.405.0" +version = "1.405.1" dependencies = [ "anyhow", "getrandom 0.2.15", @@ -10745,7 +10745,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.405.0" +version = "1.405.1" dependencies = [ "anyhow", "serde_json", @@ -10755,7 +10755,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.405.0" +version = "1.405.1" dependencies = [ "anyhow", "async-recursion", @@ -10788,7 +10788,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.405.0" +version = "1.405.1" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -10798,7 +10798,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.405.0" +version = "1.405.1" dependencies = [ "anyhow", "async-recursion", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index a5445ae3dc..dbb3bd66bb 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.405.0" +version = "1.405.1" authors.workspace = true edition.workspace = true @@ -27,7 +27,7 @@ members = [ ] [workspace.package] -version = "1.405.0" +version = "1.405.1" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index bafb0be959..ff8f0ccc28 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.405.0 + version: 1.405.1 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index eeaa3189a3..605880f41c 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.405.0"; +export const VERSION = "v1.405.1"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/main.ts b/cli/main.ts index 1e3e3c49ac..d8a6f7abbf 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -60,7 +60,7 @@ export { // } // }); -export const VERSION = "1.405.0"; +export const VERSION = "1.405.1"; const command = new Command() .name("wmill") diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 8b44e00bd6..7c90a607c5 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.405.0", + "version": "1.405.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.405.0", + "version": "1.405.1", "license": "AGPL-3.0", "dependencies": { "@aws-crypto/sha256-js": "^4.0.0", diff --git a/frontend/package.json b/frontend/package.json index 698fb2798b..e79457fa92 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.405.0", + "version": "1.405.1", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index 302cf7fec2..6ef3528699 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.405.0" -wmill_pg = ">=1.405.0" +wmill = ">=1.405.1" +wmill_pg = ">=1.405.1" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 7139849353..6aff7711fb 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.405.0 + version: 1.405.1 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index b96f2fc15d..fc98eda757 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. -ModuleVersion = '1.405.0' +ModuleVersion = '1.405.1' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 8d7317e22e..d8ab4833f5 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.405.0" +version = "1.405.1" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index 00d5fdd6ad..4582ff382a 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.405.0" +version = "1.405.1" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index f479d84579..a8e34e7682 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.405.0", + "version": "1.405.1", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index bef2616769..e4742591fc 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.405.0", + "version": "1.405.1", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index 45ef41ed49..8a7e63b8f2 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.405.0 +1.405.1 From b96cc197182488827e395b6af5a5fd2c8b619973 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Thu, 3 Oct 2024 21:17:27 +0200 Subject: [PATCH 25/38] prevent flow logo to shrink (#4481) Co-authored-by: Guilhem --- .../components/flows/pickers/WorkspaceScriptPickerQuick.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/lib/components/flows/pickers/WorkspaceScriptPickerQuick.svelte b/frontend/src/lib/components/flows/pickers/WorkspaceScriptPickerQuick.svelte index 0d9e372083..67988e8005 100644 --- a/frontend/src/lib/components/flows/pickers/WorkspaceScriptPickerQuick.svelte +++ b/frontend/src/lib/components/flows/pickers/WorkspaceScriptPickerQuick.svelte @@ -119,7 +119,7 @@ }} > {#if kind == 'flow'} - + {:else} {/if} From 26659ce37d2887d5b98dbdbdbba27bab85d4fe3f Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 3 Oct 2024 21:31:03 +0200 Subject: [PATCH 26/38] fix(cli): fix opts.yes for instance sync --- cli/instance.ts | 2 ++ cli/sync.ts | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/cli/instance.ts b/cli/instance.ts index 69a69619ea..6ad60ad113 100644 --- a/cli/instance.ts +++ b/cli/instance.ts @@ -336,6 +336,7 @@ async function instancePull(opts: GlobalOptions & InstanceSyncOptions) { includeSettings: true, includeUsers: true, includeKey: true, + yes: opts.yes, }); } @@ -489,6 +490,7 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) { includeSettings: true, includeUsers: true, includeKey: true, + yes: opts.yes, }); } diff --git a/cli/sync.ts b/cli/sync.ts index 088571362d..3a12272b6e 100644 --- a/cli/sync.ts +++ b/cli/sync.ts @@ -504,7 +504,7 @@ function ZipFSElement( if (formatExtension) { const fileContent: string = parsed["value"]["content"]; - if (typeof(fileContent) === "string") { + if (typeof fileContent === "string") { r.push({ isDirectory: false, path: @@ -948,7 +948,7 @@ export async function pull(opts: GlobalOptions & SyncOptions) { if ( !opts.yes && !(await Confirm.prompt({ - message: `Do you want to apply these ${changes.length} changes?`, + message: `Do you want to apply these ${changes.length} changes to your local files?`, default: true, })) ) { @@ -1251,7 +1251,7 @@ export async function push(opts: GlobalOptions & SyncOptions) { if ( !opts.yes && !(await Confirm.prompt({ - message: `Do you want to apply these ${changes.length} changes?`, + message: `Do you want to apply these ${changes.length} changes to the remote?`, default: true, })) ) { From 19c62ba195b1df85c38c748dab7d9f137696a5c3 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 3 Oct 2024 22:42:34 +0200 Subject: [PATCH 27/38] fix: fix uv path --- Dockerfile | 2 +- backend/windmill-worker/src/python_executor.rs | 13 +++++++++++-- frontend/src/lib/components/FlowJobResult.svelte | 10 ++++++++-- frontend/src/lib/components/FlowStatusViewer.svelte | 2 ++ .../src/lib/components/FlowStatusViewerInner.svelte | 6 +++++- frontend/src/lib/components/LogViewer.svelte | 7 ++++--- .../src/lib/components/copilot/StepGenQuick.svelte | 5 +++-- .../components/flows/map/InsertModuleButton.svelte | 2 +- frontend/src/lib/components/graph/model.ts | 1 + 9 files changed, 36 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index d5d4b8cc92..8f5ccc71e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -159,7 +159,7 @@ ENV PATH="${PATH}:/usr/local/go/bin" ENV GO_PATH=/usr/local/go/bin/go # Install UV -RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - RUN apt-get -y update && apt-get install -y curl nodejs awscli && apt-get clean \ diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index baefdfd9bc..df94b75964 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -29,6 +29,9 @@ lazy_static::lazy_static! { static ref PYTHON_PATH: String = std::env::var("PYTHON_PATH").unwrap_or_else(|_| "/usr/local/bin/python3".to_string()); + static ref UV_PATH: String = + std::env::var("UV_PATH").unwrap_or_else(|_| "/usr/local/bin/uv".to_string()); + static ref FLOCK_PATH: String = std::env::var("FLOCK_PATH").unwrap_or_else(|_| "/usr/bin/flock".to_string()); static ref NON_ALPHANUM_CHAR: Regex = regex::Regex::new(r"[^0-9A-Za-z=.-]").unwrap(); @@ -294,13 +297,19 @@ pub async fn uv_pip_compile( } tracing::debug!("uv args: {:?}", args); - let mut child_cmd = Command::new("uv"); + #[cfg(windows)] + let uv_cmd = "uv"; + + #[cfg(unix)] + let uv_cmd = UV_PATH.as_str(); + + let mut child_cmd = Command::new(uv_cmd); child_cmd .current_dir(job_dir) .args(args) .stdout(Stdio::piped()) .stderr(Stdio::piped()); - let child_process = start_child_process(child_cmd, "uv").await?; + let child_process = start_child_process(child_cmd, "/usr/local/bin/uv").await?; append_logs(&job_id, &w_id, logs, db).await; handle_child( job_id, diff --git a/frontend/src/lib/components/FlowJobResult.svelte b/frontend/src/lib/components/FlowJobResult.svelte index 0f0e4eebc0..cd83372389 100644 --- a/frontend/src/lib/components/FlowJobResult.svelte +++ b/frontend/src/lib/components/FlowJobResult.svelte @@ -22,6 +22,7 @@ export let workspaceId: string | undefined = undefined export let refreshLog: boolean = false export let durationStates: Writable> | undefined + export let downloadLogs = true let lastJobId: string | undefined = undefined let drawer: Drawer | undefined = undefined @@ -40,7 +41,6 @@ let logOffset = 0 async function getLogs() { if (jobId) { - const getUpdate = await JobService.getJobUpdates({ workspace: workspaceId ?? $workspaceStore!, id: jobId, @@ -85,6 +85,12 @@
- +
diff --git a/frontend/src/lib/components/FlowStatusViewer.svelte b/frontend/src/lib/components/FlowStatusViewer.svelte index db0460a08c..c2d4ec1b3a 100644 --- a/frontend/src/lib/components/FlowStatusViewer.svelte +++ b/frontend/src/lib/components/FlowStatusViewer.svelte @@ -18,6 +18,7 @@ export let hideDownloadInGraph = false export let hideNodeDefinition = false export let hideJobId = false + export let hideDownloadLogs = false export let isOwner = false export let wideResults = false @@ -34,6 +35,7 @@ hideNodeDefinition, hideTimeline, hideJobId + hideDownloadLogs }) function loadOwner(path: string) { diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index 2dcc2a030b..6d0343220c 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -38,7 +38,8 @@ suspendStatus, hideDownloadInGraph, hideTimeline, - hideNodeDefinition + hideNodeDefinition, + hideDownloadLogs } = getContext('FlowStatusViewer') export let jobId: string @@ -669,6 +670,7 @@ result={job.result} logs={job.logs} durationStates={localDurationStatuses} + downloadLogs={!hideDownloadLogs} />
{/if} @@ -1026,6 +1028,7 @@ result={job['result']} logs={job.logs ?? ''} durationStates={localDurationStatuses} + downloadLogs={!hideDownloadLogs} /> {:else if selectedNode == 'start'} {#if job.args} @@ -1101,6 +1104,7 @@ tag={node.tag} logs={node.logs} durationStates={localDurationStatuses} + downloadLogs={!hideDownloadLogs} /> {:else}

limit) { return content.substring(content.length - limit) } @@ -148,7 +149,7 @@ - {#if jobId} + {#if jobId && download}