diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index f4cb3ce1ba..e5f1a48803 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -23,6 +23,8 @@ jobs: env: DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} + WORKER_GROUP: main + WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow options: >- --pull always --health-interval 10s --health-timeout 5s --health-retries 5 --health-cmd "curl @@ -34,6 +36,7 @@ jobs: env: DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill DISABLE_SERVER: true + WORKER_GROUP: dedicated DEDICATED_WORKER: "admins:f/benchmarks/dedicated" LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} options: >- diff --git a/CHANGELOG.md b/CHANGELOG.md index e0288c1961..ec949e6faf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,29 @@ # Changelog +## [1.176.0](https://github.com/windmill-labs/windmill/compare/v1.175.0...v1.176.0) (2023-09-24) + + +### Features + +* add license key as superadmin setting ([#2321](https://github.com/windmill-labs/windmill/issues/2321)) ([304a259](https://github.com/windmill-labs/windmill/commit/304a2596fd29fbd9a79c5cf9fe4df7b44d5c5254)) +* add running filter ([ea364ad](https://github.com/windmill-labs/windmill/commit/ea364ad9602647cbc9e8ee78fb5f17f0012105f6)) +* ai flow trigger menu ([#2317](https://github.com/windmill-labs/windmill/issues/2317)) ([95194ab](https://github.com/windmill-labs/windmill/commit/95194abeacc42416174ee9dd79b75f2204a40d33)) +* improved dedicated benchmarks + buffer fix ([#2313](https://github.com/windmill-labs/windmill/issues/2313)) ([fc93c2a](https://github.com/windmill-labs/windmill/commit/fc93c2a7cece95c00070a3a3391ae2bcb4513e85)) +* set instance settings from UI ([#2314](https://github.com/windmill-labs/windmill/issues/2314)) ([2f0e43b](https://github.com/windmill-labs/windmill/commit/2f0e43bfdbd1e196131f126c83b1d7dd2eea98d8)) + + +### Bug Fixes + +* add ability to test this step for flow step ([3585929](https://github.com/windmill-labs/windmill/commit/3585929bb758b0cfc2cbe43f66597b184e7b8ee0)) +* benchmark worker tags ([#2319](https://github.com/windmill-labs/windmill/issues/2319)) ([481bcd5](https://github.com/windmill-labs/windmill/commit/481bcd53cb07e4520d5fd81572cad74340c4eb64)) +* change cache implementation to remove async-timer ([4911b4b](https://github.com/windmill-labs/windmill/commit/4911b4b3fd6e3a9f6bccc4c8712b736e18dcb6e1)) +* fix upto preview issue with nested flows ([6492ff6](https://github.com/windmill-labs/windmill/commit/6492ff627a800832e12a31fd89a6070703988eb9)) +* flow steps appears in all static inputs ([c043847](https://github.com/windmill-labs/windmill/commit/c0438479aa3b6dc6349df01abdd9dcc434fe8781)) +* optimize performance for bun scripts without deps ([5b33f56](https://github.com/windmill-labs/windmill/commit/5b33f563e6e83605ae72338af351dcc97beb1a55)) +* overflow on workspace script picker ([5e4db0e](https://github.com/windmill-labs/windmill/commit/5e4db0ebab616305928cfa455af6833335e0fcf9)) +* tag id as flow ([#2318](https://github.com/windmill-labs/windmill/issues/2318)) ([f68cee4](https://github.com/windmill-labs/windmill/commit/f68cee4ebddbf6e774f80e91a8c89fb8dc213f91)) + ## [1.175.0](https://github.com/windmill-labs/windmill/compare/v1.174.0...v1.175.0) (2023-09-19) diff --git a/backend/.sqlx/query-123c0608e229c29187009b7961355ddd99c4ad1f46b876dd86e372b84d806ecd.json b/backend/.sqlx/query-123c0608e229c29187009b7961355ddd99c4ad1f46b876dd86e372b84d806ecd.json index 1b8084742c..7718e05ccf 100644 --- a/backend/.sqlx/query-123c0608e229c29187009b7961355ddd99c4ad1f46b876dd86e372b84d806ecd.json +++ b/backend/.sqlx/query-123c0608e229c29187009b7961355ddd99c4ad1f46b876dd86e372b84d806ecd.json @@ -37,7 +37,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-25bef6a248f3ee0ea2cbcc376c217cbcf1013ae311c36b42d423bf6a02fa016c.json b/backend/.sqlx/query-25bef6a248f3ee0ea2cbcc376c217cbcf1013ae311c36b42d423bf6a02fa016c.json index bf591ef11c..9d082a6772 100644 --- a/backend/.sqlx/query-25bef6a248f3ee0ea2cbcc376c217cbcf1013ae311c36b42d423bf6a02fa016c.json +++ b/backend/.sqlx/query-25bef6a248f3ee0ea2cbcc376c217cbcf1013ae311c36b42d423bf6a02fa016c.json @@ -67,7 +67,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-438b5b5d29b05846c2e074cad2404e797527841cd97dba80c271cbefafae65cc.json b/backend/.sqlx/query-438b5b5d29b05846c2e074cad2404e797527841cd97dba80c271cbefafae65cc.json index a5dee163e5..1166260449 100644 --- a/backend/.sqlx/query-438b5b5d29b05846c2e074cad2404e797527841cd97dba80c271cbefafae65cc.json +++ b/backend/.sqlx/query-438b5b5d29b05846c2e074cad2404e797527841cd97dba80c271cbefafae65cc.json @@ -28,7 +28,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-5cd89ab614d3cac80fb81627267ee85b191263989b0c78b2bfce77e796e96825.json b/backend/.sqlx/query-5cd89ab614d3cac80fb81627267ee85b191263989b0c78b2bfce77e796e96825.json index 1517c8d1d4..eabf671894 100644 --- a/backend/.sqlx/query-5cd89ab614d3cac80fb81627267ee85b191263989b0c78b2bfce77e796e96825.json +++ b/backend/.sqlx/query-5cd89ab614d3cac80fb81627267ee85b191263989b0c78b2bfce77e796e96825.json @@ -42,7 +42,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-65835f2e5ad38f7cc6b147dadfef6f580f15bca96d9746c9359e98ca793f8f1f.json b/backend/.sqlx/query-65835f2e5ad38f7cc6b147dadfef6f580f15bca96d9746c9359e98ca793f8f1f.json index bfe7c41f64..c52efca4c0 100644 --- a/backend/.sqlx/query-65835f2e5ad38f7cc6b147dadfef6f580f15bca96d9746c9359e98ca793f8f1f.json +++ b/backend/.sqlx/query-65835f2e5ad38f7cc6b147dadfef6f580f15bca96d9746c9359e98ca793f8f1f.json @@ -42,7 +42,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-2de52e1f3226ca9281b6e25f74d4d05f4509cb87c875234bfc7b310a012e4d40.json b/backend/.sqlx/query-6b9ff3fbca9e825c95d14705082a10de88172c0c748a45aba4d2d03c3b58f54d.json similarity index 73% rename from backend/.sqlx/query-2de52e1f3226ca9281b6e25f74d4d05f4509cb87c875234bfc7b310a012e4d40.json rename to backend/.sqlx/query-6b9ff3fbca9e825c95d14705082a10de88172c0c748a45aba4d2d03c3b58f54d.json index 9c386c3509..73e6e17c76 100644 --- a/backend/.sqlx/query-2de52e1f3226ca9281b6e25f74d4d05f4509cb87c875234bfc7b310a012e4d40.json +++ b/backend/.sqlx/query-6b9ff3fbca9e825c95d14705082a10de88172c0c748a45aba4d2d03c3b58f54d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 FROM generate_series(1, $11)) RETURNING id", + "query": "WITH uuid_table as (\n select gen_random_uuid() as uuid from generate_series(1, $11)\n )\n INSERT INTO queue \n (id, script_hash, script_path, job_kind, language, args, tag, created_by, permissioned_as, email, scheduled_for, workspace_id)\n (SELECT uuid, $1, $2, $3, $4, ('{ \"uuid\": \"' || uuid || '\" }')::jsonb, $5, $6, $7, $8, $9, $10 FROM uuid_table) \n RETURNING id", "describe": { "columns": [ { @@ -46,7 +46,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", @@ -70,5 +69,5 @@ false ] }, - "hash": "2de52e1f3226ca9281b6e25f74d4d05f4509cb87c875234bfc7b310a012e4d40" + "hash": "6b9ff3fbca9e825c95d14705082a10de88172c0c748a45aba4d2d03c3b58f54d" } diff --git a/backend/.sqlx/query-9e8c3ff3d6b31e366e15beda1e96e03e870ccc3b353401439bc0ed8ff219249b.json b/backend/.sqlx/query-9e8c3ff3d6b31e366e15beda1e96e03e870ccc3b353401439bc0ed8ff219249b.json index 6308bf3bb2..fead4ba250 100644 --- a/backend/.sqlx/query-9e8c3ff3d6b31e366e15beda1e96e03e870ccc3b353401439bc0ed8ff219249b.json +++ b/backend/.sqlx/query-9e8c3ff3d6b31e366e15beda1e96e03e870ccc3b353401439bc0ed8ff219249b.json @@ -60,7 +60,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-b224cdd1221fc9e7227ef8e8c025eedc09bceb82bca349f3e31c8513ebbf0192.json b/backend/.sqlx/query-b224cdd1221fc9e7227ef8e8c025eedc09bceb82bca349f3e31c8513ebbf0192.json index 38f81da395..c90719118a 100644 --- a/backend/.sqlx/query-b224cdd1221fc9e7227ef8e8c025eedc09bceb82bca349f3e31c8513ebbf0192.json +++ b/backend/.sqlx/query-b224cdd1221fc9e7227ef8e8c025eedc09bceb82bca349f3e31c8513ebbf0192.json @@ -42,7 +42,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 88ec17a3f2..cfaf58feb7 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -64,9 +64,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.0.5" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783" +checksum = "ea5d730647d4fadd988536d06fecce94b7b4f2a7efdae548f1cf4b63205518ab" dependencies = [ "memchr", ] @@ -204,7 +204,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -334,7 +334,7 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -392,17 +392,6 @@ version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ecc7ab41815b3c653ccd2978ec3255c81349336702dfdf62ee6f7069b12a3aae" -[[package]] -name = "async-timer" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba5fa6ed76cb2aa820707b4eb9ec46f42da9ce70b0eafab5e5e34942b38a44d5" -dependencies = [ - "libc", - "wasm-bindgen", - "winapi", -] - [[package]] name = "async-trait" version = "0.1.73" @@ -411,7 +400,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -582,6 +571,19 @@ dependencies = [ "num-traits", ] +[[package]] +name = "bigdecimal" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "454bca3db10617b88b566f205ed190aedb0e0e6dd4cad61d3988a72e8c5594cb" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "bincode" version = "1.3.3" @@ -610,7 +612,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.34", + "syn 2.0.37", "which", ] @@ -631,7 +633,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -889,7 +891,7 @@ version = "9.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e11c675378efb449ed3ce8de78d75d0d80542fc98487c26aba28eb3b82feac72" dependencies = [ - "semver 1.0.18", + "semver 1.0.19", "serde", "toml 0.7.8", "url", @@ -979,9 +981,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.3" +version = "4.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84ed82781cea27b43c9b106a979fe450a13a31aab0500595fb3fc06616de08e6" +checksum = "b1d7b8d5ec32af0fadc644bf1fd509a688c2103b185644bb1e29d164e0703136" dependencies = [ "clap_builder", "clap_derive", @@ -989,9 +991,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.2" +version = "4.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08" +checksum = "5179bb514e4d7c2051749d8fcefa2ed6d06a9f4e6d69faf3805f5d80b8cf8d56" dependencies = [ "anstream", "anstyle", @@ -1008,7 +1010,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -1315,7 +1317,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -1337,7 +1339,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core 0.20.3", "quote", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -1395,7 +1397,7 @@ checksum = "3c65c2ffdafc1564565200967edc4851c7b55422d3913466688907efd05ea26f" dependencies = [ "deno-proc-macro-rules-macros", "proc-macro2", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -1407,7 +1409,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -1527,7 +1529,7 @@ dependencies = [ "strum", "strum_macros", "syn 1.0.109", - "syn 2.0.34", + "syn 2.0.37", "thiserror", ] @@ -1748,9 +1750,9 @@ dependencies = [ [[package]] name = "dyn-clone" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbfc4744c1b8f2a09adc0e55242f60b1af195d88596bd8700be74418c056c555" +checksum = "23d2f3407d9a573d666de4b5bdf10569d73ca9478087346697dcbae6244bfbcd" [[package]] name = "dyn-iter" @@ -1937,7 +1939,7 @@ dependencies = [ "pmutil", "proc-macro2", "swc_macros_common", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -1965,7 +1967,7 @@ checksum = "b0fa992f1656e1707946bbba340ad244f0814009ef8c0118eb7b658395f19a2e" dependencies = [ "frunk_proc_macro_helpers", "quote", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -1977,7 +1979,7 @@ dependencies = [ "frunk_core", "proc-macro2", "quote", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -1989,7 +1991,7 @@ dependencies = [ "frunk_core", "frunk_proc_macro_helpers", "quote", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -2090,7 +2092,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -2363,9 +2365,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" [[package]] name = "hex" @@ -2621,7 +2623,7 @@ dependencies = [ "pmutil", "proc-macro2", "quote", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -2631,7 +2633,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi", - "rustix 0.38.13", + "rustix 0.38.14", "windows-sys", ] @@ -3054,9 +3056,9 @@ checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" [[package]] name = "matchit" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed1202b2a6f884ae56f04cff409ab315c5ce26b5e58d7412e484f01fd52f52ef" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "md-5" @@ -3071,10 +3073,11 @@ dependencies = [ [[package]] name = "md-5" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ + "cfg-if", "digest 0.10.7", ] @@ -3149,7 +3152,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.34", + "syn 2.0.37", "termcolor", "thiserror", ] @@ -3194,7 +3197,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57349d5a326b437989b6ee4dc8f2f34b0cc131202748414712a8e7d98952fc8c" dependencies = [ "base64 0.21.4", - "bigdecimal", + "bigdecimal 0.3.1", "bindgen 0.68.1", "bitflags 2.4.0", "bitvec", @@ -3416,7 +3419,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -3675,7 +3678,7 @@ dependencies = [ "phf_shared 0.11.2", "proc-macro2", "quote", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -3713,7 +3716,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -3785,7 +3788,7 @@ checksum = "52a40bc70c2c58040d2d8b167ba9a5ff59fc9dab7ad44771cfde3dcfde7a09c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -3842,7 +3845,7 @@ dependencies = [ "bytes", "fallible-iterator", "hmac", - "md-5 0.10.5", + "md-5 0.10.6", "memchr", "rand 0.8.5", "sha2 0.10.7", @@ -3894,7 +3897,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -4014,7 +4017,7 @@ dependencies = [ "schemars", "serde", "serde_json", - "syn 2.0.34", + "syn 2.0.37", "thiserror", "typify", "unicode-ident", @@ -4034,7 +4037,7 @@ dependencies = [ "serde_json", "serde_tokenstream", "serde_yaml", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -4096,6 +4099,18 @@ dependencies = [ "serde", ] +[[package]] +name = "quick_cache" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f69f8d22fa3f34f3083d9a4375c038732c7a7e964de1beb81c544da92dfc40b8" +dependencies = [ + "ahash 0.8.3", + "equivalent", + "hashbrown 0.14.0", + "parking_lot", +] + [[package]] name = "quote" version = "1.0.33" @@ -4295,9 +4310,9 @@ dependencies = [ [[package]] name = "rend" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581008d2099240d37fb08d77ad713bcaec2c4d89d50b5b21a8bb1996bbab68ab" +checksum = "a2571463863a6bd50c32f94402933f03457a3fbaf697a707c5be741e459f08fd" dependencies = [ "bytecheck", ] @@ -4348,18 +4363,6 @@ dependencies = [ "winreg", ] -[[package]] -name = "retainer" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df8c01a8276c14d0f8d51ebcf8a48f0748f9f73f5f6b29e688126e6a52bcb145" -dependencies = [ - "async-lock", - "async-timer", - "log", - "rand 0.8.5", -] - [[package]] name = "ring" version = "0.16.20" @@ -4481,7 +4484,7 @@ dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn 2.0.34", + "syn 2.0.37", "walkdir", ] @@ -4539,7 +4542,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.18", + "semver 1.0.19", ] [[package]] @@ -4571,9 +4574,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.13" +version = "0.38.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7db8590df6dfcd144d22afd1b83b36c21a18d7cbc1dc4bb5295a8712e9eb662" +checksum = "747c788e9ce8e92b12cd485c49ddf90723550b654b32508f979b71a7b1ecda4f" dependencies = [ "bitflags 2.4.0", "errno", @@ -4590,7 +4593,7 @@ checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" dependencies = [ "log", "ring", - "rustls-webpki 0.101.5", + "rustls-webpki 0.101.6", "sct", ] @@ -4627,9 +4630,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.5" +version = "0.101.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45a27e3b59326c16e23d30aeb7a36a24cc0d29e71d68ff611cdfb4a01d013bed" +checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" dependencies = [ "ring", "untrusted", @@ -4754,9 +4757,9 @@ dependencies = [ [[package]] name = "schemars" -version = "0.8.13" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "763f8cd0d4c71ed8389c90cb8100cba87e763bd01a8e614d4f0af97bcd50a161" +checksum = "1f7b0ce13155372a76ee2e1c5ffba1fe61ede73fbea5630d61eee6fac4929c0c" dependencies = [ "chrono", "dyn-clone", @@ -4768,9 +4771,9 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "0.8.13" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0f696e21e10fa546b7ffb1c9672c6de8fbc7a81acf59524386d8639bf12737" +checksum = "e85e2a16b12bdb763244c69ab79363d71db2b4b918a2def53f80b02e0574b13c" dependencies = [ "proc-macro2", "quote", @@ -4849,9 +4852,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" +checksum = "ad977052201c6de01a8ef2aa3378c4bd23217a056337d1d6da40468d267a4fb0" dependencies = [ "serde", ] @@ -4919,7 +4922,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -4995,7 +4998,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -5052,9 +5055,9 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures", @@ -5170,9 +5173,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" [[package]] name = "smart-default" @@ -5324,7 +5327,7 @@ checksum = "dd4cef4251aabbae751a3710927945901ee1d97ee96d757f6880ebb9a79bfd53" dependencies = [ "ahash 0.8.3", "atoi", - "bigdecimal", + "bigdecimal 0.3.1", "byteorder", "bytes", "chrono", @@ -5409,7 +5412,7 @@ checksum = "8ca69bf415b93b60b80dc8fda3cb4ef52b2336614d8da2de5456cc942a110482" dependencies = [ "atoi", "base64 0.21.4", - "bigdecimal", + "bigdecimal 0.3.1", "bitflags 2.4.0", "byteorder", "bytes", @@ -5428,7 +5431,7 @@ dependencies = [ "hmac", "itoa", "log", - "md-5 0.10.5", + "md-5 0.10.6", "memchr", "once_cell", "percent-encoding", @@ -5454,7 +5457,7 @@ checksum = "a0db2df1b8731c3651e204629dd55e52adbae0462fa1bdcbed56a2302c18181e" dependencies = [ "atoi", "base64 0.21.4", - "bigdecimal", + "bigdecimal 0.3.1", "bitflags 2.4.0", "byteorder", "chrono", @@ -5471,7 +5474,7 @@ dependencies = [ "home", "itoa", "log", - "md-5 0.10.5", + "md-5 0.10.6", "memchr", "num-bigint", "once_cell", @@ -5574,7 +5577,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -5613,7 +5616,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -5695,7 +5698,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -5744,7 +5747,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -5827,7 +5830,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -5931,7 +5934,7 @@ dependencies = [ "pmutil", "proc-macro2", "quote", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -5943,7 +5946,7 @@ dependencies = [ "pmutil", "proc-macro2", "quote", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -5967,7 +5970,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -5983,9 +5986,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.34" +version = "2.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ec6cdb6a4c16306eccf52ccd8d492e4ab64705a15a5016acb205251001bf72" +checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" dependencies = [ "proc-macro2", "quote", @@ -6013,7 +6016,7 @@ dependencies = [ "cfg-if", "fastrand 2.0.0", "redox_syscall 0.3.5", - "rustix 0.38.13", + "rustix 0.38.14", "windows-sys", ] @@ -6030,9 +6033,9 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +checksum = "6093bad37da69aab9d123a8091e4be0aa4a03e4d601ec641c327398315f62b64" dependencies = [ "winapi-util", ] @@ -6063,7 +6066,7 @@ checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ "proc-macro2", "quote", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -6089,9 +6092,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" +checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe" dependencies = [ "deranged", "itoa", @@ -6102,15 +6105,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" +checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" dependencies = [ "time-core", ] @@ -6167,7 +6170,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -6267,9 +6270,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.8" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" dependencies = [ "bytes", "futures-core", @@ -6420,7 +6423,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.34", + "syn 2.0.37", ] [[package]] @@ -6558,7 +6561,7 @@ dependencies = [ "regress", "schemars", "serde_json", - "syn 2.0.34", + "syn 2.0.37", "thiserror", "unicode-ident", ] @@ -6575,7 +6578,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.34", + "syn 2.0.37", "typify-impl", ] @@ -6703,9 +6706,9 @@ checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" [[package]] name = "unicode-width" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" [[package]] name = "unicode-xid" @@ -6893,7 +6896,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.34", + "syn 2.0.37", "wasm-bindgen-shared", ] @@ -6927,7 +6930,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.34", + "syn 2.0.37", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -7019,7 +7022,7 @@ version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b291546d5d9d1eab74f069c77749f2cb8504a12caa20f0f2de93ddbf6f411888" dependencies = [ - "rustls-webpki 0.101.5", + "rustls-webpki 0.101.6", ] [[package]] @@ -7037,7 +7040,7 @@ dependencies = [ "either", "home", "once_cell", - "rustix 0.38.13", + "rustix 0.38.14", ] [[package]] @@ -7068,9 +7071,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" dependencies = [ "winapi", ] @@ -7083,7 +7086,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.175.0" +version = "1.176.0" dependencies = [ "anyhow", "axum", @@ -7098,7 +7101,6 @@ dependencies = [ "prometheus", "rand 0.8.5", "reqwest", - "rsa 0.7.2", "rsmq_async", "serde", "serde_json", @@ -7118,7 +7120,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.175.0" +version = "1.176.0" dependencies = [ "anyhow", "argon2", @@ -7138,16 +7140,17 @@ dependencies = [ "hex", "hmac", "hyper", - "itertools 0.10.5", + "itertools 0.11.0", "lazy_static", "magic-crypt", "mail-send", "mime_guess", "prometheus", + "quick_cache", "rand 0.8.5", "regex", "reqwest", - "retainer", + "rsa 0.7.2", "rsmq_async", "rust-embed", "samael", @@ -7177,7 +7180,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.175.0" +version = "1.176.0" dependencies = [ "base64 0.21.4", "chrono", @@ -7195,7 +7198,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.175.0" +version = "1.176.0" dependencies = [ "chrono", "serde", @@ -7208,7 +7211,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.175.0" +version = "1.176.0" dependencies = [ "anyhow", "axum", @@ -7216,7 +7219,7 @@ dependencies = [ "hex", "hmac", "hyper", - "itertools 0.10.5", + "itertools 0.11.0", "lazy_static", "prometheus", "rand 0.8.5", @@ -7236,7 +7239,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.175.0" +version = "1.176.0" dependencies = [ "serde", "serde_json", @@ -7244,7 +7247,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.175.0" +version = "1.176.0" dependencies = [ "anyhow", "lazy_static", @@ -7255,11 +7258,11 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.175.0" +version = "1.176.0" dependencies = [ "anyhow", "gosyn", - "itertools 0.10.5", + "itertools 0.11.0", "lazy_static", "regex", "windmill-parser", @@ -7267,7 +7270,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.175.0" +version = "1.176.0" dependencies = [ "anyhow", "lazy_static", @@ -7278,10 +7281,10 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.175.0" +version = "1.176.0" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.11.0", "rustpython-parser", "serde_json", "windmill-parser", @@ -7289,11 +7292,11 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.175.0" +version = "1.176.0" dependencies = [ "anyhow", "async-recursion", - "itertools 0.10.5", + "itertools 0.11.0", "lazy_static", "phf 0.11.2", "regex", @@ -7306,7 +7309,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.175.0" +version = "1.176.0" dependencies = [ "anyhow", "lazy_static", @@ -7317,7 +7320,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.175.0" +version = "1.176.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -7334,7 +7337,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.175.0" +version = "1.176.0" dependencies = [ "anyhow", "getrandom 0.2.10", @@ -7352,18 +7355,18 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.175.0" +version = "1.176.0" dependencies = [ "anyhow", "async-recursion", - "bigdecimal", + "bigdecimal 0.4.1", "chrono", "chrono-tz", "cron", "futures-core", "hex", "hmac", - "itertools 0.10.5", + "itertools 0.11.0", "lazy_static", "prometheus", "reqwest", @@ -7382,7 +7385,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.175.0" +version = "1.176.0" dependencies = [ "anyhow", "async-recursion", @@ -7401,7 +7404,7 @@ dependencies = [ "futures", "gcp_auth", "git-version", - "itertools 0.10.5", + "itertools 0.11.0", "jsonwebtoken", "lazy_static", "mysql_async", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 00f25adf3a..2af48f5ddd 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.175.0" +version = "1.176.0" authors.workspace = true edition.workspace = true @@ -22,7 +22,7 @@ members = [ ] [workspace.package] -version = "1.175.0" +version = "1.176.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -54,7 +54,6 @@ tokio-metrics.workspace = true rand.workspace = true chrono.workspace = true git-version.workspace = true -rsa.workspace = true base64.workspace = true sha2.workspace = true rsmq_async.workspace = true @@ -114,7 +113,7 @@ mime_guess = "^2" hex = "^0" sql-builder = "^3" argon2 = "^0" -retainer = "^0" +quick_cache = "^0" rand = "0.8.5" rand_core = { version = "^0", features = ["std"] } magic-crypt = "^3" diff --git a/backend/src/ee.rs b/backend/src/ee.rs index 865819a701..1d185b4638 100644 --- a/backend/src/ee.rs +++ b/backend/src/ee.rs @@ -1,41 +1,49 @@ #[cfg(feature = "enterprise")] -use base64::Engine; -#[cfg(feature = "enterprise")] -use rsa::{pkcs8::DecodePublicKey, signature::Verifier}; -#[cfg(feature = "enterprise")] -use sha2::Sha256; +use windmill_common::error; -#[cfg(feature = "enterprise")] -pub fn verify_license_key(license_key: Option) -> anyhow::Result<()> { - if let Some(license_key) = license_key { - let mut splitted_lk = license_key.split("."); - if splitted_lk.clone().count() != 3 { - panic!("license_key can be splitted with 2 . (..)"); - } - let id = splitted_lk.next().unwrap(); - let expiry = splitted_lk.next().unwrap(); - let signature_b64 = splitted_lk.next().unwrap(); +pub async fn set_license_key(license_key: String) -> anyhow::Result<()> { + use windmill_api::{ee::validate_license_key, LICENSE_KEY, LICENSE_KEY_ID, LICENSE_KEY_VALID}; - let expiry_nb = expiry.parse::()?; - if expiry_nb < chrono::Utc::now().timestamp() as u64 { - panic!( - "License key is expired (timestamp expiry: {expiry_nb}. Now: {}", - chrono::Utc::now().timestamp() - ); - } - const PUBLIC_KEY: &str = "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDgVShzcLSPiOi+8ET8fggob1kmi47/cE12JaidPkwfGnScZItghkqtiLsct0U4kJhlp5gO89DYTBmIKadvxwY7kMsLlZzmi2emVH7c27cByGASY8QmWDNdG4Ggy/NDflGGBdAtN6gHawZAg4zHv3qpbPQGHH1/6sXIohcXhOnouwIDAQAB"; - let pub_key = rsa::RsaPublicKey::from_public_key_der( - &base64::engine::general_purpose::STANDARD.decode(PUBLIC_KEY)?, - )?; - let signature = base64::engine::general_purpose::STANDARD.decode(signature_b64)?; - rsa::pss::VerifyingKey::::new(pub_key) - .verify( - &format!("{id}{expiry}").as_bytes(), - &rsa::pss::Signature::from(signature), - ) - .map_err(|_| anyhow::anyhow!("Invalid license key".to_string()))?; - } else { - panic!("License key is required for the enterprise edition"); + let id = validate_license_key(license_key.clone()).await?; + { + let mut l = LICENSE_KEY_ID.write().await; + *l = id.to_string() } + + { + let mut l = LICENSE_KEY.write().await; + *l = license_key + } + { + let mut l = LICENSE_KEY_VALID.write().await; + *l = true + } + + Ok(()) +} + +#[cfg(feature = "enterprise")] +pub async fn verify_license_key() -> error::Result<()> { + use windmill_api::{LICENSE_KEY, LICENSE_KEY_VALID}; + use windmill_common::error::to_anyhow; + + let expiry_nb = LICENSE_KEY + .read() + .await + .clone() + .split(".") + .nth(1) + .unwrap_or_else(|| "") + .parse::() + .map_err(to_anyhow)?; + if expiry_nb < chrono::Utc::now().timestamp() as u64 { + tracing::error!( + "License key expired: {} < {}", + expiry_nb, + chrono::Utc::now().timestamp() as u64 + ); + let mut l = LICENSE_KEY_VALID.write().await; + *l = false; + }; Ok(()) } diff --git a/backend/src/main.rs b/backend/src/main.rs index ec55eb9901..e8fba9e221 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -19,15 +19,14 @@ use tokio::{ fs::{metadata, DirBuilder}, sync::RwLock, }; -use windmill_api::LICENSE_KEY; use windmill_common::{ global_settings::{ - BASE_URL_SETTING, CUSTOM_TAGS_SETTING, ENV_SETTINGS, OAUTH_SETTING, + BASE_URL_SETTING, CUSTOM_TAGS_SETTING, ENV_SETTINGS, LICENSE_KEY_SETTING, OAUTH_SETTING, REQUEST_SIZE_LIMIT_SETTING, RETENTION_PERIOD_SECS_SETTING, }, utils::rd_string, worker::{reload_custom_tags_setting, WORKER_GROUP}, - METRICS_ADDR, + DB, METRICS_ADDR, }; use windmill_worker::{ BUN_CACHE_DIR, BUN_TMP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, @@ -37,8 +36,8 @@ use windmill_worker::{ }; use crate::monitor::{ - initial_load, monitor_db, reload_base_url_setting, reload_retention_period_setting, - reload_server_config, reload_worker_config, + initial_load, monitor_db, reload_base_url_setting, reload_license_key, + reload_retention_period_setting, reload_server_config, reload_worker_config, }; const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version"); @@ -153,10 +152,10 @@ Windmill Community Edition {GIT_VERSION} // since it's only on server mode, the port is statically defined let base_internal_url: String = format!("http://localhost:{}", port.to_string()); - monitor_db(&db, &base_internal_url, rsmq.clone(), server_mode).await; - initial_load(&db, tx.clone(), worker_mode, server_mode).await; + monitor_db(&db, &base_internal_url, rsmq.clone(), server_mode).await; + if std::env::var("BASE_INTERNAL_URL").is_ok() { tracing::warn!("BASE_INTERNAL_URL is now unecessary and ignored, you can remove it."); } @@ -202,21 +201,7 @@ Windmill Community Edition {GIT_VERSION} //monitor_db is applied at start, no need to apply it twice tokio::time::sleep(Duration::from_secs(rd_delay)).await; - let mut listener = match PgListener::connect_with(&db).await { - Ok(l) => l, - Err(e) => { - tracing::error!(error = %e, "Could not connect to database"); - return; - } - }; - - if let Err(e) = listener - .listen_all(vec!["notify_config_change", "notify_global_setting_change"]) - .await - { - tracing::error!(error = %e, "Could not listen to database"); - return; - } + let mut listener = retry_listen_pg(&db).await; loop { tokio::select! { @@ -271,6 +256,12 @@ Windmill Community Edition {GIT_VERSION} tracing::error!(error = %e, "Could not reload custom tags setting"); } }, + LICENSE_KEY_SETTING => { + tracing::info!("License Key setting change detected"); + if let Err(e) = reload_license_key(&db).await { + tracing::error!(error = %e, "Could not reload license key setting"); + } + }, RETENTION_PERIOD_SECS_SETTING => { tracing::info!("Retention period setting change detected"); reload_retention_period_setting(&db).await @@ -296,7 +287,8 @@ Windmill Community Edition {GIT_VERSION} } }, Err(e) => { - tracing::error!(error = %e, "Could not receive notification"); + tracing::error!(error = %e, "Could not receive notification, attempting to reconnect listener"); + listener = retry_listen_pg(&db).await; continue; } }; @@ -330,6 +322,40 @@ Windmill Community Edition {GIT_VERSION} Ok(()) } +async fn listen_pg(db: &DB) -> Option { + let mut listener = match PgListener::connect_with(&db).await { + Ok(l) => l, + Err(e) => { + tracing::error!(error = %e, "Could not connect to database"); + return None; + } + }; + + if let Err(e) = listener + .listen_all(vec!["notify_config_change", "notify_global_setting_change"]) + .await + { + tracing::error!(error = %e, "Could not listen to database"); + return None; + } + + return Some(listener); +} + +async fn retry_listen_pg(db: &DB) -> PgListener { + let mut listener = listen_pg(db).await; + loop { + if listener.is_none() { + tracing::info!("Retrying listening to pg listen in 5 seconds"); + tokio::time::sleep(Duration::from_secs(5)).await; + listener = listen_pg(db).await; + } else { + tracing::info!("Successfully connected to pg listen"); + return listener.unwrap(); + } + } +} + fn display_config(envs: &[&str]) { tracing::info!( "config: {}", @@ -355,14 +381,6 @@ pub async fn run_workers, ) -> anyhow::Result<()> { - #[cfg(feature = "enterprise")] - ee::verify_license_key(LICENSE_KEY.clone())?; - - #[cfg(not(feature = "enterprise"))] - if LICENSE_KEY.as_ref().is_some_and(|x| !x.is_empty()) { - panic!("License key is required ONLY for the enterprise edition"); - } - let instance_name = gethostname() .to_str() .map(|x| { diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 659f9213b9..a469da9144 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -15,10 +15,12 @@ use windmill_api::{ use windmill_common::{ error, global_settings::{ - BASE_URL_SETTING, OAUTH_SETTING, REQUEST_SIZE_LIMIT_SETTING, RETENTION_PERIOD_SECS_SETTING, + BASE_URL_SETTING, LICENSE_KEY_SETTING, OAUTH_SETTING, REQUEST_SIZE_LIMIT_SETTING, + RETENTION_PERIOD_SECS_SETTING, }, jobs::{JobKind, QueuedJob}, server::load_server_config, + users::truncate_token, worker::{load_worker_config, reload_custom_tags_setting, SERVER_CONFIG, WORKER_CONFIG}, BASE_URL, DB, METRICS_ENABLED, }; @@ -26,6 +28,14 @@ use windmill_worker::{ create_token_for_owner, handle_job_error, AuthedClient, SCRIPT_TOKEN_EXPIRY, }; +#[cfg(feature = "enterprise")] +use crate::ee::verify_license_key; + +#[cfg(feature = "enterprise")] +use windmill_api::LICENSE_KEY_VALID; + +use crate::ee::set_license_key; + lazy_static::lazy_static! { static ref ZOMBIE_JOB_TIMEOUT: String = std::env::var("ZOMBIE_JOB_TIMEOUT") .ok() @@ -79,11 +89,11 @@ pub async fn initial_load( }; let reload_base_url_f = async { - if server_mode { + if let Err(e) = reload_base_url_setting(db).await { - tracing::error!("Error reloading custom tags: {:?}", e) + tracing::error!("Error reloading base url: {:?}", e) } - } + }; let reload_server_config_f = async { @@ -102,13 +112,24 @@ pub async fn initial_load( reload_request_size(&db).await; } }; + + let reload_license_key_f = async { + if server_mode { + #[cfg(feature = "enterprise")] + if let Err(e) = reload_license_key(&db).await { + tracing::error!("Error reloading license key: {:?}", e) + } + } + }; + join!( reload_worker_config_f, reload_server_config_f, reload_custom_tags_f, reload_request_size_f, reload_base_url_f, - reload_retention_period_f + reload_retention_period_f, + reload_license_key_f ); } @@ -214,6 +235,36 @@ pub async fn reload_request_size(db: &DB) { } } +pub async fn reload_license_key(db: &DB) -> error::Result<()> { + let q = sqlx::query!( + "SELECT value FROM global_settings WHERE name = $1", + LICENSE_KEY_SETTING + ) + .fetch_optional(db) + .await?; + + let mut value = std::env::var("LICENSE_KEY") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(String::new()); + + if let Some(q) = q { + if let Ok(v) = serde_json::from_value::(q.value.clone()) { + tracing::info!( + "Loaded setting LICENSE_KEY from db config: {}", + truncate_token(&v) + ); + value = v; + } else { + tracing::error!("Could not parse LICENSE_KEY found: {:#?}", &q.value); + } + }; + + set_license_key(value).await?; + + Ok(()) +} + pub async fn reload_setting( db: &DB, setting_name: &str, @@ -271,12 +322,28 @@ pub async fn monitor_db) { diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 45b32a0b3f..b41ca8245b 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -2,7 +2,6 @@ use std::sync::Arc; #[cfg(feature = "enterprise")] use chrono::Timelike; -#[cfg(feature = "enterprise")] use futures::StreamExt; use futures::{stream, Stream}; @@ -18,11 +17,12 @@ use windmill_api_client::types::{ CreateFlowBody, RawScript }; -#[cfg(feature = "enterprise")] use sqlx::query; #[cfg(feature = "enterprise")] -use windmill_api_client::types::{EditSchedule, NewSchedule, ScriptArgs, NewScript, NewScriptLanguage}; +use windmill_api_client::types::{EditSchedule, NewSchedule, ScriptArgs}; + +use windmill_api_client::types::{NewScript, NewScriptLanguage}; use windmill_common::worker::WORKER_CONFIG; use windmill_common::{ diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index cd423b5e1d..8cdcd80f6c 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -53,7 +53,7 @@ cron.workspace = true mime_guess.workspace = true rust-embed.workspace = true tracing-subscriber.workspace = true -retainer.workspace = true +quick_cache.workspace = true rand.workspace = true time.workspace = true magic-crypt.workspace = true @@ -74,3 +74,4 @@ bytes.workspace = true mail-send.workspace = true samael = { workspace = true, optional = true } async-recursion.workspace = true +rsa.workspace = true \ No newline at end of file diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index b3b3fd77fa..6213b82128 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.175.0 + version: 1.176.0 title: Windmill API contact: @@ -624,6 +624,32 @@ paths: schema: type: string + /settings/test_license_key: + post: + summary: test license key + operationId: testLicenseKey + tags: + - setting + requestBody: + description: test license key + required: true + content: + application/json: + schema: + type: object + properties: + license_key: + type: string + required: + - license_key + responses: + "200": + description: status + content: + text/plain:: + schema: + type: string + /users/email: get: summary: get current user email (if logged in) @@ -4117,6 +4143,7 @@ paths: - $ref: "#/components/parameters/StartedBefore" - $ref: "#/components/parameters/StartedAfter" - $ref: "#/components/parameters/CreatedOrStartedBefore" + - $ref: "#/components/parameters/Running" - $ref: "#/components/parameters/CreatedOrStartedAfter" - $ref: "#/components/parameters/JobKinds" - $ref: "#/components/parameters/ArgsFilter" diff --git a/backend/windmill-api/src/ee.rs b/backend/windmill-api/src/ee.rs new file mode 100644 index 0000000000..8eda5eeafd --- /dev/null +++ b/backend/windmill-api/src/ee.rs @@ -0,0 +1,41 @@ +use anyhow::anyhow; +use base64::Engine; +use rsa::{pkcs8::DecodePublicKey, signature::Verifier}; +use sha2::Sha256; + +pub async fn validate_license_key(license_key: String) -> anyhow::Result { + let mut splitted_lk = license_key.split("."); + if splitted_lk.clone().count() != 3 { + return Err(anyhow!( + "license_key can be splitted with 2 . (..)" + )); + } + + let id = splitted_lk.next().unwrap(); + let expiry = splitted_lk.next().unwrap(); + let signature_b64 = splitted_lk.next().unwrap(); + + const PUBLIC_KEY: &str = "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDgVShzcLSPiOi+8ET8fggob1kmi47/cE12JaidPkwfGnScZItghkqtiLsct0U4kJhlp5gO89DYTBmIKadvxwY7kMsLlZzmi2emVH7c27cByGASY8QmWDNdG4Ggy/NDflGGBdAtN6gHawZAg4zHv3qpbPQGHH1/6sXIohcXhOnouwIDAQAB"; + let pub_key = rsa::RsaPublicKey::from_public_key_der( + &base64::engine::general_purpose::STANDARD.decode(PUBLIC_KEY)?, + )?; + let signature = base64::engine::general_purpose::STANDARD.decode(signature_b64)?; + rsa::pss::VerifyingKey::::new(pub_key) + .verify( + &format!("{id}{expiry}").as_bytes(), + &rsa::pss::Signature::from(signature), + ) + .map_err(|_| anyhow::anyhow!("Invalid license key".to_string()))?; + + let expiry_nb = expiry.parse::()?; + if expiry_nb < chrono::Utc::now().timestamp() as u64 { + tracing::error!( + "License key expired: {} < {}", + expiry_nb, + chrono::Utc::now().timestamp() as u64 + ); + return Err(anyhow!("License key expired".to_string())); + }; + + Ok(id.to_string()) +} diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index b97a06c747..d2f213a8f1 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -780,7 +780,14 @@ async fn list_jobs( let (per_page, offset) = paginate(pagination); let lqc = lq.clone(); - let sqlc = list_completed_jobs_query( + if lq.success.is_some() && lq.running.is_some_and(|x| x) { + return Err(error::Error::BadRequest( + "cannot specify both success and running".to_string(), + )); + + } + let sqlc = if lq.running.is_none() { + Some(list_completed_jobs_query( &w_id, per_page + offset, 0, @@ -817,7 +824,10 @@ async fn list_jobs( "null as concurrent_limit", "null as concurrency_time_window_s", ], - ); + )) + } else { + None + }; let sql = if lq.success.is_none() { let sqlq = list_queue_jobs_query( @@ -833,7 +843,7 @@ async fn list_jobs( created_after: lq.created_after, created_or_started_before: lq.created_or_started_before, created_or_started_after: lq.created_or_started_after, - running: None, + running: lq.running, parent_job: lq.parent_job, order_desc: Some(true), job_kinds: lq.job_kinds, @@ -876,6 +886,7 @@ async fn list_jobs( ], ); + if let Some(sqlc) = sqlc { format!( "{} UNION ALL {} LIMIT {} OFFSET {};", &sqlq.subquery()?, @@ -883,8 +894,11 @@ async fn list_jobs( per_page, offset ) + } else { + sqlq.query()? + } } else { - sqlc.query()? + sqlc.unwrap().query()? }; let mut tx = user_db.begin(&authed).await?; let jobs: Vec = sqlx::query_as(&sql).fetch_all(&mut *tx).await?; @@ -1636,6 +1650,19 @@ async fn check_tag_available_for_workspace(w_id: &str, tag: &Option) -> } } +#[cfg(feature = "enterprise")] +pub async fn check_license_key_valid() -> error::Result<()> { + use crate::LICENSE_KEY_VALID; + + let valid = *LICENSE_KEY_VALID.read().await; + if !valid { + return Err(error::Error::BadRequest(format!( + "License key is not valid. Go to your superadmin settings to update your license key.", + ))); + } + Ok(()) +} + pub async fn run_flow_by_path( authed: ApiAuthed, Extension(db): Extension, @@ -1646,6 +1673,8 @@ pub async fn run_flow_by_path( headers: HeaderMap, JsonOrForm(args, raw_string): JsonOrForm, ) -> error::Result<(StatusCode, String)> { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; let flow_path = flow_path.to_path(); check_scopes(&authed, || format!("run:flow/{flow_path}"))?; @@ -1699,7 +1728,11 @@ pub async fn run_job_by_path( headers: HeaderMap, JsonOrForm(args, raw_string): JsonOrForm, ) -> error::Result<(StatusCode, String)> { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + let script_path = script_path.to_path(); + check_scopes(&authed, || format!("run:script/{script_path}"))?; let (job_payload, tag) = script_path_to_payload(script_path, &db, &w_id).await?; @@ -1887,6 +1920,9 @@ pub async fn run_wait_result_job_by_path_get( Path((w_id, script_path)): Path<(String, StripPath)>, Query(run_query): Query, ) -> error::JsonResult { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + if method == http::Method::HEAD { return Ok(Json(serde_json::json!(""))); } @@ -1947,6 +1983,9 @@ pub async fn run_wait_result_flow_by_path_get( headers: HeaderMap, Query(run_query): Query, ) -> error::JsonResult { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + if method == http::Method::HEAD { return Ok(Json(serde_json::json!(""))); } @@ -1986,6 +2025,9 @@ pub async fn run_wait_result_script_by_path( headers: HeaderMap, JsonOrForm(args, raw_string): JsonOrForm, ) -> error::JsonResult { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + run_wait_result_script_by_path_internal( db, run_query, @@ -2111,6 +2153,9 @@ pub async fn run_wait_result_script_by_hash( headers: HeaderMap, JsonOrForm(args, raw_string): JsonOrForm, ) -> error::JsonResult { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + check_queue_too_long(&db, run_query.queue_limit).await?; let hash = script_hash.0; @@ -2201,6 +2246,9 @@ pub async fn run_wait_result_flow_by_path( headers: HeaderMap, JsonOrForm(args, raw_string): JsonOrForm, ) -> error::JsonResult { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + run_wait_result_flow_by_path_internal( db, run_query, flow_path, authed, rsmq, user_db, headers, args, raw_string, w_id, ) @@ -2276,6 +2324,9 @@ async fn run_preview_job( headers: HeaderMap, Json(preview): Json, ) -> error::Result<(StatusCode, String)> { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + check_scopes(&authed, || format!("runscript"))?; if authed.is_operator { return Err(error::Error::NotAuthorized( @@ -2332,7 +2383,6 @@ struct BatchInfo { kind: String, flow_value: Option, path: Option, - dedicated_worker: Option, } #[tracing::instrument(level = "trace", skip_all)] @@ -2362,7 +2412,7 @@ async fn add_batch_jobs( batch_info.path, JobKind::Script, Some(script.language), - batch_info.dedicated_worker, + script.dedicated_worker, ) } "flow" => { @@ -2433,15 +2483,22 @@ async fn add_batch_jobs( format!("{}", language.as_str()) }; - let uuids = sqlx::query_scalar!("INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 FROM generate_series(1, $11)) RETURNING id", + let uuids = sqlx::query_scalar!( + r#"WITH uuid_table as ( + select gen_random_uuid() as uuid from generate_series(1, $11) + ) + INSERT INTO queue + (id, script_hash, script_path, job_kind, language, args, tag, created_by, permissioned_as, email, scheduled_for, workspace_id) + (SELECT uuid, $1, $2, $3, $4, ('{ "uuid": "' || uuid || '" }')::jsonb, $5, $6, $7, $8, $9, $10 FROM uuid_table) + RETURNING id"#, hash.map(|h| h.0), path, job_kind.clone() as JobKind, language as ScriptLang, tag, authed.username, - authed.email, username_to_permissioned_as(&authed.username), + authed.email, Utc::now(), w_id, n @@ -2512,6 +2569,9 @@ pub async fn run_job_by_hash( headers: HeaderMap, JsonOrForm(args, raw_string): JsonOrForm, ) -> error::Result<(StatusCode, String)> { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + let hash = script_hash.0; let ( path, @@ -2715,6 +2775,7 @@ pub struct ListCompletedQuery { pub created_or_started_before: Option>, pub created_or_started_after: Option>, pub success: Option, + pub running: Option, pub parent_job: Option, pub order_desc: Option, pub job_kinds: Option, diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 960715ab9c..be71de8a38 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -47,6 +47,7 @@ mod capture; mod configs; mod db; mod drafts; +pub mod ee; mod favorite; mod flows; mod folders; @@ -101,7 +102,9 @@ lazy_static::lazy_static! { slack: None })); - pub static ref LICENSE_KEY: Option = std::env::var("LICENSE_KEY").ok(); + pub static ref LICENSE_KEY_VALID: Arc> = Arc::new(RwLock::new(true)); + pub static ref LICENSE_KEY_ID: Arc> = Arc::new(RwLock::new("".to_string())); + pub static ref LICENSE_KEY: Arc> = Arc::new(RwLock::new("".to_string())); } pub async fn run_server( @@ -268,8 +271,6 @@ pub async fn run_server( println!("Graceful shutdown of server"); }); - tokio::spawn(async move { auth_cache.monitor().await }); - server.await?; Ok(()) } @@ -317,13 +318,7 @@ async fn ee_license() -> &'static str { #[cfg(feature = "enterprise")] async fn ee_license() -> String { - LICENSE_KEY - .as_ref() - .unwrap() - .split(".") - .next() - .unwrap() - .to_string() + LICENSE_KEY_ID.read().await.clone() } async fn openapi() -> &'static str { diff --git a/backend/windmill-api/src/oauth2.rs b/backend/windmill-api/src/oauth2.rs index d81d50e88d..fe06b35450 100644 --- a/backend/windmill-api/src/oauth2.rs +++ b/backend/windmill-api/src/oauth2.rs @@ -182,7 +182,7 @@ pub fn build_oauth_clients( .as_ref() .map(|c| (x.0.clone(), (x.1, c.clone()))) })) - .map(|(k, (client_params, config))| { + .filter_map(|(k, (client_params, config))| { let named_client = build_basic_client( k.clone(), config.clone(), @@ -191,17 +191,25 @@ pub fn build_oauth_clients( base_url, None, ); - ( - named_client.0, - ClientWithScopes { - client: named_client.1, - scopes: config.scopes.unwrap_or(vec![]), - extra_params: config.extra_params, - extra_params_callback: config.extra_params_callback, - allowed_domains: client_params.allowed_domains.clone(), - userinfo_url: config.userinfo_url, - }, - ) + named_client + .map(|named_client| { + ( + named_client.0, + ClientWithScopes { + client: named_client.1, + scopes: config.scopes.unwrap_or(vec![]), + extra_params: config.extra_params, + extra_params_callback: config.extra_params_callback, + allowed_domains: client_params.allowed_domains.clone(), + userinfo_url: config.userinfo_url, + }, + ) + }) + .map_err(|e| { + tracing::error!("Error building oauth client {k}: {e}"); + e + }) + .ok() }) .collect(); @@ -213,7 +221,7 @@ pub fn build_oauth_clients( .as_ref() .map(|c| (x.0.clone(), (x.1, c.clone()))) })) - .map(|(k, (client_params, config))| { + .filter_map(|(k, (client_params, config))| { let named_client = build_basic_client( k.clone(), config.clone(), @@ -226,43 +234,61 @@ pub fn build_oauth_clients( None }, ); - ( - named_client.0, - ClientWithScopes { - client: named_client.1, - scopes: config.scopes.unwrap_or(vec![]), - extra_params: config.extra_params, - extra_params_callback: config.extra_params_callback, - allowed_domains: None, - userinfo_url: None, - }, - ) + named_client + .map(|named_client| { + ( + named_client.0, + ClientWithScopes { + client: named_client.1, + scopes: config.scopes.unwrap_or(vec![]), + extra_params: config.extra_params, + extra_params_callback: config.extra_params_callback, + allowed_domains: None, + userinfo_url: None, + }, + ) + }) + .map_err(|e| { + tracing::error!("Error building oauth client {k}: {e}"); + e + }) + .ok() }) .collect(); - let slack = oauths.get("slack").map(|v| { - build_basic_client( - "slack".to_string(), - OAuthConfig { - auth_url: "https://slack.com/oauth/authorize".to_string(), - token_url: "https://slack.com/api/oauth.access".to_string(), - userinfo_url: None, - scopes: None, - extra_params: None, - extra_params_callback: None, - req_body_auth: None, - }, - v.clone(), - false, - base_url, - Some(format!("{base_url}/oauth/callback_slack")), - ) - .1 - }); - - Ok(AllClients { logins, connects, slack }) + let slack = oauths + .get("slack") + .map(|v| { + build_basic_client( + "slack".to_string(), + OAuthConfig { + auth_url: "https://slack.com/oauth/authorize".to_string(), + token_url: "https://slack.com/api/oauth.access".to_string(), + userinfo_url: None, + scopes: None, + extra_params: None, + extra_params_callback: None, + req_body_auth: None, + }, + v.clone(), + false, + base_url, + Some(format!("{base_url}/oauth/callback_slack")), + ) + .map(|x| x.1) + .map_err(|e| { + tracing::error!("Error building oauth slack client: {e}"); + e + }) + .ok() + }) + .flatten(); + let all_clients = AllClients { logins, connects, slack }; + tracing::info!("Final oauth config: {all_clients:#?}"); + Ok(all_clients) } +use anyhow::anyhow; pub fn build_basic_client( name: String, config: OAuthConfig, @@ -270,9 +296,11 @@ pub fn build_basic_client( login: bool, base_url: &str, override_callback: Option, -) -> (String, OClient) { - let auth_url = Url::parse(&config.auth_url).expect("Invalid authorization endpoint URL"); - let token_url = Url::parse(&config.token_url).expect("Invalid token endpoint URL"); +) -> error::Result<(String, OClient)> { + let auth_url = Url::parse(&config.auth_url) + .map_err(|e| anyhow!("Invalid authorization endpoint URL: {e}"))?; + let token_url = + Url::parse(&config.token_url).map_err(|e| anyhow!("Invalid token endpoint URL: {e}"))?; let redirect_url = if login { format!("{base_url}/user/login_callback/{name}") @@ -287,9 +315,12 @@ pub fn build_basic_client( client.set_auth_type(AuthType::RequestBody); } client.set_client_secret(client_params.secret.clone()); - client.set_redirect_url(Url::parse(&redirect_url).expect("Invalid redirect URL")); + client.set_redirect_url( + Url::parse(&redirect_url).map_err(|e| anyhow!("Invalid redirect URL: {e}"))?, + ); + // Set up the config for the Github OAuth2 process. - (name.to_string(), client) + Ok((name.to_string(), client)) } #[derive(Clone, Debug, Deserialize, Serialize)] diff --git a/backend/windmill-api/src/settings.rs b/backend/windmill-api/src/settings.rs index 69864a6b80..99da96777c 100644 --- a/backend/windmill-api/src/settings.rs +++ b/backend/windmill-api/src/settings.rs @@ -10,6 +10,7 @@ use std::time::Duration; use crate::{ db::{ApiAuthed, DB}, + ee::validate_license_key, utils::require_super_admin, }; @@ -36,6 +37,7 @@ pub fn global_service() -> Router { post(set_global_setting).get(get_global_setting), ) .route("/test_smtp", post(test_email)) + .route("/test_license_key", post(test_license_key)) } #[derive(Deserialize)] @@ -72,6 +74,22 @@ pub async fn test_email( Ok("Sent test email".to_string()) } +#[derive(Deserialize)] +pub struct TestKey { + pub license_key: String, +} + + +pub async fn test_license_key( + Extension(db): Extension, + authed: ApiAuthed, + Json(TestKey { license_key }): Json, +) -> error::Result { + require_super_admin(&db, &authed.email).await?; + validate_license_key(license_key).await?; + Ok("Sent test email".to_string()) +} + pub async fn get_local_settings( Extension(db): Extension, authed: ApiAuthed, diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index a56ddf3e62..3284c1d964 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -8,7 +8,7 @@ #![allow(non_snake_case)] -use std::{sync::Arc, time::Duration}; +use std::sync::Arc; use crate::db::ApiAuthed; @@ -33,15 +33,16 @@ use hyper::{header::LOCATION, StatusCode}; use lazy_static::lazy_static; use mail_send::mail_builder::MessageBuilder; use mail_send::SmtpClientBuilder; +use quick_cache::sync::Cache; use rand::rngs::OsRng; use regex::Regex; -use retainer::Cache; use serde::{Deserialize, Serialize}; use sqlx::FromRow; use time::OffsetDateTime; use tower_cookies::{Cookie, Cookies}; use tracing::{Instrument, Span}; use windmill_audit::{audit_log, ActionKind}; +use windmill_common::users::truncate_token; use windmill_common::worker::{CLOUD_HOSTED, SERVER_CONFIG}; use windmill_common::{ db::UserDB, @@ -50,7 +51,6 @@ use windmill_common::{ utils::{not_found_if_none, rd_string, require_admin, Pagination, StripPath}, }; -const TTL_TOKEN_CACHE_S: u64 = 60; // 60s pub const TTL_TOKEN_DB_H: u32 = 72; const COOKIE_NAME: &str = "token"; @@ -103,19 +103,24 @@ pub fn make_unauthed_service() -> Router { .route("/logout", get(logout)) } +#[derive(Clone)] +pub struct ExpiringAuthCache { + pub authed: ApiAuthed, + pub expiry: Option>, +} pub struct AuthCache { - cache: Cache<(String, String), ApiAuthed>, + cache: Cache<(String, String), ExpiringAuthCache>, db: DB, superadmin_secret: Option, } impl AuthCache { pub fn new(db: DB, superadmin_secret: Option) -> Self { - AuthCache { cache: Cache::new(), db, superadmin_secret } + AuthCache { cache: Cache::new(300), db, superadmin_secret } } pub async fn invalidate(&self, w_id: &str, token: String) { - self.cache.remove(&(w_id.to_string(), token)).await; + self.cache.remove(&(w_id.to_string(), token)); } pub async fn get_authed(&self, w_id: Option, token: &str) -> Option { @@ -123,13 +128,17 @@ impl AuthCache { w_id.as_ref().unwrap_or(&"".to_string()).to_string(), token.to_string(), ); - let s = self.cache.get(&key).await.map(|c| c.to_owned()); + let s = self.cache.get(&key).map(|c| c.to_owned()); match s { - a @ Some(_) => a, - None => { - let user_o = sqlx::query_as::<_, (Option, Option, bool, Option>)>( + Some(ExpiringAuthCache { authed, expiry }) + if expiry.is_none() || expiry.unwrap() > chrono::Utc::now() => + { + Some(authed) + } + _ => { + let user_o = sqlx::query_as::<_, (Option, Option, bool, Option>, Option>)>( "UPDATE token SET last_used_at = now() WHERE token = $1 AND (expiration > NOW() \ - OR expiration IS NULL) RETURNING owner, email, super_admin, scopes", + OR expiration IS NULL) RETURNING owner, email, super_admin, scopes, expiration", ) .bind(token) .fetch_optional(&self.db) @@ -140,7 +149,7 @@ impl AuthCache { if let Some(user) = user_o { let authed_o = { match user { - (Some(owner), Some(email), super_admin, _) if w_id.is_some() => { + (Some(owner), Some(email), super_admin, _, _) if w_id.is_some() => { if let Some((prefix, name)) = owner.split_once('/') { if prefix == "u" { let (is_admin, is_operator) = if super_admin { @@ -219,7 +228,7 @@ impl AuthCache { }) } } - (_, Some(email), super_admin, scopes) => { + (_, Some(email), super_admin, scopes, _) => { if w_id.is_some() { let row_o = sqlx::query_as::<_, (String, bool, bool)>( "SELECT username, is_admin, operator FROM usr where email = $1 AND \ @@ -289,9 +298,10 @@ impl AuthCache { } }; if let Some(authed) = authed_o.as_ref() { - self.cache - .insert(key, authed.clone(), Duration::from_secs(TTL_TOKEN_CACHE_S)) - .await; + self.cache.insert( + key, + ExpiringAuthCache { authed: authed.clone(), expiry: user.4 }, + ); } authed_o } else if self @@ -315,10 +325,6 @@ impl AuthCache { } } } - - pub async fn monitor(&self) { - self.cache.monitor(20, 0.25, Duration::from_secs(10)).await; - } } async fn extract_token(parts: &mut Parts, state: &S) -> Option { @@ -2359,9 +2365,3 @@ pub async fn login_externally( tx.commit().await?; Ok(()) } - -pub fn truncate_token(token: &str) -> String { - let mut s = token[..10].to_owned(); - s.push_str("*****"); - s -} diff --git a/backend/windmill-api/src/webhook_util.rs b/backend/windmill-api/src/webhook_util.rs index f312448701..1be01a1222 100644 --- a/backend/windmill-api/src/webhook_util.rs +++ b/backend/windmill-api/src/webhook_util.rs @@ -1,7 +1,8 @@ use std::time::Duration; +use quick_cache::sync::Cache; use serde::Serialize; -use tokio::{select, sync::mpsc, time::interval}; +use tokio::{select, sync::mpsc}; use windmill_common::METRICS_ENABLED; use crate::db::DB; @@ -78,8 +79,7 @@ impl WebhookShared { .timeout(Duration::from_secs(5)) .build() .unwrap(); - let cache = retainer::Cache::new(); - let mut cache_purge_interval = interval(Duration::from_secs(30)); + let cache = Cache::new(100); loop { select! { @@ -87,7 +87,7 @@ impl WebhookShared { _ = shutdown_rx.recv() => break, r = rx.recv() => match r { Some(WebhookPayload::WorkspaceEvent(workspace_id, message)) => { - let url_guard = match cache.get(&workspace_id).await { + let webhook_opt = match cache.get(&workspace_id) { Some(guard) => { guard }, @@ -104,16 +104,14 @@ impl WebhookShared { tracing::error!("Webhook Message to send - but cannot get workspace settings! Workspace: {workspace_id}"); continue; }; - cache.insert(workspace_id.clone(), webook_opt, Duration::from_secs(30)).await; - cache.get(&workspace_id).await.unwrap() + cache.insert(workspace_id, webook_opt.clone()); + webook_opt } }; - let webook_opt = url_guard.value(); - if let Some(url) = webook_opt { + if let Some(url) = webhook_opt { let timer = if *METRICS_ENABLED { Some(WEBHOOK_REQUEST_COUNT.start_timer()) } else { None }; let _ = client.post(url).json(&message).send().await; timer.map(|x| x.stop_and_record()); - drop(url_guard); } }, Some(WebhookPayload::InstanceEvent(event)) => { @@ -125,10 +123,6 @@ impl WebhookShared { }, None => break, }, - _ = futures::future::poll_fn(|cx| cache_purge_interval.poll_tick(cx)) => { - tracing::trace!("Purging Webhook Cache"); - cache.purge(10, 0.50).await; - }, } } }); diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 83837c55a1..8119f783bd 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -4,6 +4,7 @@ pub const BASE_URL_SETTING: &str = "base_url"; pub const OAUTH_SETTING: &str = "oauths"; pub const RETENTION_PERIOD_SECS_SETTING: &str = "retention_period_secs"; pub const REQUEST_SIZE_LIMIT_SETTING: &str = "request_size_limit_mb"; +pub const LICENSE_KEY_SETTING: &str = "license_key"; pub const ENV_SETTINGS: [&str; 54] = [ "DISABLE_NSJAIL", diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 960cdf3a98..af17eae543 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -161,6 +161,7 @@ pub struct Script { pub concurrent_limit: Option, #[serde(skip_serializing_if = "Option::is_none")] pub concurrency_time_window_s: Option, + pub dedicated_worker: Option, } #[derive(Serialize)] diff --git a/backend/windmill-common/src/users.rs b/backend/windmill-common/src/users.rs index 772670af74..688e0ba8e2 100644 --- a/backend/windmill-common/src/users.rs +++ b/backend/windmill-common/src/users.rs @@ -15,3 +15,9 @@ pub fn username_to_permissioned_as(user: &str) -> String { format!("u/{}", user) } } + +pub fn truncate_token(token: &str) -> String { + let mut s = token[..10].to_owned(); + s.push_str("*****"); + s +} diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 5c3fcbb939..0f561f7220 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -78,15 +78,12 @@ pub async fn reload_custom_tags_setting(db: &DB) -> error::Result<()> { let custom_tags = process_custom_tags(tags); - { - let l = CUSTOM_TAGS_PER_WORKSPACE.read().await; - if l.clone() == custom_tags { - tracing::info!("Custom tags setting unchanged, skipping update"); - return Ok(()); - } else { - tracing::info!("Custom tags setting changed, updating"); - } - } + tracing::info!( + "Loaded setting custom tags, common: {:?}, per-workspace: {:?}", + custom_tags.0, + custom_tags.1, + ); + { let mut l = CUSTOM_TAGS_PER_WORKSPACE.write().await; *l = custom_tags.clone() diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index ca9cabb890..764efc3dba 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -1744,11 +1744,11 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>( tag = None; } let default = || { - if job_kind == JobKind::Flow || job_kind == JobKind::FlowPreview { + if job_kind == JobKind::Flow + || job_kind == JobKind::FlowPreview + || job_kind == JobKind::Identity + { "flow".to_string() - } else if job_kind == JobKind::Identity { - // identity is a light script, nativets is too - "nativets".to_string() } else if job_kind == JobKind::Dependencies || job_kind == JobKind::FlowDependencies { "dependency".to_string() } else { diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index d46ab7bbb6..f7c033d357 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -219,18 +219,19 @@ pub async fn handle_bun_job( })?, ) .await?; + + install_lockfile( + logs, + &job.id, + &job.workspace_id, + db, + job_dir, + worker_name, + common_bun_proc_envs.clone(), + ) + .await?; + remove_dir_all(format!("{}/node_modules", job_dir)).await?; } - install_lockfile( - logs, - &job.id, - &job.workspace_id, - db, - job_dir, - worker_name, - common_bun_proc_envs.clone(), - ) - .await?; - remove_dir_all(format!("{}/node_modules", job_dir)).await?; } else if !*DISABLE_NSJAIL { logs.push_str("\n\n--- BUN INSTALL ---\n"); set_logs(&logs, &job.id, &db).await; @@ -464,6 +465,10 @@ pub async fn start_worker( mut jobs_rx: Receiver, mut killpill_rx: tokio::sync::broadcast::Receiver<()>, ) -> Result<()> { + use std::task::Poll; + + use futures::{future, Future}; + let mut logs = "".to_string(); let _ = write_file(job_dir, "main.ts", inner_content).await?; let common_bun_proc_envs: HashMap = @@ -684,6 +689,21 @@ plugin(p) // let mut i = 0; // let mut j = 0; let mut alive = true; + + fn conditional_polling( + fut: impl Future, + predicate: bool, + ) -> impl Future { + let mut fut = Box::pin(fut); + future::poll_fn(move |cx| { + if predicate { + fut.as_mut().poll(cx) + } else { + Poll::Pending + } + }) + } + loop { tokio::select! { biased; @@ -711,8 +731,8 @@ plugin(p) tracing::info!("dedicated worker process exited"); break; } - } - job = jobs_rx.recv(), if alive && jobs.len() < MAX_BUFFERED_DEDICATED_JOBS => { + }, + job = conditional_polling(jobs_rx.recv(), alive && jobs.len() < MAX_BUFFERED_DEDICATED_JOBS) => { // i += 1; if let Some(job) = job { tracing::debug!("received job"); diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index b3fa1fe62e..cb717233ac 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -10,12 +10,16 @@ use anyhow::Result; use const_format::concatcp; use itertools::Itertools; use once_cell::sync::OnceCell; +use prometheus::core::{AtomicU64, GenericCounter}; #[cfg(feature = "benchmark")] use serde::Serialize; use sqlx::{Pool, Postgres}; use std::{ collections::HashMap, - sync::{atomic::Ordering, Arc}, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, time::Duration, }; use windmill_api_client::Client; @@ -337,6 +341,52 @@ macro_rules! add_time { }; } +async fn handle_receive_completed_job< + R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static, +>( + jc: JobCompleted, + worker_execution_failed: HashMap, GenericCounter>, + base_internal_url: String, + db: Pool, + worker_dir: String, + same_worker_tx: Sender, + rsmq: Option, +) { + let metrics = build_language_metrics(&worker_execution_failed.clone(), &jc.job.language); + let token = jc.token.clone(); + let workspace = jc.job.workspace_id.clone(); + let client = AuthedClient { + base_internal_url: base_internal_url.to_string(), + workspace, + token, + client: OnceCell::new(), + }; + if let Err(err) = process_completed_job( + &jc, + &client, + &db, + &worker_dir, + metrics.clone(), + same_worker_tx.clone(), + rsmq.clone(), + ) + .await + { + handle_job_error( + &db, + &client, + &jc.job, + err, + metrics, + false, + same_worker_tx.clone(), + &worker_dir, + rsmq.clone(), + ) + .await; + } +} + pub async fn run_worker( db: &Pool, worker_instance: &str, @@ -512,8 +562,6 @@ pub async fn run_worker> = None; - tracing::info!(worker = %worker_name, "starting worker"); - #[cfg(feature = "enterprise")] let mut last_sync = Instant::now() + Duration::from_secs(rand::thread_rng().gen_range(0..*GLOBAL_CACHE_INTERVAL)); @@ -549,43 +597,168 @@ pub async fn run_worker, + None::, + JobKind::Noop as JobKind, + ScriptLang::Deno as ScriptLang, + "deno", + "admin", + "u/admin", + "admin@windmill.dev", + chrono::Utc::now(), + "admins", + jobs + ) + .execute(db) + .await.unwrap_or_else(|_e| panic!("failed to insert noop jobs")); + } + } + + #[cfg(feature = "benchmark")] + let completed_jobs = Arc::new(AtomicUsize::new(0)); + #[cfg(feature = "benchmark")] + let start = Instant::now(); + #[cfg(feature = "benchmark")] + let main_duration = Arc::new(AtomicUsize::new(0)); + #[cfg(feature = "benchmark")] + let send_duration = Arc::new(AtomicUsize::new(0)); + #[cfg(feature = "benchmark")] + let process_duration = Arc::new(AtomicUsize::new(0)); + + #[cfg(feature = "benchmark")] + let main_duration2 = main_duration.clone(); + #[cfg(feature = "benchmark")] + let send_duration2 = send_duration.clone(); + let send_result = tokio::spawn(async move { while let Some(jc) = job_completed_rx.recv().await { - let metrics = build_language_metrics(&worker_execution_failed2, &jc.job.language); - let token = jc.token.clone(); - let workspace = jc.job.workspace_id.clone(); - let client = AuthedClient { - base_internal_url: base_internal_url2.to_string(), - workspace, - token, - client: OnceCell::new(), - }; - if let Err(err) = process_completed_job( - &jc, - &client, - &db2, - &worker_dir2, - metrics.clone(), - same_worker_tx2.clone(), - rsmq2.clone(), - ) - .await - { - handle_job_error( - &db2, - &client, - &jc.job, - err, - metrics, - false, - same_worker_tx2.clone(), - &worker_dir2, - rsmq2.clone(), + let base_internal_url2 = base_internal_url2.clone(); + let worker_execution_failed2 = worker_execution_failed2.clone(); + let worker_dir2 = worker_dir2.clone(); + let db2 = db2.clone(); + let same_worker_tx2 = same_worker_tx2.clone(); + let rsmq2 = rsmq2.clone(); + + if matches!(jc.job.job_kind, JobKind::Noop) || is_dedicated_worker { + thread_count.fetch_add(1, Ordering::SeqCst); + let thread_count = thread_count.clone(); + + #[cfg(feature = "benchmark")] + let send_duration = send_duration2.clone(); + #[cfg(feature = "benchmark")] + let process_duration = process_duration.clone(); + #[cfg(feature = "benchmark")] + let completed_jobs = completed_jobs.clone(); + #[cfg(feature = "benchmark")] + let main_duration = main_duration2.clone(); + + tokio::spawn(async move { + #[cfg(feature = "benchmark")] + let process_start = Instant::now(); + + handle_receive_completed_job( + jc, + worker_execution_failed2, + base_internal_url2, + db2, + worker_dir2, + same_worker_tx2, + rsmq2, + ) + .await; + #[cfg(feature = "benchmark")] + { + let n = completed_jobs.fetch_add(1, Ordering::SeqCst); + if (n + 1) % 1000 == 0 || n == (jobs - 1) as usize { + let duration_s = start.elapsed().as_secs_f64(); + let jobs_per_sec = n as f64 / duration_s; + tracing::info!( + "completed {} jobs in {}s, {} jobs/s", + n + 1, + duration_s, + jobs_per_sec + ); + + tracing::info!( + "main loop without send {}s", + main_duration.load(Ordering::SeqCst) as f64 / 1000.0 + ); + + tracing::info!( + "send job completed / send dedicated job duration {}s", + send_duration.load(Ordering::SeqCst) as f64 / 1000.0 + ); + + tracing::info!( + "job completed process duration {}s", + process_duration.load(Ordering::SeqCst) as f64 / 1000.0 + ); + } + + process_duration.fetch_add( + process_start.elapsed().as_millis() as usize, + Ordering::SeqCst, + ); + } + + thread_count.fetch_sub(1, Ordering::SeqCst); + }); + } else { + handle_receive_completed_job( + jc, + worker_execution_failed2, + base_internal_url2, + db2, + worker_dir2, + same_worker_tx2, + rsmq2, ) .await; } } + tracing::info!("stopped processing new completed jobs"); + while thread_count.load(Ordering::SeqCst) > 0 { + tokio::time::sleep(Duration::from_millis(100)).await; + } + tracing::info!("finished processing all completed jobs"); + // if let Err(e) = // add_completed_job(&db2, &job, success, false, result, logs, rsmq2.clone()).await // { @@ -719,6 +892,9 @@ pub async fn run_worker>, Option>) }; + #[cfg(feature = "benchmark")] + tracing::info!("pre loop time {}s", start.elapsed().as_secs_f64()); + loop { #[cfg(feature = "benchmark")] let loop_start = Instant::now(); @@ -889,11 +1065,27 @@ pub async fn run_worker new TextEncoder().encode(s); + async function getQueueCount() { + return ( + await ( + await fetch( + config.server + "/api/w/" + config.workspace_id + "/jobs/queue/count", + { headers: { ["Authorization"]: "Bearer " + config.token } } + ) + ).json() + ).database_length; + } + + let pastJobs = 0; + async function getCompletedJobsCount(): Promise { + const completedJobs = ( + await ( + await fetch( + host + "/api/w/" + config.workspace_id + "/jobs/completed/count", + { headers: { ["Authorization"]: "Bearer " + config.token } } + ) + ).json() + ).database_length; + return completedJobs - pastJobs; + } + if (["deno", "python", "go", "bash", "dedicated", "bun"].includes(kind)) { await createBenchScript(kind, workspace); } - let jobsSent = jobs; + pastJobs = await getCompletedJobsCount(); + + const jobsSent = jobs; console.log(`Bulk creating ${jobsSent} jobs`); const start_create = Date.now(); @@ -84,9 +140,8 @@ export async function main({ body = JSON.stringify({ kind: "script", path: "f/benchmarks/" + kind, - dedicated_worker: kind === "dedicated", }); - } else if (["2steps", "onebranch", "branchallparrallel"].includes(kind)) { + } else if (["2steps"].includes(kind)) { const payload = getFlowPayload(kind); body = JSON.stringify({ kind: "flow", @@ -113,6 +168,7 @@ export async function main({ if (!response.ok) { throw new Error("Failed to create jobs: " + response.statusText); } + const uuids = await response.json(); const end_create = Date.now(); const create_duration = end_create - start_create; console.log( @@ -122,69 +178,66 @@ export async function main({ ); let start = Date.now(); - let queue_length = jobsSent; + let completedJobs = 0; let lastElapsed = 0; - let lastQueueLength = queue_length; - const updateState = setInterval(async () => { - const elapsed = start ? Date.now() - start : 0; - queue_length = ( - await ( - await fetch( - host + "/api/w/" + config.workspace_id + "/jobs/queue/count", - { headers: { ["Authorization"]: "Bearer " + config.token } } + let lastCompletedJobs = 0; + + let didStart = false; + while (completedJobs < jobsSent) { + const loopStart = Date.now(); + if (!didStart) { + const actual_queue = await getQueueCount(); + if (actual_queue < jobsSent) { + start = Date.now(); + didStart = true; + } + } else { + const elapsed = start ? Date.now() - start : 0; + completedJobs = await getCompletedJobsCount(); + if (kind === "2steps") { + completedJobs = Math.floor(completedJobs / 3); + } + const avgThr = ((completedJobs / elapsed) * 1000).toFixed(2); + const instThr = + lastElapsed > 0 + ? ( + ((completedJobs - lastCompletedJobs) / (elapsed - lastElapsed)) * + 1000 + ).toFixed(2) + : 0; + + lastElapsed = elapsed; + lastCompletedJobs = completedJobs; + + await Deno.stdout.write( + enc( + `elapsed: ${(elapsed / 1000).toFixed( + 2 + )} | jobs executed: ${completedJobs}/${jobsSent} (thr: inst ${instThr} - avg ${avgThr}) | remaining: ${ + jobsSent - completedJobs + } \r` ) - ).json() - ).database_length; - const avgThr = (((jobsSent - queue_length) / elapsed) * 1000).toFixed(2); - const instThr = - lastElapsed > 0 - ? ( - ((lastQueueLength - queue_length) / (elapsed - lastElapsed)) * - 1000 - ).toFixed(2) - : 0; - - lastElapsed = elapsed; - lastQueueLength = queue_length; - - await Deno.stdout.write( - enc( - `elapsed: ${(elapsed / 1000).toFixed(2)} | jobs executed: ${ - jobsSent - queue_length - }/${jobsSent} (thr: inst ${instThr} - avg ${avgThr}) | queue: ${queue_length} \r` - ) - ); - }, 10); - - while (queue_length > 0) { - if (queue_length < jobsSent && jobsSent === jobs) { - // reset start time to when the first job was picked up - start = Date.now(); - jobsSent = queue_length; + ); + } + const loopDuration = (Date.now() - loopStart) / 1000.0; + if (loopDuration < 0.05) { + await sleep(0.05 - loopDuration); } - await sleep(0.01); } - clearInterval(updateState); - const total_duration_sec = (Date.now() - start) / 1000.0; - await sleep(0.1); console.log(`\njobs: ${jobsSent}`); console.log(`duration: ${total_duration_sec}s`); console.log(`avg. throughput (jobs/time): ${jobsSent / total_duration_sec}`); - console.log( - "queue length:", - ( - await ( - await fetch( - host + "/api/w/" + config.workspace_id + "/jobs/queue/count", - { headers: { ["Authorization"]: "Bearer " + config.token } } - ) - ).json() - ).database_length - ); + console.log("completed jobs", completedJobs); + console.log("queue length:", await getQueueCount()); + + if (!noVerify && kind !== "noop") { + await verifyOutputs(uuids, config.workspace_id); + } + console.log("done"); return { @@ -229,7 +282,7 @@ if (import.meta.main) { ) .option( "--kind ", - "Specifiy the benchmark kind among: deno, identity, python, go, bash, dedicated, bun, noop, 2steps, onebranch, branchallparrallel", + "Specifiy the benchmark kind among: deno, identity, python, go, bash, dedicated, bun, noop, 2steps", { required: true, } @@ -237,6 +290,9 @@ if (import.meta.main) { .option("-j --jobs ", "Number of jobs to create.", { default: 10000, }) + .option("--no-verify", "Do not verify the output of the jobs.", { + default: false, + }) .action(main) .command( "upgrade", diff --git a/benchmarks/benchmark_suite.ts b/benchmarks/benchmark_suite.ts index 1572f510bf..0b9e85429b 100644 --- a/benchmarks/benchmark_suite.ts +++ b/benchmarks/benchmark_suite.ts @@ -23,6 +23,25 @@ type Config = { ]; }; +async function warmUp( + host: string, + email: string | undefined, + password: string | undefined, + token: string | undefined, + workspace: string +) { + console.log("%cWarming up...", "font-weight: bold;"); + await runBenchmark({ + host, + email, + password, + token, + workspace, + kind: "noop", + jobs: 5000, + }); +} + async function main({ host, email, @@ -47,6 +66,8 @@ async function main({ } } + await warmUp(host, email, password, token, workspace); + try { const config = await getConfig(configPath); for (const benchmark of config.benchmarks) { diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index e40040c351..ffe3cec953 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.175.0"; +export const VERSION = "v1.176.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ @@ -32,16 +32,16 @@ async function waitForDedicatedWorker(workspace: string, path: string) { const query = windmill.JobService.runWaitResultScriptByPath({ workspace, path, - requestBody: { - args: {}, - }, + requestBody: {}, }); - const timeout = new Promise((_, reject) => { - setTimeout(() => { + let timeout; + const timeoutPromise = new Promise((_, reject) => { + timeout = setTimeout(() => { reject("Timeout"); }, 15000); }); - await Promise.race([query, timeout]); + await Promise.race([query, timeoutPromise]); + clearTimeout(timeout); } export async function createBenchScript( @@ -63,6 +63,7 @@ export async function createBenchScript( let scriptContent: string; let language: string; + let schemaProperties = {}; if (scriptPattern === "python") { scriptContent = 'import os\n\ndef main():\n return os.environ.get("WM_JOB_ID")'; @@ -74,9 +75,15 @@ export async function createBenchScript( } else if (scriptPattern === "bash") { scriptContent = "echo $WM_JOB_ID"; language = "bash"; - } else if (scriptPattern === "dedicated" || scriptPattern === "bun") { + } else if (scriptPattern === "bun") { scriptContent = 'export function main(){ return Bun.env["WM_JOB_ID"]; }'; language = "bun"; + } else if (scriptPattern === "dedicated") { + scriptContent = "export function main(uuid){ return uuid; }"; + language = "bun"; + schemaProperties = { + uuid: { default: null, description: "", type: "string" }, + }; } else if (scriptPattern === "deno") { scriptContent = 'export function main(){ return Deno.env.get("WM_JOB_ID"); }'; @@ -96,6 +103,12 @@ export async function createBenchScript( description: "", language: language as api.NewScript.language, dedicated_worker: scriptPattern === "dedicated", + schema: { + $schema: "https://json-schema.org/draft/2020-12/schema", + properties: schemaProperties, + required: [], + type: "object", + }, }, }); diff --git a/benchmarks/suite_config.json b/benchmarks/suite_config.json index 8ccc26fb0f..f973b907b3 100644 --- a/benchmarks/suite_config.json +++ b/benchmarks/suite_config.json @@ -3,7 +3,7 @@ { "graph_title": "noop throughput benchmark (single worker)", "kind": "noop", - "jobs": 5000 + "jobs": 30000 }, { "graph_title": "flow throughput benchmark (single worker)", @@ -13,7 +13,7 @@ { "graph_title": "dedicated throughput benchmark (single worker)", "kind": "dedicated", - "jobs": 2000 + "jobs": 30000 }, { "graph_title": "deno throughput benchmark (single worker)", diff --git a/cli/main.ts b/cli/main.ts index 106df7dd78..4f37dfdb51 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -31,7 +31,7 @@ addEventListener("error", (event) => { } }); -export const VERSION = "v1.175.0"; +export const VERSION = "v1.176.0"; let command: any = new Command() .name("wmill") diff --git a/docker-compose.yml b/docker-compose.yml index 4f236ee163..e997e0e0e5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,12 +41,15 @@ services: db: condition: service_healthy - windmill_worker: image: ${WM_IMAGE} pull_policy: always deploy: replicas: 3 + resources: + limits: + cpus: "1" + memory: 2048M restart: unless-stopped environment: - DATABASE_URL=${DATABASE_URL} @@ -66,30 +69,28 @@ services: - /var/run/docker.sock:/var/run/docker.sock - worker_dependency_cache:/tmp/windmill/cache - ## This worker is specialized for "native" jobs. They run in-process and can thus be parallelized to more than 1 at a time on a given worker which is why NUM_WORKERS is set to 4 + ## This worker is specialized for "native" jobs. Native jobs run in-process and thus are much more lightweight than other jobs windmill_worker_native: # Use ghcr.io/windmill-labs/windmill-ee:main for the ee image: ${WM_IMAGE} pull_policy: always deploy: - replicas: 1 + replicas: 2 resources: - limits: - cpus: "0.25" - memory: 512M + limits: + cpus: "0.1" + memory: 128M restart: unless-stopped environment: - DATABASE_URL=${DATABASE_URL} - RUST_LOG=info - DISABLE_SERVER=true - METRICS_ADDR=false # (ee only, if set to true, metrics will be exposed on port 8001) - - NUM_WORKERS=4 - WORKER_GROUP=native depends_on: db: condition: service_healthy - lsp: image: ghcr.io/windmill-labs/windmill-lsp:latest restart: unless-stopped diff --git a/frontend/package-lock.json b/frontend/package-lock.json index b433b07487..789855e6f0 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.175.0", + "version": "1.176.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.175.0", + "version": "1.176.0", "license": "AGPL-3.0", "dependencies": { "@aws-crypto/sha256-js": "^4.0.0", diff --git a/frontend/package.json b/frontend/package.json index d0d7e50d80..4c174f081c 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.175.0", + "version": "1.176.0", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/frontend/src/lib/components/AppConnect.svelte b/frontend/src/lib/components/AppConnect.svelte index 9e74269f47..c36a17fa3e 100644 --- a/frontend/src/lib/components/AppConnect.svelte +++ b/frontend/src/lib/components/AppConnect.svelte @@ -57,12 +57,10 @@ + + + + + + + diff --git a/frontend/src/lib/components/FlowBuilder.svelte b/frontend/src/lib/components/FlowBuilder.svelte index 1c4251403c..8387e59f3a 100644 --- a/frontend/src/lib/components/FlowBuilder.svelte +++ b/frontend/src/lib/components/FlowBuilder.svelte @@ -421,7 +421,7 @@ let abortController: AbortController | undefined = undefined let copilotLoading = false - let flowCopilotMode: 'trigger' | 'sequence' = 'trigger' + let flowCopilotMode: 'trigger' | 'sequence' = 'sequence' let copilotStatus: string = '' let copilotFlowInputs: Record = {} let copilotFlowRequiredInputs: string[] = [] diff --git a/frontend/src/lib/components/FlowPreviewContent.svelte b/frontend/src/lib/components/FlowPreviewContent.svelte index 4361dc4e0d..eb6f300fcf 100644 --- a/frontend/src/lib/components/FlowPreviewContent.svelte +++ b/frontend/src/lib/components/FlowPreviewContent.svelte @@ -34,6 +34,9 @@ return modules .filter((x) => idOrders.indexOf(x.id) <= upTo) .map((m) => { + if (idOrders.indexOf(m.id) == upTo) { + return m + } if (m.value.type === 'forloopflow') { m.value.modules = sliceModules(m.value.modules, upTo, idOrders) } else if (m.value.type === 'branchone') { diff --git a/frontend/src/lib/components/InputTransformForm.svelte b/frontend/src/lib/components/InputTransformForm.svelte index b72c11d3f5..fbb5393464 100644 --- a/frontend/src/lib/components/InputTransformForm.svelte +++ b/frontend/src/lib/components/InputTransformForm.svelte @@ -15,11 +15,11 @@ import type VariableEditor from './VariableEditor.svelte' import type ItemPicker from './ItemPicker.svelte' - import { ResourceService, type InputTransform } from '$lib/gen' + import type { InputTransform } from '$lib/gen' import TemplateEditor from './TemplateEditor.svelte' import { setInputCat as computeInputCat, isCodeInjection } from '$lib/utils' import { FunctionSquare, Plug } from 'lucide-svelte' - import { workspaceStore } from '$lib/stores' + import { getResourceTypes } from './resourceTypesStore' export let schema: Schema export let arg: InputTransform | any @@ -141,8 +141,9 @@ let resourceTypes: string[] | undefined = undefined async function loadResourceTypes() { - resourceTypes = await ResourceService.listResourceTypeNames({ workspace: $workspaceStore! }) + resourceTypes = await getResourceTypes() } + loadResourceTypes() @@ -263,7 +264,7 @@ {/if} {#if isStaticTemplate(inputCat) && propertyType == 'static' && !noDynamicToggle} -
+
{#if arg} = { Core: [ @@ -50,6 +53,14 @@ fieldType: 'seconds', placeholder: '60', storage: 'config' + }, + { + label: 'License Key', + description: 'License Key required to use the EE (switch image for windmill-ee)', + key: 'license_key', + fieldType: 'license_key', + placeholder: 'only needed to prepare upgrade to EE', + storage: 'setting' } ], SMTP: [ @@ -117,7 +128,7 @@ } } initialOauths = (await SettingService.getGlobal({ key: 'oauths' })) ?? {} - oauths = { ...initialOauths } + oauths = JSON.parse(JSON.stringify(initialOauths)) initialValues = Object.fromEntries( ( await Promise.all( @@ -128,7 +139,7 @@ ) ).flat() ) - values = { ...initialValues } + values = JSON.parse(JSON.stringify(initialValues)) if (values['retention_period_secs'] == undefined) { values['retention_period_secs'] = 60 * 60 * 24 * 60 } @@ -150,7 +161,7 @@ name: 'server', requestBody: newServerConfig }) - serverConfig = { ...newServerConfig } + serverConfig = JSON.parse(JSON.stringify(newServerConfig)) } await Promise.all( allSettings @@ -166,7 +177,7 @@ await SettingService.setGlobal({ key: x.key, requestBody: { value: values?.[x.key] } }) }) ) - initialValues = { ...values } + initialValues = JSON.parse(JSON.stringify(initialValues)) if (!deepEqual(initialOauths, oauths)) { await SettingService.setGlobal({ @@ -175,7 +186,7 @@ value: oauths } }) - initialOauths = { ...oauths } + initialOauths = JSON.parse(JSON.stringify(oauths)) } } else { console.error('Values not loaded') @@ -187,7 +198,36 @@ let resourceName = '' let tab: 'Core' | 'SMTP' | 'OAuth' = 'Core' + function parseDate(license_key: string): string | undefined { + let splitted = license_key.split('.') + if (splitted.length >= 3) { + try { + let i = parseInt(splitted[1]) + let date = new Date(i * 1000) + return date.toDateString() + } catch {} + } + return undefined + } + let to: string = '' + + const windmillBuiltins = [ + 'github', + 'gitlab', + 'bitbucket', + 'slack', + 'gsheets', + 'gdrive', + 'gmail', + 'gcal', + 'gcloud', + 'gworkspace', + 'basecamp', + 'linkedin' + ] + + let oauth_name = 'custom'
@@ -226,6 +266,37 @@ placeholder={setting.placeholder} bind:value={values[setting.key]} /> + {:else if setting.fieldType == 'textarea'} +