From 79c4ee5e6fc27fb15648c8d4b45b40260e62f3db Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 20 Sep 2023 23:15:19 +0200 Subject: [PATCH] feat: set instance settings from UI (#2314) --- Dockerfile | 2 +- LICENSE | 27 +- ...d278f2070a939ad8a0680cedd0216d53505a.json} | 4 +- ...55ddd99c4ad1f46b876dd86e372b84d806ecd.json | 1 + ...17cbcf1013ae311c36b42d423bf6a02fa016c.json | 1 + ...4d05f4509cb87c875234bfc7b310a012e4d40.json | 1 + ...04e797527841cd97dba80c271cbefafae65cc.json | 1 + ...ee85b191263989b0c78b2bfce77e796e96825.json | 1 + ...f6f580f15bca96d9746c9359e98ca793f8f1f.json | 1 + ...e2d5200f3cef9dedb83b76c8a6a433d2a4de3.json | 20 + ...bb9e132d310d19ca744531a42ca2c7ac56f56.json | 15 - ...6e03e870ccc3b353401439bc0ed8ff219249b.json | 1 + ...82ba09e47db50ba93af322649ccddbb05cc49.json | 15 + ...5eedc09bceb82bca349f3e31c8513ebbf0192.json | 1 + ...74fd4874f8f74d9f06c018db8867af18e429.json} | 4 +- ...f93f78658e57b79dc5295553cacc328eb2b7.json} | 4 +- ...d78d592c00ffb6f42c3d072f56305b40e50f9.json | 28 ++ backend/Cargo.lock | 163 ++++---- backend/Cargo.toml | 2 + .../20230917075933_rename_config.down.sql | 2 + .../20230917075933_rename_config.up.sql | 29 ++ backend/src/main.rs | 149 +++++-- backend/src/monitor.rs | 331 ++++++++++++++-- backend/windmill-api/openapi.yaml | 90 ++++- backend/windmill-api/src/configs.rs | 114 ++++++ backend/windmill-api/src/jobs.rs | 49 +-- backend/windmill-api/src/lib.rs | 77 +--- backend/windmill-api/src/oauth2.rs | 101 +++-- backend/windmill-api/src/saml.rs | 2 +- backend/windmill-api/src/settings.rs | 45 ++- backend/windmill-api/src/variables.rs | 1 + backend/windmill-api/src/workers.rs | 88 +---- backend/windmill-api/src/workspaces.rs | 11 +- backend/windmill-common/Cargo.toml | 2 +- .../windmill-common/src/global_settings.rs | 4 + backend/windmill-common/src/lib.rs | 6 +- backend/windmill-common/src/server.rs | 112 ++++++ backend/windmill-common/src/variables.rs | 4 +- backend/windmill-common/src/worker.rs | 13 +- backend/windmill-worker/src/bun_executor.rs | 4 +- backend/windmill-worker/src/common.rs | 1 + backend/windmill-worker/src/deno_executor.rs | 10 +- .../lib/components/InstanceSettings.svelte | 372 ++++++++++++++++++ .../src/lib/components/KeycloakSetting.svelte | 64 +++ .../src/lib/components/OAuthSetting.svelte | 85 ++++ .../src/lib/components/OktaSetting.svelte | 85 ++++ .../lib/components/SuperadminSettings.svelte | 160 ++++---- .../src/lib/components/WorkspaceGroup.svelte | 18 +- .../common/seconds/SecondsInput.svelte | 6 +- .../components/details/WebhooksPanel.svelte | 119 +++--- frontend/src/lib/components/icons/index.ts | 4 +- .../src/lib/components/instanceSettings.ts | 20 + .../user/(user)/first-time/+page.svelte | 18 +- .../(user)/instance_settings/+page.svelte | 9 + .../user/(user)/workspaces/+page.svelte | 1 + .../(root)/(logged)/workers/+page.svelte | 20 +- 56 files changed, 1921 insertions(+), 597 deletions(-) rename backend/.sqlx/{query-210b7fa50246d9b8fd1a24ade787bad6882c86e9422b715844aa92b67ed05174.json => query-0d407b9e18a6ee7f4be4fb017e7ed278f2070a939ad8a0680cedd0216d53505a.json} (63%) create mode 100644 backend/.sqlx/query-8ca76fcadc659d7069b97930cdde2d5200f3cef9dedb83b76c8a6a433d2a4de3.json delete mode 100644 backend/.sqlx/query-903f2f62f3829274f5dfa0cb1ebbb9e132d310d19ca744531a42ca2c7ac56f56.json create mode 100644 backend/.sqlx/query-af00c212f509076e37538be52f582ba09e47db50ba93af322649ccddbb05cc49.json rename backend/.sqlx/{query-6c0136f7965f1e01620a7d5efd7edbc78f0bf3f815676d8b343152bfce2dfc4c.json => query-bbd4136dfcfac26296fb6c52be1a74fd4874f8f74d9f06c018db8867af18e429.json} (64%) rename backend/.sqlx/{query-7998b23eb72f5967a0fd376fa1015bf29ba5150cd59732288ceb72ce9cecf987.json => query-ce9e56ff451bae10af2c396352f5f93f78658e57b79dc5295553cacc328eb2b7.json} (71%) create mode 100644 backend/.sqlx/query-d233e07d19e8e339e1378c1bfc5d78d592c00ffb6f42c3d072f56305b40e50f9.json create mode 100644 backend/migrations/20230917075933_rename_config.down.sql create mode 100644 backend/migrations/20230917075933_rename_config.up.sql create mode 100644 backend/windmill-api/src/configs.rs create mode 100644 backend/windmill-common/src/server.rs create mode 100644 frontend/src/lib/components/InstanceSettings.svelte create mode 100644 frontend/src/lib/components/KeycloakSetting.svelte create mode 100644 frontend/src/lib/components/OAuthSetting.svelte create mode 100644 frontend/src/lib/components/OktaSetting.svelte create mode 100644 frontend/src/lib/components/instanceSettings.ts create mode 100644 frontend/src/routes/(root)/(logged)/user/(user)/instance_settings/+page.svelte diff --git a/Dockerfile b/Dockerfile index a68af51b26..64a35afcfd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,7 @@ RUN apt-get update && apt-get install -y git libssl-dev pkg-config npm RUN apt-get -y update \ && apt-get install -y \ - curl nodejs npm + curl nodejs RUN rustup component add rustfmt diff --git a/LICENSE b/LICENSE index 61849398f0..e8b9062f61 100644 --- a/LICENSE +++ b/LICENSE @@ -1,17 +1,22 @@ - Source code in this repository is variously licensed under the Apache License -Version 2.0 (see file ./LICENSE-APACHE), or the AGPLv3 License (see file ./LICENSE-AGPL) +Version 2.0 (see file ./LICENSE-APACHE), or the AGPLv3 License (see file +./LICENSE-AGPL) -Every file is under copyright (c) Windmill Labs, Inc 2022 unless otherwise specified. -Every file is under License AGPL unless otherwise specified -or belonging to one of the below cases: +Every file is under copyright (c) Windmill Labs, Inc 2022 unless otherwise +specified. Every file is under License AGPL unless otherwise specified or +belonging to one of the below cases: -The files under backend/ are AGPLv3 Licensed, except any snippets of code under the compile flag "enterprise". Those snippets and files are under a proprietary and commerccial license. -The files under frontend/ are AGPLv3 Licensed, except any snippers of code that require a positive license check to be activated. Those snippets and files are under a proprietary and commercial license. -Private and public forks MUST not include any of the above proprietary and commercial code. Windmill Labs, Inc. provide tools to clean the codebase from those snippets upon demand. -The files under python-client/ deno-client/ go-client/ are Apache 2.0 Licensed. +The files under backend/ are AGPLv3 Licensed, except any snippets of code under +the compile flag "enterprise". Those snippets and files are under a proprietary +and commercial license. The files under frontend/ are AGPLv3 Licensed, except +any snippets of code that require a positive license check to be activated. +Those snippets and files are under a proprietary and commercial license. Private +and public forks MUST not include any of the above proprietary and commercial +code. Windmill Labs, Inc. provide tools to clean the codebase from those +snippets upon demand. The files under python-client/ deno-client/ go-client/ are +Apache 2.0 Licensed. The openapi files, including the OpenFlow spec is Apache 2.0 Licensed. -All third party components incorporated into the Windmill Software are licensed under the -original license provided by the owner of the applicable component. +All third party components incorporated into the Windmill Software are licensed +under the original license provided by the owner of the applicable component. diff --git a/backend/.sqlx/query-210b7fa50246d9b8fd1a24ade787bad6882c86e9422b715844aa92b67ed05174.json b/backend/.sqlx/query-0d407b9e18a6ee7f4be4fb017e7ed278f2070a939ad8a0680cedd0216d53505a.json similarity index 63% rename from backend/.sqlx/query-210b7fa50246d9b8fd1a24ade787bad6882c86e9422b715844aa92b67ed05174.json rename to backend/.sqlx/query-0d407b9e18a6ee7f4be4fb017e7ed278f2070a939ad8a0680cedd0216d53505a.json index ae14447a5e..5b0bcc44c8 100644 --- a/backend/.sqlx/query-210b7fa50246d9b8fd1a24ade787bad6882c86e9422b715844aa92b67ed05174.json +++ b/backend/.sqlx/query-0d407b9e18a6ee7f4be4fb017e7ed278f2070a939ad8a0680cedd0216d53505a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM worker_group_config WHERE name = $1 RETURNING name", + "query": "DELETE FROM config WHERE name = $1 RETURNING name", "describe": { "columns": [ { @@ -18,5 +18,5 @@ false ] }, - "hash": "210b7fa50246d9b8fd1a24ade787bad6882c86e9422b715844aa92b67ed05174" + "hash": "0d407b9e18a6ee7f4be4fb017e7ed278f2070a939ad8a0680cedd0216d53505a" } diff --git a/backend/.sqlx/query-123c0608e229c29187009b7961355ddd99c4ad1f46b876dd86e372b84d806ecd.json b/backend/.sqlx/query-123c0608e229c29187009b7961355ddd99c4ad1f46b876dd86e372b84d806ecd.json index 7718e05ccf..1b8084742c 100644 --- a/backend/.sqlx/query-123c0608e229c29187009b7961355ddd99c4ad1f46b876dd86e372b84d806ecd.json +++ b/backend/.sqlx/query-123c0608e229c29187009b7961355ddd99c4ad1f46b876dd86e372b84d806ecd.json @@ -37,6 +37,7 @@ "bash", "postgresql", "nativets", + "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-25bef6a248f3ee0ea2cbcc376c217cbcf1013ae311c36b42d423bf6a02fa016c.json b/backend/.sqlx/query-25bef6a248f3ee0ea2cbcc376c217cbcf1013ae311c36b42d423bf6a02fa016c.json index 9d082a6772..bf591ef11c 100644 --- a/backend/.sqlx/query-25bef6a248f3ee0ea2cbcc376c217cbcf1013ae311c36b42d423bf6a02fa016c.json +++ b/backend/.sqlx/query-25bef6a248f3ee0ea2cbcc376c217cbcf1013ae311c36b42d423bf6a02fa016c.json @@ -67,6 +67,7 @@ "bash", "postgresql", "nativets", + "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-2de52e1f3226ca9281b6e25f74d4d05f4509cb87c875234bfc7b310a012e4d40.json b/backend/.sqlx/query-2de52e1f3226ca9281b6e25f74d4d05f4509cb87c875234bfc7b310a012e4d40.json index 0174706d37..9c386c3509 100644 --- a/backend/.sqlx/query-2de52e1f3226ca9281b6e25f74d4d05f4509cb87c875234bfc7b310a012e4d40.json +++ b/backend/.sqlx/query-2de52e1f3226ca9281b6e25f74d4d05f4509cb87c875234bfc7b310a012e4d40.json @@ -46,6 +46,7 @@ "bash", "postgresql", "nativets", + "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-438b5b5d29b05846c2e074cad2404e797527841cd97dba80c271cbefafae65cc.json b/backend/.sqlx/query-438b5b5d29b05846c2e074cad2404e797527841cd97dba80c271cbefafae65cc.json index 1166260449..a5dee163e5 100644 --- a/backend/.sqlx/query-438b5b5d29b05846c2e074cad2404e797527841cd97dba80c271cbefafae65cc.json +++ b/backend/.sqlx/query-438b5b5d29b05846c2e074cad2404e797527841cd97dba80c271cbefafae65cc.json @@ -28,6 +28,7 @@ "bash", "postgresql", "nativets", + "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-5cd89ab614d3cac80fb81627267ee85b191263989b0c78b2bfce77e796e96825.json b/backend/.sqlx/query-5cd89ab614d3cac80fb81627267ee85b191263989b0c78b2bfce77e796e96825.json index eabf671894..1517c8d1d4 100644 --- a/backend/.sqlx/query-5cd89ab614d3cac80fb81627267ee85b191263989b0c78b2bfce77e796e96825.json +++ b/backend/.sqlx/query-5cd89ab614d3cac80fb81627267ee85b191263989b0c78b2bfce77e796e96825.json @@ -42,6 +42,7 @@ "bash", "postgresql", "nativets", + "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-65835f2e5ad38f7cc6b147dadfef6f580f15bca96d9746c9359e98ca793f8f1f.json b/backend/.sqlx/query-65835f2e5ad38f7cc6b147dadfef6f580f15bca96d9746c9359e98ca793f8f1f.json index c52efca4c0..bfe7c41f64 100644 --- a/backend/.sqlx/query-65835f2e5ad38f7cc6b147dadfef6f580f15bca96d9746c9359e98ca793f8f1f.json +++ b/backend/.sqlx/query-65835f2e5ad38f7cc6b147dadfef6f580f15bca96d9746c9359e98ca793f8f1f.json @@ -42,6 +42,7 @@ "bash", "postgresql", "nativets", + "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-8ca76fcadc659d7069b97930cdde2d5200f3cef9dedb83b76c8a6a433d2a4de3.json b/backend/.sqlx/query-8ca76fcadc659d7069b97930cdde2d5200f3cef9dedb83b76c8a6a433d2a4de3.json new file mode 100644 index 0000000000..f05a1b36ac --- /dev/null +++ b/backend/.sqlx/query-8ca76fcadc659d7069b97930cdde2d5200f3cef9dedb83b76c8a6a433d2a4de3.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT config FROM config WHERE name = 'server'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "config", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + true + ] + }, + "hash": "8ca76fcadc659d7069b97930cdde2d5200f3cef9dedb83b76c8a6a433d2a4de3" +} diff --git a/backend/.sqlx/query-903f2f62f3829274f5dfa0cb1ebbb9e132d310d19ca744531a42ca2c7ac56f56.json b/backend/.sqlx/query-903f2f62f3829274f5dfa0cb1ebbb9e132d310d19ca744531a42ca2c7ac56f56.json deleted file mode 100644 index e24ca9042f..0000000000 --- a/backend/.sqlx/query-903f2f62f3829274f5dfa0cb1ebbb9e132d310d19ca744531a42ca2c7ac56f56.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO worker_group_config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "903f2f62f3829274f5dfa0cb1ebbb9e132d310d19ca744531a42ca2c7ac56f56" -} diff --git a/backend/.sqlx/query-9e8c3ff3d6b31e366e15beda1e96e03e870ccc3b353401439bc0ed8ff219249b.json b/backend/.sqlx/query-9e8c3ff3d6b31e366e15beda1e96e03e870ccc3b353401439bc0ed8ff219249b.json index fead4ba250..6308bf3bb2 100644 --- a/backend/.sqlx/query-9e8c3ff3d6b31e366e15beda1e96e03e870ccc3b353401439bc0ed8ff219249b.json +++ b/backend/.sqlx/query-9e8c3ff3d6b31e366e15beda1e96e03e870ccc3b353401439bc0ed8ff219249b.json @@ -60,6 +60,7 @@ "bash", "postgresql", "nativets", + "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-af00c212f509076e37538be52f582ba09e47db50ba93af322649ccddbb05cc49.json b/backend/.sqlx/query-af00c212f509076e37538be52f582ba09e47db50ba93af322649ccddbb05cc49.json new file mode 100644 index 0000000000..2c3c76ca31 --- /dev/null +++ b/backend/.sqlx/query-af00c212f509076e37538be52f582ba09e47db50ba93af322649ccddbb05cc49.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "af00c212f509076e37538be52f582ba09e47db50ba93af322649ccddbb05cc49" +} diff --git a/backend/.sqlx/query-b224cdd1221fc9e7227ef8e8c025eedc09bceb82bca349f3e31c8513ebbf0192.json b/backend/.sqlx/query-b224cdd1221fc9e7227ef8e8c025eedc09bceb82bca349f3e31c8513ebbf0192.json index c90719118a..38f81da395 100644 --- a/backend/.sqlx/query-b224cdd1221fc9e7227ef8e8c025eedc09bceb82bca349f3e31c8513ebbf0192.json +++ b/backend/.sqlx/query-b224cdd1221fc9e7227ef8e8c025eedc09bceb82bca349f3e31c8513ebbf0192.json @@ -42,6 +42,7 @@ "bash", "postgresql", "nativets", + "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-6c0136f7965f1e01620a7d5efd7edbc78f0bf3f815676d8b343152bfce2dfc4c.json b/backend/.sqlx/query-bbd4136dfcfac26296fb6c52be1a74fd4874f8f74d9f06c018db8867af18e429.json similarity index 64% rename from backend/.sqlx/query-6c0136f7965f1e01620a7d5efd7edbc78f0bf3f815676d8b343152bfce2dfc4c.json rename to backend/.sqlx/query-bbd4136dfcfac26296fb6c52be1a74fd4874f8f74d9f06c018db8867af18e429.json index 9796d12680..d43b1e23fd 100644 --- a/backend/.sqlx/query-6c0136f7965f1e01620a7d5efd7edbc78f0bf3f815676d8b343152bfce2dfc4c.json +++ b/backend/.sqlx/query-bbd4136dfcfac26296fb6c52be1a74fd4874f8f74d9f06c018db8867af18e429.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT config FROM worker_group_config WHERE name = $1", + "query": "SELECT config FROM config WHERE name = $1", "describe": { "columns": [ { @@ -18,5 +18,5 @@ true ] }, - "hash": "6c0136f7965f1e01620a7d5efd7edbc78f0bf3f815676d8b343152bfce2dfc4c" + "hash": "bbd4136dfcfac26296fb6c52be1a74fd4874f8f74d9f06c018db8867af18e429" } diff --git a/backend/.sqlx/query-7998b23eb72f5967a0fd376fa1015bf29ba5150cd59732288ceb72ce9cecf987.json b/backend/.sqlx/query-ce9e56ff451bae10af2c396352f5f93f78658e57b79dc5295553cacc328eb2b7.json similarity index 71% rename from backend/.sqlx/query-7998b23eb72f5967a0fd376fa1015bf29ba5150cd59732288ceb72ce9cecf987.json rename to backend/.sqlx/query-ce9e56ff451bae10af2c396352f5f93f78658e57b79dc5295553cacc328eb2b7.json index a3846f69f7..b64a1eb468 100644 --- a/backend/.sqlx/query-7998b23eb72f5967a0fd376fa1015bf29ba5150cd59732288ceb72ce9cecf987.json +++ b/backend/.sqlx/query-ce9e56ff451bae10af2c396352f5f93f78658e57b79dc5295553cacc328eb2b7.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT * FROM worker_group_config", + "query": "SELECT * FROM config WHERE name LIKE 'worker__%'", "describe": { "columns": [ { @@ -22,5 +22,5 @@ true ] }, - "hash": "7998b23eb72f5967a0fd376fa1015bf29ba5150cd59732288ceb72ce9cecf987" + "hash": "ce9e56ff451bae10af2c396352f5f93f78658e57b79dc5295553cacc328eb2b7" } diff --git a/backend/.sqlx/query-d233e07d19e8e339e1378c1bfc5d78d592c00ffb6f42c3d072f56305b40e50f9.json b/backend/.sqlx/query-d233e07d19e8e339e1378c1bfc5d78d592c00ffb6f42c3d072f56305b40e50f9.json new file mode 100644 index 0000000000..b235a60c1a --- /dev/null +++ b/backend/.sqlx/query-d233e07d19e8e339e1378c1bfc5d78d592c00ffb6f42c3d072f56305b40e50f9.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT * FROM config WHERE name = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "config", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "d233e07d19e8e339e1378c1bfc5d78d592c00ffb6f42c3d072f56305b40e50f9" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 730b208ae3..88ec17a3f2 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -64,9 +64,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.0" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f2135563fb5c609d2b2b87c1e8ce7bc41b0b45430fa9661f457981503dd5bf0" +checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783" dependencies = [ "memchr", ] @@ -204,7 +204,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -334,7 +334,7 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -411,7 +411,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -582,19 +582,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "bigdecimal" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "454bca3db10617b88b566f205ed190aedb0e0e6dd4cad61d3988a72e8c5594cb" -dependencies = [ - "autocfg", - "libm", - "num-bigint", - "num-integer", - "num-traits", -] - [[package]] name = "bincode" version = "1.3.3" @@ -623,7 +610,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.37", + "syn 2.0.34", "which", ] @@ -644,7 +631,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -992,9 +979,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.4" +version = "4.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d7b8d5ec32af0fadc644bf1fd509a688c2103b185644bb1e29d164e0703136" +checksum = "84ed82781cea27b43c9b106a979fe450a13a31aab0500595fb3fc06616de08e6" dependencies = [ "clap_builder", "clap_derive", @@ -1002,9 +989,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.4" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5179bb514e4d7c2051749d8fcefa2ed6d06a9f4e6d69faf3805f5d80b8cf8d56" +checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08" dependencies = [ "anstream", "anstyle", @@ -1021,7 +1008,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -1328,7 +1315,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -1350,7 +1337,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core 0.20.3", "quote", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -1408,7 +1395,7 @@ checksum = "3c65c2ffdafc1564565200967edc4851c7b55422d3913466688907efd05ea26f" dependencies = [ "deno-proc-macro-rules-macros", "proc-macro2", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -1420,7 +1407,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -1540,7 +1527,7 @@ dependencies = [ "strum", "strum_macros", "syn 1.0.109", - "syn 2.0.37", + "syn 2.0.34", "thiserror", ] @@ -1761,9 +1748,9 @@ dependencies = [ [[package]] name = "dyn-clone" -version = "1.0.14" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d2f3407d9a573d666de4b5bdf10569d73ca9478087346697dcbae6244bfbcd" +checksum = "bbfc4744c1b8f2a09adc0e55242f60b1af195d88596bd8700be74418c056c555" [[package]] name = "dyn-iter" @@ -1950,7 +1937,7 @@ dependencies = [ "pmutil", "proc-macro2", "swc_macros_common", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -1978,7 +1965,7 @@ checksum = "b0fa992f1656e1707946bbba340ad244f0814009ef8c0118eb7b658395f19a2e" dependencies = [ "frunk_proc_macro_helpers", "quote", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -1990,7 +1977,7 @@ dependencies = [ "frunk_core", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -2002,7 +1989,7 @@ dependencies = [ "frunk_core", "frunk_proc_macro_helpers", "quote", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -2103,7 +2090,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -2376,9 +2363,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" [[package]] name = "hex" @@ -2634,7 +2621,7 @@ dependencies = [ "pmutil", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -3162,7 +3149,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.34", "termcolor", "thiserror", ] @@ -3207,7 +3194,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57349d5a326b437989b6ee4dc8f2f34b0cc131202748414712a8e7d98952fc8c" dependencies = [ "base64 0.21.4", - "bigdecimal 0.3.1", + "bigdecimal", "bindgen 0.68.1", "bitflags 2.4.0", "bitvec", @@ -3429,7 +3416,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -3688,7 +3675,7 @@ dependencies = [ "phf_shared 0.11.2", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -3726,7 +3713,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -3798,7 +3785,7 @@ checksum = "52a40bc70c2c58040d2d8b167ba9a5ff59fc9dab7ad44771cfde3dcfde7a09c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -3907,7 +3894,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -4027,7 +4014,7 @@ dependencies = [ "schemars", "serde", "serde_json", - "syn 2.0.37", + "syn 2.0.34", "thiserror", "typify", "unicode-ident", @@ -4047,7 +4034,7 @@ dependencies = [ "serde_json", "serde_tokenstream", "serde_yaml", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -4494,7 +4481,7 @@ dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn 2.0.37", + "syn 2.0.34", "walkdir", ] @@ -4767,9 +4754,9 @@ dependencies = [ [[package]] name = "schemars" -version = "0.8.15" +version = "0.8.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f7b0ce13155372a76ee2e1c5ffba1fe61ede73fbea5630d61eee6fac4929c0c" +checksum = "763f8cd0d4c71ed8389c90cb8100cba87e763bd01a8e614d4f0af97bcd50a161" dependencies = [ "chrono", "dyn-clone", @@ -4781,9 +4768,9 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "0.8.15" +version = "0.8.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e85e2a16b12bdb763244c69ab79363d71db2b4b918a2def53f80b02e0574b13c" +checksum = "ec0f696e21e10fa546b7ffb1c9672c6de8fbc7a81acf59524386d8639bf12737" dependencies = [ "proc-macro2", "quote", @@ -4932,7 +4919,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -5008,7 +4995,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -5337,7 +5324,7 @@ checksum = "dd4cef4251aabbae751a3710927945901ee1d97ee96d757f6880ebb9a79bfd53" dependencies = [ "ahash 0.8.3", "atoi", - "bigdecimal 0.3.1", + "bigdecimal", "byteorder", "bytes", "chrono", @@ -5422,7 +5409,7 @@ checksum = "8ca69bf415b93b60b80dc8fda3cb4ef52b2336614d8da2de5456cc942a110482" dependencies = [ "atoi", "base64 0.21.4", - "bigdecimal 0.3.1", + "bigdecimal", "bitflags 2.4.0", "byteorder", "bytes", @@ -5467,7 +5454,7 @@ checksum = "a0db2df1b8731c3651e204629dd55e52adbae0462fa1bdcbed56a2302c18181e" dependencies = [ "atoi", "base64 0.21.4", - "bigdecimal 0.3.1", + "bigdecimal", "bitflags 2.4.0", "byteorder", "chrono", @@ -5587,7 +5574,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -5626,7 +5613,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -5708,7 +5695,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -5757,7 +5744,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -5840,7 +5827,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -5944,7 +5931,7 @@ dependencies = [ "pmutil", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -5956,7 +5943,7 @@ dependencies = [ "pmutil", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -5980,7 +5967,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -5996,9 +5983,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.37" +version = "2.0.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" +checksum = "88ec6cdb6a4c16306eccf52ccd8d492e4ab64705a15a5016acb205251001bf72" dependencies = [ "proc-macro2", "quote", @@ -6043,9 +6030,9 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.3.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6093bad37da69aab9d123a8091e4be0aa4a03e4d601ec641c327398315f62b64" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" dependencies = [ "winapi-util", ] @@ -6076,7 +6063,7 @@ checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -6180,7 +6167,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -6433,7 +6420,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.34", ] [[package]] @@ -6571,7 +6558,7 @@ dependencies = [ "regress", "schemars", "serde_json", - "syn 2.0.37", + "syn 2.0.34", "thiserror", "unicode-ident", ] @@ -6588,7 +6575,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.37", + "syn 2.0.34", "typify-impl", ] @@ -6716,9 +6703,9 @@ checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" [[package]] name = "unicode-width" -version = "0.1.11" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" [[package]] name = "unicode-xid" @@ -6906,7 +6893,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.34", "wasm-bindgen-shared", ] @@ -6940,7 +6927,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.34", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -7151,7 +7138,7 @@ dependencies = [ "hex", "hmac", "hyper", - "itertools 0.11.0", + "itertools 0.10.5", "lazy_static", "magic-crypt", "mail-send", @@ -7229,7 +7216,7 @@ dependencies = [ "hex", "hmac", "hyper", - "itertools 0.11.0", + "itertools 0.10.5", "lazy_static", "prometheus", "rand 0.8.5", @@ -7272,7 +7259,7 @@ version = "1.175.0" dependencies = [ "anyhow", "gosyn", - "itertools 0.11.0", + "itertools 0.10.5", "lazy_static", "regex", "windmill-parser", @@ -7294,7 +7281,7 @@ name = "windmill-parser-py" version = "1.175.0" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.10.5", "rustpython-parser", "serde_json", "windmill-parser", @@ -7306,7 +7293,7 @@ version = "1.175.0" dependencies = [ "anyhow", "async-recursion", - "itertools 0.11.0", + "itertools 0.10.5", "lazy_static", "phf 0.11.2", "regex", @@ -7369,14 +7356,14 @@ version = "1.175.0" dependencies = [ "anyhow", "async-recursion", - "bigdecimal 0.4.1", + "bigdecimal", "chrono", "chrono-tz", "cron", "futures-core", "hex", "hmac", - "itertools 0.11.0", + "itertools 0.10.5", "lazy_static", "prometheus", "reqwest", @@ -7414,7 +7401,7 @@ dependencies = [ "futures", "gcp_auth", "git-version", - "itertools 0.11.0", + "itertools 0.10.5", "jsonwebtoken", "lazy_static", "mysql_async", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 516d63f6d6..00f25adf3a 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -64,6 +64,8 @@ once_cell.workspace = true prometheus.workspace = true uuid.workspace = true gethostname.workspace = true +serde_json.workspace = true +serde.workspace = true [dev-dependencies] serde_json.workspace = true diff --git a/backend/migrations/20230917075933_rename_config.down.sql b/backend/migrations/20230917075933_rename_config.down.sql new file mode 100644 index 0000000000..2dedc82b56 --- /dev/null +++ b/backend/migrations/20230917075933_rename_config.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +ALTER TABLE IF exists config RENAME TO worker_group_config ; diff --git a/backend/migrations/20230917075933_rename_config.up.sql b/backend/migrations/20230917075933_rename_config.up.sql new file mode 100644 index 0000000000..75d8e7655e --- /dev/null +++ b/backend/migrations/20230917075933_rename_config.up.sql @@ -0,0 +1,29 @@ +-- Add up migration script here +ALTER TABLE IF exists worker_group_config RENAME TO config; +UPDATE config SET name = 'worker__' || name; + +CREATE FUNCTION "notify_config_change" () +RETURNS TRIGGER AS $$ +BEGIN + PERFORM pg_notify('notify_config_change', NEW.name::text); + RETURN NEW; +END; +$$ LANGUAGE PLPGSQL; + + CREATE TRIGGER "notify_config_change" + AFTER INSERT OR UPDATE ON "config" + FOR EACH ROW +EXECUTE FUNCTION "notify_config_change" (); + +CREATE FUNCTION "notify_global_setting_change" () +RETURNS TRIGGER AS $$ +BEGIN + PERFORM pg_notify('notify_global_setting_change', NEW.name::text); + RETURN NEW; +END; +$$ LANGUAGE PLPGSQL; + + CREATE TRIGGER "notify_global_setting_change" + AFTER INSERT OR UPDATE ON "global_settings" + FOR EACH ROW +EXECUTE FUNCTION "notify_global_setting_change" (); diff --git a/backend/src/main.rs b/backend/src/main.rs index b6a9c12503..ec55eb9901 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -9,7 +9,7 @@ use gethostname::gethostname; use git_version::git_version; use rand::Rng; -use sqlx::{Pool, Postgres}; +use sqlx::{postgres::PgListener, Pool, Postgres}; use std::{ net::{IpAddr, Ipv4Addr, SocketAddr}, sync::Arc, @@ -19,8 +19,16 @@ use tokio::{ fs::{metadata, DirBuilder}, sync::RwLock, }; -use windmill_api::{LICENSE_KEY, OAUTH_CLIENTS, SMTP_CLIENT}; -use windmill_common::{global_settings::ENV_SETTINGS, utils::rd_string, METRICS_ADDR}; +use windmill_api::LICENSE_KEY; +use windmill_common::{ + global_settings::{ + BASE_URL_SETTING, CUSTOM_TAGS_SETTING, ENV_SETTINGS, OAUTH_SETTING, + REQUEST_SIZE_LIMIT_SETTING, RETENTION_PERIOD_SECS_SETTING, + }, + utils::rd_string, + worker::{reload_custom_tags_setting, WORKER_GROUP}, + METRICS_ADDR, +}; use windmill_worker::{ BUN_CACHE_DIR, BUN_TMP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, DENO_TMP_CACHE_DIR, DENO_TMP_CACHE_DIR_DEPS, DENO_TMP_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, @@ -28,7 +36,10 @@ use windmill_worker::{ PIP_CACHE_DIR, ROOT_TMP_CACHE_DIR, TAR_PIP_TMP_CACHE_DIR, }; -use crate::monitor::monitor_db; +use crate::monitor::{ + initial_load, monitor_db, reload_base_url_setting, reload_retention_period_setting, + reload_server_config, reload_worker_config, +}; const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version"); const DEFAULT_NUM_WORKERS: usize = 1; @@ -128,16 +139,6 @@ Windmill Community Edition {GIT_VERSION} display_config(&ENV_SETTINGS); - tracing::info!("Loading OAuth providers...: {:#?}", *OAUTH_CLIENTS); - if let Some(ref smtp) = *SMTP_CLIENT { - tracing::info!("Smtp client defined. Testing connection..."); - if let Err(e) = smtp.connect().await { - tracing::error!("Failed to connect to smtp server: {}", e); - } else { - tracing::info!("Smtp client connected."); - } - } - let worker_mode = num_workers > 0; if server_mode || worker_mode { @@ -152,15 +153,9 @@ Windmill Community Edition {GIT_VERSION} // since it's only on server mode, the port is statically defined let base_internal_url: String = format!("http://localhost:{}", port.to_string()); - monitor_db( - &db, - tx.clone(), - &base_internal_url, - rsmq.clone(), - worker_mode, - server_mode, - ) - .await; + monitor_db(&db, &base_internal_url, rsmq.clone(), server_mode).await; + + initial_load(&db, tx.clone(), worker_mode, server_mode).await; if std::env::var("BASE_INTERNAL_URL").is_ok() { tracing::warn!("BASE_INTERNAL_URL is now unecessary and ignored, you can remove it."); @@ -206,18 +201,106 @@ Windmill Community Edition {GIT_VERSION} tokio::spawn(async move { //monitor_db is applied at start, no need to apply it twice tokio::time::sleep(Duration::from_secs(rd_delay)).await; + + let mut listener = match PgListener::connect_with(&db).await { + Ok(l) => l, + Err(e) => { + tracing::error!(error = %e, "Could not connect to database"); + return; + } + }; + + if let Err(e) = listener + .listen_all(vec!["notify_config_change", "notify_global_setting_change"]) + .await + { + tracing::error!(error = %e, "Could not listen to database"); + return; + } + loop { - monitor_db( - &db, - tx.clone(), - &base_internal_url, - rsmq.clone(), - worker_mode, - server_mode, - ) - .await; tokio::select! { - _ = tokio::time::sleep(Duration::from_secs(30)) => (), + _ = tokio::time::sleep(Duration::from_secs(30)) => { + monitor_db( + &db, + &base_internal_url, + rsmq.clone(), + server_mode, + ) + .await; + }, + notification = listener.recv() => { + match notification { + Ok(n) => { + tracing::info!("Received new pg notification: {n:?}"); + match n.channel() { + "notify_config_change" => { + tracing::info!("Config change detected"); + match n.payload() { + "server" if server_mode => { + tracing::info!("Server config change detected"); + reload_server_config(&db).await; + }, + a@ _ if worker_mode && a == format!("worker__{}", *WORKER_GROUP) => { + tracing::info!("Worker config change detected"); + reload_worker_config(&db, tx.clone(), true).await; + }, + _ => { + () + } + } + }, + "notify_global_setting_change" => { + tracing::info!("Global setting change detected"); + match n.payload() { + BASE_URL_SETTING => { + tracing::info!("Base URL setting change detected"); + if let Err(e) = reload_base_url_setting(&db).await { + tracing::error!(error = %e, "Could not reload base url setting"); + } + }, + OAUTH_SETTING => { + tracing::info!("OAuth setting change detected"); + if let Err(e) = reload_base_url_setting(&db).await { + tracing::error!(error = %e, "Could not reload oauth setting"); + } + }, + CUSTOM_TAGS_SETTING => { + tracing::info!("Custom tags setting change detected"); + if let Err(e) = reload_custom_tags_setting(&db).await { + tracing::error!(error = %e, "Could not reload custom tags setting"); + } + }, + RETENTION_PERIOD_SECS_SETTING => { + tracing::info!("Retention period setting change detected"); + reload_retention_period_setting(&db).await + }, + REQUEST_SIZE_LIMIT_SETTING => { + tracing::info!("Request limit size change detected, killing server expecting to be restarted"); + // we wait a bit randomly to avoid having all servers shutdown at same time + let rd_delay = rand::thread_rng().gen_range(0..4); + tokio::time::sleep(Duration::from_secs(rd_delay)).await; + if let Err(e) = tx.send(()) { + tracing::error!(error = %e, "Could not send killpill to server"); + } + } + a @_ => { + tracing::info!("Unrecognized Global Setting Change Payload: {:?}", a); + } + } + }, + _ => { + tracing::warn!("Unknown notification received"); + continue; + } + } + }, + Err(e) => { + tracing::error!(error = %e, "Could not receive notification"); + continue; + } + }; + }, _ = rx.recv() => { println!("received killpill for monitor job"); break; diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 57c7f9766d..659f9213b9 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -1,12 +1,26 @@ +use std::{collections::HashMap, fmt::Display, ops::Mul, str::FromStr, sync::Arc}; + use once_cell::sync::OnceCell; +use serde::de::DeserializeOwned; use sqlx::{Pool, Postgres}; -use tokio::{join, sync::mpsc}; +use tokio::{ + join, + sync::{mpsc, RwLock}, +}; use uuid::Uuid; +use windmill_api::{ + oauth2::{build_oauth_clients, OAuthClient}, + DEFAULT_BODY_LIMIT, IS_SECURE, OAUTH_CLIENTS, REQUEST_SIZE_LIMIT, +}; use windmill_common::{ error, + global_settings::{ + BASE_URL_SETTING, OAUTH_SETTING, REQUEST_SIZE_LIMIT_SETTING, RETENTION_PERIOD_SECS_SETTING, + }, jobs::{JobKind, QueuedJob}, - worker::{load_worker_config, reload_custom_tags_setting, WORKER_CONFIG}, - METRICS_ENABLED, + server::load_server_config, + worker::{load_worker_config, reload_custom_tags_setting, SERVER_CONFIG, WORKER_CONFIG}, + BASE_URL, DB, METRICS_ENABLED, }; use windmill_worker::{ create_token_for_owner, handle_job_error, AuthedClient, SCRIPT_TOKEN_EXPIRY, @@ -40,29 +54,20 @@ lazy_static::lazy_static! { "Number of jobs in the queue", &["tag"] ).unwrap(); + + static ref JOB_RETENTION_SECS: Arc> = Arc::new(RwLock::new(0)); + } -pub async fn monitor_db( +pub async fn initial_load( db: &Pool, tx: tokio::sync::broadcast::Sender<()>, - base_internal_url: &str, - rsmq: Option, worker_mode: bool, server_mode: bool, ) { - let zombie_jobs_f = async { - if server_mode { - handle_zombie_jobs(db, base_internal_url, rsmq.clone()).await; - } - }; - let expired_items_f = async { - if server_mode { - windmill_api::delete_expired_items(&db).await; - } - }; let reload_worker_config_f = async { if worker_mode { - reload_worker_config(&db, tx).await; + reload_worker_config(&db, tx, false).await; } }; let reload_custom_tags_f = async { @@ -72,18 +77,206 @@ pub async fn monitor_db () { + let tokens_deleted_r: std::result::Result, _> = sqlx::query_scalar( + "DELETE FROM token WHERE expiration <= now() + RETURNING concat(substring(token for 10), '*****')", + ) + .fetch_all(db) + .await; + + match tokens_deleted_r { + Ok(tokens) => { + if tokens.len() > 0 { + tracing::info!("deleted {} tokens: {:?}", tokens.len(), tokens) + } + } + Err(e) => tracing::error!("Error deleting token: {}", e.to_string()), + } + + let pip_resolution_r = sqlx::query_scalar!( + "DELETE FROM pip_resolution_cache WHERE expiration <= now() RETURNING hash", + ) + .fetch_all(db) + .await; + + match pip_resolution_r { + Ok(res) => { + if res.len() > 0 { + tracing::info!("deleted {} pip_resolution: {:?}", res.len(), res) + } + } + Err(e) => tracing::error!("Error deleting pip_resolution: {}", e.to_string()), + } + + let deleted_cache = sqlx::query_scalar!( + "DELETE FROM resource WHERE resource_type = 'cache' AND to_timestamp((value->>'expire')::int) < now() RETURNING path", + ) + .fetch_all(db) + .await; + + match deleted_cache { + Ok(res) => { + if res.len() > 0 { + tracing::info!("deleted {} cache resource: {:?}", res.len(), res) + } + } + Err(e) => tracing::error!("Error deleting cache resource {}", e.to_string()), + } + + let job_retention_secs = *JOB_RETENTION_SECS.read().await; + if job_retention_secs > 0 { + let deleted_jobs = sqlx::query_scalar!( + "DELETE FROM completed_job WHERE started_at + ((duration_ms/1000 + $1) || ' s')::interval <= now() RETURNING id", + job_retention_secs + ) + .fetch_all(db) + .await; + + match deleted_jobs { + Ok(deleted_jobs) => { + if deleted_jobs.len() > 0 { + tracing::info!( + "deleted {} jobs completed JOB_RETENTION_SECS {} ago: {:?}", + deleted_jobs.len(), + job_retention_secs, + deleted_jobs, + ) + } + } + Err(e) => tracing::error!("Error deleting jobs: {}", e.to_string()), + } + } +} + +pub async fn reload_retention_period_setting(db: &DB) { + if let Err(e) = reload_setting( + db, + RETENTION_PERIOD_SECS_SETTING, + "JOB_RETENTION_SECS", + 60 * 60 * 24 * 60, + JOB_RETENTION_SECS.clone(), + |x| x, + ) + .await + { + tracing::error!("Error reloading retention period: {:?}", e) + } +} + +pub async fn reload_request_size(db: &DB) { + if let Err(e) = reload_setting( + db, + REQUEST_SIZE_LIMIT_SETTING, + "REQUEST_SIZE_LIMIT", + DEFAULT_BODY_LIMIT, + REQUEST_SIZE_LIMIT.clone(), + |x| x.mul(1024 * 1024), + ) + .await + { + tracing::error!("Error reloading retention period: {:?}", e) + } +} + +pub async fn reload_setting( + db: &DB, + setting_name: &str, + std_env_var: &str, + default: T, + lock: Arc>, + transformer: fn(T) -> T, +) -> error::Result<()> { + let q = sqlx::query!( + "SELECT value FROM global_settings WHERE name = $1", + setting_name + ) + .fetch_optional(db) + .await?; + + let mut value = std::env::var(std_env_var) + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(default); + + if let Some(q) = q { + if let Ok(v) = serde_json::from_value::(q.value.clone()) { + tracing::info!( + "Loaded setting {setting_name} from db config: {:#?}", + &q.value + ); + value = transformer(v); + } else { + tracing::error!("Could not parse {setting_name} found: {:#?}", &q.value); + } + }; + + { + let mut l = lock.write().await; + *l = value; + } + + Ok(()) +} + +pub async fn monitor_db( + db: &Pool, + base_internal_url: &str, + rsmq: Option, + server_mode: bool, +) { + let zombie_jobs_f = async { + if server_mode { + handle_zombie_jobs(db, base_internal_url, rsmq.clone()).await; + } + }; + let expired_items_f = async { + if server_mode { + delete_expired_items(&db).await; + } + }; + let expose_queue_metrics_f = async { if *METRICS_ENABLED && server_mode { expose_queue_metrics(&db).await; } }; - join!( - expired_items_f, - zombie_jobs_f, - reload_worker_config_f, - reload_custom_tags_f, - expose_queue_metrics_f - ); + join!(expired_items_f, zombie_jobs_f, expose_queue_metrics_f); } pub async fn expose_queue_metrics(db: &Pool) { @@ -104,7 +297,22 @@ pub async fn expose_queue_metrics(db: &Pool) { } } -pub async fn reload_worker_config(db: &Pool, tx: tokio::sync::broadcast::Sender<()>) { +pub async fn reload_server_config(db: &Pool) { + let config = load_server_config(&db).await; + if let Err(e) = config { + tracing::error!("Error reloading server config: {:?}", e) + } else { + let mut wc = SERVER_CONFIG.write().await; + tracing::info!("Reloading server config..."); + *wc = config.unwrap() + } +} + +pub async fn reload_worker_config( + db: &DB, + tx: tokio::sync::broadcast::Sender<()>, + kill_if_change: bool, +) { let config = load_worker_config(&db).await; if let Err(e) = config { tracing::error!("Error reloading worker config: {:?}", e) @@ -112,7 +320,7 @@ pub async fn reload_worker_config(db: &Pool, tx: tokio::sync::broadcas let wc = WORKER_CONFIG.read().await; let config = config.unwrap(); if *wc != config { - if (*wc).dedicated_worker != config.dedicated_worker { + if kill_if_change && (*wc).dedicated_worker != config.dedicated_worker { tracing::info!("Dedicated worker config changed, sending killpill. Expecting to be restarted by supervisor."); let _ = tx.send(()); } @@ -125,6 +333,77 @@ pub async fn reload_worker_config(db: &Pool, tx: tokio::sync::broadcas } } +pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> { + let q_base_url = sqlx::query!( + "SELECT value FROM global_settings WHERE name = $1", + BASE_URL_SETTING + ) + .fetch_optional(db) + .await?; + + let base_url = if let Some(q) = q_base_url { + if let Ok(v) = serde_json::from_value::(q.value.clone()) { + v + } else { + tracing::error!( + "Could not parse base_url setting as a string, found: {:#?}", + &q.value + ); + std::env::var("BASE_URL") + .ok() + .unwrap_or_else(|| "http://localhost".to_string()) + } + } else { + std::env::var("BASE_URL") + .ok() + .unwrap_or_else(|| "http://localhost".to_string()) + }; + + let q_oauth = sqlx::query!( + "SELECT value FROM global_settings WHERE name = $1", + OAUTH_SETTING + ) + .fetch_optional(db) + .await?; + + let oauths = if let Some(q) = q_oauth { + if let Ok(v) = + serde_json::from_value::>>(q.value.clone()) + { + v + } else { + tracing::error!( + "Could not parse oauth setting as a json, found: {:#?}", + &q.value + ); + None + } + } else { + None + }; + + let is_secure = base_url.starts_with("https://"); + + { + let mut l = OAUTH_CLIENTS.write().await; + *l = build_oauth_clients(&base_url, oauths) + .map_err(|e| tracing::error!("Error building oauth clients (is the oauth.json mounted and in correct format? Use '{}' as minimal oauth.json): {}", "{}", e)) + .unwrap(); + } + + { + let mut l = BASE_URL.write().await; + *l = base_url + } + + { + let mut l = IS_SECURE.write().await; + *l = is_secure; + } + + Ok(()) +} + async fn handle_zombie_jobs( db: &Pool, base_internal_url: &str, diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 65ea950432..b3b3fd77fa 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -575,6 +575,55 @@ paths: application/json: schema: {} + /settings/test_smtp: + post: + summary: test smtp + operationId: testSmtp + tags: + - setting + requestBody: + description: test smtp payload + required: true + content: + application/json: + schema: + type: object + properties: + to: + type: string + smtp: + type: object + properties: + host: + type: string + username: + type: string + password: + type: string + port: + type: integer + from: + type: string + tls_implicit: + type: boolean + required: + - host + - username + - password + - port + - from + - tls_implicit + required: + - to + - smtp + responses: + "200": + description: status + content: + text/plain:: + schema: + type: string + /users/email: get: summary: get current user email (if logged in) @@ -5268,15 +5317,15 @@ paths: items: $ref: "#/components/schemas/WorkerPing" - /workers/list_worker_groups: + /configs/list_worker_groups: get: - summary: list workers + summary: list worker groups operationId: listWorkerGroups tags: - - worker + - config responses: "200": - description: a list of workers + description: a list of worker group configs content: application/json: schema: @@ -5291,12 +5340,27 @@ paths: - name - config - /workers/worker_group/{name}: - post: - summary: Update Worker Group - operationId: updateWorkerGroup + /configs/get/{name}: + get: + summary: get config + operationId: get config tags: - - worker + - config + parameters: + - $ref: "#/components/parameters/Name" + responses: + "200": + description: a config + content: + application/json: + schema: {} + + /configs/update/{name}: + post: + summary: Update config + operationId: updateConfig + tags: + - config parameters: - $ref: "#/components/parameters/Name" requestBody: @@ -5313,15 +5377,15 @@ paths: schema: type: string delete: - summary: Delete Worker Group - operationId: deleteWorkerGroup + summary: Delete Config + operationId: deleteConfig tags: - - worker + - config parameters: - $ref: "#/components/parameters/Name" responses: "200": - description: Delete a worker group + description: Delete config content: text/plain: schema: diff --git a/backend/windmill-api/src/configs.rs b/backend/windmill-api/src/configs.rs new file mode 100644 index 0000000000..000db77bf9 --- /dev/null +++ b/backend/windmill-api/src/configs.rs @@ -0,0 +1,114 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use axum::{ + extract::{Extension, Path}, + routing::{get, post}, + Json, Router, +}; + +use serde::{Deserialize, Serialize}; +use sqlx::FromRow; +use windmill_common::{ + db::UserDB, + error::{self}, + DB, +}; + +use crate::{db::ApiAuthed, utils::require_super_admin}; + +pub fn global_service() -> Router { + Router::new() + .route("/list_worker_groups", get(list_worker_groups)) + .route("/update/:name", post(update_config).delete(delete_config)) + .route("/get/:name", get(get_config)) +} + +#[derive(Serialize, Deserialize, FromRow)] +struct Config { + name: String, + config: serde_json::Value, +} + +async fn list_worker_groups( + authed: ApiAuthed, + Extension(db): Extension, +) -> error::JsonResult> { + require_super_admin(&db, &authed.email).await?; + + let rows = sqlx::query_as!(Config, "SELECT * FROM config WHERE name LIKE 'worker__%'") + .fetch_all(&db) + .await?; + Ok(Json(rows)) +} + +async fn get_config( + authed: ApiAuthed, + Path(name): Path, + Extension(db): Extension, +) -> error::JsonResult> { + require_super_admin(&db, &authed.email).await?; + + let config = sqlx::query_as!(Config, "SELECT * FROM config WHERE name = $1", name) + .fetch_optional(&db) + .await? + .map(|c| c.config); + + Ok(Json(config)) +} + +async fn update_config( + Path(name): Path, + Extension(db): Extension, + authed: ApiAuthed, + Json(config): Json, +) -> error::Result { + require_super_admin(&db, &authed.email).await?; + + #[cfg(not(feature = "enterprise"))] + if name.starts_with("worker__") { + return Err(error::Error::BadRequest( + "Worker groups configurable from UI available only in the enterprise version" + .to_string(), + )); + } + + sqlx::query!( + "INSERT INTO config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = $2", + &name, + config + ) + .execute(&db) + .await?; + + Ok(format!("Updated config {name}")) +} + +async fn delete_config( + Path(name): Path, + Extension(db): Extension, + Extension(user_db): Extension, + authed: ApiAuthed, +) -> error::Result { + let tx = user_db.begin(&authed).await?; + + require_super_admin(&db, &authed.email).await?; + tx.commit().await?; + + let deleted = sqlx::query!("DELETE FROM config WHERE name = $1 RETURNING name", name) + .fetch_all(&db) + .await?; + + if deleted.len() == 0 { + return Err(error::Error::NotFound(format!( + "Config {name} not found", + name = name + ))); + } + Ok(format!("Deleted config {name}")) +} diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index e2f017e3d0..b97a06c747 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -13,7 +13,6 @@ use crate::{ users::{check_scopes, require_owner_of_path, OptAuthed}, utils::require_super_admin, variables::get_workspace_key, - BASE_URL, }; use anyhow::Context; use axum::{ @@ -34,7 +33,8 @@ use sqlx::{query_scalar, types::Uuid, FromRow, Postgres, Transaction}; use tower_http::cors::{Any, CorsLayer}; use urlencoding::encode; use windmill_audit::{audit_log, ActionKind}; -use windmill_common::worker::CUSTOM_TAGS_PER_WORKSPACE; +use windmill_common::worker::{CUSTOM_TAGS_PER_WORKSPACE, SERVER_CONFIG}; +use windmill_common::BASE_URL; use windmill_common::{ db::UserDB, error::{self, to_anyhow, Error}, @@ -1296,7 +1296,8 @@ pub async fn get_resume_urls( .map(|x| format!("?approver={}", encode(x))) .unwrap_or_else(String::new); - let base_url = BASE_URL.as_str(); + let base_url_str = BASE_URL.read().await.clone(); + let base_url = base_url_str.as_str(); let res = ResumeUrls { approvalPage: format!( "{base_url}/approve/{w_id}/{job_id}/{resume_id}/{signature}{approver}" @@ -1775,11 +1776,11 @@ impl Drop for Guard { async fn run_wait_result( authed: ApiAuthed, Extension(user_db): Extension, - timeout: i32, uuid: Uuid, Path((w_id, _)): Path<(String, T)>, ) -> error::JsonResult { let mut result; + let timeout = SERVER_CONFIG.read().await.timeout_wait_result.clone(); let timeout_ms = if timeout <= 0 { 2000 } else { @@ -1863,10 +1864,6 @@ lazy_static::lazy_static! { pub static ref QUEUE_LIMIT_WAIT_RESULT: Option = std::env::var("QUEUE_LIMIT_WAIT_RESULT") .ok() .and_then(|x| x.parse().ok()); - pub static ref TIMEOUT_WAIT_RESULT: i32 = std::env::var("TIMEOUT_WAIT_RESULT") - .ok() - .and_then(|x| x.parse().ok()) - .unwrap_or(20); pub static ref WAIT_RESULT_FAST_POLL_INTERVAL_MS: u64 = std::env::var("WAIT_RESULT_FAST_POLL_INTERVAL_MS") .ok() .and_then(|x| x.parse().ok()) @@ -1937,14 +1934,7 @@ pub async fn run_wait_result_job_by_path_get( .await?; tx.commit().await?; - run_wait_result( - authed, - Extension(user_db), - *TIMEOUT_WAIT_RESULT, - uuid, - Path((w_id, script_path)), - ) - .await + run_wait_result(authed, Extension(user_db), uuid, Path((w_id, script_path))).await } pub async fn run_wait_result_flow_by_path_get( @@ -2108,14 +2098,7 @@ async fn run_wait_result_script_by_path_internal( .await?; tx.commit().await?; - run_wait_result( - authed, - Extension(user_db), - *TIMEOUT_WAIT_RESULT, - uuid, - Path((w_id, script_path)), - ) - .await + run_wait_result(authed, Extension(user_db), uuid, Path((w_id, script_path))).await } pub async fn run_wait_result_script_by_hash( @@ -2180,14 +2163,7 @@ pub async fn run_wait_result_script_by_hash( .await?; tx.commit().await?; - run_wait_result( - authed, - Extension(user_db), - *TIMEOUT_WAIT_RESULT, - uuid, - Path((w_id, script_hash)), - ) - .await + run_wait_result(authed, Extension(user_db), uuid, Path((w_id, script_hash))).await } pub async fn openai_sync_flow_by_path( @@ -2287,14 +2263,7 @@ async fn run_wait_result_flow_by_path_internal( .await?; tx.commit().await?; - run_wait_result( - authed, - Extension(user_db), - *TIMEOUT_WAIT_RESULT, - uuid, - Path((w_id, flow_path)), - ) - .await + run_wait_result(authed, Extension(user_db), uuid, Path((w_id, flow_path))).await } async fn run_preview_job( diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index f76800fb4a..960715ab9c 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -12,7 +12,7 @@ use crate::saml::{SamlSsoLogin, ServiceProviderExt}; use crate::scim::has_scim_token; use crate::tracing_init::MyOnFailure; use crate::{ - oauth2::{build_oauth_clients, SlackVerifier}, + oauth2::SlackVerifier, tracing_init::{MyMakeSpan, MyOnResponse}, users::OptAuthed, webhook_util::WebhookShared, @@ -24,9 +24,10 @@ use axum::{middleware::from_extractor, routing::get, Extension, Router}; use db::DB; use git_version::git_version; use hyper::{http, Method}; -use mail_send::SmtpClientBuilder; use reqwest::Client; +use std::collections::HashMap; use std::{net::SocketAddr, sync::Arc}; +use tokio::sync::RwLock; use tower::ServiceBuilder; use tower_cookies::CookieManagerLayer; use tower_http::{ @@ -36,12 +37,14 @@ use tower_http::{ use windmill_common::db::UserDB; use windmill_common::utils::rd_string; use windmill_common::worker::ALL_TAGS; +use windmill_common::BASE_URL; use windmill_common::error::AppError; mod apps; mod audit; mod capture; +mod configs; mod db; mod drafts; mod favorite; @@ -51,7 +54,7 @@ mod granular_acls; mod groups; mod inputs; pub mod jobs; -mod oauth2; +pub mod oauth2; mod openai; mod raw_apps; mod resources; @@ -72,16 +75,11 @@ mod workspaces; pub const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version"); -pub use users::delete_expired_items; +pub const DEFAULT_BODY_LIMIT: usize = 2097152 * 100; // 200MB -pub const DEFAULT_BODY_LIMIT: usize = 2097152; // 2MB lazy_static::lazy_static! { - pub static ref BASE_URL: String = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string()); - pub static ref REQUEST_SIZE_LIMIT: usize = std::env::var("REQUEST_SIZE_LIMIT") - .ok() - .and_then(|x| x.parse::().ok()) - .unwrap_or(DEFAULT_BODY_LIMIT); + pub static ref REQUEST_SIZE_LIMIT: Arc> = Arc::new(RwLock::new(DEFAULT_BODY_LIMIT)); pub static ref COOKIE_DOMAIN: Option = std::env::var("COOKIE_DOMAIN").ok(); @@ -90,61 +88,22 @@ lazy_static::lazy_static! { .ok() .map(|x| SlackVerifier::new(x).unwrap()); - static ref IS_SECURE: bool = BASE_URL.starts_with("https://"); + pub static ref IS_SECURE: Arc> = Arc::new(RwLock::new(false)); pub static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new() .user_agent("windmill/beta") .danger_accept_invalid_certs(std::env::var("ACCEPT_INVALID_CERTS").is_ok()) .build().unwrap(); - pub static ref OAUTH_CLIENTS: AllClients = build_oauth_clients(&BASE_URL) - .map_err(|e| tracing::error!("Error building oauth clients (is the oauth.json mounted and in correct format? Use '{}' as minimal oauth.json): {}", "{}", e)) - .unwrap(); - - pub static ref SMTP_CLIENT: Option> = { - let smtp = parse_smtp(); - if let Some(smtp) = smtp { - match smtp { - Ok(smtp) => Some(smtp), - Err(e) => { - tracing::error!("SMTP is not configured correctly, emails will not be sent: {}", e); - None - } - } - } else { - tracing::warn!("SMTP is not configured, emails will not be sent"); - None - } - }; - - pub static ref SMTP_FROM: String = std::env::var("SMTP_FROM").unwrap_or_else(|_| "noreply@getwindmill.com".to_string()); + pub static ref OAUTH_CLIENTS: Arc> = Arc::new(RwLock::new(AllClients { + logins: HashMap::new(), + connects: HashMap::new(), + slack: None + })); pub static ref LICENSE_KEY: Option = std::env::var("LICENSE_KEY").ok(); } -pub fn parse_smtp() -> Option>> { - let username = std::env::var("SMTP_USERNAME").ok(); - let port = std::env::var("SMTP_PORT") - .ok() - .and_then(|p| p.parse().ok()) - .unwrap_or(587); - let password = std::env::var("SMTP_PASSWORD").ok(); - let host = std::env::var("SMTP_HOST").ok(); - let tls_implicit = std::env::var("SMTP_TLS_IMPLICIT") - .ok() - .and_then(|p| p.parse().ok()) - .unwrap_or(false); - - if username.is_some() && password.is_some() && host.is_some() { - let smtp = SmtpClientBuilder::new(host.unwrap(), port) - .implicit_tls(tls_implicit) - .credentials((username.unwrap(), password.unwrap())); - Some(Ok(smtp)) - } else { - None - } -} - pub async fn run_server( db: DB, rsmq: Option, @@ -185,7 +144,9 @@ pub async fn run_server( .layer(Extension(auth_cache.clone())) .layer(CookieManagerLayer::new()) .layer(Extension(WebhookShared::new(rx.resubscribe(), db.clone()))) - .layer(DefaultBodyLimit::max(*REQUEST_SIZE_LIMIT)); + .layer(DefaultBodyLimit::max( + REQUEST_SIZE_LIMIT.read().await.clone(), + )); let cors = CorsLayer::new() .allow_methods([Method::GET, Method::POST]) @@ -239,6 +200,7 @@ pub async fn run_server( .nest("/settings", settings::global_service()) .nest("/jobs", jobs::global_root_service()) .nest("/workers", workers::global_service()) + .nest("/configs", configs::global_service()) .nest("/scripts", scripts::global_service()) .nest("/groups", groups::global_service()) .nest("/flows", flows::global_service()) @@ -287,15 +249,16 @@ pub async fn run_server( let instance_name = rd_string(5); - tracing::info!(addr = %addr.to_string(), instance = %instance_name, "server started listening"); let server = axum::Server::bind(&addr).serve(app.into_make_service()); let port = server.local_addr().port(); tracing::info!( + instance = %instance_name, "server started on port={} and addr={}", port, server.local_addr().ip() ); + port_tx .send(server.local_addr().port()) .expect("Failed to send port"); diff --git a/backend/windmill-api/src/oauth2.rs b/backend/windmill-api/src/oauth2.rs index 6aef1eca6f..d81d50e88d 100644 --- a/backend/windmill-api/src/oauth2.rs +++ b/backend/windmill-api/src/oauth2.rs @@ -81,7 +81,7 @@ pub fn workspaced_service() -> Router { .route("/connect_slack_callback", post(connect_slack_callback)) } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ClientWithScopes { client: OClient, scopes: Vec, @@ -120,7 +120,10 @@ pub struct AllClients { pub slack: Option, } -pub fn build_oauth_clients(base_url: &str) -> anyhow::Result { +pub fn build_oauth_clients( + base_url: &str, + oauths_from_config: Option>, +) -> anyhow::Result { let connect_configs = serde_json::from_str::>(include_str!( "../../oauth_connect.json" ))?; @@ -128,26 +131,36 @@ pub fn build_oauth_clients(base_url: &str) -> anyhow::Result { "../../oauth_login.json" ))?; - let path = "./oauth.json"; - let content: String = if let Ok(e) = std::env::var("OAUTH_JSON_AS_BASE64") { - str::from_utf8( - &base64::engine::general_purpose::STANDARD - .decode(e) - .map_err(to_anyhow)?, - )? - .to_string() - } else if std::path::Path::new(path).exists() { - fs::read_to_string(path).map_err(to_anyhow)? + let oauths = if let Some(oauths) = oauths_from_config { + oauths } else { - tracing::warn!("oauth.json not found, no OAuth clients loaded"); - return Ok(AllClients { logins: HashMap::new(), connects: HashMap::new(), slack: None }); - }; + let path = "./oauth.json"; + let content: String = if let Ok(e) = std::env::var("OAUTH_JSON_AS_BASE64") { + str::from_utf8( + &base64::engine::general_purpose::STANDARD + .decode(e) + .map_err(to_anyhow)?, + )? + .to_string() + } else if std::path::Path::new(path).exists() { + fs::read_to_string(path).map_err(to_anyhow)? + } else { + tracing::warn!("oauth.json not found, no OAuth clients loaded"); + return Ok(AllClients { + logins: HashMap::new(), + connects: HashMap::new(), + slack: None, + }); + }; - if content.is_empty() { - tracing::warn!("oauth.json is empty, no OAuth clients loaded"); - return Ok(AllClients { logins: HashMap::new(), connects: HashMap::new(), slack: None }); - }; - let oauths: HashMap = + if content.is_empty() { + tracing::warn!("oauth.json is empty, no OAuth clients loaded"); + return Ok(AllClients { + logins: HashMap::new(), + connects: HashMap::new(), + slack: None, + }); + }; match serde_json::from_str::>(&content) { Ok(clients) => clients, Err(e) => { @@ -156,7 +169,8 @@ pub fn build_oauth_clients(base_url: &str) -> anyhow::Result { } } .into_iter() - .collect(); + .collect() + }; tracing::info!("OAuth loaded clients: {}", oauths.keys().join(", ")); @@ -314,7 +328,7 @@ async fn connect( cookies: Cookies, ) -> error::Result { let mut query = query.clone(); - let connects = &OAUTH_CLIENTS.connects; + let connects = &OAUTH_CLIENTS.read().await.connects; let scopes = query .get("scopes") .map(|x| x.split('+').map(|x| x.to_owned()).collect()); @@ -330,7 +344,7 @@ async fn connect( cookies, scopes, extra_params, - *IS_SECURE, + IS_SECURE.read().await.clone(), ) } @@ -406,6 +420,8 @@ struct Logins { async fn list_logins(Extension(sso): Extension>) -> error::JsonResult { Ok(Json(Logins { oauth: OAUTH_CLIENTS + .read() + .await .logins .keys() .map(|x| x.to_owned()) @@ -421,7 +437,7 @@ struct ScopesAndParams { } async fn list_connects() -> error::JsonResult> { Ok(Json( - (&OAUTH_CLIENTS.connects) + (&OAUTH_CLIENTS.read().await.connects) .into_iter() .map(|(k, v)| { ( @@ -438,6 +454,8 @@ async fn list_connects() -> error::JsonResult> async fn connect_slack(cookies: Cookies) -> error::Result { let mut client = OAUTH_CLIENTS + .read() + .await .slack .as_ref() .ok_or_else(|| { @@ -453,7 +471,7 @@ async fn connect_slack(cookies: Cookies) -> error::Result { client.add_scope("commands"); let url = client.authorize_url(&state); - set_cookie(&state, cookies, *IS_SECURE); + set_cookie(&state, cookies, IS_SECURE.read().await.clone()); Ok(Redirect::to(url.as_str())) } @@ -496,8 +514,15 @@ async fn disconnect_slack( } async fn login(Path(client_name): Path, cookies: Cookies) -> error::Result { - let clients = &OAUTH_CLIENTS.logins; - oauth_redirect(clients, client_name, cookies, None, None, *IS_SECURE) + let clients = &OAUTH_CLIENTS.read().await.logins; + oauth_redirect( + clients, + client_name, + cookies, + None, + None, + IS_SECURE.read().await.clone(), + ) } #[derive(Deserialize)] @@ -532,6 +557,8 @@ pub async fn _refresh_token<'c>( .await?; let account = not_found_if_none(account, "Account", &id.to_string())?; let client = (&OAUTH_CLIENTS + .read() + .await .connects .get(&account.client) .ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))? @@ -622,11 +649,10 @@ async fn connect_callback( Path(client_name): Path, Json(callback): Json, ) -> error::JsonResult { - let client_w_scopes = OAUTH_CLIENTS - .connects + let connects = &OAUTH_CLIENTS.read().await.connects; + let client_w_scopes = connects .get(&client_name) .ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?; - let client = client_w_scopes.client.to_owned(); let extra_params = client_w_scopes.extra_params_callback.clone(); let token_response = @@ -644,6 +670,8 @@ async fn connect_slack_callback( Json(callback): Json, ) -> error::Result { let client = OAUTH_CLIENTS + .read() + .await .slack .as_ref() .ok_or_else(|| { @@ -858,7 +886,7 @@ async fn slack_command( None, ) .await?; - let url = BASE_URL.to_owned(); + let url = BASE_URL.read().await.clone(); tx.commit().await?; return Ok(format!( "Job launched. See details at {url}/run/{uuid}?workspace={}", @@ -880,10 +908,13 @@ async fn login_callback( Extension(webhook): Extension, Json(callback): Json, ) -> error::Result { - let client_w_config = &OAUTH_CLIENTS - .logins - .get(&client_name) - .ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?; + let client_w_config = { + let clients = OAUTH_CLIENTS.read().await.logins.clone(); + clients + .get(&client_name) + .ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))? + .clone() + }; let client = client_w_config.client.to_owned(); let token_res = exchange_code::(callback, &cookies, client, &HTTP_CLIENT, None).await; diff --git a/backend/windmill-api/src/saml.rs b/backend/windmill-api/src/saml.rs index 51d4c6fa1f..8bf41e82aa 100644 --- a/backend/windmill-api/src/saml.rs +++ b/backend/windmill-api/src/saml.rs @@ -71,7 +71,7 @@ pub async fn build_sp_extension() -> anyhow::Result<(ServiceProviderExt, SamlSso ..ContactPerson::default() }) .idp_metadata(idp_metadata) - .acs_url(format!("{}/api/saml/acs", *BASE_URL)) + .acs_url(format!("{}/api/saml/acs", BASE_URL.read().await.clone())) .build()?; tracing::info!("SAML Configured, sso login link at: {:?}", url); diff --git a/backend/windmill-api/src/settings.rs b/backend/windmill-api/src/settings.rs index c5e7546fbf..69864a6b80 100644 --- a/backend/windmill-api/src/settings.rs +++ b/backend/windmill-api/src/settings.rs @@ -6,6 +6,8 @@ * LICENSE-AGPL for a copy of the license. */ +use std::time::Duration; + use crate::{ db::{ApiAuthed, DB}, utils::require_super_admin, @@ -17,18 +19,57 @@ use axum::{ Json, Router, }; +use mail_send::{mail_builder::MessageBuilder, SmtpClientBuilder}; +use serde::Deserialize; +use tokio::time::timeout; use windmill_common::{ - error::{self, JsonResult}, + error::{self, to_anyhow, JsonResult}, global_settings::ENV_SETTINGS, + server::Smtp, }; pub fn global_service() -> Router { Router::new() - .route("/local", get(get_local_settings)) + .route("/envs", get(get_local_settings)) .route( "/global/:key", post(set_global_setting).get(get_global_setting), ) + .route("/test_smtp", post(test_email)) +} + +#[derive(Deserialize)] +pub struct TestEmail { + pub to: String, + pub smtp: Smtp, +} + +pub async fn test_email( + Extension(db): Extension, + authed: ApiAuthed, + Json(test_email): Json, +) -> error::Result { + require_super_admin(&db, &authed.email).await?; + let smtp = test_email.smtp; + let to = test_email.to; + let client = SmtpClientBuilder::new(smtp.host, smtp.port) + .implicit_tls(smtp.tls_implicit) + .credentials((smtp.username, smtp.password)); + let message = MessageBuilder::new() + .from(("Windmill", smtp.from.as_str())) + .to(to.clone()) + .subject("Test email from Windmill") + .text_body("Test email content"); + let dur = Duration::from_secs(3); + timeout(dur, client.connect()) + .await + .map_err(to_anyhow)? + .map_err(to_anyhow)? + .send(message) + .await + .map_err(to_anyhow)?; + tracing::info!("Sent test email to {to}"); + Ok("Sent test email".to_string()) } pub async fn get_local_settings( diff --git a/backend/windmill-api/src/variables.rs b/backend/windmill-api/src/variables.rs index 9d6bb1d2db..99a8987cc9 100644 --- a/backend/windmill-api/src/variables.rs +++ b/backend/windmill-api/src/variables.rs @@ -68,6 +68,7 @@ async fn list_contextual_variables( Some("u/user/triggering_flow_path".to_string()), Some("c".to_string()), ) + .await .to_vec(), )) } diff --git a/backend/windmill-api/src/workers.rs b/backend/windmill-api/src/workers.rs index d444312fc6..7385bb33df 100644 --- a/backend/windmill-api/src/workers.rs +++ b/backend/windmill-api/src/workers.rs @@ -7,7 +7,7 @@ */ use axum::{ - extract::{Extension, Path, Query}, + extract::{Extension, Query}, routing::get, Json, Router, }; @@ -16,27 +16,17 @@ use serde::{Deserialize, Serialize}; use sqlx::FromRow; use windmill_common::{ db::UserDB, - error::{self, JsonResult}, + error::JsonResult, utils::{paginate, Pagination}, worker::ALL_TAGS, - DB, }; -use crate::{db::ApiAuthed, utils::require_super_admin}; +use crate::db::ApiAuthed; pub fn global_service() -> Router { - use axum::routing::post; - - let router = Router::new() + Router::new() .route("/list", get(list_worker_pings)) .route("/custom_tags", get(get_custom_tags)) - .route("/list_worker_groups", get(get_worker_groups)) - .route( - "/worker_group/:name", - post(update_worker_group).delete(delete_worker_group), - ); - - return router; } #[derive(FromRow, Serialize, Deserialize)] @@ -80,73 +70,3 @@ async fn list_worker_pings( async fn get_custom_tags() -> Json> { Json(ALL_TAGS.read().await.clone().into()) } - -#[derive(Serialize, Deserialize, FromRow)] -struct WorkerGroup { - name: String, - config: serde_json::Value, -} - -async fn get_worker_groups( - authed: ApiAuthed, - Extension(db): Extension, -) -> error::JsonResult> { - require_super_admin(&db, &authed.email).await?; - - let rows = sqlx::query_as!(WorkerGroup, "SELECT * FROM worker_group_config") - .fetch_all(&db) - .await?; - Ok(Json(rows)) -} - -#[cfg(feature = "enterprise")] -async fn update_worker_group( - Path(name): Path, - Extension(db): Extension, - authed: ApiAuthed, - Json(config): Json, -) -> error::Result { - require_super_admin(&db, &authed.email).await?; - - sqlx::query!( - "INSERT INTO worker_group_config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = $2", - &name, - config - ) - .execute(&db) - .await?; - - Ok(format!("Updated worker group {name}")) -} - -#[cfg(not(feature = "enterprise"))] -async fn update_worker_group() -> String { - "Worker groups available only in enterprise version".to_string() -} - -async fn delete_worker_group( - Path(name): Path, - Extension(db): Extension, - Extension(user_db): Extension, - authed: ApiAuthed, -) -> error::Result { - let tx = user_db.begin(&authed).await?; - - require_super_admin(&db, &authed.email).await?; - tx.commit().await?; - - let deleted = sqlx::query!( - "DELETE FROM worker_group_config WHERE name = $1 RETURNING name", - name, - ) - .fetch_all(&db) - .await?; - - if deleted.len() == 0 { - return Err(error::Error::NotFound(format!( - "Worker group {name} not found", - name = name - ))); - } - Ok(format!("Deleted worker group {name}")) -} diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 0a6d60a590..a54d74df95 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -284,8 +284,9 @@ async fn stripe_checkout( require_admin(authed.is_admin, &authed.username)?; let client = stripe::Client::new(std::env::var("STRIPE_KEY").expect("STRIPE_KEY")); - let success_rd = format!("{}/workspace_settings/checkout?success=true", *BASE_URL); - let failure_rd = format!("{}/workspace_settings/checkout?success=false", *BASE_URL); + let base_url = BASE_URL.read().await.clone(); + let success_rd = format!("{}/workspace_settings/checkout?success=true", base_url); + let failure_rd = format!("{}/workspace_settings/checkout?success=false", base_url); let checkout_session = { let mut params = stripe::CreateCheckoutSession::new(&failure_rd, &success_rd); params.mode = Some(stripe::CheckoutSessionMode::Subscription); @@ -332,7 +333,7 @@ async fn stripe_portal( .await? .ok_or_else(|| Error::InternalErr(format!("no customer id for workspace {}", w_id)))?; let client = stripe::Client::new(std::env::var("STRIPE_KEY").expect("STRIPE_KEY")); - let success_rd = format!("{}/workspace_settings?tab=premium", *BASE_URL); + let success_rd = format!("{}/workspace_settings?tab=premium", BASE_URL.read().await.clone()); let portal_session = { let customer_id = CustomerId::from_str(&customer_id).unwrap(); let mut params = stripe::CreateBillingPortalSession::new(customer_id); @@ -1221,7 +1222,7 @@ async fn invite_user( "You have been granted access to Windmill's workspace {w_id} If you do not have an account on {}, login with SSO or ask an admin to create an account for you.", - *BASE_URL + BASE_URL.read().await.clone() ), &nu.email, ); @@ -1285,7 +1286,7 @@ async fn add_user( "You have been granted access to Windmill's workspace {w_id} by {email} If you do not have an account on {}, login with SSO or ask an admin to create an account for you.", - *BASE_URL + BASE_URL.read().await.clone() ), &nu.email, ); diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index caccedb1d5..1f75867d12 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -45,4 +45,4 @@ tracing-subscriber = { workspace = true, optional = true } lazy_static.workspace = true tracing-flame = { version = "^0", optional = true } itertools.workspace = true -regex.workspace = true \ No newline at end of file +regex.workspace = true diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 490bd47338..83837c55a1 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -1,5 +1,9 @@ pub const WORKER_S3_BUCKET_SYNC: &str = "worker_s3_bucket_sync"; pub const CUSTOM_TAGS_SETTING: &str = "custom_tags"; +pub const BASE_URL_SETTING: &str = "base_url"; +pub const OAUTH_SETTING: &str = "oauths"; +pub const RETENTION_PERIOD_SECS_SETTING: &str = "retention_period_secs"; +pub const REQUEST_SIZE_LIMIT_SETTING: &str = "request_size_limit_mb"; pub const ENV_SETTINGS: [&str; 54] = [ "DISABLE_NSJAIL", diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 2451930ab1..1601461e42 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -6,7 +6,7 @@ * LICENSE-AGPL for a copy of the license. */ -use std::net::SocketAddr; +use std::{net::SocketAddr, sync::Arc}; use error::Error; use scripts::ScriptLang; @@ -24,6 +24,7 @@ pub mod more_serde; pub mod oauth2; pub mod schedule; pub mod scripts; +pub mod server; pub mod users; pub mod utils; pub mod variables; @@ -47,7 +48,7 @@ lazy_static::lazy_static! { .flatten() .flatten(); pub static ref METRICS_ENABLED: bool = METRICS_ADDR.is_some(); - pub static ref BASE_URL: String = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string()); + pub static ref BASE_URL: Arc> = Arc::new(RwLock::new("".to_string())); pub static ref IS_READY: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); } @@ -78,6 +79,7 @@ pub async fn shutdown_signal( Ok(()) } +use tokio::sync::RwLock; #[cfg(feature = "prometheus")] use tokio::task::JoinHandle; diff --git a/backend/windmill-common/src/server.rs b/backend/windmill-common/src/server.rs new file mode 100644 index 0000000000..6591377f63 --- /dev/null +++ b/backend/windmill-common/src/server.rs @@ -0,0 +1,112 @@ +use serde::{Deserialize, Serialize}; + +use crate::{error, DB}; + +#[derive(Serialize, Deserialize, PartialEq, Clone, Debug)] +pub struct Smtp { + pub host: String, + pub username: String, + pub password: String, + pub port: u16, + pub from: String, + pub tls_implicit: bool, +} + +#[derive(Serialize, Deserialize, PartialEq)] +pub struct ServerConfigOpt { + pub smtp_host: Option, + pub smtp_username: Option, + pub smtp_password: Option, + pub smtp_port: Option, + pub smtp_from: Option, + pub smtp_tls_implicit: Option, + pub timeout_wait_result: Option, +} + +pub async fn load_server_config(db: &DB) -> error::Result { + let config: ServerConfigOpt = + sqlx::query_scalar!("SELECT config FROM config WHERE name = 'server'",) + .fetch_optional(db) + .await? + .flatten() + .map(|x| serde_json::from_value(x).ok()) + .flatten() + .unwrap_or_default(); + + let config_smtp = if let (Some(host), Some(username), Some(password)) = + (config.smtp_host, config.smtp_username, config.smtp_password) + { + Some(Smtp { + host, + username, + password, + tls_implicit: config.smtp_tls_implicit.unwrap_or(false), + port: config.smtp_port.unwrap_or(587), + from: config + .smtp_from + .unwrap_or_else(|| "noreply@getwindmill.com".to_string()), + }) + } else { + None + }; + let smtp = config_smtp.or( + if let (Some(host), Some(username), Some(password)) = ( + std::env::var("SMTP_HOST").ok(), + std::env::var("SMTP_USERNAME").ok(), + std::env::var("SMTP_PASSWORD").ok(), + ) { + Some(Smtp { + host, + username, + password, + tls_implicit: std::env::var("SMTP_TLS_IMPLICIT") + .ok() + .and_then(|p| p.parse().ok()) + .unwrap_or(false), + port: std::env::var("SMTP_PORT") + .ok() + .and_then(|p| p.parse().ok()) + .unwrap_or(587), + from: std::env::var("SMTP_FROM") + .unwrap_or_else(|_| "noreply@getwindmill.com".to_string()), + }) + } else { + None + }, + ); + if smtp.is_none() { + tracing::warn!("SMTP not configured"); + } + + Ok(ServerConfig { + smtp, + timeout_wait_result: config + .timeout_wait_result + .ok_or( + std::env::var("TIMEOUT_WAIT_RESULT") + .ok() + .and_then(|x| x.parse::().ok()), + ) + .unwrap_or(600), + }) +} + +impl Default for ServerConfigOpt { + fn default() -> Self { + Self { + smtp_from: None, + smtp_host: None, + smtp_password: None, + smtp_port: None, + smtp_tls_implicit: None, + smtp_username: None, + timeout_wait_result: Default::default(), + } + } +} + +#[derive(PartialEq, Clone, Debug)] +pub struct ServerConfig { + pub smtp: Option, + pub timeout_wait_result: u64, +} diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index 00c1846ada..98d641724d 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -61,7 +61,7 @@ pub struct CreateVariable { pub is_oauth: Option, } -pub fn get_reserved_variables( +pub async fn get_reserved_variables( w_id: &str, token: &str, email: &str, @@ -163,7 +163,7 @@ pub fn get_reserved_variables( }, ContextualVariable { name: "WM_BASE_URL".to_string(), - value: BASE_URL.clone(), + value: BASE_URL.read().await.clone(), description: "base url of this instance".to_string(), }, ContextualVariable { diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index bc0a92b407..5c3fcbb939 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -5,7 +5,7 @@ use regex::Regex; use serde::{Deserialize, Serialize}; use tokio::sync::RwLock; -use crate::{error, global_settings::CUSTOM_TAGS_SETTING, DB}; +use crate::{error, global_settings::CUSTOM_TAGS_SETTING, server::ServerConfig, DB}; lazy_static::lazy_static! { pub static ref WORKER_GROUP: String = std::env::var("WORKER_GROUP").unwrap_or_else(|_| "default".to_string()); @@ -35,6 +35,9 @@ lazy_static::lazy_static! { dedicated_worker: Default::default(), })); + pub static ref SERVER_CONFIG: Arc> = Arc::new(RwLock::new(ServerConfig { smtp: Default::default(), timeout_wait_result: 20 })); + + pub static ref CLOUD_HOSTED: bool = std::env::var("CLOUD_HOSTED").is_ok(); @@ -97,10 +100,6 @@ pub async fn reload_custom_tags_setting(db: &DB) -> error::Result<()> { .concat(); } Ok(()) - // pub static ref CUSTOM_TAGS_PER_WORKSPACE: (Vec, HashMap>) = process_custom_tags(std::env::var("CUSTOM_TAGS") - // .ok()); - - // pub static ref ALL_TAGS: Vec = [CUSTOM_TAGS_PER_WORKSPACE.0.clone(), CUSTOM_TAGS_PER_WORKSPACE.1.keys().map(|x| x.to_string()).collect_vec()].concat(); } fn process_custom_tags(tags: Vec) -> (Vec, HashMap>) { @@ -144,8 +143,8 @@ pub async fn update_ping(worker_instance: &str, worker_name: &str, ip: &str, db: pub async fn load_worker_config(db: &DB) -> error::Result { let mut config: WorkerConfigOpt = sqlx::query_scalar!( - "SELECT config FROM worker_group_config WHERE name = $1", - *WORKER_GROUP + "SELECT config FROM config WHERE name = $1", + format!("worker__{}", *WORKER_GROUP) ) .fetch_optional(db) .await? diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 8eac5bcb58..d46ab7bbb6 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -481,6 +481,7 @@ pub async fn start_worker( None, None, ) + .await .to_vec(); let context_envs = build_envs_map(context); if let Some(reqs) = requirements_o { @@ -603,7 +604,8 @@ for await (const chunk of Bun.stdin.stream()) {{ None, None, None, - ); + ) + .await; let _ = write_file( &job_dir, diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index fd9ea8db24..2e2827d6c7 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -192,6 +192,7 @@ pub async fn get_reserved_variables( job.schedule_path.clone(), job.flow_step_id.clone(), ) + .await .to_vec(); Ok(build_envs_map(variables)) diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 716c582f72..faf7a5e043 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -41,8 +41,12 @@ lazy_static::lazy_static! { } -fn get_common_deno_proc_envs(token: &str, base_internal_url: &str) -> HashMap { - let hostname_base = BASE_URL.split("://").last().unwrap_or("localhost"); +async fn get_common_deno_proc_envs( + token: &str, + base_internal_url: &str, +) -> HashMap { + let hostname = BASE_URL.read().await.clone(); + let hostname_base = hostname.split("://").last().unwrap_or("localhost"); let hostname_internal = base_internal_url.split("://").last().unwrap_or("localhost"); let deno_auth_tokens_base = DENO_AUTH_TOKENS.as_str(); let deno_auth_tokens = @@ -259,7 +263,7 @@ run().catch(async (e) => {{ write_import_map_f )?; - let common_deno_proc_envs = get_common_deno_proc_envs(&token, base_internal_url); + let common_deno_proc_envs = get_common_deno_proc_envs(&token, base_internal_url).await; //do not cache local dependencies let reload = format!("--reload={base_internal_url}"); diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte new file mode 100644 index 0000000000..fb34131b02 --- /dev/null +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -0,0 +1,372 @@ + + +
+ + {#each Object.keys(settings) as category} + {category} + {/each} + SSO/OAuth + + +
+ {#each Object.keys(settings) as category} + + {#if category == 'SMTP'} +
Setting SMTP unlock sending emails upon adding new users to the workspace or the + instance.
+ {/if} +
+
+ {#each settings[category] as setting} + {#if !setting.cloudonly || isCloudHosted()} + + {/if} + {/each} +
+
+ {#if category == 'SMTP'} +
+
+ {/if} + + {/each} + +
+

SSO

+ + Without EE, the number of SSO users is limited to 50. SCIM/SAML is available on EE + +
+ + + + + + + +
+

OAuth

+ +
+ + {#each Object.keys(oauths) as k} + {#if !['google', 'microsoft', 'github', 'gitlab', 'jumpcloud', 'okta', 'keycloak'].includes(k)} + {#if oauths[k]} +
+
+ + { + delete oauths[k] + oauths = { ...oauths } + }} + /> +
+
+ + +
+
+ {/if} + {/if} + {/each} + +
+ + +
+
+ + + +
+
+ + +
diff --git a/frontend/src/lib/components/KeycloakSetting.svelte b/frontend/src/lib/components/KeycloakSetting.svelte new file mode 100644 index 0000000000..d302c17f6f --- /dev/null +++ b/frontend/src/lib/components/KeycloakSetting.svelte @@ -0,0 +1,64 @@ + + +
+ + {#if enabled} +
+ + + +
+ {/if} +
diff --git a/frontend/src/lib/components/OAuthSetting.svelte b/frontend/src/lib/components/OAuthSetting.svelte new file mode 100644 index 0000000000..8de1c5b07c --- /dev/null +++ b/frontend/src/lib/components/OAuthSetting.svelte @@ -0,0 +1,85 @@ + + +
+ + {#if enabled} +
+ + + {#if login} + + {/if} + {#if name == 'google'} + +
+ Create a new OAuth 2.0 Client in google console + and set the redirect URI to BASE_URL/user/login_callback/google + where BASE_URL is what you configured as core BASE_URL +
+ + {:else if name == 'slack'} + Read more about Slack OAuth on the docs + {:else if name == 'microsoft'} + +
+ Create a new OAuth 2.0 Client in microsoft portal + and in the "Authentication" tab, set the redirect URI to + BASE_URL/user/login_callback/microsoft, the logout channel to + BASE_URL/auth/logoutwhere BASE_URL is what you configured as core BASE_URL. + Also set "Accounts in any organizational directory (Any Microsoft Entra ID tenant - + Multitenant) and personal Microsoft accounts (e.g. Skype, Xbox)", you can restrict the + emails directly in windmill using the "allowed_domains" setting. +
+ + {/if} +
+ {/if} +
diff --git a/frontend/src/lib/components/OktaSetting.svelte b/frontend/src/lib/components/OktaSetting.svelte new file mode 100644 index 0000000000..99348d2940 --- /dev/null +++ b/frontend/src/lib/components/OktaSetting.svelte @@ -0,0 +1,85 @@ + + +
+ + {#if enabled} +
+ + + + +
+ From your Admin page, setup windmill using the service flow Create a new app integration + a. For "sign-in method" select "OIDC - Open ID Connect" b. For "application type" select + "Web Appliction" Select all of the following options for Grant type of "Client acting on + behalf of a user" Authorization Code Refresh Token Implicit (hybrid) Allow ID Token with + implicit grant type Allow Access Token with implicit grant type For Refresh Token, select + "Rotate token after every use" Under "LOGIN", set the following: "Sign-in redirect URIs" + BASE_URL/user/login_callback/okta "Sign-out redirect URIs" BASE_URL/auth/logout "Login + initiated by" App Only "Initiate login URI" BASE_URL/user/login +
+ +
+ {/if} +
diff --git a/frontend/src/lib/components/SuperadminSettings.svelte b/frontend/src/lib/components/SuperadminSettings.svelte index 1319e63530..82527817b4 100644 --- a/frontend/src/lib/components/SuperadminSettings.svelte +++ b/frontend/src/lib/components/SuperadminSettings.svelte @@ -1,15 +1,16 @@ x.email + ' ' + x.name + ' ' + x.company} /> - - + +
-
-
Windmill
-
- - - -
- -
-
- - +
+
+
Windmill
+
+
-
- - - email - auth - name - company - - - - - {#if filteredUsers && users} - {#each filteredUsers as { email, super_admin, login_type, name, company } (email)} - - {email} - {login_type} - {name ?? ''} - {company ?? ''} - {#if super_admin}Superadmin{/if} - -
- - | - -
- - - {/each} - {/if} - - +
+ + Global Users + Instance Settings + +
+ +
+
+ +
+
+
+ +
+
+ + + email + auth + name + company + + + + + {#if filteredUsers && users} + {#each filteredUsers as { email, super_admin, login_type, name, company } (email)} + + {email} + {login_type} + {name ?? ''} + {company ?? ''} + {#if super_admin}Superadmin{/if} + +
+ + | + +
+ + + {/each} + {/if} + + +
+
+ + +
+ + +
-
- +
diff --git a/frontend/src/lib/components/WorkspaceGroup.svelte b/frontend/src/lib/components/WorkspaceGroup.svelte index 20dd6c3f93..f0be0cdd2a 100644 --- a/frontend/src/lib/components/WorkspaceGroup.svelte +++ b/frontend/src/lib/components/WorkspaceGroup.svelte @@ -3,7 +3,7 @@ import { Button, Popup } from './common' import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' - import { WorkerService } from '$lib/gen' + import { ConfigService } from '$lib/gen' import ConfirmationModal from './common/confirmationModal/ConfirmationModal.svelte' import { createEventDispatcher } from 'svelte' import { sendUserToast } from '$lib/toast' @@ -49,7 +49,7 @@ const dispatch = createEventDispatcher() async function deleteWorkerGroup() { - await WorkerService.deleteWorkerGroup({ name }) + await ConfigService.deleteConfig({ name: 'worker__' + name }) dispatch('reload') } let dirty = false @@ -120,10 +120,10 @@ {#if selected == 'normal'} {#if nconfig?.worker_tags != undefined} -
+
{#each nconfig.worker_tags as tag} -
- {tag}
+
{tag}
{/each} @@ -221,10 +221,8 @@ color="dark" size="xs" on:click={async () => { - await WorkerService.updateWorkerGroup({ name, requestBody: nconfig }) - sendUserToast( - 'Setting configuration, it can take up to 30s to get propagated to all workers' - ) + await ConfigService.updateConfig({ name: 'worker__' + name, requestBody: nconfig }) + sendUserToast('Configuration set') dispatch('reload') }} disabled={!dirty || !$enterpriseLicense} diff --git a/frontend/src/lib/components/common/seconds/SecondsInput.svelte b/frontend/src/lib/components/common/seconds/SecondsInput.svelte index 156e3b278a..fc5641e0a7 100644 --- a/frontend/src/lib/components/common/seconds/SecondsInput.svelte +++ b/frontend/src/lib/components/common/seconds/SecondsInput.svelte @@ -44,7 +44,11 @@
{#if !hideDisplay}
+ {#key token} + + + REST + {#if SCRIPT_VIEW_SHOW_EXAMPLE_CURL} + Curl + {/if} + Fetch - - - REST - {#if SCRIPT_VIEW_SHOW_EXAMPLE_CURL} - Curl - {/if} - Fetch + + +
+ - - -
- - - {#if requestType !== 'get_path'} - - {/if} - {#key requestType} - {#key tokenType} - + {#if requestType !== 'get_path'} + + {/if} + {#key requestType} + {#key tokenType} + + {/key} {/key} - {/key} -
- - -
+
+ + +
+ {#key args} + {#key requestType} + {#key webhookType} + {#key tokenType} +
{ + e.preventDefault() + copyToClipboard(curlCode()) + }} + > + + +
+ {/key} + {/key} + {/key} + {/key} +
+ + {#key args} {#key requestType} {#key webhookType} {#key tokenType} -
{ - e.preventDefault() - copyToClipboard(curlCode()) - }} - > - - -
- {/key} - {/key} - {/key} + {#key token} +
{ + e.preventDefault() + copyToClipboard(fetchCode()) + }} + > + + +
+ {/key}{/key}{/key}{/key} {/key} -
- - - {#key args} - {#key requestType} - {#key webhookType} - {#key tokenType} - {#key token} -
{ - e.preventDefault() - copyToClipboard(fetchCode()) - }} - > - - -
- {/key}{/key}{/key}{/key} - {/key} - - - + + + + {/key}
diff --git a/frontend/src/lib/components/icons/index.ts b/frontend/src/lib/components/icons/index.ts index 22a45bbda9..d0538b84fb 100644 --- a/frontend/src/lib/components/icons/index.ts +++ b/frontend/src/lib/components/icons/index.ts @@ -132,7 +132,7 @@ export const APP_TO_ICON_COMPONENT = { bigquery: BigQueryIcon, graphql: GraphqlIcon, nocodb: NocoDbIcon, - azure: AzureIcon, + azure: AzureIcon } as const export { @@ -196,5 +196,5 @@ export { BigQueryIcon, GraphqlIcon, NocoDbIcon, - AzureIcon, + AzureIcon } diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts new file mode 100644 index 0000000000..ed9ad5204a --- /dev/null +++ b/frontend/src/lib/components/instanceSettings.ts @@ -0,0 +1,20 @@ +export interface Setting { + label: string + description?: string + placeholder?: string + cloudonly?: boolean + tooltip?: string + key: string + fieldType: + | 'text' + | 'number' + | 'boolean' + | 'password' + | 'select' + | 'textarea' + | 'seconds' + | 'email' + storage: SettingStorage +} + +export type SettingStorage = 'setting' | 'config' diff --git a/frontend/src/routes/(root)/(logged)/user/(user)/first-time/+page.svelte b/frontend/src/routes/(root)/(logged)/user/(user)/first-time/+page.svelte index bf04614fcf..01a0554092 100644 --- a/frontend/src/routes/(root)/(logged)/user/(user)/first-time/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/user/(user)/first-time/+page.svelte @@ -7,7 +7,7 @@ async function startSetup(): Promise { $workspaceStore = 'admins' - goto('/apps/get/g/all/setup_app') + goto('/user/instance_settings') } async function decline(): Promise { @@ -16,18 +16,12 @@ -

- This is a brand new instance. Go the superadmins app to setup the default superadmin user and - enable hub resource type sync? +

+ This is a brand new instance. Setup the instance settings, then set the default superadmin user + and enable hub resource type sync

- - + +
diff --git a/frontend/src/routes/(root)/(logged)/user/(user)/instance_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/user/(user)/instance_settings/+page.svelte new file mode 100644 index 0000000000..627496666d --- /dev/null +++ b/frontend/src/routes/(root)/(logged)/user/(user)/instance_settings/+page.svelte @@ -0,0 +1,9 @@ + + + + goto('/apps/get/g/all/setup_app')} /> + diff --git a/frontend/src/routes/(root)/(logged)/user/(user)/workspaces/+page.svelte b/frontend/src/routes/(root)/(logged)/user/(user)/workspaces/+page.svelte index 914085b860..fad59c1dd8 100644 --- a/frontend/src/routes/(root)/(logged)/user/(user)/workspaces/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/user/(user)/workspaces/+page.svelte @@ -224,6 +224,7 @@ User settings + -
+ +
{/each}