From 7851e932eca9904c1e192a9bea9ae4002a46fdf2 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 29 Jun 2023 10:34:44 +0200 Subject: [PATCH] feat: smtp support to invite users (#1777) * email support * everyhting * smtp * update * update * update * setup backend test --- .github/workflows/backend-test.yml | 1 + README.md | 19 + backend/Cargo.lock | 380 +++++++++++- backend/Cargo.toml | 10 +- .../migrations/20230627235827_noop.down.sql | 1 + backend/migrations/20230627235827_noop.up.sql | 2 + .../migrations/20230628003451_noop.down.sql | 1 + backend/migrations/20230628003451_noop.up.sql | 1 + backend/sqlx-data.json | 62 +- backend/src/main.rs | 16 +- backend/windmill-api/Cargo.toml | 1 + backend/windmill-api/openapi.yaml | 20 + backend/windmill-api/src/jobs.rs | 42 ++ backend/windmill-api/src/lib.rs | 40 ++ backend/windmill-api/src/oauth2.rs | 3 + backend/windmill-api/src/users.rs | 72 ++- backend/windmill-api/src/workspaces.rs | 16 +- backend/windmill-common/src/jobs.rs | 2 + backend/windmill-common/src/tracing_init.rs | 27 +- backend/windmill-queue/src/jobs.rs | 14 +- backend/windmill-worker/src/worker.rs | 14 +- benchmarks/main.ts | 129 ++-- benchmarks/worker.ts | 549 +++++++++++------- frontend/package.json | 12 +- frontend/src/lib/components/AddUser.svelte | 30 +- .../lib/components/InviteGlobalUser.svelte | 68 ++- frontend/src/lib/components/InviteUser.svelte | 30 +- .../lib/components/SuperadminSettings.svelte | 7 +- frontend/src/lib/stores.ts | 1 + frontend/src/lib/utils.ts | 4 +- .../(logged)/user/(user)/login/+page.svelte | 10 +- react-sdk/package-lock.json | 4 +- react-sdk/src/main.tsx | 1 - 33 files changed, 1201 insertions(+), 388 deletions(-) create mode 100644 backend/migrations/20230627235827_noop.down.sql create mode 100644 backend/migrations/20230627235827_noop.up.sql create mode 100644 backend/migrations/20230628003451_noop.down.sql create mode 100644 backend/migrations/20230628003451_noop.up.sql diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index 7b89d6aaa5..ff220203b2 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -33,6 +33,7 @@ jobs: --health-retries 5 steps: - uses: actions/checkout@v3 + - uses: actions-rust-lang/setup-rust-toolchain@v1 - uses: Swatinem/rust-cache@v2 with: workspaces: | diff --git a/README.md b/README.md index a3d17e2262..4875831cc6 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ https://github.com/windmill-labs/windmill/assets/122811744/0b132cd1-ee67-4505-82 - [Postgres without superuser](#postgres-without-superuser) - [Commercial license](#commercial-license) - [OAuth for self-hosting](#oauth-for-self-hosting) + - [smtp for self-hostring](#smtp-for-self-hostring) - [Resource types](#resource-types) - [Environment Variables](#environment-variables) - [Run a local dev setup](#run-a-local-dev-setup) @@ -296,6 +297,19 @@ You may also add your own custom OAuth2 IdP and OAuth2 Resource provider: } ``` +### smtp for self-hostring + +For users to receive emails when you invite them to workspaces or add them to +the instances using their emails, configure the SMTP env variables in the +servers: + +``` +SMTP_HOST=smtp.gmail.com +SMTP_PORT=587 +SMTP_USERNAME=ruben@windmill.dev +SMTP_PASSWORD=yourpasswordapp +``` + ### Resource types You will also want to import all the approved resource types from @@ -365,6 +379,11 @@ it being synced automatically everyday. | HTTP_PROXY | None | http_proxy | Server + Worker | | HTTPS_PROXY | None | https_proxy | Server + Worker | | NO_PROXY | None | no_proxy | Server + Worker | +| SMTP_HOST | None | host for the smtp server to send invite emails | Server | +| SMTP_PORT | 587 | port for the smtp server to send invite emails | Server | +| SMTP_USERNAME | None | username for the smtp server to send invite emails | Server | +| SMTP_PASSWORD | None | password for the smtp server to send invite emails | Server | +| SMTP_TLS_IMPLICIT | false | https://docs.rs/mail-send/latest/mail_send/struct.SmtpClientBuilder.html#method.implicit_tlsemails | Server | ## Run a local dev setup diff --git a/backend/Cargo.lock b/backend/Cargo.lock index b026224fc7..b1abf92303 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -25,11 +25,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" dependencies = [ "cfg-if", - "cipher", + "cipher 0.3.0", "cpufeatures", "opaque-debug", ] +[[package]] +name = "aes" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" +dependencies = [ + "cfg-if", + "cipher 0.4.4", + "cpufeatures", +] + [[package]] name = "ahash" version = "0.7.6" @@ -146,7 +157,7 @@ checksum = "95c2fcf79ad1932ac6269a738109997a83c227c09b75842ae564dc8ede6a861c" dependencies = [ "base64ct", "blake2", - "password-hash", + "password-hash 0.5.0", ] [[package]] @@ -462,6 +473,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5" + [[package]] name = "base64" version = "0.21.2" @@ -567,7 +584,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2cb03d1bed155d89dce0f845b7899b18a9a163e148fd004e1c28421a783e2d8e" dependencies = [ "block-padding", - "cipher", + "cipher 0.3.0", ] [[package]] @@ -725,6 +742,16 @@ dependencies = [ "generic-array", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "4.3.5" @@ -833,6 +860,12 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + [[package]] name = "convert_case" version = "0.4.0" @@ -1058,7 +1091,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac41dd49fb554432020d52c875fc290e110113f864c6b1b525cd62c7e7747a5d" dependencies = [ "byteorder", - "cipher", + "cipher 0.3.0", "opaque-debug", ] @@ -1181,6 +1214,18 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enum-as-inner" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "errno" version = "0.3.1" @@ -1663,6 +1708,17 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "hostname" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +dependencies = [ + "libc", + "match_cfg", + "winapi", +] + [[package]] name = "http" version = "0.2.9" @@ -1784,6 +1840,17 @@ dependencies = [ "cc", ] +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "idna" version = "0.4.0" @@ -1817,6 +1884,15 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac" +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "generic-array", +] + [[package]] name = "instant" version = "0.1.12" @@ -1837,6 +1913,18 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "ipconfig" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" +dependencies = [ + "socket2 0.5.3", + "widestring", + "windows-sys 0.48.0", + "winreg 0.50.0", +] + [[package]] name = "ipnet" version = "2.7.2" @@ -2106,6 +2194,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + [[package]] name = "linux-raw-sys" version = "0.3.8" @@ -2132,6 +2226,15 @@ dependencies = [ "value-bag", ] +[[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "lz4_flex" version = "0.9.5" @@ -2158,7 +2261,7 @@ version = "3.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0196bd5c76f5f51d7d6563545f86262fef4c82d75466ba6f6d359c40a523318d" dependencies = [ - "aes", + "aes 0.7.5", "base64 0.13.1", "block-modes", "crc-any", @@ -2169,6 +2272,70 @@ dependencies = [ "tiger", ] +[[package]] +name = "mail-auth" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6b0969bac270a60560d3a6f89c812b3e39b2f4b3ca8d866063f482030d14f19" +dependencies = [ + "ahash 0.8.3", + "flate2", + "lru-cache", + "mail-builder", + "mail-parser", + "parking_lot 0.12.1", + "quick-xml", + "ring", + "rustls-pemfile", + "serde", + "serde_json", + "trust-dns-resolver", + "zip", +] + +[[package]] +name = "mail-builder" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "765969f4385f88a62738e8ed63e2fa630571d7ed6fd96ca6932d699513dd8c28" +dependencies = [ + "gethostname", +] + +[[package]] +name = "mail-parser" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4158a1c18963244e083888b21465846dfb68d6170850ed1ab4742edd57c9d47" +dependencies = [ + "encoding_rs", +] + +[[package]] +name = "mail-send" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6d2b8d0cb56f199d36f527ff96453cf3b1cdfdacb5e4d154ac1d8fcd89873c2" +dependencies = [ + "base64 0.20.0", + "gethostname", + "mail-auth", + "mail-builder", + "md5", + "rand 0.8.5", + "rustls 0.21.2", + "smtp-proto", + "tokio", + "tokio-rustls 0.24.1", + "webpki-roots 0.23.1", +] + +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + [[package]] name = "matchers" version = "0.1.0" @@ -2210,6 +2377,12 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "memchr" version = "2.5.0" @@ -2519,6 +2692,17 @@ dependencies = [ "regex", ] +[[package]] +name = "password-hash" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" +dependencies = [ + "base64ct", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "password-hash" version = "0.5.0" @@ -2536,6 +2720,18 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" +[[package]] +name = "pbkdf2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +dependencies = [ + "digest 0.10.7", + "hmac", + "password-hash 0.4.2", + "sha2 0.10.7", +] + [[package]] name = "pem-rfc7468" version = "0.6.0" @@ -2939,6 +3135,21 @@ dependencies = [ "cc", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quick-xml" +version = "0.28.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce5e73202a820a31f8a0ee32ada5e21029c81fd9e3ebf668a40832e4219d9d1" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.28" @@ -3153,7 +3364,17 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "winreg", + "winreg 0.10.1", +] + +[[package]] +name = "resolv-conf" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" +dependencies = [ + "hostname", + "quick-error", ] [[package]] @@ -3316,6 +3537,18 @@ dependencies = [ "webpki", ] +[[package]] +name = "rustls" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e32ca28af694bc1bbf399c33a516dbdf1c90090b8ab23c2bc24f834aa2247f5f" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + [[package]] name = "rustls-pemfile" version = "1.0.2" @@ -3325,6 +3558,16 @@ dependencies = [ "base64 0.21.2", ] +[[package]] +name = "rustls-webpki" +version = "0.100.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustpython-ast" version = "0.2.0" @@ -3810,6 +4053,12 @@ dependencies = [ "serde", ] +[[package]] +name = "smtp-proto" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b756ac662e92a0e5b360349bea5f0b0784d4be4541eff2972049dfdfd7f862" + [[package]] name = "socket2" version = "0.4.9" @@ -3929,7 +4178,7 @@ dependencies = [ "paste", "percent-encoding", "rand 0.8.5", - "rustls", + "rustls 0.20.8", "rustls-pemfile", "serde", "serde_json", @@ -3943,7 +4192,7 @@ dependencies = [ "tokio-stream", "url", "uuid 1.3.4", - "webpki-roots", + "webpki-roots 0.22.6", "whoami", ] @@ -3977,7 +4226,7 @@ checksum = "804d3f245f894e61b1e6263c84b23ca675d96753b5abfd5cc8597d86806e8024" dependencies = [ "once_cell", "tokio", - "tokio-rustls", + "tokio-rustls 0.23.4", ] [[package]] @@ -4451,11 +4700,21 @@ version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ - "rustls", + "rustls 0.20.8", "tokio", "webpki", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.2", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.14" @@ -4702,6 +4961,59 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "trust-dns-proto" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f7f83d1e4a0e4358ac54c5c3681e5d7da5efc5a7a632c90bb6d6669ddd9bc26" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "enum-as-inner", + "futures-channel", + "futures-io", + "futures-util", + "idna 0.2.3", + "ipnet", + "lazy_static", + "rand 0.8.5", + "ring", + "rustls 0.20.8", + "rustls-pemfile", + "smallvec", + "thiserror", + "tinyvec", + "tokio", + "tokio-rustls 0.23.4", + "tracing", + "url", + "webpki", +] + +[[package]] +name = "trust-dns-resolver" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aff21aa4dcefb0a1afbfac26deb0adc93888c7d295fb63ab273ef276ba2b7cfe" +dependencies = [ + "cfg-if", + "futures-util", + "ipconfig", + "lazy_static", + "lru-cache", + "parking_lot 0.12.1", + "resolv-conf", + "rustls 0.20.8", + "smallvec", + "thiserror", + "tokio", + "tokio-rustls 0.23.4", + "tracing", + "trust-dns-proto", + "webpki-roots 0.22.6", +] + [[package]] name = "try-lock" version = "0.2.4" @@ -4939,7 +5251,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" dependencies = [ "form_urlencoded", - "idna", + "idna 0.4.0", "percent-encoding", "serde", ] @@ -5190,6 +5502,15 @@ dependencies = [ "webpki", ] +[[package]] +name = "webpki-roots" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338" +dependencies = [ + "rustls-webpki", +] + [[package]] name = "which" version = "4.4.0" @@ -5211,6 +5532,12 @@ dependencies = [ "web-sys", ] +[[package]] +name = "widestring" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "653f141f39ec16bba3c5abe400a0c60da7468261cc2cbf36805022876bc721a8" + [[package]] name = "winapi" version = "0.3.9" @@ -5301,6 +5628,7 @@ dependencies = [ "itertools 0.11.0", "lazy_static", "magic-crypt", + "mail-send", "mime_guess", "prometheus", "rand 0.8.5", @@ -5692,6 +6020,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "xattr" version = "0.2.3" @@ -5716,6 +6054,26 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" +[[package]] +name = "zip" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +dependencies = [ + "aes 0.8.3", + "byteorder", + "bzip2", + "constant_time_eq", + "crc32fast", + "crossbeam-utils", + "flate2", + "hmac", + "pbkdf2", + "sha1", + "time 0.3.22", + "zstd", +] + [[package]] name = "zstd" version = "0.11.2+zstd.1.5.2" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 01f4536c16..c8bf5e8f94 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -114,13 +114,7 @@ magic-crypt = "^3" git-version = "^0" rustpython-parser = "0.2.0" cron = "^0" -lettre = { version = "^0", features = [ - "rustls-tls", - "tokio1", - "tokio1-rustls-tls", - "builder", - "smtp-transport", -], default-features = false } +mail-send = "0.4.0" urlencoding = "^2" url = "^2" async-oauth2 = "^0" @@ -176,4 +170,4 @@ serde-wasm-bindgen = "0.4" wasm-bindgen-test = "0.3.0" convert_case = "0.6.0" getrandom = "0.2" -tokio-postgres = {version = "^0.7", features = ["array-impls", "with-serde_json-1"]} \ No newline at end of file +tokio-postgres = {version = "^0.7", features = ["array-impls", "with-serde_json-1"]} diff --git a/backend/migrations/20230627235827_noop.down.sql b/backend/migrations/20230627235827_noop.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20230627235827_noop.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20230627235827_noop.up.sql b/backend/migrations/20230627235827_noop.up.sql new file mode 100644 index 0000000000..d90665db72 --- /dev/null +++ b/backend/migrations/20230627235827_noop.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +ALTER TYPE JOB_KIND ADD VALUE IF NOT EXISTS 'noop'; \ No newline at end of file diff --git a/backend/migrations/20230628003451_noop.down.sql b/backend/migrations/20230628003451_noop.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20230628003451_noop.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20230628003451_noop.up.sql b/backend/migrations/20230628003451_noop.up.sql new file mode 100644 index 0000000000..0da0a538a3 --- /dev/null +++ b/backend/migrations/20230628003451_noop.up.sql @@ -0,0 +1 @@ +-- Add up migration script here diff --git a/backend/sqlx-data.json b/backend/sqlx-data.json index f95064d7cf..6aa85670ab 100644 --- a/backend/sqlx-data.json +++ b/backend/sqlx-data.json @@ -443,6 +443,26 @@ }, "query": "DELETE FROM workspace_invite WHERE\n workspace_id = $1 AND email = $2 AND is_admin = $3 AND operator = $4" }, + "0cf42f7e76fe01e6a9a20499b2228d76a1919b8b4050afedb2459be083a4ad4d": { + "describe": { + "columns": [ + { + "name": "exists", + "ordinal": 0, + "type_info": "Bool" + } + ], + "nullable": [ + null + ], + "parameters": { + "Left": [ + "Text" + ] + } + }, + "query": "SELECT EXISTS(SELECT 1 FROM password WHERE email = $1)" + }, "0d6412bc3ebb1d58bdd9cbcef774dacf9016fa402af5c1b4e339b9a3d7163d5e": { "describe": { "columns": [ @@ -803,7 +823,11 @@ "flowpreview", "script_hub", "identity", - "flowdependencies" + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop" ] }, "name": "job_kind" @@ -1174,7 +1198,11 @@ "flowpreview", "script_hub", "identity", - "flowdependencies" + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop" ] }, "name": "job_kind" @@ -1193,7 +1221,8 @@ "python3", "deno", "go", - "bash" + "bash", + "postgresql" ] }, "name": "script_lang" @@ -2055,7 +2084,11 @@ "flowpreview", "script_hub", "identity", - "flowdependencies" + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop" ] }, "name": "job_kind" @@ -2072,7 +2105,8 @@ "python3", "deno", "go", - "bash" + "bash", + "postgresql" ] }, "name": "script_lang" @@ -2739,7 +2773,8 @@ "python3", "deno", "go", - "bash" + "bash", + "postgresql" ] }, "name": "script_lang" @@ -3846,7 +3881,8 @@ "python3", "deno", "go", - "bash" + "bash", + "postgresql" ] }, "name": "script_lang" @@ -7326,7 +7362,11 @@ "flowpreview", "script_hub", "identity", - "flowdependencies" + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop" ] }, "name": "job_kind" @@ -7343,7 +7383,11 @@ "flowpreview", "script_hub", "identity", - "flowdependencies" + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop" ] }, "name": "job_kind" diff --git a/backend/src/main.rs b/backend/src/main.rs index 295216ef63..a7ac2bdada 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -19,7 +19,7 @@ use tokio::{ join, sync::RwLock, }; -use windmill_api::LICENSE_KEY; +use windmill_api::{LICENSE_KEY, OAUTH_CLIENTS, SMTP_CLIENT}; use windmill_common::{utils::rd_string, METRICS_ADDR}; use windmill_worker::{ DENO_CACHE_DIR, DENO_TMP_CACHE_DIR, GO_CACHE_DIR, GO_TMP_CACHE_DIR, HUB_CACHE_DIR, @@ -90,7 +90,6 @@ async fn main() -> anyhow::Result<()> { .unwrap_or(std::borrow::Cow::Borrowed("rsmq")) .into_owned(); config.port = url.port().unwrap_or(6379).to_string(); - config }); @@ -181,8 +180,21 @@ Windmill Community Edition {GIT_VERSION} "WAIT_RESULT_FAST_POLL_INTERVAL_MS", "EXIT_AFTER_NO_JOB_FOR_SECS", "REQUEST_SIZE_LIMIT", + "SMTP_HOST", + "SMTP_USERNAME", + "SMTP_PORT", + "SMTP_TLS_IMPLICIT", ]); + tracing::info!("Loading OAuth providers...: {:#?}", *OAUTH_CLIENTS); + if let Some(ref smtp) = *SMTP_CLIENT { + tracing::info!("Smtp client defined. Testing connection..."); + if let Err(e) = smtp.connect().await { + tracing::error!("Failed to connect to smtp server: {}", e); + } else { + tracing::info!("Smtp client connected."); + } + } if server_mode || num_workers > 0 { let addr = SocketAddr::from((server_bind_address, port)); diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index f4ba7f4945..8756692058 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -70,3 +70,4 @@ async_zip.workspace = true rsmq_async.workspace = true regex.workspace = true bytes.workspace = true +mail-send.workspace = true \ No newline at end of file diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 96b1a138a8..55e2ee33df 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -877,6 +877,26 @@ paths: schema: $ref: "#/components/schemas/User" + /users/exists/{email}: + get: + summary: exists email + operationId: existsEmail + tags: + - user + parameters: + - name: email + in: path + required: true + schema: + type: string + responses: + "200": + description: user + content: + application/json: + schema: + type: boolean + /users/list_as_super_admin: get: summary: list all users as super admin (require to be super amdin) diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index d7b6a658f6..cf5a3f9c6e 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -9,6 +9,7 @@ use crate::{ db::{UserDB, DB}, users::{check_scopes, require_owner_of_path, Authed, OptAuthed}, + utils::require_super_admin, variables::get_workspace_key, BASE_URL, }; @@ -76,6 +77,7 @@ pub fn workspaced_service() -> Router { ) .route("/run/h/:hash", post(run_job_by_hash).head(|| async { "" })) .route("/run/preview", post(run_preview_job)) + .route("/add_noop_jobs/:n", post(add_noop_jobs)) .route("/run/preview_flow", post(run_preview_flow_job)) .route("/list", get(list_jobs)) .route("/queue/list", get(list_queue_jobs)) @@ -1241,6 +1243,7 @@ enum PreviewKind { Identity, Http, Graphql, + Noop, } #[derive(Deserialize)] struct Preview { @@ -1976,6 +1979,7 @@ async fn run_preview_job( Some(PreviewKind::Identity) => JobPayload::Identity, Some(PreviewKind::Http) => JobPayload::Http, Some(PreviewKind::Graphql) => JobPayload::Graphql, + Some(PreviewKind::Noop) => JobPayload::Noop, _ => JobPayload::Code(RawCode { content: preview.content.unwrap_or_default(), path: preview.path, @@ -2004,6 +2008,44 @@ async fn run_preview_job( Ok((StatusCode::CREATED, uuid.to_string())) } +async fn add_noop_jobs( + authed: Authed, + Extension(db): Extension, + Extension(rsmq): Extension>, + Path((w_id, n)): Path<(String, i32)>, +) -> error::JsonResult> { + require_super_admin(&mut db.begin().await?, &authed.email).await?; + let mut tx: QueueTransaction<'_, _> = (rsmq, db.begin().await?).into(); + + let mut uuids: Vec = Vec::new(); + for _ in 0..n { + let (uuid, ntx) = push( + tx, + &w_id, + JobPayload::Noop, + serde_json::Map::new(), + &authed.username, + &authed.email, + username_to_permissioned_as(&authed.username), + None, + None, + None, + None, + None, + false, + false, + None, + true, + None, + ) + .await?; + tx = ntx; + uuids.push(uuid.to_string()); + } + tx.commit().await?; + + Ok(Json(uuids)) +} async fn run_preview_flow_job( authed: Authed, Extension(user_db): Extension, diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 3c530d53ea..b41f6e8e51 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -20,6 +20,7 @@ use axum::{middleware::from_extractor, routing::get, Extension, Router}; use db::DB; use git_version::git_version; use hyper::Method; +use mail_send::SmtpClientBuilder; use reqwest::Client; use std::{net::SocketAddr, sync::Arc}; use tower::ServiceBuilder; @@ -90,9 +91,48 @@ lazy_static::lazy_static! { .map_err(|e| tracing::error!("Error building oauth clients: {}", e)) .unwrap(); + pub static ref SMTP_CLIENT: Option> = { + let smtp = parse_smtp(); + if let Some(smtp) = smtp { + match smtp { + Ok(smtp) => Some(smtp), + Err(e) => { + tracing::error!("SMTP is not configured correctly, emails will not be sent: {}", e); + None + } + } + } else { + tracing::warn!("SMTP is not configured, emails will not be sent"); + None + } + }; + pub static ref LICENSE_KEY: Option = std::env::var("LICENSE_KEY").ok(); } +pub fn parse_smtp() -> Option>> { + let username = std::env::var("SMTP_USERNAME").ok(); + let port = std::env::var("SMTP_PORT") + .ok() + .and_then(|p| p.parse().ok()) + .unwrap_or(587); + let password = std::env::var("SMTP_PASSWORD").ok(); + let host = std::env::var("SMTP_HOST").ok(); + let tls_implicit = std::env::var("SMTP_TLS_IMPLICIT") + .ok() + .and_then(|p| p.parse().ok()) + .unwrap_or(false); + + if username.is_some() && password.is_some() && host.is_some() { + let smtp = SmtpClientBuilder::new(host.unwrap(), port) + .implicit_tls(tls_implicit) + .credentials((username.unwrap(), password.unwrap())); + Some(Ok(smtp)) + } else { + None + } +} + pub async fn run_server( db: DB, rsmq: Option, diff --git a/backend/windmill-api/src/oauth2.rs b/backend/windmill-api/src/oauth2.rs index b6ee665c03..ecca619b03 100644 --- a/backend/windmill-api/src/oauth2.rs +++ b/backend/windmill-api/src/oauth2.rs @@ -75,6 +75,7 @@ pub fn workspaced_service() -> Router { .route("/connect_slack_callback", post(connect_slack_callback)) } +#[derive(Debug)] pub struct ClientWithScopes { client: OClient, scopes: Vec, @@ -105,6 +106,8 @@ pub struct OAuthClient { connect_config: Option, login_config: Option, } + +#[derive(Debug)] pub struct AllClients { pub logins: BasicClientsMap, pub connects: BasicClientsMap, diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 75666ba167..cc60bb0ff9 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -14,7 +14,7 @@ use crate::{ utils::require_super_admin, webhook_util::{InstanceEvent, WebhookShared}, workspaces::invite_user_to_all_auto_invite_worspaces, - COOKIE_DOMAIN, IS_SECURE, + BASE_URL, COOKIE_DOMAIN, IS_SECURE, SMTP_CLIENT, }; use argon2::{password_hash::SaltString, Argon2, PasswordHash, PasswordHasher, PasswordVerifier}; use axum::{ @@ -27,6 +27,7 @@ use axum::{ }; use hyper::{header::LOCATION, StatusCode}; use lazy_static::lazy_static; +use mail_send::mail_builder::MessageBuilder; use rand::rngs::OsRng; use regex::Regex; use retainer::Cache; @@ -37,7 +38,7 @@ use tower_cookies::{Cookie, Cookies}; use tracing::{Instrument, Span}; use windmill_audit::{audit_log, ActionKind}; use windmill_common::{ - error::{self, Error, JsonResult, Result}, + error::{self, to_anyhow, Error, JsonResult, Result}, users::SUPERADMIN_SECRET_EMAIL, utils::{not_found_if_none, rd_string, require_admin, Pagination, StripPath}, }; @@ -57,13 +58,14 @@ pub fn workspaced_service() -> Router { .route("/update/:user", post(update_workspace_user)) .route("/delete/:user", delete(delete_workspace_user)) .route("/is_owner/*path", get(is_owner_of_path)) - .route("/whois/:email", get(whois)) + .route("/whois/:username", get(whois)) .route("/whoami", get(whoami)) .route("/leave", post(leave_workspace)) } pub fn global_service() -> Router { Router::new() + .route("/exists/:email", get(exists_email)) .route("/email", get(get_email)) .route("/whoami", get(global_whoami)) .route("/list_invites", get(list_invites)) @@ -882,6 +884,17 @@ async fn global_whoami( } } +async fn exists_email(Extension(db): Extension, Path(email): Path) -> JsonResult { + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM password WHERE email = $1)", + email + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + Ok(Json(exists)) +} + async fn get_email(Authed { email, .. }: Authed) -> Result { Ok(email) } @@ -1078,6 +1091,7 @@ lazy_static! { .ok() .and_then(|x| x.parse::().ok()) .unwrap_or(60 * 60 * 24 * 60); // 60 days + } async fn accept_invite( @@ -1222,6 +1236,16 @@ async fn add_user_to_workspace<'c>( None, ) .await?; + send_email_if_possible( + &format!("Added to Windmill's workspace: {w_id}"), + &format!( + "You have been granted access to Windmill's workspace {w_id} + +If you do not have an account on {}, login with SSO or ask an admin to create an account for you.", + *BASE_URL + ), + &email, + ); Ok(tx) } @@ -1368,6 +1392,7 @@ async fn delete_user( lazy_static::lazy_static! { pub static ref NEW_USER_WEBHOOK: Option = std::env::var("NEW_USER_WEBHOOK").ok(); + } async fn create_user( @@ -1394,7 +1419,7 @@ async fn create_user( VALUES ($1, $2, $3, 'password', $4, $5, $6)", &nu.email, true, - &hash_password(argon2, nu.password)?, + &hash_password(argon2, nu.password.clone())?, &nu.super_admin, nu.name, nu.company @@ -1415,11 +1440,50 @@ async fn create_user( tx.commit().await?; invite_user_to_all_auto_invite_worspaces(&db, &nu.email).await?; + send_email_if_possible( + "Invited to Windmill", + &format!( + "You have been granted access to Windmill by {email}. +Login and change your password: {}/user/login?email={}&password={}&rd=%2F%23user-settings + +You can then join or create a workspace. Happy building!", + *BASE_URL, &nu.email, &nu.password + ), + &nu.email, + ); webhook.send_instance_event(InstanceEvent::UserAdded { email: nu.email.clone() }); Ok((StatusCode::CREATED, format!("email {} created", nu.email))) } +pub fn send_email_if_possible(subject: &str, content: &str, to: &str) { + let subject = subject.to_string(); + let content = content.to_string(); + let to = to.to_string(); + tokio::spawn(async move { + if let Err(e) = send_email_if_possible_intern(&subject, &content, &to).await { + tracing::error!("Failed to send email to {}: {}", &to, e); + } + }); +} + +pub async fn send_email_if_possible_intern(subject: &str, content: &str, to: &str) -> Result<()> { + if let Some(ref smtp) = *SMTP_CLIENT { + let message = MessageBuilder::new() + .from(("Windmill", "noreply@getwindmill.com")) + .to(to) + .subject(subject) + .text_body(content); + smtp.connect() + .await + .map_err(to_anyhow)? + .send(message) + .await + .map_err(to_anyhow)?; + } + return Ok(()); +} + async fn delete_workspace_user( Authed { username, is_admin, .. }: Authed, Extension(db): Extension, diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index ec1a2cb00b..cb2e794ad6 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -9,17 +9,16 @@ #[cfg(feature = "enterprise")] use std::str::FromStr; -#[cfg(feature = "enterprise")] use crate::BASE_URL; use crate::{ apps::AppWithLastVersion, db::{UserDB, DB}, folders::Folder, resources::{Resource, ResourceType}, - users::{Authed, WorkspaceInvite, VALID_USERNAME}, + users::{Authed, WorkspaceInvite, VALID_USERNAME, send_email_if_possible}, utils::require_super_admin, variables::build_crypt, - webhook_util::{InstanceEvent, WebhookShared}, + webhook_util::{InstanceEvent, WebhookShared} }; #[cfg(feature = "enterprise")] use axum::response::Redirect; @@ -1021,6 +1020,17 @@ async fn invite_user( tx.commit().await?; + send_email_if_possible( + &format!("Invited to Windmill's workspace: {w_id}"), + &format!( + "You have been granted access to Windmill's workspace {w_id} + +If you do not have an account on {}, login with SSO or ask an admin to create an account for you.", + *BASE_URL + ), + &nu.email, + ); + webhook.send_instance_event(InstanceEvent::UserInvitedWorkspace { email: nu.email.clone(), workspace: w_id, diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 5f1ce7cdfe..4bea7ebe1d 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -25,6 +25,7 @@ pub enum JobKind { FlowDependencies, Http, Graphql, + Noop, } #[derive(Debug, sqlx::FromRow, Serialize, Clone)] @@ -160,6 +161,7 @@ pub enum JobPayload { Identity, Http, Graphql, + Noop, } #[derive(Clone, Serialize, Deserialize, Debug)] diff --git a/backend/windmill-common/src/tracing_init.rs b/backend/windmill-common/src/tracing_init.rs index 1531350ada..14b98651db 100644 --- a/backend/windmill-common/src/tracing_init.rs +++ b/backend/windmill-common/src/tracing_init.rs @@ -6,9 +6,7 @@ * LICENSE-AGPL for a copy of the license. */ -use tracing::Metadata; use tracing_subscriber::{ - filter::filter_fn, fmt::{format, Layer}, prelude::*, EnvFilter, @@ -26,34 +24,27 @@ fn compact_layer() -> Layer bool { - meta.target().starts_with("windmill") -} - pub fn initialize_tracing() { let style = std::env::var("RUST_LOG_STYLE").unwrap_or_else(|_| "auto".into()); let json_fmt = std::env::var("JSON_FMT") .map(|x| x == "true") .unwrap_or(false); + if std::env::var("RUST_LOG").is_ok_and(|x| x == "debug" || x == "info") { + std::env::set_var( + "RUST_LOG", + &format!("windmill={}", std::env::var("RUST_LOG").unwrap()), + ) + } + let env_filter = EnvFilter::from_default_env(); let ts_base = tracing_subscriber::registry().with(env_filter); match json_fmt { - true => ts_base - .with( - json_layer() - .flatten_event(true) - .with_filter(filter_fn(filter_metadata)), - ) - .init(), + true => ts_base.with(json_layer().flatten_event(true)).init(), false => ts_base - .with( - compact_layer() - .with_ansi(style.to_lowercase() != "never") - .with_filter(filter_fn(filter_metadata)), - ) + .with(compact_layer().with_ansi(style.to_lowercase() != "never")) .init(), } } diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index dd54611a9c..99e96a7f37 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -6,7 +6,11 @@ * LICENSE-AGPL for a copy of the license. */ -use std::{collections::HashMap, vec}; +use std::{ + collections::HashMap, + time::{Instant, SystemTime}, + vec, +}; use async_recursion::async_recursion; use itertools::Itertools; @@ -28,6 +32,7 @@ use windmill_common::{ schedule::{schedule_to_user, Schedule}, scripts::{ScriptHash, ScriptLang}, users::username_to_permissioned_as, + utils::rd_string, METRICS_ENABLED, }; @@ -543,12 +548,16 @@ pub async fn pull( } } + // let rs = rd_string(2); + // let instant = Instant::now(); + let job: Option = if let Some(mut rsmq) = rsmq { // TODO: REDIS: Race conditions / replace last_ping let msg = rsmq .pop_message::>(RSMQ_MAIN_QUEUE) .await .map_err(|e| anyhow::anyhow!(e))?; + // println!("3.1: {:?} {rs}", instant.elapsed()); if let Some(msg) = msg { let uuid = Uuid::from_bytes_le( @@ -606,6 +615,7 @@ pub async fn pull( .fetch_optional(db) .await? }; + // println!("3.2: {:?} {rs}", instant.elapsed()); if job.is_some() && *METRICS_ENABLED { QUEUE_PULL_COUNT.inc(); @@ -948,6 +958,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>( JobPayload::Identity => (None, None, None, JobKind::Identity, None, None), JobPayload::Graphql => (None, None, None, JobKind::Graphql, None, None), JobPayload::Http => (None, None, None, JobKind::Http, None, None), + JobPayload::Noop => (None, None, None, JobKind::Noop, None, None), }; let is_running = same_worker; @@ -1081,6 +1092,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>( JobKind::Dependencies => "jobs.run.dependencies", JobKind::Identity => "jobs.run.identity", JobKind::Http => "jobs.run.http", + JobKind::Noop => "jobs.run.noop", JobKind::Graphql => "jobs.run.graphql", JobKind::FlowDependencies => "jobs.run.flow_dependencies", }; diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index da3dc4b9a2..36df748f31 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -17,7 +17,7 @@ use windmill_api_client::{Client, types::CompletedJob}; use windmill_parser::Typ; use std::{ borrow::Borrow, collections::HashMap, io, os::unix::process::ExitStatusExt, panic, - process::Stdio, time::{Duration}, + process::Stdio, time::{Duration, SystemTime}, sync::{Arc, atomic::Ordering}, collections::hash_map::DefaultHasher, hash::{Hasher, Hash}, @@ -449,7 +449,7 @@ pub async fn run_worker(5); - let (job_completed_tx, mut job_completed_rx) = mpsc::channel::(10); + let (job_completed_tx, mut job_completed_rx) = mpsc::channel::(1000); let db2 = db.clone(); let rsmq2 = rsmq.clone(); @@ -468,6 +468,7 @@ pub async fn run_worker = None; loop { + // let instant: Instant = Instant::now(); if *METRICS_ENABLED { worker_busy.set(0); uptime_metric.inc_by( @@ -541,6 +542,7 @@ pub async fn run_worker { - // println!("{:?}", SystemTime::now()); - last_executed_job = None; - + if matches!(job.job_kind, JobKind::Noop) { + job_completed_tx.send(JobCompleted { job, success: true, result: json!({}), logs: String::new()}).await.expect("send job completed"); + return false + } let token = create_token_for_owner_in_bg(&db, &job).await; let language = job.language.clone(); let _timer = worker_execution_duration diff --git a/benchmarks/main.ts b/benchmarks/main.ts index b1a25266c2..7e3b8ee6b2 100644 --- a/benchmarks/main.ts +++ b/benchmarks/main.ts @@ -174,21 +174,23 @@ await new Command() exportSimple = []; } - let metrics_worker: Worker; + let metrics_worker: Worker | undefined = undefined; if (!continous) { - metrics_worker = new Worker( - new URL("./scraper.ts", import.meta.url).href, - { - type: "module", - } - ); + if (exportJson || exportCsv) { + metrics_worker = new Worker( + new URL("./scraper.ts", import.meta.url).href, + { + type: "module", + } + ); - metrics_worker.postMessage({ - exportHistograms, - histogramBuckets, - exportSimple, - host: metrics, - }); + metrics_worker.postMessage({ + exportHistograms, + histogramBuckets, + exportSimple, + host: metrics, + }); + } } console.log( @@ -287,12 +289,12 @@ await new Command() }, 100); workers.forEach((worker, i) => { - worker.postMessage({ ...shared_config, i }); worker.addEventListener("message", (evt: MessageEvent) => { if (evt.data.type === "jobs_sent") { jobsSent[i] = evt.data.jobs_sent; } }); + worker.postMessage({ ...shared_config, i }); }); start = Date.now(); @@ -307,7 +309,7 @@ await new Command() clearInterval(updateState); - const sum = jobsSent.reduce((a, b) => a + b, 0); + let sum = jobsSent.reduce((a, b) => a + b, 0); await Deno.stdout.write( enc(" ".padStart(30) + `\rduration: ${seconds} | jobs sent: ${sum}\n`) ); @@ -315,13 +317,14 @@ await new Command() const shutdown_start = Date.now(); let zombie_jobs = 0; let incorrect_results = 0; - workers.forEach((worker) => { + workers.forEach((worker, i) => { const l = (evt: MessageEvent) => { if (evt.data.type === "zombie_jobs") { zombie_jobs += evt.data.zombie_jobs; incorrect_results += evt.data.incorrect_results; worker.removeEventListener("message", l); workers = workers.filter((w) => w != worker); + jobsSent[i] = evt.data.jobs_sent; worker.terminate(); } }; @@ -335,6 +338,8 @@ await new Command() while (workers.length > 0) { await sleep(0.1); } + sum = jobsSent.reduce((a, b) => a + b, 0); + const tts = (Date.now() - shutdown_start) / 1000; const time = seconds + tts; console.log("\ntime to shutdown:", tts); @@ -356,53 +361,59 @@ await new Command() ).database_length ); - metrics_worker!.postMessage("stop"); - console.log("waiting for metrics"); - const { columns, transfer_values } = await new Promise<{ - columns: string[]; - transfer_values: ArrayBufferLike[]; - }>((resolve, _reject) => { - metrics_worker.onmessage = (e) => { - resolve(e.data); - metrics_worker.terminate(); - }; - }); - const values = transfer_values.map((x) => new Float32Array(x)); - - if (exportJson) { - console.log("exporting mean & stdev to json"); - const obj: any = {}; - for (let i = 0; i < columns.length; i++) { - const name = columns[i]!; - const value = values[i]!; - const mean = value.reduce((acc, e) => acc + e, 0) / values.length; - const stdev = Math.sqrt( - value.reduce((acc, e) => acc + (e - mean) ** 2) / values.length - ); - obj[name] = { mean, stdev }; - } - - await Deno.writeTextFile(exportJson, JSON.stringify(obj)); - } - - if (exportCsv) { - const f = await Deno.open(exportCsv, { - write: true, - create: true, - truncate: true, + if (metrics_worker) { + metrics_worker.postMessage("stop"); + console.log("waiting for metrics"); + const { columns, transfer_values } = await new Promise<{ + columns: string[]; + transfer_values: ArrayBufferLike[]; + }>((resolve, _reject) => { + if (metrics_worker) { + metrics_worker.onmessage = (e) => { + resolve(e.data); + metrics_worker?.terminate(); + }; + } }); - const encoder = new TextEncoder(); - const newline = new Uint8Array(1); - newline[0] = 0x0a; - await f.write(encoder.encode(columns.join(","))); - await f.write(newline); + const values = transfer_values.map((x) => new Float32Array(x)); - for (let i = 0; i < values.length; i++) { - await f.write(encoder.encode(values[i].join(","))); - await f.write(newline); + if (exportJson) { + console.log("exporting mean & stdev to json"); + const obj: any = {}; + for (let i = 0; i < columns.length; i++) { + const name = columns[i]!; + const value = values[i]!; + const mean = value.reduce((acc, e) => acc + e, 0) / values.length; + const stdev = Math.sqrt( + value.reduce((acc, e) => acc + (e - mean) ** 2) / values.length + ); + obj[name] = { mean, stdev }; + } + + await Deno.writeTextFile(exportJson, JSON.stringify(obj)); } - f.close(); + if (exportCsv) { + const f = await Deno.open(exportCsv, { + write: true, + create: true, + truncate: true, + }); + const encoder = new TextEncoder(); + const newline = new Uint8Array(1); + newline[0] = 0x0a; + await f.write(encoder.encode(columns.join(","))); + await f.write(newline); + + for (let i = 0; i < values.length; i++) { + await f.write(encoder.encode(values[i].join(","))); + await f.write(newline); + } + + f.close(); + } + } else { + return; } console.log("done"); } diff --git a/benchmarks/worker.ts b/benchmarks/worker.ts index 26910a4ad1..ea96b1754a 100644 --- a/benchmarks/worker.ts +++ b/benchmarks/worker.ts @@ -42,19 +42,295 @@ const config = await promise; const outstanding: string[] = []; let cont = true; let total_spawned = 0; -const start_time = Date.now(); + +let start_time: number; let complete_timeout = Infinity; -self.onmessage = (evt) => { - cont = false; - complete_timeout = evt.data; -}; -const updateStatusInterval = setInterval(() => { +if (config.scriptPattern == "noop") { + const n = 10000; + const res = await fetch( + config.server + + "/api/w/" + + config.workspace_id + + `/jobs/add_noop_jobs/${n}`, + { + method: "POST", + headers: { ["Authorization"]: "Bearer " + config.token }, + } + ); + const uuids = await res.json(); + outstanding.push(...uuids); + total_spawned += n; self.postMessage({ type: "jobs_sent", jobs_sent: total_spawned }); -}, 100); + cont = false; + self.onmessage = (evt) => { + cont = false; + complete_timeout = evt.data; + start_time = Date.now(); + }; +} else { + start_time = Date.now(); -while (cont) { - const queue_length = ( + self.onmessage = (evt) => { + cont = false; + complete_timeout = evt.data; + }; + + const updateStatusInterval = setInterval(() => { + self.postMessage({ type: "jobs_sent", jobs_sent: total_spawned }); + }, 100); + + while (cont) { + const queue_length = ( + await ( + await fetch( + config.server + "/api/w/" + config.workspace_id + "/jobs/queue/count", + { headers: { ["Authorization"]: "Bearer " + config.token } } + ) + ).json() + ).database_length; + if (queue_length > 2500) { + console.log( + `queue length: ${queue_length} > 2500. waiting... ` + ); + await sleep(0.5); + continue; + } + + if ( + (total_spawned * 1000) / (Date.now() - start_time) > + config.per_worker_throughput + ) { + console.log("at maximum throughput. waiting..."); + await sleep(0.1); + continue; + } + total_spawned++; + if (total_spawned > config.max_per_worker) { + break; + } + let uuid: string; + if (config.custom) { + await evaluate(config.custom); + continue; + } else if (config.useFlows) { + let payload: api.FlowPreview; + if (config.flowPattern == "branchone") { + payload = { + path: "branchone", + args: {}, + value: { + modules: [ + { + id: "a", + value: { + input_transforms: {}, + language: api.RawScript.language.DENO, + type: "rawscript", + content: + 'export function main(){ return Deno.env.get("WM_FLOW_JOB_ID"); }', + }, + }, + { + id: "b", + value: { + type: "branchone", + branches: [], + default: [ + { + id: "c", + value: { + input_transforms: { + x: { + type: "javascript", + expr: "results.a", + }, + }, + language: api.RawScript.language.DENO, + type: "rawscript", + content: "export function main(x: string){ return x; }", + }, + }, + ], + }, + }, + ], + }, + }; + } else if (config.flowPattern == "branchallparrallel") { + payload = { + path: "branchall", + args: {}, + value: { + modules: [ + { + id: "a", + value: { + input_transforms: {}, + language: api.RawScript.language.DENO, + type: "rawscript", + content: + 'export function main(){ return Deno.env.get("WM_FLOW_JOB_ID"); }', + }, + }, + { + id: "b", + value: { + type: "branchall", + parallel: true, + branches: [ + { + modules: [ + { + id: "c", + value: { + input_transforms: { + x: { + type: "javascript", + expr: "results.a", + }, + }, + language: api.RawScript.language.DENO, + type: "rawscript", + content: + "export function main(x: string){ return x; }", + }, + }, + ], + }, + { + modules: [ + { + id: "d", + value: { + input_transforms: { + x: { + type: "javascript", + expr: "results.a", + }, + }, + language: api.RawScript.language.DENO, + type: "rawscript", + content: + "export function main(x: string){ return x; }", + }, + }, + ], + }, + ], + }, + }, + ], + }, + }; + } else { + payload = { + path: "2steps", + args: {}, + value: { + modules: [ + { + id: "a", + value: { + input_transforms: {}, + language: api.RawScript.language.DENO, + type: "rawscript", + content: + 'export function main(){ return Deno.env.get("WM_JOB_ID"); }', + }, + }, + { + id: "b", + value: { + input_transforms: {}, + language: api.RawScript.language.DENO, + type: "rawscript", + content: + 'export function main(){ return Deno.env.get("WM_JOB_ID"); }', + }, + }, + ], + }, + }; + } + uuid = await windmill.JobService.runFlowPreview({ + workspace: config.workspace_id, + requestBody: payload, + }); + } else { + let payload: api.Preview; + if (config.scriptPattern == "httpversion") { + payload = { + path: "httpversion", + kind: "http", + args: { + url: "http://localhost:8000/api/version", + }, + }; + } else if (config.scriptPattern == "httpslow") { + payload = { + path: "httpversion", + kind: "http", + args: { + url: "https://hub.dummyapis.com/delay?seconds=10", + }, + }; + } else if (config.scriptPattern == "noop") { + payload = { + path: "noop", + kind: "noop", + args: {}, + }; + } else if (config.scriptPattern == "identity") { + payload = { + path: "identity", + kind: "identity", + args: { + identity: "itsme", + }, + }; + } else if (config.scriptPattern == "postgresql") { + payload = { + path: "postgresql", + language: "postgresql", + args: { + query: "SELECT email FROM usr", + database_url: + "postgres://postgres:changeme@localhost:5432/windmill", + }, + }; + } else { + payload = { + path: "denosimple", + language: api.Preview.language.DENO, + content: + 'export function main(){ return Deno.env.get("WM_JOB_ID"); }', + args: {}, + }; + } + try { + uuid = await windmill.JobService.runScriptPreview({ + workspace: config.workspace_id, + requestBody: payload, + }); + } catch (e) { + console.error("error running script: " + e.body); + Deno.exit(1); + } + } + if (!config.continous) outstanding.push(uuid); + } + + clearInterval(updateStatusInterval); +} + +const end_time = Date.now() + complete_timeout; + +let incorrect_results = 0; +const enc = (s: string) => new TextEncoder().encode(s); + +async function getQueueCount() { + return ( await ( await fetch( config.server + "/api/w/" + config.workspace_id + "/jobs/queue/count", @@ -62,221 +338,61 @@ while (cont) { ) ).json() ).database_length; - if (queue_length > 2500) { - console.log( - `queue length: ${queue_length} > 2500. waiting... ` - ); - await sleep(0.5); - continue; - } - if ( - (total_spawned * 1000) / (Date.now() - start_time) > - config.per_worker_throughput - ) { - console.log("at maximum throughput. waiting..."); - await sleep(0.1); - continue; - } - total_spawned++; - if (total_spawned > config.max_per_worker) { - break; - } - let uuid: string; - if (config.custom) { - await evaluate(config.custom); - continue; - } else if (config.useFlows) { - let payload: api.FlowPreview; - if (config.flowPattern == "branchone") { - payload = { - path: "branchone", - args: {}, - value: { - modules: [ - { - id: "a", - value: { - input_transforms: {}, - language: api.RawScript.language.DENO, - type: "rawscript", - content: - 'export function main(){ return Deno.env.get("WM_FLOW_JOB_ID"); }', - }, - }, - { - id: "b", - value: { - type: "branchone", - branches: [], - default: [ - { - id: "c", - value: { - input_transforms: { - x: { - type: "javascript", - expr: "results.a", - }, - }, - language: api.RawScript.language.DENO, - type: "rawscript", - content: "export function main(x: string){ return x; }", - }, - }, - ], - }, - }, - ], - }, - }; - } else { - payload = { - path: "2steps", - args: {}, - value: { - modules: [ - { - id: "a", - value: { - input_transforms: {}, - language: api.RawScript.language.DENO, - type: "rawscript", - content: - 'export function main(){ return Deno.env.get("WM_JOB_ID"); }', - }, - }, - { - id: "b", - value: { - input_transforms: {}, - language: api.RawScript.language.DENO, - type: "rawscript", - content: - 'export function main(){ return Deno.env.get("WM_JOB_ID"); }', - }, - }, - ], - }, - }; - } - uuid = await windmill.JobService.runFlowPreview({ - workspace: config.workspace_id, - requestBody: payload, - }); - } else { - let payload: api.Preview; - if (config.scriptPattern == "httpversion") { - payload = { - path: "httpversion", - kind: "http", - args: { - url: "http://localhost:8000/api/version", - }, - }; - } - if (config.scriptPattern == "httpslow") { - payload = { - path: "httpversion", - kind: "http", - args: { - url: "https://hub.dummyapis.com/delay?seconds=10", - }, - }; - } else if (config.scriptPattern == "identity") { - payload = { - path: "identity", - kind: "identity", - args: { - identity: "itsme", - }, - }; - } else if (config.scriptPattern == "postgresql") { - payload = { - path: "postgresql", - language: "postgresql", - args: { - query: "SELECT email FROM usr", - database_url: "postgres://postgres:changeme@localhost:5432/windmill", - }, - }; - } else { - payload = { - path: "denosimple", - language: api.Preview.language.DENO, - content: 'export function main(){ return Deno.env.get("WM_JOB_ID"); }', - args: {}, - }; - } - uuid = await windmill.JobService.runScriptPreview({ - workspace: config.workspace_id, - requestBody: payload, - }); - } - if (!config.continous) outstanding.push(uuid); } -clearInterval(updateStatusInterval); - -const end_time = Date.now() + complete_timeout; -let incorrect_results = 0; -const enc = (s: string) => new TextEncoder().encode(s); - -while (outstanding.length > 0 && Date.now() < end_time) { - const uuid = outstanding.shift()!; - - let r: Job; - try { - r = await windmill.JobService.getJob({ - workspace: config.workspace_id, - id: uuid, - }); - } catch (e) { - console.log("job not found: " + uuid + " " + e.message); - continue; +if (config.scriptPattern == "noop") { + let queue_length = await getQueueCount(); + while (queue_length > 0 && Date.now() < end_time) { + await Deno.stdout.write(enc(`queue length: ${queue_length}\r`)); + queue_length = await getQueueCount(); } - if (r.type == "QueuedJob") { - outstanding.push(uuid); +} else { + while (outstanding.length > 0 && Date.now() < end_time) { await Deno.stdout.write( - enc( - `uuid: ${uuid}, queue length: ${ - ( - await ( - await fetch( - config.server + - "/api/w/" + - config.workspace_id + - "/jobs/queue/count", - { headers: { ["Authorization"]: "Bearer " + config.token } } - ) - ).json() - ).database_length - } \r` - ) + enc("\rwaiting for jobs to complete: " + outstanding.length + "\n") ); - } else { - r = r as api.CompletedJob; + const uuid = outstanding.shift()!; + + let r: Job; try { - if ( - !["httpversion", "identity", "httpslow"].includes( - config.scriptPattern - ) && - r.result != uuid - ) { - console.log( - "job did not return correct UUID: " + - r.result + - " != " + - uuid + - "job: \n" + - JSON.stringify(r, null, 2) - ); - incorrect_results++; - } else { - // console.log(r.result); - } + r = await windmill.JobService.getJob({ + workspace: config.workspace_id, + id: uuid, + }); } catch (e) { - console.log("error during wait: ", e); + console.log("job not found: " + uuid + " " + e.message); + continue; + } + if (r.type == "QueuedJob") { outstanding.push(uuid); + await Deno.stdout.write( + enc(`uuid: ${uuid}, queue length: ${await getQueueCount()}\r`) + ); + } else { + r = r as api.CompletedJob; + try { + if ( + !["httpversion", "identity", "httpslow", "noop"].includes( + config.scriptPattern + ) && + r.result != uuid + ) { + console.log( + "job did not return correct UUID: " + + r.result + + " != " + + uuid + + "job: \n" + + JSON.stringify(r, null, 2) + ); + incorrect_results++; + } else { + // console.log(r.result); + } + } catch (e) { + console.log("error during wait: ", e); + outstanding.push(uuid); + } } } } @@ -285,4 +401,5 @@ self.postMessage({ type: "zombie_jobs", zombie_jobs: outstanding.length, incorrect_results, + jobs_sent: total_spawned, }); diff --git a/frontend/package.json b/frontend/package.json index 7165147eac..796c42d7d9 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,5 +1,5 @@ { - "name": "windmill", + "name": "windmill-components", "version": "1.122.0", "scripts": { "dev": "vite dev", @@ -246,9 +246,9 @@ "types": "./package/components/apps/editor/inlineScriptsPanel/utils.d.ts", "default": "./package/components/apps/editor/inlineScriptsPanel/utils.js" }, - "./gen/OpenAPI": { - "types": "./package/gen/OpenAPI.d.ts", - "default": "./package/gen/OpenAPI.js" + "./gen/core/OpenAPI": { + "types": "./package/gen/core/OpenAPI.d.ts", + "default": "./package/gen/core/OpenAPI.js" } }, "files": [ @@ -343,8 +343,8 @@ "components/apps/editor/inlineScriptsPanel/utils": [ "./package/components/apps/editor/inlineScriptsPanel/utils.d.ts" ], - "gen/OpenAPI": [ - "./package/gen/OpenAPI.d.ts" + "gen/core/OpenAPI": [ + "./package/gen/core/OpenAPI.d.ts" ] } } diff --git a/frontend/src/lib/components/AddUser.svelte b/frontend/src/lib/components/AddUser.svelte index 46066896f4..4192442ca0 100644 --- a/frontend/src/lib/components/AddUser.svelte +++ b/frontend/src/lib/components/AddUser.svelte @@ -1,10 +1,12 @@ -
- +
+ + - - - - - - + +
+
+ +
+
+
+
+
+ +
+
+ +
+
Email will be sent if SMTP configured
diff --git a/frontend/src/lib/components/InviteUser.svelte b/frontend/src/lib/components/InviteUser.svelte index 8bec67d41f..5c43fd7185 100644 --- a/frontend/src/lib/components/InviteUser.svelte +++ b/frontend/src/lib/components/InviteUser.svelte @@ -1,10 +1,12 @@