starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-09 08:03:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: smtp support to invite users (#1777)

Browse Source 
* email support

* everyhting

* smtp

* update

* update

* update

* setup backend test
This commit is contained in:
Ruben Fiszel
2023-06-29 10:34:44 +02:00
committed by GitHub
parent a8af158b9f
commit 7851e932ec
33 changed files with 1201 additions and 388 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/backend-test.yml
+1
View File
@@ -33,6 +33,7 @@ jobs:
--health-retries 5
steps:
- uses: actions/checkout@v3
- uses: actions-rust-lang/setup-rust-toolchain@v1
- uses: Swatinem/rust-cache@v2
with:
workspaces: |
README.md
+19
View File
@@ -62,6 +62,7 @@ https://github.com/windmill-labs/windmill/assets/122811744/0b132cd1-ee67-4505-82
- [Postgres without superuser](#postgres-without-superuser)
- [Commercial license](#commercial-license)
- [OAuth for self-hosting](#oauth-for-self-hosting)
- [smtp for self-hostring](#smtp-for-self-hostring)
- [Resource types](#resource-types)
- [Environment Variables](#environment-variables)
- [Run a local dev setup](#run-a-local-dev-setup)
@@ -296,6 +297,19 @@ You may also add your own custom OAuth2 IdP and OAuth2 Resource provider:
}
```
### smtp for self-hostring
For users to receive emails when you invite them to workspaces or add them to
the instances using their emails, configure the SMTP env variables in the
servers:
```
SMTP_HOST=smtp.gmail.com
SMTP_PORT=587
SMTP_USERNAME=ruben@windmill.dev
SMTP_PASSWORD=yourpasswordapp
```
### Resource types
You will also want to import all the approved resource types from
@@ -365,6 +379,11 @@ it being synced automatically everyday.
| HTTP_PROXY | None | http_proxy | Server + Worker |
| HTTPS_PROXY | None | https_proxy | Server + Worker |
| NO_PROXY | None | no_proxy | Server + Worker |
| SMTP_HOST | None | host for the smtp server to send invite emails | Server |
| SMTP_PORT | 587 | port for the smtp server to send invite emails | Server |
| SMTP_USERNAME | None | username for the smtp server to send invite emails | Server |
| SMTP_PASSWORD | None | password for the smtp server to send invite emails | Server |
| SMTP_TLS_IMPLICIT | false | https://docs.rs/mail-send/latest/mail_send/struct.SmtpClientBuilder.html#method.implicit_tlsemails | Server |
## Run a local dev setup
backend/Cargo.lock
Generated
+369 -11
View File
@@ -25,11 +25,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8"
dependencies = [
"cfg-if",
"cipher",
"cipher 0.3.0",
"cpufeatures",
"opaque-debug",
]
[[package]]
name = "aes"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2"
dependencies = [
"cfg-if",
"cipher 0.4.4",
"cpufeatures",
]
[[package]]
name = "ahash"
version = "0.7.6"
@@ -146,7 +157,7 @@ checksum = "95c2fcf79ad1932ac6269a738109997a83c227c09b75842ae564dc8ede6a861c"
dependencies = [
"base64ct",
"blake2",
"password-hash",
"password-hash 0.5.0",
]
[[package]]
@@ -462,6 +473,12 @@ version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
[[package]]
name = "base64"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5"
[[package]]
name = "base64"
version = "0.21.2"
@@ -567,7 +584,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2cb03d1bed155d89dce0f845b7899b18a9a163e148fd004e1c28421a783e2d8e"
dependencies = [
"block-padding",
"cipher",
"cipher 0.3.0",
]
[[package]]
@@ -725,6 +742,16 @@ dependencies = [
"generic-array",
]
[[package]]
name = "cipher"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
dependencies = [
"crypto-common",
"inout",
]
[[package]]
name = "clap"
version = "4.3.5"
@@ -833,6 +860,12 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "constant_time_eq"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
[[package]]
name = "convert_case"
version = "0.4.0"
@@ -1058,7 +1091,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac41dd49fb554432020d52c875fc290e110113f864c6b1b525cd62c7e7747a5d"
dependencies = [
"byteorder",
"cipher",
"cipher 0.3.0",
"opaque-debug",
]
@@ -1181,6 +1214,18 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "enum-as-inner"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116"
dependencies = [
"heck",
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "errno"
version = "0.3.1"
@@ -1663,6 +1708,17 @@ dependencies = [
"windows-sys 0.48.0",
]
[[package]]
name = "hostname"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867"
dependencies = [
"libc",
"match_cfg",
"winapi",
]
[[package]]
name = "http"
version = "0.2.9"
@@ -1784,6 +1840,17 @@ dependencies = [
"cc",
]
[[package]]
name = "idna"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8"
dependencies = [
"matches",
"unicode-bidi",
"unicode-normalization",
]
[[package]]
name = "idna"
version = "0.4.0"
@@ -1817,6 +1884,15 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac"
[[package]]
name = "inout"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5"
dependencies = [
"generic-array",
]
[[package]]
name = "instant"
version = "0.1.12"
@@ -1837,6 +1913,18 @@ dependencies = [
"windows-sys 0.48.0",
]
[[package]]
name = "ipconfig"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f"
dependencies = [
"socket2 0.5.3",
"widestring",
"windows-sys 0.48.0",
"winreg 0.50.0",
]
[[package]]
name = "ipnet"
version = "2.7.2"
@@ -2106,6 +2194,12 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "linked-hash-map"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
[[package]]
name = "linux-raw-sys"
version = "0.3.8"
@@ -2132,6 +2226,15 @@ dependencies = [
"value-bag",
]
[[package]]
name = "lru-cache"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c"
dependencies = [
"linked-hash-map",
]
[[package]]
name = "lz4_flex"
version = "0.9.5"
@@ -2158,7 +2261,7 @@ version = "3.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0196bd5c76f5f51d7d6563545f86262fef4c82d75466ba6f6d359c40a523318d"
dependencies = [
"aes",
"aes 0.7.5",
"base64 0.13.1",
"block-modes",
"crc-any",
@@ -2169,6 +2272,70 @@ dependencies = [
"tiger",
]
[[package]]
name = "mail-auth"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6b0969bac270a60560d3a6f89c812b3e39b2f4b3ca8d866063f482030d14f19"
dependencies = [
"ahash 0.8.3",
"flate2",
"lru-cache",
"mail-builder",
"mail-parser",
"parking_lot 0.12.1",
"quick-xml",
"ring",
"rustls-pemfile",
"serde",
"serde_json",
"trust-dns-resolver",
"zip",
]
[[package]]
name = "mail-builder"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "765969f4385f88a62738e8ed63e2fa630571d7ed6fd96ca6932d699513dd8c28"
dependencies = [
"gethostname",
]
[[package]]
name = "mail-parser"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4158a1c18963244e083888b21465846dfb68d6170850ed1ab4742edd57c9d47"
dependencies = [
"encoding_rs",
]
[[package]]
name = "mail-send"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6d2b8d0cb56f199d36f527ff96453cf3b1cdfdacb5e4d154ac1d8fcd89873c2"
dependencies = [
"base64 0.20.0",
"gethostname",
"mail-auth",
"mail-builder",
"md5",
"rand 0.8.5",
"rustls 0.21.2",
"smtp-proto",
"tokio",
"tokio-rustls 0.24.1",
"webpki-roots 0.23.1",
]
[[package]]
name = "match_cfg"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4"
[[package]]
name = "matchers"
version = "0.1.0"
@@ -2210,6 +2377,12 @@ dependencies = [
"digest 0.10.7",
]
[[package]]
name = "md5"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
[[package]]
name = "memchr"
version = "2.5.0"
@@ -2519,6 +2692,17 @@ dependencies = [
"regex",
]
[[package]]
name = "password-hash"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700"
dependencies = [
"base64ct",
"rand_core 0.6.4",
"subtle",
]
[[package]]
name = "password-hash"
version = "0.5.0"
@@ -2536,6 +2720,18 @@ version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79"
[[package]]
name = "pbkdf2"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917"
dependencies = [
"digest 0.10.7",
"hmac",
"password-hash 0.4.2",
"sha2 0.10.7",
]
[[package]]
name = "pem-rfc7468"
version = "0.6.0"
@@ -2939,6 +3135,21 @@ dependencies = [
"cc",
]
[[package]]
name = "quick-error"
version = "1.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
[[package]]
name = "quick-xml"
version = "0.28.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ce5e73202a820a31f8a0ee32ada5e21029c81fd9e3ebf668a40832e4219d9d1"
dependencies = [
"memchr",
]
[[package]]
name = "quote"
version = "1.0.28"
@@ -3153,7 +3364,17 @@ dependencies = [
"wasm-bindgen-futures",
"wasm-streams",
"web-sys",
"winreg",
"winreg 0.10.1",
]
[[package]]
name = "resolv-conf"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00"
dependencies = [
"hostname",
"quick-error",
]
[[package]]
@@ -3316,6 +3537,18 @@ dependencies = [
"webpki",
]
[[package]]
name = "rustls"
version = "0.21.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e32ca28af694bc1bbf399c33a516dbdf1c90090b8ab23c2bc24f834aa2247f5f"
dependencies = [
"log",
"ring",
"rustls-webpki",
"sct",
]
[[package]]
name = "rustls-pemfile"
version = "1.0.2"
@@ -3325,6 +3558,16 @@ dependencies = [
"base64 0.21.2",
]
[[package]]
name = "rustls-webpki"
version = "0.100.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b"
dependencies = [
"ring",
"untrusted",
]
[[package]]
name = "rustpython-ast"
version = "0.2.0"
@@ -3810,6 +4053,12 @@ dependencies = [
"serde",
]
[[package]]
name = "smtp-proto"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4b756ac662e92a0e5b360349bea5f0b0784d4be4541eff2972049dfdfd7f862"
[[package]]
name = "socket2"
version = "0.4.9"
@@ -3929,7 +4178,7 @@ dependencies = [
"paste",
"percent-encoding",
"rand 0.8.5",
"rustls",
"rustls 0.20.8",
"rustls-pemfile",
"serde",
"serde_json",
@@ -3943,7 +4192,7 @@ dependencies = [
"tokio-stream",
"url",
"uuid 1.3.4",
"webpki-roots",
"webpki-roots 0.22.6",
"whoami",
]
@@ -3977,7 +4226,7 @@ checksum = "804d3f245f894e61b1e6263c84b23ca675d96753b5abfd5cc8597d86806e8024"
dependencies = [
"once_cell",
"tokio",
"tokio-rustls",
"tokio-rustls 0.23.4",
]
[[package]]
@@ -4451,11 +4700,21 @@ version = "0.23.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59"
dependencies = [
"rustls",
"rustls 0.20.8",
"tokio",
"webpki",
]
[[package]]
name = "tokio-rustls"
version = "0.24.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"
dependencies = [
"rustls 0.21.2",
"tokio",
]
[[package]]
name = "tokio-stream"
version = "0.1.14"
@@ -4702,6 +4961,59 @@ dependencies = [
"stable_deref_trait",
]
[[package]]
name = "trust-dns-proto"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f7f83d1e4a0e4358ac54c5c3681e5d7da5efc5a7a632c90bb6d6669ddd9bc26"
dependencies = [
"async-trait",
"cfg-if",
"data-encoding",
"enum-as-inner",
"futures-channel",
"futures-io",
"futures-util",
"idna 0.2.3",
"ipnet",
"lazy_static",
"rand 0.8.5",
"ring",
"rustls 0.20.8",
"rustls-pemfile",
"smallvec",
"thiserror",
"tinyvec",
"tokio",
"tokio-rustls 0.23.4",
"tracing",
"url",
"webpki",
]
[[package]]
name = "trust-dns-resolver"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aff21aa4dcefb0a1afbfac26deb0adc93888c7d295fb63ab273ef276ba2b7cfe"
dependencies = [
"cfg-if",
"futures-util",
"ipconfig",
"lazy_static",
"lru-cache",
"parking_lot 0.12.1",
"resolv-conf",
"rustls 0.20.8",
"smallvec",
"thiserror",
"tokio",
"tokio-rustls 0.23.4",
"tracing",
"trust-dns-proto",
"webpki-roots 0.22.6",
]
[[package]]
name = "try-lock"
version = "0.2.4"
@@ -4939,7 +5251,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb"
dependencies = [
"form_urlencoded",
"idna",
"idna 0.4.0",
"percent-encoding",
"serde",
]
@@ -5190,6 +5502,15 @@ dependencies = [
"webpki",
]
[[package]]
name = "webpki-roots"
version = "0.23.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338"
dependencies = [
"rustls-webpki",
]
[[package]]
name = "which"
version = "4.4.0"
@@ -5211,6 +5532,12 @@ dependencies = [
"web-sys",
]
[[package]]
name = "widestring"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "653f141f39ec16bba3c5abe400a0c60da7468261cc2cbf36805022876bc721a8"
[[package]]
name = "winapi"
version = "0.3.9"
@@ -5301,6 +5628,7 @@ dependencies = [
"itertools 0.11.0",
"lazy_static",
"magic-crypt",
"mail-send",
"mime_guess",
"prometheus",
"rand 0.8.5",
@@ -5692,6 +6020,16 @@ dependencies = [
"winapi",
]
[[package]]
name = "winreg"
version = "0.50.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1"
dependencies = [
"cfg-if",
"windows-sys 0.48.0",
]
[[package]]
name = "xattr"
version = "0.2.3"
@@ -5716,6 +6054,26 @@ version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9"
[[package]]
name = "zip"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261"
dependencies = [
"aes 0.8.3",
"byteorder",
"bzip2",
"constant_time_eq",
"crc32fast",
"crossbeam-utils",
"flate2",
"hmac",
"pbkdf2",
"sha1",
"time 0.3.22",
"zstd",
]
[[package]]
name = "zstd"
version = "0.11.2+zstd.1.5.2"
backend/Cargo.toml
+2 -8
View File
@@ -114,13 +114,7 @@ magic-crypt = "^3"
git-version = "^0"
rustpython-parser = "0.2.0"
cron = "^0"
lettre = { version = "^0", features = [
"rustls-tls",
"tokio1",
"tokio1-rustls-tls",
"builder",
"smtp-transport",
], default-features = false }
mail-send = "0.4.0"
urlencoding = "^2"
url = "^2"
async-oauth2 = "^0"
@@ -176,4 +170,4 @@ serde-wasm-bindgen = "0.4"
wasm-bindgen-test = "0.3.0"
convert_case = "0.6.0"
getrandom = "0.2"
tokio-postgres = {version = "^0.7", features = ["array-impls", "with-serde_json-1"]}
tokio-postgres = {version = "^0.7", features = ["array-impls", "with-serde_json-1"]}
backend/migrations/20230627235827_noop.down.sql
+1
View File
@@ -0,0 +1 @@
-- Add down migration script here
backend/migrations/20230627235827_noop.up.sql
+2
View File
@@ -0,0 +1,2 @@
-- Add up migration script here
ALTER TYPE JOB_KIND ADD VALUE IF NOT EXISTS 'noop';
backend/migrations/20230628003451_noop.down.sql
+1
View File
@@ -0,0 +1 @@
-- Add down migration script here
backend/migrations/20230628003451_noop.up.sql
+1
View File
@@ -0,0 +1 @@
-- Add up migration script here
backend/sqlx-data.json
+53 -9
View File
@@ -443,6 +443,26 @@
},
"query": "DELETE FROM workspace_invite WHERE\n workspace_id = $1 AND email = $2 AND is_admin = $3 AND operator = $4"
},
"0cf42f7e76fe01e6a9a20499b2228d76a1919b8b4050afedb2459be083a4ad4d": {
"describe": {
"columns": [
{
"name": "exists",
"ordinal": 0,
"type_info": "Bool"
}
],
"nullable": [
null
],
"parameters": {
"Left": [
"Text"
]
}
},
"query": "SELECT EXISTS(SELECT 1 FROM password WHERE email = $1)"
},
"0d6412bc3ebb1d58bdd9cbcef774dacf9016fa402af5c1b4e339b9a3d7163d5e": {
"describe": {
"columns": [
@@ -803,7 +823,11 @@
"flowpreview",
"script_hub",
"identity",
"flowdependencies"
"flowdependencies",
"http",
"graphql",
"postgresql",
"noop"
]
},
"name": "job_kind"
@@ -1174,7 +1198,11 @@
"flowpreview",
"script_hub",
"identity",
"flowdependencies"
"flowdependencies",
"http",
"graphql",
"postgresql",
"noop"
]
},
"name": "job_kind"
@@ -1193,7 +1221,8 @@
"python3",
"deno",
"go",
"bash"
"bash",
"postgresql"
]
},
"name": "script_lang"
@@ -2055,7 +2084,11 @@
"flowpreview",
"script_hub",
"identity",
"flowdependencies"
"flowdependencies",
"http",
"graphql",
"postgresql",
"noop"
]
},
"name": "job_kind"
@@ -2072,7 +2105,8 @@
"python3",
"deno",
"go",
"bash"
"bash",
"postgresql"
]
},
"name": "script_lang"
@@ -2739,7 +2773,8 @@
"python3",
"deno",
"go",
"bash"
"bash",
"postgresql"
]
},
"name": "script_lang"
@@ -3846,7 +3881,8 @@
"python3",
"deno",
"go",
"bash"
"bash",
"postgresql"
]
},
"name": "script_lang"
@@ -7326,7 +7362,11 @@
"flowpreview",
"script_hub",
"identity",
"flowdependencies"
"flowdependencies",
"http",
"graphql",
"postgresql",
"noop"
]
},
"name": "job_kind"
@@ -7343,7 +7383,11 @@
"flowpreview",
"script_hub",
"identity",
"flowdependencies"
"flowdependencies",
"http",
"graphql",
"postgresql",
"noop"
]
},
"name": "job_kind"
backend/src/main.rs
+14 -2
View File
@@ -19,7 +19,7 @@ use tokio::{
join,
sync::RwLock,
};
use windmill_api::LICENSE_KEY;
use windmill_api::{LICENSE_KEY, OAUTH_CLIENTS, SMTP_CLIENT};
use windmill_common::{utils::rd_string, METRICS_ADDR};
use windmill_worker::{
DENO_CACHE_DIR, DENO_TMP_CACHE_DIR, GO_CACHE_DIR, GO_TMP_CACHE_DIR, HUB_CACHE_DIR,
@@ -90,7 +90,6 @@ async fn main() -> anyhow::Result<()> {
.unwrap_or(std::borrow::Cow::Borrowed("rsmq"))
.into_owned();
config.port = url.port().unwrap_or(6379).to_string();
config
});
@@ -181,8 +180,21 @@ Windmill Community Edition {GIT_VERSION}
"WAIT_RESULT_FAST_POLL_INTERVAL_MS",
"EXIT_AFTER_NO_JOB_FOR_SECS",
"REQUEST_SIZE_LIMIT",
"SMTP_HOST",
"SMTP_USERNAME",
"SMTP_PORT",
"SMTP_TLS_IMPLICIT",
]);
tracing::info!("Loading OAuth providers...: {:#?}", *OAUTH_CLIENTS);
if let Some(ref smtp) = *SMTP_CLIENT {
tracing::info!("Smtp client defined. Testing connection...");
if let Err(e) = smtp.connect().await {
tracing::error!("Failed to connect to smtp server: {}", e);
} else {
tracing::info!("Smtp client connected.");
}
}
if server_mode || num_workers > 0 {
let addr = SocketAddr::from((server_bind_address, port));
backend/windmill-api/Cargo.toml
+1
View File
@@ -70,3 +70,4 @@ async_zip.workspace = true
rsmq_async.workspace = true
regex.workspace = true
bytes.workspace = true
mail-send.workspace = true
backend/windmill-api/openapi.yaml
+20
View File
@@ -877,6 +877,26 @@ paths:
schema:
$ref: "#/components/schemas/User"
/users/exists/{email}:
get:
summary: exists email
operationId: existsEmail
tags:
- user
parameters:
- name: email
in: path
required: true
schema:
type: string
responses:
"200":
description: user
content:
application/json:
schema:
type: boolean
/users/list_as_super_admin:
get:
summary: list all users as super admin (require to be super amdin)
backend/windmill-api/src/jobs.rs
+42
View File
@@ -9,6 +9,7 @@
use crate::{
db::{UserDB, DB},
users::{check_scopes, require_owner_of_path, Authed, OptAuthed},
utils::require_super_admin,
variables::get_workspace_key,
BASE_URL,
};
@@ -76,6 +77,7 @@ pub fn workspaced_service() -> Router {
)
.route("/run/h/:hash", post(run_job_by_hash).head(|| async { "" }))
.route("/run/preview", post(run_preview_job))
.route("/add_noop_jobs/:n", post(add_noop_jobs))
.route("/run/preview_flow", post(run_preview_flow_job))
.route("/list", get(list_jobs))
.route("/queue/list", get(list_queue_jobs))
@@ -1241,6 +1243,7 @@ enum PreviewKind {
Identity,
Http,
Graphql,
Noop,
}
#[derive(Deserialize)]
struct Preview {
@@ -1976,6 +1979,7 @@ async fn run_preview_job(
Some(PreviewKind::Identity) => JobPayload::Identity,
Some(PreviewKind::Http) => JobPayload::Http,
Some(PreviewKind::Graphql) => JobPayload::Graphql,
Some(PreviewKind::Noop) => JobPayload::Noop,
_ => JobPayload::Code(RawCode {
content: preview.content.unwrap_or_default(),
path: preview.path,
@@ -2004,6 +2008,44 @@ async fn run_preview_job(
Ok((StatusCode::CREATED, uuid.to_string()))
}
async fn add_noop_jobs(
authed: Authed,
Extension(db): Extension<DB>,
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
Path((w_id, n)): Path<(String, i32)>,
) -> error::JsonResult<Vec<String>> {
require_super_admin(&mut db.begin().await?, &authed.email).await?;
let mut tx: QueueTransaction<'_, _> = (rsmq, db.begin().await?).into();
let mut uuids: Vec<String> = Vec::new();
for _ in 0..n {
let (uuid, ntx) = push(
tx,
&w_id,
JobPayload::Noop,
serde_json::Map::new(),
&authed.username,
&authed.email,
username_to_permissioned_as(&authed.username),
None,
None,
None,
None,
None,
false,
false,
None,
true,
None,
)
.await?;
tx = ntx;
uuids.push(uuid.to_string());
}
tx.commit().await?;
Ok(Json(uuids))
}
async fn run_preview_flow_job(
authed: Authed,
Extension(user_db): Extension<UserDB>,
backend/windmill-api/src/lib.rs
+40
View File
@@ -20,6 +20,7 @@ use axum::{middleware::from_extractor, routing::get, Extension, Router};
use db::DB;
use git_version::git_version;
use hyper::Method;
use mail_send::SmtpClientBuilder;
use reqwest::Client;
use std::{net::SocketAddr, sync::Arc};
use tower::ServiceBuilder;
@@ -90,9 +91,48 @@ lazy_static::lazy_static! {
.map_err(|e| tracing::error!("Error building oauth clients: {}", e))
.unwrap();
pub static ref SMTP_CLIENT: Option<SmtpClientBuilder<String>> = {
let smtp = parse_smtp();
if let Some(smtp) = smtp {
match smtp {
Ok(smtp) => Some(smtp),
Err(e) => {
tracing::error!("SMTP is not configured correctly, emails will not be sent: {}", e);
None
}
}
} else {
tracing::warn!("SMTP is not configured, emails will not be sent");
None
}
};
pub static ref LICENSE_KEY: Option<String> = std::env::var("LICENSE_KEY").ok();
}
pub fn parse_smtp() -> Option<windmill_common::error::Result<SmtpClientBuilder<String>>> {
let username = std::env::var("SMTP_USERNAME").ok();
let port = std::env::var("SMTP_PORT")
.ok()
.and_then(|p| p.parse().ok())
.unwrap_or(587);
let password = std::env::var("SMTP_PASSWORD").ok();
let host = std::env::var("SMTP_HOST").ok();
let tls_implicit = std::env::var("SMTP_TLS_IMPLICIT")
.ok()
.and_then(|p| p.parse().ok())
.unwrap_or(false);
if username.is_some() && password.is_some() && host.is_some() {
let smtp = SmtpClientBuilder::new(host.unwrap(), port)
.implicit_tls(tls_implicit)
.credentials((username.unwrap(), password.unwrap()));
Some(Ok(smtp))
} else {
None
}
}
pub async fn run_server(
db: DB,
rsmq: Option<rsmq_async::MultiplexedRsmq>,
backend/windmill-api/src/oauth2.rs
+3
View File
@@ -75,6 +75,7 @@ pub fn workspaced_service() -> Router {
.route("/connect_slack_callback", post(connect_slack_callback))
}
#[derive(Debug)]
pub struct ClientWithScopes {
client: OClient,
scopes: Vec<String>,
@@ -105,6 +106,8 @@ pub struct OAuthClient {
connect_config: Option<OAuthConfig>,
login_config: Option<OAuthConfig>,
}
#[derive(Debug)]
pub struct AllClients {
pub logins: BasicClientsMap,
pub connects: BasicClientsMap,
backend/windmill-api/src/users.rs
+68 -4
View File
@@ -14,7 +14,7 @@ use crate::{
utils::require_super_admin,
webhook_util::{InstanceEvent, WebhookShared},
workspaces::invite_user_to_all_auto_invite_worspaces,
COOKIE_DOMAIN, IS_SECURE,
BASE_URL, COOKIE_DOMAIN, IS_SECURE, SMTP_CLIENT,
};
use argon2::{password_hash::SaltString, Argon2, PasswordHash, PasswordHasher, PasswordVerifier};
use axum::{
@@ -27,6 +27,7 @@ use axum::{
};
use hyper::{header::LOCATION, StatusCode};
use lazy_static::lazy_static;
use mail_send::mail_builder::MessageBuilder;
use rand::rngs::OsRng;
use regex::Regex;
use retainer::Cache;
@@ -37,7 +38,7 @@ use tower_cookies::{Cookie, Cookies};
use tracing::{Instrument, Span};
use windmill_audit::{audit_log, ActionKind};
use windmill_common::{
error::{self, Error, JsonResult, Result},
error::{self, to_anyhow, Error, JsonResult, Result},
users::SUPERADMIN_SECRET_EMAIL,
utils::{not_found_if_none, rd_string, require_admin, Pagination, StripPath},
};
@@ -57,13 +58,14 @@ pub fn workspaced_service() -> Router {
.route("/update/:user", post(update_workspace_user))
.route("/delete/:user", delete(delete_workspace_user))
.route("/is_owner/*path", get(is_owner_of_path))
.route("/whois/:email", get(whois))
.route("/whois/:username", get(whois))
.route("/whoami", get(whoami))
.route("/leave", post(leave_workspace))
}
pub fn global_service() -> Router {
Router::new()
.route("/exists/:email", get(exists_email))
.route("/email", get(get_email))
.route("/whoami", get(global_whoami))
.route("/list_invites", get(list_invites))
@@ -882,6 +884,17 @@ async fn global_whoami(
}
}
async fn exists_email(Extension(db): Extension<DB>, Path(email): Path<String>) -> JsonResult<bool> {
let exists = sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM password WHERE email = $1)",
email
)
.fetch_one(&db)
.await?
.unwrap_or(false);
Ok(Json(exists))
}
async fn get_email(Authed { email, .. }: Authed) -> Result<String> {
Ok(email)
}
@@ -1078,6 +1091,7 @@ lazy_static! {
.ok()
.and_then(|x| x.parse::<u32>().ok())
.unwrap_or(60 * 60 * 24 * 60); // 60 days
}
async fn accept_invite(
@@ -1222,6 +1236,16 @@ async fn add_user_to_workspace<'c>(
None,
)
.await?;
send_email_if_possible(
&format!("Added to Windmill's workspace: {w_id}"),
&format!(
"You have been granted access to Windmill's workspace {w_id}
If you do not have an account on {}, login with SSO or ask an admin to create an account for you.",
*BASE_URL
),
&email,
);
Ok(tx)
}
@@ -1368,6 +1392,7 @@ async fn delete_user(
lazy_static::lazy_static! {
pub static ref NEW_USER_WEBHOOK: Option<String> = std::env::var("NEW_USER_WEBHOOK").ok();
}
async fn create_user(
@@ -1394,7 +1419,7 @@ async fn create_user(
VALUES ($1, $2, $3, 'password', $4, $5, $6)",
&nu.email,
true,
&hash_password(argon2, nu.password)?,
&hash_password(argon2, nu.password.clone())?,
&nu.super_admin,
nu.name,
nu.company
@@ -1415,11 +1440,50 @@ async fn create_user(
tx.commit().await?;
invite_user_to_all_auto_invite_worspaces(&db, &nu.email).await?;
send_email_if_possible(
"Invited to Windmill",
&format!(
"You have been granted access to Windmill by {email}.
Login and change your password: {}/user/login?email={}&password={}&rd=%2F%23user-settings
You can then join or create a workspace. Happy building!",
*BASE_URL, &nu.email, &nu.password
),
&nu.email,
);
webhook.send_instance_event(InstanceEvent::UserAdded { email: nu.email.clone() });
Ok((StatusCode::CREATED, format!("email {} created", nu.email)))
}
pub fn send_email_if_possible(subject: &str, content: &str, to: &str) {
let subject = subject.to_string();
let content = content.to_string();
let to = to.to_string();
tokio::spawn(async move {
if let Err(e) = send_email_if_possible_intern(&subject, &content, &to).await {
tracing::error!("Failed to send email to {}: {}", &to, e);
}
});
}
pub async fn send_email_if_possible_intern(subject: &str, content: &str, to: &str) -> Result<()> {
if let Some(ref smtp) = *SMTP_CLIENT {
let message = MessageBuilder::new()
.from(("Windmill", "noreply@getwindmill.com"))
.to(to)
.subject(subject)
.text_body(content);
smtp.connect()
.await
.map_err(to_anyhow)?
.send(message)
.await
.map_err(to_anyhow)?;
}
return Ok(());
}
async fn delete_workspace_user(
Authed { username, is_admin, .. }: Authed,
Extension(db): Extension<DB>,
backend/windmill-api/src/workspaces.rs
+13 -3
View File
@@ -9,17 +9,16 @@
#[cfg(feature = "enterprise")]
use std::str::FromStr;
#[cfg(feature = "enterprise")]
use crate::BASE_URL;
use crate::{
apps::AppWithLastVersion,
db::{UserDB, DB},
folders::Folder,
resources::{Resource, ResourceType},
users::{Authed, WorkspaceInvite, VALID_USERNAME},
users::{Authed, WorkspaceInvite, VALID_USERNAME, send_email_if_possible},
utils::require_super_admin,
variables::build_crypt,
webhook_util::{InstanceEvent, WebhookShared},
webhook_util::{InstanceEvent, WebhookShared}
};
#[cfg(feature = "enterprise")]
use axum::response::Redirect;
@@ -1021,6 +1020,17 @@ async fn invite_user(
tx.commit().await?;
send_email_if_possible(
&format!("Invited to Windmill's workspace: {w_id}"),
&format!(
"You have been granted access to Windmill's workspace {w_id}
If you do not have an account on {}, login with SSO or ask an admin to create an account for you.",
*BASE_URL
),
&nu.email,
);
webhook.send_instance_event(InstanceEvent::UserInvitedWorkspace {
email: nu.email.clone(),
workspace: w_id,
backend/windmill-common/src/jobs.rs
+2
View File
@@ -25,6 +25,7 @@ pub enum JobKind {
FlowDependencies,
Http,
Graphql,
Noop,
}
#[derive(Debug, sqlx::FromRow, Serialize, Clone)]
@@ -160,6 +161,7 @@ pub enum JobPayload {
Identity,
Http,
Graphql,
Noop,
}
#[derive(Clone, Serialize, Deserialize, Debug)]
backend/windmill-common/src/tracing_init.rs
+9 -18
View File
@@ -6,9 +6,7 @@
* LICENSE-AGPL for a copy of the license.
*/
use tracing::Metadata;
use tracing_subscriber::{
filter::filter_fn,
fmt::{format, Layer},
prelude::*,
EnvFilter,
@@ -26,34 +24,27 @@ fn compact_layer<S>() -> Layer<S, format::DefaultFields, format::Format<format::
tracing_subscriber::fmt::layer().compact()
}
fn filter_metadata(meta: &Metadata) -> bool {
meta.target().starts_with("windmill")
}
pub fn initialize_tracing() {
let style = std::env::var("RUST_LOG_STYLE").unwrap_or_else(|_| "auto".into());
let json_fmt = std::env::var("JSON_FMT")
.map(|x| x == "true")
.unwrap_or(false);
if std::env::var("RUST_LOG").is_ok_and(|x| x == "debug" || x == "info") {
std::env::set_var(
"RUST_LOG",
&format!("windmill={}", std::env::var("RUST_LOG").unwrap()),
)
}
let env_filter = EnvFilter::from_default_env();
let ts_base = tracing_subscriber::registry().with(env_filter);
match json_fmt {
true => ts_base
.with(
json_layer()
.flatten_event(true)
.with_filter(filter_fn(filter_metadata)),
)
.init(),
true => ts_base.with(json_layer().flatten_event(true)).init(),
false => ts_base
.with(
compact_layer()
.with_ansi(style.to_lowercase() != "never")
.with_filter(filter_fn(filter_metadata)),
)
.with(compact_layer().with_ansi(style.to_lowercase() != "never"))
.init(),
}
}
backend/windmill-queue/src/jobs.rs
+13 -1
View File
@@ -6,7 +6,11 @@
* LICENSE-AGPL for a copy of the license.
*/
use std::{collections::HashMap, vec};
use std::{
collections::HashMap,
time::{Instant, SystemTime},
vec,
};
use async_recursion::async_recursion;
use itertools::Itertools;
@@ -28,6 +32,7 @@ use windmill_common::{
schedule::{schedule_to_user, Schedule},
scripts::{ScriptHash, ScriptLang},
users::username_to_permissioned_as,
utils::rd_string,
METRICS_ENABLED,
};
@@ -543,12 +548,16 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Clone>(
}
}
// let rs = rd_string(2);
// let instant = Instant::now();
let job: Option<QueuedJob> = if let Some(mut rsmq) = rsmq {
// TODO: REDIS: Race conditions / replace last_ping
let msg = rsmq
.pop_message::<Vec<u8>>(RSMQ_MAIN_QUEUE)
.await
.map_err(|e| anyhow::anyhow!(e))?;
// println!("3.1: {:?} {rs}", instant.elapsed());
if let Some(msg) = msg {
let uuid = Uuid::from_bytes_le(
@@ -606,6 +615,7 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Clone>(
.fetch_optional(db)
.await?
};
// println!("3.2: {:?} {rs}", instant.elapsed());
if job.is_some() && *METRICS_ENABLED {
QUEUE_PULL_COUNT.inc();
@@ -948,6 +958,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
JobPayload::Identity => (None, None, None, JobKind::Identity, None, None),
JobPayload::Graphql => (None, None, None, JobKind::Graphql, None, None),
JobPayload::Http => (None, None, None, JobKind::Http, None, None),
JobPayload::Noop => (None, None, None, JobKind::Noop, None, None),
};
let is_running = same_worker;
@@ -1081,6 +1092,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
JobKind::Dependencies => "jobs.run.dependencies",
JobKind::Identity => "jobs.run.identity",
JobKind::Http => "jobs.run.http",
JobKind::Noop => "jobs.run.noop",
JobKind::Graphql => "jobs.run.graphql",
JobKind::FlowDependencies => "jobs.run.flow_dependencies",
};
backend/windmill-worker/src/worker.rs
+9 -5
View File
@@ -17,7 +17,7 @@ use windmill_api_client::{Client, types::CompletedJob};
use windmill_parser::Typ;
use std::{
borrow::Borrow, collections::HashMap, io, os::unix::process::ExitStatusExt, panic,
process::Stdio, time::{Duration},
process::Stdio, time::{Duration, SystemTime},
sync::{Arc, atomic::Ordering},
collections::hash_map::DefaultHasher,
hash::{Hasher, Hash},
@@ -449,7 +449,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
let (same_worker_tx, mut same_worker_rx) = mpsc::channel::<Uuid>(5);
let (job_completed_tx, mut job_completed_rx) = mpsc::channel::<JobCompleted>(10);
let (job_completed_tx, mut job_completed_rx) = mpsc::channel::<JobCompleted>(1000);
let db2 = db.clone();
let rsmq2 = rsmq.clone();
@@ -468,6 +468,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
let mut last_executed_job: Option<Instant> = None;
loop {
// let instant: Instant = Instant::now();
if *METRICS_ENABLED {
worker_busy.set(0);
uptime_metric.inc_by(
@@ -541,6 +542,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
let (do_break, next_job) = if first_run {
(false, Ok(Some(QueuedJob::default())))
} else {
// println!("2: {:?}", instant.elapsed());
async {
tokio::select! {
biased;
@@ -589,6 +591,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
let duration_pull_s = timer.stop_and_record();
worker_pull_duration_counter.inc_by(duration_pull_s);
});
// println!("Pull: {:?}", instant.elapsed());
(false, job)
},
}
@@ -606,10 +609,11 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
}
match next_job {
Ok(Some(job)) => {
// println!("{:?}", SystemTime::now());
last_executed_job = None;
if matches!(job.job_kind, JobKind::Noop) {
job_completed_tx.send(JobCompleted { job, success: true, result: json!({}), logs: String::new()}).await.expect("send job completed");
return false
}
let token = create_token_for_owner_in_bg(&db, &job).await;
let language = job.language.clone();
let _timer = worker_execution_duration
benchmarks/main.ts
+70 -59
View File
@@ -174,21 +174,23 @@ await new Command()
exportSimple = [];
}
let metrics_worker: Worker;
let metrics_worker: Worker | undefined = undefined;
if (!continous) {
metrics_worker = new Worker(
new URL("./scraper.ts", import.meta.url).href,
{
type: "module",
}
);
if (exportJson || exportCsv) {
metrics_worker = new Worker(
new URL("./scraper.ts", import.meta.url).href,
{
type: "module",
}
);
metrics_worker.postMessage({
exportHistograms,
histogramBuckets,
exportSimple,
host: metrics,
});
metrics_worker.postMessage({
exportHistograms,
histogramBuckets,
exportSimple,
host: metrics,
});
}
}
console.log(
@@ -287,12 +289,12 @@ await new Command()
}, 100);
workers.forEach((worker, i) => {
worker.postMessage({ ...shared_config, i });
worker.addEventListener("message", (evt: MessageEvent<any>) => {
if (evt.data.type === "jobs_sent") {
jobsSent[i] = evt.data.jobs_sent;
}
});
worker.postMessage({ ...shared_config, i });
});
start = Date.now();
@@ -307,7 +309,7 @@ await new Command()
clearInterval(updateState);
const sum = jobsSent.reduce((a, b) => a + b, 0);
let sum = jobsSent.reduce((a, b) => a + b, 0);
await Deno.stdout.write(
enc(" ".padStart(30) + `\rduration: ${seconds} | jobs sent: ${sum}\n`)
);
@@ -315,13 +317,14 @@ await new Command()
const shutdown_start = Date.now();
let zombie_jobs = 0;
let incorrect_results = 0;
workers.forEach((worker) => {
workers.forEach((worker, i) => {
const l = (evt: MessageEvent<any>) => {
if (evt.data.type === "zombie_jobs") {
zombie_jobs += evt.data.zombie_jobs;
incorrect_results += evt.data.incorrect_results;
worker.removeEventListener("message", l);
workers = workers.filter((w) => w != worker);
jobsSent[i] = evt.data.jobs_sent;
worker.terminate();
}
};
@@ -335,6 +338,8 @@ await new Command()
while (workers.length > 0) {
await sleep(0.1);
}
sum = jobsSent.reduce((a, b) => a + b, 0);
const tts = (Date.now() - shutdown_start) / 1000;
const time = seconds + tts;
console.log("\ntime to shutdown:", tts);
@@ -356,53 +361,59 @@ await new Command()
).database_length
);
metrics_worker!.postMessage("stop");
console.log("waiting for metrics");
const { columns, transfer_values } = await new Promise<{
columns: string[];
transfer_values: ArrayBufferLike[];
}>((resolve, _reject) => {
metrics_worker.onmessage = (e) => {
resolve(e.data);
metrics_worker.terminate();
};
});
const values = transfer_values.map((x) => new Float32Array(x));
if (exportJson) {
console.log("exporting mean & stdev to json");
const obj: any = {};
for (let i = 0; i < columns.length; i++) {
const name = columns[i]!;
const value = values[i]!;
const mean = value.reduce((acc, e) => acc + e, 0) / values.length;
const stdev = Math.sqrt(
value.reduce((acc, e) => acc + (e - mean) ** 2) / values.length
);
obj[name] = { mean, stdev };
}
await Deno.writeTextFile(exportJson, JSON.stringify(obj));
}
if (exportCsv) {
const f = await Deno.open(exportCsv, {
write: true,
create: true,
truncate: true,
if (metrics_worker) {
metrics_worker.postMessage("stop");
console.log("waiting for metrics");
const { columns, transfer_values } = await new Promise<{
columns: string[];
transfer_values: ArrayBufferLike[];
}>((resolve, _reject) => {
if (metrics_worker) {
metrics_worker.onmessage = (e) => {
resolve(e.data);
metrics_worker?.terminate();
};
}
});
const encoder = new TextEncoder();
const newline = new Uint8Array(1);
newline[0] = 0x0a;
await f.write(encoder.encode(columns.join(",")));
await f.write(newline);
const values = transfer_values.map((x) => new Float32Array(x));
for (let i = 0; i < values.length; i++) {
await f.write(encoder.encode(values[i].join(",")));
await f.write(newline);
if (exportJson) {
console.log("exporting mean & stdev to json");
const obj: any = {};
for (let i = 0; i < columns.length; i++) {
const name = columns[i]!;
const value = values[i]!;
const mean = value.reduce((acc, e) => acc + e, 0) / values.length;
const stdev = Math.sqrt(
value.reduce((acc, e) => acc + (e - mean) ** 2) / values.length
);
obj[name] = { mean, stdev };
}
await Deno.writeTextFile(exportJson, JSON.stringify(obj));
}
f.close();
if (exportCsv) {
const f = await Deno.open(exportCsv, {
write: true,
create: true,
truncate: true,
});
const encoder = new TextEncoder();
const newline = new Uint8Array(1);
newline[0] = 0x0a;
await f.write(encoder.encode(columns.join(",")));
await f.write(newline);
for (let i = 0; i < values.length; i++) {
await f.write(encoder.encode(values[i].join(",")));
await f.write(newline);
}
f.close();
}
} else {
return;
}
console.log("done");
}
benchmarks/worker.ts
+333 -216
View File
@@ -42,19 +42,295 @@ const config = await promise;
const outstanding: string[] = [];
let cont = true;
let total_spawned = 0;
const start_time = Date.now();
let start_time: number;
let complete_timeout = Infinity;
self.onmessage = (evt) => {
cont = false;
complete_timeout = evt.data;
};
const updateStatusInterval = setInterval(() => {
if (config.scriptPattern == "noop") {
const n = 10000;
const res = await fetch(
config.server +
"/api/w/" +
config.workspace_id +
`/jobs/add_noop_jobs/${n}`,
{
method: "POST",
headers: { ["Authorization"]: "Bearer " + config.token },
}
);
const uuids = await res.json();
outstanding.push(...uuids);
total_spawned += n;
self.postMessage({ type: "jobs_sent", jobs_sent: total_spawned });
}, 100);
cont = false;
self.onmessage = (evt) => {
cont = false;
complete_timeout = evt.data;
start_time = Date.now();
};
} else {
start_time = Date.now();
while (cont) {
const queue_length = (
self.onmessage = (evt) => {
cont = false;
complete_timeout = evt.data;
};
const updateStatusInterval = setInterval(() => {
self.postMessage({ type: "jobs_sent", jobs_sent: total_spawned });
}, 100);
while (cont) {
const queue_length = (
await (
await fetch(
config.server + "/api/w/" + config.workspace_id + "/jobs/queue/count",
{ headers: { ["Authorization"]: "Bearer " + config.token } }
)
).json()
).database_length;
if (queue_length > 2500) {
console.log(
`queue length: ${queue_length} > 2500. waiting... `
);
await sleep(0.5);
continue;
}
if (
(total_spawned * 1000) / (Date.now() - start_time) >
config.per_worker_throughput
) {
console.log("at maximum throughput. waiting...");
await sleep(0.1);
continue;
}
total_spawned++;
if (total_spawned > config.max_per_worker) {
break;
}
let uuid: string;
if (config.custom) {
await evaluate(config.custom);
continue;
} else if (config.useFlows) {
let payload: api.FlowPreview;
if (config.flowPattern == "branchone") {
payload = {
path: "branchone",
args: {},
value: {
modules: [
{
id: "a",
value: {
input_transforms: {},
language: api.RawScript.language.DENO,
type: "rawscript",
content:
'export function main(){ return Deno.env.get("WM_FLOW_JOB_ID"); }',
},
},
{
id: "b",
value: {
type: "branchone",
branches: [],
default: [
{
id: "c",
value: {
input_transforms: {
x: {
type: "javascript",
expr: "results.a",
},
},
language: api.RawScript.language.DENO,
type: "rawscript",
content: "export function main(x: string){ return x; }",
},
},
],
},
},
],
},
};
} else if (config.flowPattern == "branchallparrallel") {
payload = {
path: "branchall",
args: {},
value: {
modules: [
{
id: "a",
value: {
input_transforms: {},
language: api.RawScript.language.DENO,
type: "rawscript",
content:
'export function main(){ return Deno.env.get("WM_FLOW_JOB_ID"); }',
},
},
{
id: "b",
value: {
type: "branchall",
parallel: true,
branches: [
{
modules: [
{
id: "c",
value: {
input_transforms: {
x: {
type: "javascript",
expr: "results.a",
},
},
language: api.RawScript.language.DENO,
type: "rawscript",
content:
"export function main(x: string){ return x; }",
},
},
],
},
{
modules: [
{
id: "d",
value: {
input_transforms: {
x: {
type: "javascript",
expr: "results.a",
},
},
language: api.RawScript.language.DENO,
type: "rawscript",
content:
"export function main(x: string){ return x; }",
},
},
],
},
],
},
},
],
},
};
} else {
payload = {
path: "2steps",
args: {},
value: {
modules: [
{
id: "a",
value: {
input_transforms: {},
language: api.RawScript.language.DENO,
type: "rawscript",
content:
'export function main(){ return Deno.env.get("WM_JOB_ID"); }',
},
},
{
id: "b",
value: {
input_transforms: {},
language: api.RawScript.language.DENO,
type: "rawscript",
content:
'export function main(){ return Deno.env.get("WM_JOB_ID"); }',
},
},
],
},
};
}
uuid = await windmill.JobService.runFlowPreview({
workspace: config.workspace_id,
requestBody: payload,
});
} else {
let payload: api.Preview;
if (config.scriptPattern == "httpversion") {
payload = {
path: "httpversion",
kind: "http",
args: {
url: "http://localhost:8000/api/version",
},
};
} else if (config.scriptPattern == "httpslow") {
payload = {
path: "httpversion",
kind: "http",
args: {
url: "https://hub.dummyapis.com/delay?seconds=10",
},
};
} else if (config.scriptPattern == "noop") {
payload = {
path: "noop",
kind: "noop",
args: {},
};
} else if (config.scriptPattern == "identity") {
payload = {
path: "identity",
kind: "identity",
args: {
identity: "itsme",
},
};
} else if (config.scriptPattern == "postgresql") {
payload = {
path: "postgresql",
language: "postgresql",
args: {
query: "SELECT email FROM usr",
database_url:
"postgres://postgres:changeme@localhost:5432/windmill",
},
};
} else {
payload = {
path: "denosimple",
language: api.Preview.language.DENO,
content:
'export function main(){ return Deno.env.get("WM_JOB_ID"); }',
args: {},
};
}
try {
uuid = await windmill.JobService.runScriptPreview({
workspace: config.workspace_id,
requestBody: payload,
});
} catch (e) {
console.error("error running script: " + e.body);
Deno.exit(1);
}
}
if (!config.continous) outstanding.push(uuid);
}
clearInterval(updateStatusInterval);
}
const end_time = Date.now() + complete_timeout;
let incorrect_results = 0;
const enc = (s: string) => new TextEncoder().encode(s);
async function getQueueCount() {
return (
await (
await fetch(
config.server + "/api/w/" + config.workspace_id + "/jobs/queue/count",
@@ -62,221 +338,61 @@ while (cont) {
)
).json()
).database_length;
if (queue_length > 2500) {
console.log(
`queue length: ${queue_length} > 2500. waiting... `
);
await sleep(0.5);
continue;
}
if (
(total_spawned * 1000) / (Date.now() - start_time) >
config.per_worker_throughput
) {
console.log("at maximum throughput. waiting...");
await sleep(0.1);
continue;
}
total_spawned++;
if (total_spawned > config.max_per_worker) {
break;
}
let uuid: string;
if (config.custom) {
await evaluate(config.custom);
continue;
} else if (config.useFlows) {
let payload: api.FlowPreview;
if (config.flowPattern == "branchone") {
payload = {
path: "branchone",
args: {},
value: {
modules: [
{
id: "a",
value: {
input_transforms: {},
language: api.RawScript.language.DENO,
type: "rawscript",
content:
'export function main(){ return Deno.env.get("WM_FLOW_JOB_ID"); }',
},
},
{
id: "b",
value: {
type: "branchone",
branches: [],
default: [
{
id: "c",
value: {
input_transforms: {
x: {
type: "javascript",
expr: "results.a",
},
},
language: api.RawScript.language.DENO,
type: "rawscript",
content: "export function main(x: string){ return x; }",
},
},
],
},
},
],
},
};
} else {
payload = {
path: "2steps",
args: {},
value: {
modules: [
{
id: "a",
value: {
input_transforms: {},
language: api.RawScript.language.DENO,
type: "rawscript",
content:
'export function main(){ return Deno.env.get("WM_JOB_ID"); }',
},
},
{
id: "b",
value: {
input_transforms: {},
language: api.RawScript.language.DENO,
type: "rawscript",
content:
'export function main(){ return Deno.env.get("WM_JOB_ID"); }',
},
},
],
},
};
}
uuid = await windmill.JobService.runFlowPreview({
workspace: config.workspace_id,
requestBody: payload,
});
} else {
let payload: api.Preview;
if (config.scriptPattern == "httpversion") {
payload = {
path: "httpversion",
kind: "http",
args: {
url: "http://localhost:8000/api/version",
},
};
}
if (config.scriptPattern == "httpslow") {
payload = {
path: "httpversion",
kind: "http",
args: {
url: "https://hub.dummyapis.com/delay?seconds=10",
},
};
} else if (config.scriptPattern == "identity") {
payload = {
path: "identity",
kind: "identity",
args: {
identity: "itsme",
},
};
} else if (config.scriptPattern == "postgresql") {
payload = {
path: "postgresql",
language: "postgresql",
args: {
query: "SELECT email FROM usr",
database_url: "postgres://postgres:changeme@localhost:5432/windmill",
},
};
} else {
payload = {
path: "denosimple",
language: api.Preview.language.DENO,
content: 'export function main(){ return Deno.env.get("WM_JOB_ID"); }',
args: {},
};
}
uuid = await windmill.JobService.runScriptPreview({
workspace: config.workspace_id,
requestBody: payload,
});
}
if (!config.continous) outstanding.push(uuid);
}
clearInterval(updateStatusInterval);
const end_time = Date.now() + complete_timeout;
let incorrect_results = 0;
const enc = (s: string) => new TextEncoder().encode(s);
while (outstanding.length > 0 && Date.now() < end_time) {
const uuid = outstanding.shift()!;
let r: Job;
try {
r = await windmill.JobService.getJob({
workspace: config.workspace_id,
id: uuid,
});
} catch (e) {
console.log("job not found: " + uuid + " " + e.message);
continue;
if (config.scriptPattern == "noop") {
let queue_length = await getQueueCount();
while (queue_length > 0 && Date.now() < end_time) {
await Deno.stdout.write(enc(`queue length: ${queue_length}\r`));
queue_length = await getQueueCount();
}
if (r.type == "QueuedJob") {
outstanding.push(uuid);
} else {
while (outstanding.length > 0 && Date.now() < end_time) {
await Deno.stdout.write(
enc(
`uuid: ${uuid}, queue length: ${
(
await (
await fetch(
config.server +
"/api/w/" +
config.workspace_id +
"/jobs/queue/count",
{ headers: { ["Authorization"]: "Bearer " + config.token } }
)
).json()
).database_length
} \r`
)
enc("\rwaiting for jobs to complete: " + outstanding.length + "\n")
);
} else {
r = r as api.CompletedJob;
const uuid = outstanding.shift()!;
let r: Job;
try {
if (
!["httpversion", "identity", "httpslow"].includes(
config.scriptPattern
) &&
r.result != uuid
) {
console.log(
"job did not return correct UUID: " +
r.result +
" != " +
uuid +
"job: \n" +
JSON.stringify(r, null, 2)
);
incorrect_results++;
} else {
// console.log(r.result);
}
r = await windmill.JobService.getJob({
workspace: config.workspace_id,
id: uuid,
});
} catch (e) {
console.log("error during wait: ", e);
console.log("job not found: " + uuid + " " + e.message);
continue;
}
if (r.type == "QueuedJob") {
outstanding.push(uuid);
await Deno.stdout.write(
enc(`uuid: ${uuid}, queue length: ${await getQueueCount()}\r`)
);
} else {
r = r as api.CompletedJob;
try {
if (
!["httpversion", "identity", "httpslow", "noop"].includes(
config.scriptPattern
) &&
r.result != uuid
) {
console.log(
"job did not return correct UUID: " +
r.result +
" != " +
uuid +
"job: \n" +
JSON.stringify(r, null, 2)
);
incorrect_results++;
} else {
// console.log(r.result);
}
} catch (e) {
console.log("error during wait: ", e);
outstanding.push(uuid);
}
}
}
}
@@ -285,4 +401,5 @@ self.postMessage({
type: "zombie_jobs",
zombie_jobs: outstanding.length,
incorrect_results,
jobs_sent: total_spawned,
});
frontend/package.json
+6 -6
View File
@@ -1,5 +1,5 @@
{
"name": "windmill",
"name": "windmill-components",
"version": "1.122.0",
"scripts": {
"dev": "vite dev",
@@ -246,9 +246,9 @@
"types": "./package/components/apps/editor/inlineScriptsPanel/utils.d.ts",
"default": "./package/components/apps/editor/inlineScriptsPanel/utils.js"
},
"./gen/OpenAPI": {
"types": "./package/gen/OpenAPI.d.ts",
"default": "./package/gen/OpenAPI.js"
"./gen/core/OpenAPI": {
"types": "./package/gen/core/OpenAPI.d.ts",
"default": "./package/gen/core/OpenAPI.js"
}
},
"files": [
@@ -343,8 +343,8 @@
"components/apps/editor/inlineScriptsPanel/utils": [
"./package/components/apps/editor/inlineScriptsPanel/utils.d.ts"
],
"gen/OpenAPI": [
"./package/gen/OpenAPI.d.ts"
"gen/core/OpenAPI": [
"./package/gen/core/OpenAPI.d.ts"
]
}
}
frontend/src/lib/components/AddUser.svelte
+28 -2
View File
@@ -1,10 +1,12 @@
<script lang="ts">
import { createEventDispatcher } from 'svelte'
import { workspaceStore } from '$lib/stores'
import { WorkspaceService } from '$lib/gen'
import { globalEmailInvite, superadmin, workspaceStore } from '$lib/stores'
import { UserService, WorkspaceService } from '$lib/gen'
import { Button, ToggleButton, ToggleButtonGroup } from './common'
import Tooltip from './Tooltip.svelte'
import { sendUserToast } from '$lib/toast'
import { isCloudHosted } from '$lib/cloud'
import { goto } from '$app/navigation'
const dispatch = createEventDispatcher()
@@ -30,6 +32,30 @@
}
})
sendUserToast(`Added ${email}`)
if (!(await UserService.existsEmail({ email }))) {
let isSuperadmin = $superadmin
if (!isCloudHosted()) {
sendUserToast(
`User ${email} is not registered yet on the instance. ${
!isSuperadmin
? `If not using SSO, ask an administrator to add ${email} to the instance`
: ''
}`,
true,
isSuperadmin
? [
{
label: 'Add user to the instance',
callback: () => {
$globalEmailInvite = email
goto('#superadmin-settings')
}
}
]
: []
)
}
}
dispatch('new')
}
frontend/src/lib/components/InviteGlobalUser.svelte
+39 -29
View File
@@ -4,54 +4,64 @@
import { UserService } from '$lib/gen'
import { Button } from './common'
import Toggle from './Toggle.svelte'
import { generateRandomString } from '$lib/utils'
import { globalEmailInvite } from '$lib/stores'
const dispatch = createEventDispatcher()
let email: string
let is_super_admin = false
let password: string
let password: string = generateRandomString(10)
let name: string | undefined
let company: string | undefined
function handleKeyUp(event: KeyboardEvent) {
const key = event.key
if (key === 'Enter') {
event.preventDefault()
addUser()
}
}
async function addUser() {
await UserService.createUserGlobally({
requestBody: {
email,
email: $globalEmailInvite,
password,
super_admin: is_super_admin,
name,
company
}
})
sendUserToast(`Added ${email}`)
sendUserToast(`Added ${$globalEmailInvite}`)
$globalEmailInvite = ''
password = generateRandomString(10)
dispatch('new')
}
</script>
<div class="flex flex-row space-x-1">
<input type="email" on:keyup={handleKeyUp} placeholder="email" bind:value={email} />
<div class="flex flex-row gap-2 mb-2 items-end">
<label class="block shrink min-w-0">
<span class="text-gray-700 text-sm">Email</span>
<input type="email" placeholder="email" bind:value={$globalEmailInvite} />
</label>
<label class="block shrink min-w-0">
<span class="text-gray-700 text-sm">Password</span>
<input bind:value={password} />
</label>
<Toggle class="mx-2" bind:checked={is_super_admin} options={{ right: 'superadmin' }} />
<input on:keyup={handleKeyUp} type="password" placeholder="password" bind:value={password} />
<input type="text" on:keyup={handleKeyUp} placeholder="name" bind:value={name} />
<input type="text" on:keyup={handleKeyUp} placeholder="company" bind:value={company} />
<Button
variant="contained"
color="blue"
size="sm"
btnClasses="!ml-4 !w-40"
on:click={addUser}
disabled={email == undefined || password == undefined}
>
Add
</Button>
<Toggle class="mx-2" bind:checked={is_super_admin} options={{ right: 'Superadmin' }} />
<div class="flex flex-row-reverse grow">
<div class="flex">
<Button
variant="contained"
color="dark"
size="sm"
on:click={addUser}
disabled={$globalEmailInvite == '' || password == undefined}
>
Add user to instance
</Button>
</div>
</div>
</div>
<div class="flex gap-2 items-end">
<div>
<input type="text" placeholder="name (optional)" bind:value={name} />
</div>
<div>
<input type="text" placeholder="company (optional)" bind:value={company} />
</div>
<div class="text-xs text-gray-600 grow text-right"> Email will be sent if SMTP configured </div>
</div>
frontend/src/lib/components/InviteUser.svelte
+28 -2
View File
@@ -1,10 +1,12 @@
<script lang="ts">
import { sendUserToast } from '$lib/toast'
import { createEventDispatcher } from 'svelte'
import { workspaceStore } from '$lib/stores'
import { WorkspaceService } from '$lib/gen'
import { globalEmailInvite, superadmin, workspaceStore } from '$lib/stores'
import { UserService, WorkspaceService } from '$lib/gen'
import { Button, ToggleButton, ToggleButtonGroup } from './common'
import Tooltip from './Tooltip.svelte'
import { isCloudHosted } from '$lib/cloud'
import { goto } from '$app/navigation'
const dispatch = createEventDispatcher()
@@ -28,6 +30,30 @@
}
})
sendUserToast(`Invited ${email}`)
if (!(await UserService.existsEmail({ email }))) {
let isSuperadmin = $superadmin
if (!isCloudHosted()) {
sendUserToast(
`User ${email} is not registered yet on the instance. ${
!isSuperadmin
? `If not using SSO, ask an administrator to add ${email} to the instance`
: ''
}`,
true,
isSuperadmin
? [
{
label: 'Add user to the instance',
callback: () => {
$globalEmailInvite = email
goto('#superadmin-settings')
}
}
]
: []
)
}
}
dispatch('new')
}
frontend/src/lib/components/SuperadminSettings.svelte
+4 -3
View File
@@ -56,8 +56,9 @@
<PageHeader title="All global users" primary={false} />
<div class="pb-1" />
<InviteGlobalUser on:new={listUsers} />
<div class="p-2 border mb-4">
<InviteGlobalUser on:new={listUsers} />
</div>
<div class="pb-1" />
<input placeholder="Search users" bind:value={filter} class="input mt-1" />
@@ -96,7 +97,7 @@
})
sendUserToast('User updated')
listUsers()
}}>{super_admin ? 'non-superadmin' : 'superadmin'}</button
}}>make {super_admin ? 'non-superadmin' : 'superadmin'}</button
>
|
<button
frontend/src/lib/stores.ts
+1
View File
@@ -18,6 +18,7 @@ export interface UserExt {
const persistedWorkspace = BROWSER && localStorage.getItem('workspace')
export const globalEmailInvite = writable<string>('')
export const awarenessStore = writable<Record<string, string>>(undefined)
export const enterpriseLicense = writable<string | undefined>(undefined)
export const workerTags = writable<string[] | undefined>(undefined)
frontend/src/lib/utils.ts
+2 -2
View File
@@ -468,11 +468,11 @@ export function sortObject<T>(o: T & object): T {
}, {}) as T
}
export function generateRandomString(): string {
export function generateRandomString(len: number = 24): string {
let chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'
let result = ''
for (let i = 0; i < 24; i++) {
for (let i = 0; i < len; i++) {
result += chars.charAt(Math.floor(Math.random() * chars.length))
}
frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte
+5 -5
View File
@@ -104,11 +104,11 @@
if (allWorkspaces?.length == 1) {
$workspaceStore = allWorkspaces[0].id
goto('/')
return
}
if (rd?.startsWith('/user/workspaces')) {
goto(rd ?? '/')
} else if (rd?.startsWith('/user/workspaces')) {
goto(rd)
} else if (rd == '/#user-settings') {
goto(`/user/workspaces#user-settings`)
} else {
goto(`/user/workspaces${rd ? `?rd=${encodeURIComponent(rd)}` : ''}`)
}
@@ -117,7 +117,7 @@
async function loadLogins() {
logins = await OauthService.listOAuthLogins()
showPassword = logins.length == 0
showPassword = logins.length == 0 || (email != undefined && email.length > 0)
}
onMount(async () => {
react-sdk/package-lock.json
Generated
+2 -2
View File
@@ -29,8 +29,8 @@
}
},
"../frontend": {
"name": "windmill",
"version": "1.121.0",
"name": "windmill-components",
"version": "1.122.0",
"license": "AGPL-3.0",
"dependencies": {
"@aws-crypto/sha256-js": "^4.0.0",
react-sdk/src/main.tsx
-1
View File
@@ -6,7 +6,6 @@ import { workspaceStore, userStore } from "windmill-components/stores";
import React from "react";
import ReactDOM from "react-dom/client";
import { writable } from "svelte/store";
// To be able to test flows and run scripts, or get the app
// you will need to:
// 1. have a backend