starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-04 16:03:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into alp/slack_interactive_approvals

Browse Source
This commit is contained in:
Alexander Petric
2024-12-11 10:35:01 +01:00
parent 410fb25309 343cd02ad5
commit 6b3881bd00
117 changed files with 4456 additions and 2424 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/docker-image.yml
+3 -3
View File
@@ -76,7 +76,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=embedding,parquet,openidconnect,jemalloc,deno_core
features=embedding,parquet,openidconnect,jemalloc,deno_core,dind
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }}
${{ steps.meta-public.outputs.tags }}
@@ -138,7 +138,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,otel,dind
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }}
${{ steps.meta-ee-public.outputs.tags }}
@@ -200,7 +200,7 @@ jobs:
platforms: linux/amd64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,otel,dind
PYTHON_IMAGE=python:3.12.2-slim-bookworm
tags: |
${{ steps.meta-ee-public-py312.outputs.tags }}
CHANGELOG.md
+37
View File
@@ -1,5 +1,42 @@
# Changelog
## [1.437.1](https://github.com/windmill-labs/windmill/compare/v1.437.0...v1.437.1) (2024-12-10)
### Bug Fixes
* **frontend:** fix newly deployed apps with frontend scripts ([bcd2cfe](https://github.com/windmill-labs/windmill/commit/bcd2cfe674038042f08f22d05286596b3442caf9))
## [1.437.0](https://github.com/windmill-labs/windmill/compare/v1.436.0...v1.437.0) (2024-12-10)
### Features
* add ctx.name to apps [#4885](https://github.com/windmill-labs/windmill/issues/4885) ([2387503](https://github.com/windmill-labs/windmill/commit/23875033dd19d6d585f2433fa7b18a703175e329))
### Bug Fixes
* **parsers:** more robust pwsh param parsing ([#4884](https://github.com/windmill-labs/windmill/issues/4884)) ([7014389](https://github.com/windmill-labs/windmill/commit/7014389d677fba563bec02e08815452b195ecd0a))
* **python:** do not follow symlinks when copying recursively in overlapping sites-package ([c173e46](https://github.com/windmill-labs/windmill/commit/c173e46a724b50d27416bbc169bf2a57fe68808c))
## [1.436.0](https://github.com/windmill-labs/windmill/compare/v1.435.2...v1.436.0) (2024-12-09)
### Features
* add db storage for app inline scripts ([#4837](https://github.com/windmill-labs/windmill/issues/4837)) ([2bc4934](https://github.com/windmill-labs/windmill/commit/2bc4934c4f38ca2631eb99140d89a0b3d3b3a2cf))
* add otlp support ([#4869](https://github.com/windmill-labs/windmill/issues/4869)) ([6d04744](https://github.com/windmill-labs/windmill/commit/6d047449e200d51f060d4107ff5ad0e10af99a66))
* **cache:** refurbish fs backed cache ([#4863](https://github.com/windmill-labs/windmill/issues/4863)) ([3c4408e](https://github.com/windmill-labs/windmill/commit/3c4408e3dbf8a37c0b862805b1991af3d39ac054))
* docker runtime ([2c5d07f](https://github.com/windmill-labs/windmill/commit/2c5d07f3d9069b21b877ea2d085f0a851b1975c1))
* **frontend:** render new job kinds ([#4864](https://github.com/windmill-labs/windmill/issues/4864)) ([691ef64](https://github.com/windmill-labs/windmill/commit/691ef6468823c99ab129c53ae699de1a91f77c97))
### Bug Fixes
* fix `flow_node` uniqueness ([#4850](https://github.com/windmill-labs/windmill/issues/4850)) ([667167a](https://github.com/windmill-labs/windmill/commit/667167a022ed31cb67a6c189cc5bd54b09473f4f))
* handle flow & workspace renames for `flow_node` ([#4861](https://github.com/windmill-labs/windmill/issues/4861)) ([f175158](https://github.com/windmill-labs/windmill/commit/f175158b9ffcc0d9c35938e61dfe3ce56d3b64c2))
## [1.435.2](https://github.com/windmill-labs/windmill/compare/v1.435.1...v1.435.2) (2024-12-05)
Dockerfile
+2 -2
View File
@@ -175,9 +175,9 @@ RUN /usr/local/bin/python3 -m pip install pip-tools
COPY --from=builder /frontend/build /static_frontend
COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
COPY --from=denoland/deno:2.0.4 --chmod=755 /usr/bin/deno /usr/bin/deno
COPY --from=denoland/deno:2.1.2 --chmod=755 /usr/bin/deno /usr/bin/deno
COPY --from=oven/bun:1.1.34 /usr/local/bin/bun /usr/bin/bun
COPY --from=oven/bun:1.1.38 /usr/local/bin/bun /usr/bin/bun
COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php
COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer
backend/.sqlx/query-0ad36c1598ff4ece0c325eaeb9a9177a87e1accd192402e21db5ae09c3498ab0.json
Generated
+2 -1
View File
@@ -44,7 +44,8 @@
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode"
"flownode",
"appscript"
]
}
}
backend/.sqlx/query-0c6c80746733be8f561ab0b631854799f5e8122adaf35465cb16c3dc795bdc3b.json
Generated
+26
View File
@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO app_script (app, hash, lock, code, code_sha256)\n VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (hash) DO UPDATE SET app = EXCLUDED.app -- trivial update to return the id\n RETURNING id\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Int8",
"Bpchar",
"Text",
"Text",
"Bpchar"
]
},
"nullable": [
false
]
},
"hash": "0c6c80746733be8f561ab0b631854799f5e8122adaf35465cb16c3dc795bdc3b"
}
backend/.sqlx/query-1bae415f9440cc1334f24ce3009242cf3a6287e7b4548c7f01ad888230c27013.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT format('rawscript/%s', code_sha256) as \"path!: String\"\n FROM app_script WHERE id = $1 LIMIT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path!: String",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
null
]
},
"hash": "1bae415f9440cc1334f24ce3009242cf3a6287e7b4548c7f01ad888230c27013"
}
backend/.sqlx/query-337f31c2172194cd594042c561998a03f751b246c40daf056fced0fd91f6dd73.json
Generated
+2 -1
View File
@@ -34,7 +34,8 @@
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode"
"flownode",
"appscript"
]
}
}
backend/.sqlx/query-83cc9e432aea1450f79e9fce04eca0e75f30faa8c39ad4754bd568ae6f210806.json → backend/.sqlx/query-5af51d5bf7614274ade044120045893edb73694601544fea7cf20f45dd25a88d.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO flow_node (path, workspace_id, hash_v2, lock, code, flow)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (hash_v2) DO UPDATE SET path = EXCLUDED.path -- trivial update to return the id\n RETURNING id\n ",
"query": "\n INSERT INTO flow_node (path, workspace_id, hash_v2, lock, code, flow)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (path, workspace_id, hash_v2) DO UPDATE SET path = EXCLUDED.path -- trivial update to return the id\n RETURNING id\n ",
"describe": {
"columns": [
{
@@ -23,5 +23,5 @@
false
]
},
"hash": "83cc9e432aea1450f79e9fce04eca0e75f30faa8c39ad4754bd568ae6f210806"
"hash": "5af51d5bf7614274ade044120045893edb73694601544fea7cf20f45dd25a88d"
}
backend/.sqlx/query-5d7081a9ba0d702f63ed9d44be5ffe0ba043565790865c6a6f52fab6ce340d2c.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT policy as \"policy: sqlx::types::Json<Box<RawValue>>\"\n FROM app WHERE app.path = $1 AND app.workspace_id = $2 LIMIT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "policy: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "5d7081a9ba0d702f63ed9d44be5ffe0ba043565790865c6a6f52fab6ce340d2c"
}
backend/.sqlx/query-a51007ca7b509b92faa6fe7aa59fa738594a2b7eaab8e38f3ca30e548f9a49a7.json → backend/.sqlx/query-6a72df33cf12824c54b29dbf011f2390af9d02efc7112a32a84e1a1247a95973.json
Generated
+8 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT worker, worker_instance, vcpus, memory, ping_at, started_at FROM worker_ping WHERE ping_at > now() - interval '30 days' ORDER BY started_at",
"query": "SELECT worker, worker_instance, vcpus, memory, ping_at, started_at, worker_group FROM worker_ping WHERE ping_at > now() - interval '30 days' ORDER BY started_at",
"describe": {
"columns": [
{
@@ -32,6 +32,11 @@
"ordinal": 5,
"name": "started_at",
"type_info": "Timestamptz"
},
{
"ordinal": 6,
"name": "worker_group",
"type_info": "Varchar"
}
],
"parameters": {
@@ -43,8 +48,9 @@
true,
true,
false,
false,
false
]
},
"hash": "a51007ca7b509b92faa6fe7aa59fa738594a2b7eaab8e38f3ca30e548f9a49a7"
"hash": "6a72df33cf12824c54b29dbf011f2390af9d02efc7112a32a84e1a1247a95973"
}
backend/.sqlx/query-3d363466d79075df3f74f946eff43ca89faefca3bcdf2c533425ca3868b0369a.json → backend/.sqlx/query-6aabe704395c9be30c86d15a5d22f3509b4fcea56227b019588837132b64d58b.json
Generated
+14 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT * FROM usr where username = $1 AND workspace_id = $2",
"query": "SELECT usr.*, password.super_admin, password.name FROM usr LEFT JOIN password ON usr.email = password.email Where usr.username = $1 AND workspace_id = $2\n ",
"describe": {
"columns": [
{
@@ -42,6 +42,16 @@
"ordinal": 7,
"name": "role",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "super_admin",
"type_info": "Bool"
},
{
"ordinal": 9,
"name": "name",
"type_info": "Varchar"
}
],
"parameters": {
@@ -58,8 +68,10 @@
false,
false,
false,
true,
false,
true
]
},
"hash": "3d363466d79075df3f74f946eff43ca89faefca3bcdf2c533425ca3868b0369a"
"hash": "6aabe704395c9be30c86d15a5d22f3509b4fcea56227b019588837132b64d58b"
}
backend/.sqlx/query-6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561.json
Generated
+2 -1
View File
@@ -28,7 +28,8 @@
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode"
"flownode",
"appscript"
]
}
}
backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json
Generated
+2 -1
View File
@@ -28,7 +28,8 @@
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode"
"flownode",
"appscript"
]
}
}
backend/.sqlx/query-dc32e9e81eab9cda88875020acc414271b96d251bcf54c153c45370d917eea9d.json
Generated
-27
View File
@@ -1,27 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n WITH existing AS (\n SELECT id FROM flow_node\n WHERE hash = $1 AND path = $2 AND workspace_id = $3\n AND (code IS NOT DISTINCT FROM $4)\n AND (lock IS NOT DISTINCT FROM $5)\n AND (flow IS NOT DISTINCT FROM $6)\n LIMIT 1\n ),\n inserted AS (\n INSERT INTO flow_node (hash, path, workspace_id, code, lock, flow)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT DO NOTHING\n RETURNING id\n )\n SELECT id FROM existing\n UNION ALL\n SELECT id FROM inserted\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Int8",
"Text",
"Text",
"Text",
"Text",
"Jsonb"
]
},
"nullable": [
null
]
},
"hash": "dc32e9e81eab9cda88875020acc414271b96d251bcf54c153c45370d917eea9d"
}
backend/.sqlx/query-29fce7e56bb4a874b4c1e3a6454273203aab7344abd03df4d86e4c5f5b9a394a.json → backend/.sqlx/query-ea9bbb972217bab4d7e8f4c08e331899161e80c239d56eb657d73bbf4272939b.json
Generated
+8 -2
View File
@@ -1,10 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "SELECT value FROM app_version WHERE id = $1",
"query": "SELECT app_id, value FROM app_version WHERE id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "app_id",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "value",
"type_info": "Json"
}
@@ -15,8 +20,9 @@
]
},
"nullable": [
false,
false
]
},
"hash": "29fce7e56bb4a874b4c1e3a6454273203aab7344abd03df4d86e4c5f5b9a394a"
"hash": "ea9bbb972217bab4d7e8f4c08e331899161e80c239d56eb657d73bbf4272939b"
}
backend/.sqlx/query-ee16199b4af456198fae062e948914fca6fc0a8787e4f8d520766b1c89f23602.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO app_version_lite (id, value) VALUES ($1, $2)\n ON CONFLICT (id) DO UPDATE SET value = EXCLUDED.value",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8",
"Jsonb"
]
},
"nullable": []
},
"hash": "ee16199b4af456198fae062e948914fca6fc0a8787e4f8d520766b1c89f23602"
}
backend/.sqlx/query-f9fc0084fe086ef80005bb64a8bb6b493e53583017c18e2ab44f44125c52d548.json
Generated
+2 -1
View File
@@ -48,7 +48,8 @@
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode"
"flownode",
"appscript"
]
}
}
backend/.sqlx/query-ffa86babfcab107caffb8dda31a66b142fa628b8983b966357deb3d5e0a1df3c.json
Generated
+28
View File
@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "SELECT lock, code FROM app_script WHERE id = $1 LIMIT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "lock",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "code",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
true,
false
]
},
"hash": "ffa86babfcab107caffb8dda31a66b142fa628b8983b966357deb3d5e0a1df3c"
}
backend/Cargo.lock
Generated
+482 -178
View File
File diff suppressed because it is too large Load Diff
backend/Cargo.toml
+26 -13
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.435.2"
version = "1.437.1"
authors.workspace = true
edition.workspace = true
@@ -29,7 +29,7 @@ members = [
]
[workspace.package]
version = "1.435.2"
version = "1.437.1"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -63,6 +63,8 @@ tantivy = ["dep:windmill-indexer", "windmill-api/tantivy"]
sqlx = ["windmill-worker/sqlx"]
deno_core = ["windmill-worker/deno_core", "dep:deno_core"]
kafka = ["windmill-api/kafka"]
otel = ["windmill-common/otel", "windmill-worker/otel"]
dind = ["windmill-worker/dind"]
[dependencies]
anyhow.workspace = true
@@ -95,7 +97,6 @@ deno_core = { workspace = true, optional = true }
object_store = { workspace = true, optional = true }
quote.workspace = true
[target.'cfg(not(target_env = "msvc"))'.dependencies]
tikv-jemallocator = { optional = true, workspace = true }
tikv-jemalloc-sys = { optional = true, workspace = true }
@@ -137,7 +138,7 @@ windmill-api-client = { path = "./windmill-api-client" }
axum = { version = "^0.7", features = ["multipart"] }
headers = "^0"
hyper = { version = "^1", features = ["full"] }
tokio = { version = "^1", features = ["full", "tracing"] }
tokio = { version = "^1.42.0", features = ["full", "tracing"] }
tower = "^0"
tower-http = { version = "^0.6", features = ["trace", "cors"] }
tower-cookies = "^0.10"
@@ -182,15 +183,16 @@ itertools = "^0"
regex = "^1"
semver = "^1"
deno_fetch = "0.195.0"
deno_tls = "0.158.0"
deno_console = "0.171.0"
deno_url = "0.171.0"
deno_webidl = "0.171.0"
deno_web = "0.202.0"
deno_net = "0.163.0"
deno_core = "0.311.0"
deno_ast = { version = "=0.42.2", features = ["transpiling"] }
deno_fetch = "0.203.0"
deno_tls = "0.166.0"
deno_console = "0.179.0"
deno_url = "0.179.0"
deno_webidl = "0.179.0"
deno_web = "0.210.0"
deno_net = "0.171.0"
deno_core = "0.321.0"
deno_ast = { version = "=0.43.3", features = ["transpiling"] }
deno_permissions = "0.39.0"
swc_common = "=0.37.5"
swc_ecma_parser = "=0.149.1"
@@ -279,6 +281,17 @@ tar = "^0"
http = "^1"
async-stream = "^0"
opentelemetry = "0.27.0"
tracing-opentelemetry = "0.28.0"
opentelemetry_sdk = { version = "*", features = ["rt-tokio"] }
opentelemetry-otlp = { version = "0.27.0", features = ["grpc-tonic", "tls"] }
opentelemetry-appender-tracing = "0.27.0"
opentelemetry-semantic-conventions = { version = "*", features = ["semconv_experimental"] }
bollard = "0.18.1"
tonic = { version = "^0", features = ["tls-native-roots"] }
tikv-jemallocator = { version = "0.5" }
tikv-jemalloc-sys = { version = "^0.5" }
tikv-jemalloc-ctl = { version = "^0.5" }
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
8606d98a692d11b09a387c5efbd6b4335c533fd3
a07bc62582c809457f1c945d6cda145770b94d04
backend/migrations/20241202095902_app_script.down.sql
+3
View File
@@ -0,0 +1,3 @@
-- Add down migration script here
DROP TABLE IF EXISTS app_version_lite;
DROP TABLE IF EXISTS app_script;
backend/migrations/20241202095902_app_script.up.sql
+26
View File
@@ -0,0 +1,26 @@
-- Add up migration script here
ALTER TYPE JOB_KIND ADD VALUE IF NOT EXISTS 'appscript';
-- Same as `app_version` but with a "lite" value (w/ `inlineScript.{code,lock}`).
CREATE TABLE app_version_lite (
id BIGSERIAL PRIMARY KEY,
value JSONB,
FOREIGN KEY (id) REFERENCES app_version (id) ON DELETE CASCADE
);
GRANT ALL ON app_version_lite TO windmill_user;
GRANT ALL ON app_version_lite TO windmill_admin;
-- App `inlineScript`.
CREATE TABLE app_script (
id BIGSERIAL PRIMARY KEY,
app BIGSERIAL NOT NULL,
hash CHAR(64) NOT NULL UNIQUE, -- sha256 of `app`, `lock`, `code`.
lock TEXT,
code TEXT NOT NULL,
code_sha256 CHAR(64) NOT NULL, -- used to retrieve the policy.
FOREIGN KEY (app) REFERENCES app (id) ON DELETE CASCADE
);
GRANT ALL ON app_script TO windmill_user;
GRANT ALL ON app_script TO windmill_admin;
backend/migrations/20241206075559_flow_node_unique_2.down.sql
+3
View File
@@ -0,0 +1,3 @@
-- Add down migration script here
ALTER TABLE flow_node DROP CONSTRAINT IF EXISTS flow_node_unique_2;
ALTER TABLE flow_node ADD CONSTRAINT flow_node_hash_v2_key UNIQUE (hash_v2);
backend/migrations/20241206075559_flow_node_unique_2.up.sql
+3
View File
@@ -0,0 +1,3 @@
-- Add up migration script here
ALTER TABLE flow_node ADD CONSTRAINT flow_node_unique_2 UNIQUE (path, workspace_id, hash_v2);
ALTER TABLE flow_node DROP CONSTRAINT IF EXISTS flow_node_hash_v2_key;
backend/migrations/20241210184049_drop_app_lite.down.sql
+1
View File
@@ -0,0 +1 @@
-- Add down migration script here
backend/migrations/20241210184049_drop_app_lite.up.sql
+2
View File
@@ -0,0 +1,2 @@
-- Add up migration script here
delete from app_version_lite;
backend/oauth_connect.json
+8
View File
@@ -172,5 +172,13 @@
"user-library-read"
]
},
"xero": {
"auth_url": "https://login.xero.com/identity/connect/authorize",
"token_url": "https://identity.xero.com/connect/token",
"scopes": [
"offline_access",
"accounting.transactions"
]
},
"snowflake_oauth": {}
}
backend/parsers/windmill-parser-bash/src/lib.rs
+156 -25
View File
@@ -9,7 +9,7 @@ use regex_lite::Regex;
use serde_json::json;
use std::collections::HashMap;
use std::{collections::HashMap, str::FromStr};
use windmill_parser::{Arg, MainArgSignature, Typ};
pub fn parse_bash_sig(code: &str) -> anyhow::Result<MainArgSignature> {
@@ -84,36 +84,94 @@ fn parse_bash_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
Ok(Some(args))
}
enum ParserState {
Normal,
InSingleQuote,
InDoubleQuote,
}
fn split_pwsh_args(code: &str) -> Vec<&str> {
let mut chars = code.char_indices().peekable();
let mut state = ParserState::Normal;
let mut splits = vec![];
let mut last_idx = 0;
while let Some((idx, char)) = chars.next() {
match (&state, char) {
(ParserState::Normal, '\'') => {
state = ParserState::InSingleQuote;
}
(ParserState::Normal, '"') => {
state = ParserState::InDoubleQuote;
}
(ParserState::InSingleQuote, '\'') => {
state = ParserState::Normal;
}
(ParserState::InDoubleQuote, '"') => {
state = ParserState::Normal;
}
(ParserState::Normal, ',') => {
splits.push(&code[last_idx..idx]);
last_idx = idx + 1; // skip the comma
}
_ => {}
}
}
if last_idx < code.len() {
splits.push(&code[last_idx..]);
}
splits
}
fn parse_powershell_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
let param_wrapper = RE_POWERSHELL_PARAM.captures(code);
let mut args = vec![];
if let Some(param_wrapper) = param_wrapper {
let param_wrapper = param_wrapper.get(1).unwrap().as_str();
for cap in RE_POWERSHELL_ARGS.captures_iter(param_wrapper) {
let typ = cap
.get(1)
.map(|x| x.as_str().to_string())
.unwrap_or("string".to_string());
let name = cap.get(2).unwrap().as_str().to_string();
let default = cap
.get(3)
.or(cap.get(4))
.map(|x| json!(x.as_str().to_string()));
let params = split_pwsh_args(param_wrapper);
for param in params {
if let Some(cap) = RE_POWERSHELL_ARGS.captures(param) {
let typ = cap.get(1).map(|x| x.as_str().to_string());
let name = cap.get(2).unwrap().as_str().to_string();
args.push(Arg {
name: name,
typ: match typ.as_str() {
"string" => Typ::Str(None),
"int" | "long" => Typ::Int,
"decimal" | "double" | "single" => Typ::Float,
"datetime" | "DateTime" => Typ::Datetime,
_ => Typ::Str(None),
},
default: default.clone(),
otyp: None,
has_default: default.is_some(),
oidx: None,
});
let mut parsed_typ = if let Some(typ) = typ {
match typ.as_str() {
"string" => Some(Typ::Str(None)),
"int" | "long" => Some(Typ::Int),
"decimal" | "double" | "single" => Some(Typ::Float),
"datetime" | "DateTime" => Some(Typ::Datetime),
_ => None,
}
} else {
None
};
let default = if let Some(x) = cap.get(3) {
Some(json!(x.as_str().to_string()))
} else if let Some(x) = cap.get(4) {
if parsed_typ.is_none() {
if x.as_str().parse::<i64>().is_ok() {
parsed_typ = Some(Typ::Int);
} else if x.as_str().parse::<f64>().is_ok() {
parsed_typ = Some(Typ::Float);
}
}
serde_json::Number::from_str(x.as_str())
.ok()
.map(serde_json::Value::Number)
} else {
None
};
args.push(Arg {
name: name,
typ: parsed_typ.unwrap_or(Typ::Str(None)),
default: default.clone(),
otyp: None,
has_default: default.is_some(),
oidx: None,
});
}
}
}
Ok(Some(args))
@@ -192,4 +250,77 @@ non_required="${5:-}"
Ok(())
}
#[test]
fn test_parse_powershell_sig() -> anyhow::Result<()> {
let code = r#"param($Msg, [string]$Msg2, $Dflt = "default value, with comma", [int]$Nb = 3 , $Nb2 = 5.0, $Nb3 = 5, $Wahoo = $env:WAHOO)"#;
assert_eq!(
parse_powershell_sig(code)?,
MainArgSignature {
star_args: false,
star_kwargs: false,
args: vec![
Arg {
otyp: None,
name: "Msg".to_string(),
typ: Typ::Str(None),
default: None,
has_default: false,
oidx: None
},
Arg {
otyp: None,
name: "Msg2".to_string(),
typ: Typ::Str(None),
default: None,
has_default: false,
oidx: None
},
Arg {
otyp: None,
name: "Dflt".to_string(),
typ: Typ::Str(None),
default: Some(json!("default value, with comma")),
has_default: true,
oidx: None
},
Arg {
otyp: None,
name: "Nb".to_string(),
typ: Typ::Int,
default: Some(json!(3)),
has_default: true,
oidx: None
},
Arg {
otyp: None,
name: "Nb2".to_string(),
typ: Typ::Float,
default: Some(json!(5.0)),
has_default: true,
oidx: None
},
Arg {
otyp: None,
name: "Nb3".to_string(),
typ: Typ::Int,
default: Some(json!(5)),
has_default: true,
oidx: None
},
Arg {
otyp: None,
name: "Wahoo".to_string(),
typ: Typ::Str(None),
default: None,
has_default: false,
oidx: None
}
],
no_main_func: None,
has_preprocessor: None
}
);
Ok(())
}
}
backend/src/main.rs
+90 -68
View File
@@ -8,7 +8,7 @@
use anyhow::Context;
use monitor::{
reload_delete_logs_periodically_setting, reload_indexer_config,
load_base_url, load_otel, reload_delete_logs_periodically_setting, reload_indexer_config,
reload_timeout_wait_result_setting, send_current_log_file_to_object_store,
send_logs_to_object_store,
};
@@ -37,7 +37,7 @@ use windmill_common::{
EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING,
JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING,
LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING,
OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
OAUTH_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
},
@@ -221,11 +221,66 @@ async fn windmill_main() -> anyhow::Result<()> {
let hostname = hostname();
#[cfg(not(feature = "flamegraph"))]
let _guard = windmill_common::tracing_init::initialize_tracing(&hostname);
let mut enable_standalone_indexer: bool = false;
let mode = std::env::var("MODE")
.map(|x| x.to_lowercase())
.map(|x| {
if &x == "server" {
println!("Binary is in 'server' mode");
Mode::Server
} else if &x == "worker" {
tracing::info!("Binary is in 'worker' mode");
#[cfg(windows)]
{
println!("It is highly recommended to use the agent mode instead on windows (MODE=agent) and to pass a BASE_INTERNAL_URL");
}
Mode::Worker
} else if &x == "agent" {
println!("Binary is in 'agent' mode");
if std::env::var("BASE_INTERNAL_URL").is_err() {
panic!("BASE_INTERNAL_URL is required in agent mode")
}
if std::env::var("JOB_TOKEN").is_err() {
println!("JOB_TOKEN is not passed, hence workers will still need to create permissions for each job and the DATABASE_URL needs to be of a role that can INSERT into the job_perms table")
}
#[cfg(not(feature = "enterprise"))]
{
panic!("Agent mode is only available in the EE, ignoring...");
}
#[cfg(feature = "enterprise")]
Mode::Agent
} else if &x == "indexer" {
tracing::info!("Binary is in 'indexer' mode");
#[cfg(not(feature = "tantivy"))]
{
eprintln!("Cannot start the indexer because tantivy is not included in this binary/image. Make sure you are using the EE image if you want to access the full text search features.");
panic!("Indexer mode requires compiling with the tantivy feature flag.");
}
#[cfg(feature = "tantivy")]
Mode::Indexer
} else if &x == "standalone+search"{
enable_standalone_indexer = true;
println!("Binary is in 'standalone' mode with search enabled");
Mode::Standalone
}
else {
if &x != "standalone" {
eprintln!("mode not recognized, defaulting to standalone: {x}");
} else {
println!("Binary is in 'standalone' mode");
}
Mode::Standalone
}
})
.unwrap_or_else(|_| {
tracing::info!("Mode not specified, defaulting to standalone");
Mode::Standalone
});
#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
tracing::info!("jemalloc enabled");
println!("jemalloc enabled");
#[cfg(feature = "flamegraph")]
let _guard = windmill_common::tracing_init::setup_flamegraph();
@@ -236,13 +291,13 @@ async fn windmill_main() -> anyhow::Result<()> {
"cache" => {
#[cfg(feature = "embedding")]
{
tracing::info!("Caching embedding model...");
println!("Caching embedding model...");
windmill_api::embeddings::ModelInstance::load_model_files().await?;
tracing::info!("Cached embedding model");
println!("Cached embedding model");
}
#[cfg(not(feature = "embedding"))]
{
tracing::warn!("Embeddings are not enabled, ignoring...");
println!("Embeddings are not enabled, ignoring...");
}
cache_hub_scripts(std::env::args().nth(2)).await?;
@@ -256,64 +311,6 @@ async fn windmill_main() -> anyhow::Result<()> {
_ => {}
}
let mut enable_standalone_indexer: bool = false;
let mode = std::env::var("MODE")
.map(|x| x.to_lowercase())
.map(|x| {
if &x == "server" {
tracing::info!("Binary is in 'server' mode");
Mode::Server
} else if &x == "worker" {
tracing::info!("Binary is in 'worker' mode");
#[cfg(windows)]
{
tracing::warn!("It is highly recommended to use the agent mode instead on windows (MODE=agent) and to pass a BASE_INTERNAL_URL");
}
Mode::Worker
} else if &x == "agent" {
tracing::info!("Binary is in 'agent' mode");
if std::env::var("BASE_INTERNAL_URL").is_err() {
panic!("BASE_INTERNAL_URL is required in agent mode")
}
if std::env::var("JOB_TOKEN").is_err() {
tracing::warn!("JOB_TOKEN is not passed, hence workers will still need to create permissions for each job and the DATABASE_URL needs to be of a role that can INSERT into the job_perms table")
}
#[cfg(not(feature = "enterprise"))]
{
panic!("Agent mode is only available in the EE, ignoring...");
}
#[cfg(feature = "enterprise")]
Mode::Agent
} else if &x == "indexer" {
tracing::info!("Binary is in 'indexer' mode");
#[cfg(not(feature = "tantivy"))]
{
tracing::error!("Cannot start the indexer because tantivy is not included in this binary/image. Make sure you are using the EE image if you want to access the full text search features.");
panic!("Indexer mode requires compiling with the tantivy feature flag.");
}
#[cfg(feature = "tantivy")]
Mode::Indexer
} else if &x == "standalone+search"{
enable_standalone_indexer = true;
tracing::info!("Binary is in 'standalone' mode with search enabled");
Mode::Standalone
}
else {
if &x != "standalone" {
tracing::error!("mode not recognized, defaulting to standalone: {x}");
} else {
tracing::info!("Binary is in 'standalone' mode");
}
Mode::Standalone
}
})
.unwrap_or_else(|_| {
tracing::info!("Mode not specified, defaulting to standalone");
Mode::Standalone
});
#[allow(unused_mut)]
let mut num_workers = if mode == Mode::Server || mode == Mode::Indexer {
0
@@ -325,7 +322,7 @@ async fn windmill_main() -> anyhow::Result<()> {
};
if num_workers > 1 {
tracing::warn!(
println!(
"We STRONGLY recommend using at most 1 worker per container, use at your own risks"
);
}
@@ -347,10 +344,26 @@ async fn windmill_main() -> anyhow::Result<()> {
IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))
};
tracing::info!("Connecting to database...");
println!("Connecting to database...");
let db = windmill_common::connect_db(server_mode, indexer_mode).await?;
load_otel(&db).await;
tracing::info!("Database connected");
let environment = load_base_url(&db)
.await
.unwrap_or_else(|_| "local".to_string())
.trim_start_matches("https://")
.trim_start_matches("http://")
.split(".")
.next()
.unwrap_or_else(|| "local")
.to_string();
#[cfg(not(feature = "flamegraph"))]
let _guard = windmill_common::tracing_init::initialize_tracing(&hostname, &mode, &environment);
let num_version = sqlx::query_scalar!("SELECT version()").fetch_one(&db).await;
tracing::info!(
@@ -776,6 +789,15 @@ Windmill Community Edition {GIT_VERSION}
tracing::error!(error = %e, "Could not reload debug metrics setting");
}
},
OTEL_SETTING => {
tracing::info!("OTEL setting changed, restarting");
// we wait a bit randomly to avoid having all servers and workers shutdown at same time
let rd_delay = rand::thread_rng().gen_range(0..4);
tokio::time::sleep(Duration::from_secs(rd_delay)).await;
if let Err(e) = tx.send(()) {
tracing::error!(error = %e, "Could not send killpill");
}
},
REQUEST_SIZE_LIMIT_SETTING => {
if server_mode {
tracing::info!("Request limit size change detected, killing server expecting to be restarted");
backend/src/monitor.rs
+73 -9
View File
@@ -12,7 +12,7 @@ use std::{
use chrono::{NaiveDateTime, Utc};
use futures::{stream::FuturesUnordered, StreamExt};
use serde::de::DeserializeOwned;
use serde::{de::DeserializeOwned, Deserializer};
use sqlx::{Pool, Postgres};
use tokio::{
join,
@@ -40,7 +40,7 @@ use windmill_common::{
EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING,
JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING,
MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING,
PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
},
@@ -58,7 +58,8 @@ use windmill_common::{
},
BASE_URL, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL,
HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED,
MONITOR_LOGS_ON_OBJECT_STORE, SERVICE_LOG_RETENTION_SECS,
MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED,
SERVICE_LOG_RETENTION_SECS,
};
use windmill_queue::cancel_job;
use windmill_worker::{
@@ -199,6 +200,65 @@ pub async fn load_metrics_enabled(db: &DB) -> error::Result<()> {
Ok(())
}
fn empty_string_as_none<'de, D>(deserializer: D) -> Result<Option<String>, D::Error>
where
D: Deserializer<'de>,
{
let option = <Option<String> as serde::Deserialize>::deserialize(deserializer)?;
Ok(option.filter(|s| !s.is_empty()))
}
#[derive(serde::Deserialize)]
struct OtelSetting {
metrics_enabled: Option<bool>,
logs_enabled: Option<bool>,
tracing_enabled: Option<bool>,
#[serde(default, deserialize_with = "empty_string_as_none")]
otel_exporter_otlp_endpoint: Option<String>,
#[serde(default, deserialize_with = "empty_string_as_none")]
otel_exporter_otlp_headers: Option<String>,
#[serde(default, deserialize_with = "empty_string_as_none")]
otel_exporter_otlp_protocol: Option<String>,
#[serde(default, deserialize_with = "empty_string_as_none")]
otel_exporter_otlp_compression: Option<String>,
}
pub async fn load_otel(db: &DB) {
let otel = load_value_from_global_settings(db, OTEL_SETTING).await;
if let Ok(v) = otel {
if let Some(v) = v {
let deser = serde_json::from_value::<OtelSetting>(v);
if let Ok(o) = deser {
let metrics_enabled = o.metrics_enabled.unwrap_or(false);
let logs_enabled = o.logs_enabled.unwrap_or(false);
let tracing_enabled = o.tracing_enabled.unwrap_or(false);
OTEL_METRICS_ENABLED.store(metrics_enabled, Ordering::Relaxed);
OTEL_LOGS_ENABLED.store(logs_enabled, Ordering::Relaxed);
OTEL_TRACING_ENABLED.store(tracing_enabled, Ordering::Relaxed);
if let Some(endpoint) = o.otel_exporter_otlp_endpoint.as_ref() {
std::env::set_var("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint);
}
if let Some(headers) = o.otel_exporter_otlp_headers.as_ref() {
std::env::set_var("OTEL_EXPORTER_OTLP_HEADERS", headers);
}
if let Some(protocol) = o.otel_exporter_otlp_protocol {
std::env::set_var("OTEL_EXPORTER_OTLP_PROTOCOL", protocol);
}
if let Some(compression) = o.otel_exporter_otlp_compression {
std::env::set_var("OTEL_EXPORTER_OTLP_COMPRESSION", compression);
}
println!("OTEL settings loaded: tracing ({tracing_enabled}), logs ({logs_enabled}), metrics ({metrics_enabled}), endpoint ({:?}), headers defined: ({})",
o.otel_exporter_otlp_endpoint, o.otel_exporter_otlp_headers.is_some());
} else {
tracing::error!("Error deserializing otel settings");
}
}
} else {
tracing::error!("Error loading otel settings: {}", otel.unwrap_err());
}
}
pub async fn load_tag_per_workspace_enabled(db: &DB) -> error::Result<()> {
let metrics_enabled =
load_value_from_global_settings(db, DEFAULT_TAGS_PER_WORKSPACE_SETTING).await;
@@ -1337,7 +1397,7 @@ pub async fn reload_worker_config(
}
}
pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
pub async fn load_base_url(db: &DB) -> error::Result<String> {
let q_base_url = load_value_from_global_settings(db, BASE_URL_SETTING).await?;
let std_base_url = std::env::var("BASE_URL")
@@ -1361,6 +1421,14 @@ pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
std_base_url
};
{
let mut l = BASE_URL.write().await;
*l = base_url.clone();
}
Ok(base_url)
}
pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
let q_oauth = load_value_from_global_settings(db, OAUTH_SETTING).await?;
let oauths = if let Some(q) = q_oauth {
@@ -1374,6 +1442,7 @@ pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
None
};
let base_url = load_base_url(db).await?;
let is_secure = base_url.starts_with("https://");
{
@@ -1383,11 +1452,6 @@ pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
.unwrap();
}
{
let mut l = BASE_URL.write().await;
*l = base_url
}
{
let mut l = IS_SECURE.write().await;
*l = is_secure;
backend/tests/worker.rs
+5 -1
View File
@@ -77,7 +77,11 @@ async fn initialize_tracing() {
static ONCE: Once = Once::new();
ONCE.call_once(|| {
let _ = windmill_common::tracing_init::initialize_tracing("test");
let _ = windmill_common::tracing_init::initialize_tracing(
"test",
&windmill_common::utils::Mode::Standalone,
"test",
);
});
}
backend/windmill-api/openapi-deref.yaml
+6
View File
@@ -8709,6 +8709,9 @@ paths:
- identity
- deploymentcallback
- singlescriptflow
- flowscript
- flownode
- appscript
schedule_path:
type: string
permissioned_as:
@@ -9267,6 +9270,9 @@ paths:
- identity
- deploymentcallback
- singlescriptflow
- flowscript
- flownode
- appscript
schedule_path:
type: string
permissioned_as:
backend/windmill-api/openapi.yaml
+28 -1
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.435.2
version: 1.437.1
title: Windmill API
contact:
@@ -5467,6 +5467,23 @@ paths:
schema:
$ref: "#/components/schemas/AppWithLastVersion"
/w/{workspace}/apps/get/lite/{path}:
get:
summary: get app lite by path
operationId: getAppLiteByPath
tags:
- app
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/ScriptPath"
responses:
"200":
description: app lite details
content:
application/json:
schema:
$ref: "#/components/schemas/AppWithLastVersion"
/w/{workspace}/apps/get/draft/{path}:
get:
summary: get app by path with draft
@@ -5791,6 +5808,8 @@ paths:
#flow: flow/<path>
path:
type: string
version:
type: integer
args: {}
raw_code:
type: object
@@ -5808,6 +5827,8 @@ paths:
required:
- content
- language
id:
type: integer
force_viewer_static_fields:
type: object
force_viewer_one_of_fields:
@@ -10999,6 +11020,8 @@ components:
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode",
"appscript",
]
schedule_path:
type: string
@@ -11119,6 +11142,8 @@ components:
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode",
"appscript",
]
schedule_path:
type: string
@@ -11228,6 +11253,8 @@ components:
type: string
is_admin:
type: boolean
name:
type: string
is_super_admin:
type: boolean
created_at:
backend/windmill-api/src/apps.rs
+200 -160
View File
@@ -1,4 +1,4 @@
use std::collections::HashMap;
use std::{collections::HashMap, sync::Arc};
/*
* Author: Ruben Fiszel
@@ -31,6 +31,7 @@ use axum::{
routing::{delete, get, post},
Router,
};
use futures::future::{FutureExt, TryFutureExt};
use hyper::StatusCode;
#[cfg(feature = "parquet")]
use itertools::Itertools;
@@ -50,13 +51,15 @@ use windmill_audit::ActionKind;
#[cfg(feature = "parquet")]
use windmill_common::s3_helpers::build_object_store_client;
use windmill_common::{
apps::ListAppQuery,
apps::{AppScriptId, ListAppQuery},
cache::{self, future::FutureCachedExt},
db::UserDB,
error::{to_anyhow, Error, JsonResult, Result},
jobs::{get_payload_tag_from_prefixed_path, JobPayload, RawCode},
users::username_to_permissioned_as,
utils::{
http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath
http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, require_admin,
Pagination, StripPath,
},
variables::{build_crypt, build_crypt_with_key_suffix},
worker::{to_raw_value, CLOUD_HOSTED},
@@ -71,6 +74,7 @@ pub fn workspaced_service() -> Router {
.route("/list", get(list_apps))
.route("/list_search", get(list_search_apps))
.route("/get/p/*path", get(get_app))
.route("/get/lite/*path", get(get_app_lite))
.route("/get/draft/*path", get(get_app_w_draft))
.route("/secret_of/*path", get(get_secret_id))
.route("/get/v/*id", get(get_app_by_id))
@@ -191,15 +195,16 @@ pub type StaticFields = HashMap<String, Box<RawValue>>;
pub type OneOfFields = HashMap<String, Vec<Box<RawValue>>>;
pub type AllowUserResources = Vec<String>;
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)]
#[serde(rename_all = "lowercase")]
pub enum ExecutionMode {
#[default]
Anonymous,
Publisher,
Viewer,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
pub struct PolicyTriggerableInputs {
static_inputs: StaticFields,
one_of_inputs: OneOfFields,
@@ -215,7 +220,7 @@ pub struct S3Input {
file_key_regex: String,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
pub struct Policy {
pub on_behalf_of: Option<String>,
pub on_behalf_of_email: Option<String>,
@@ -410,6 +415,33 @@ async fn get_app(
Ok(Json(app))
}
async fn get_app_lite(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<AppWithLastVersion> {
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
let app_o = sqlx::query_as::<_, AppWithLastVersion>(
"SELECT app.id, app.path, app.summary, app.versions, app.policy,
app.extra_perms, coalesce(app_version_lite.value::json, app_version.value) as value,
app_version.created_at, app_version.created_by, NULL as starred
FROM app, app_version
LEFT JOIN app_version_lite ON app_version_lite.id = app_version.id
WHERE app.path = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]",
)
.bind(path.to_owned())
.bind(&w_id)
.fetch_optional(&mut *tx)
.await?;
tx.commit().await?;
let app = not_found_if_none(app_o, "App", path)?;
Ok(Json(app))
}
async fn get_app_w_draft(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
@@ -527,12 +559,11 @@ async fn update_app_history(
return Ok(());
}
async fn custom_path_exists(
Extension(db): Extension<DB>,
Path((w_id, custom_path)): Path<(String, String)>,
) -> JsonResult<bool> {
let exists =
let exists =
sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2))",
custom_path,
@@ -583,8 +614,9 @@ async fn get_public_app_by_secret(
let app_o = sqlx::query_as::<_, AppWithLastVersion>(
"SELECT app.id, app.path, app.summary, app.versions, app.policy,
null as extra_perms, app_version.value,
null as extra_perms, coalesce(app_version_lite.value::json, app_version.value::json) as value,
app_version.created_at, app_version.created_by from app, app_version
LEFT JOIN app_version_lite ON app_version_lite.id = app_version.id
WHERE app.id = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]")
.bind(&id)
.bind(&w_id)
@@ -626,7 +658,6 @@ async fn get_public_app_by_secret(
Ok(Json(app))
}
async fn get_public_resource(
Extension(db): Extension<DB>,
Path((w_id, path)): Path<(String, StripPath)>,
@@ -710,7 +741,6 @@ async fn create_app(
}
if let Some(custom_path) = &app.custom_path {
require_admin(authed.is_admin, &authed.username)?;
let exists = sqlx::query_scalar!(
@@ -746,7 +776,9 @@ async fn create_app(
app.summary,
json!(app.policy),
app.draft_only,
app.custom_path,
app.custom_path
.map(|s| if s.is_empty() { None } else { Some(s) })
.flatten()
)
.fetch_one(&mut *tx)
.await?;
@@ -987,26 +1019,29 @@ async fn update_app(
}
if let Some(ncustom_path) = &ns.custom_path {
require_admin(authed.is_admin, &authed.username)?;
let exists = sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2) AND NOT (path = $3 AND workspace_id = $4))",
ncustom_path,
if *CLOUD_HOSTED { Some(&w_id) } else { None },
path,
w_id
)
.fetch_one(&mut *tx)
.await?.unwrap_or(false);
if ncustom_path.is_empty() {
sqlb.set("custom_path", "NULL");
} else {
let exists = sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2) AND NOT (path = $3 AND workspace_id = $4))",
ncustom_path,
if *CLOUD_HOSTED { Some(&w_id) } else { None },
path,
w_id
)
.fetch_one(&mut *tx)
.await?.unwrap_or(false);
if exists {
return Err(Error::BadRequest(format!(
"App with custom path {} already exists",
ncustom_path
)));
if exists {
return Err(Error::BadRequest(format!(
"App with custom path {} already exists",
ncustom_path
)));
}
sqlb.set_str("custom_path", ncustom_path);
}
sqlb.set_str("custom_path", ncustom_path);
}
if let Some(mut npolicy) = ns.policy {
@@ -1144,6 +1179,10 @@ async fn update_app(
#[derive(Debug, Deserialize, Clone)]
pub struct ExecuteApp {
/// The app version to execute. Fallback to `path` if not provided.
pub version: Option<i64>,
/// The app script id (from the `app_script` table) to execute.
pub id: Option<i64>,
pub args: HashMap<String, Box<RawValue>>,
// - script: script/<path>
// - flow: flow/<path>
@@ -1206,6 +1245,22 @@ async fn get_on_behalf_details_from_policy_and_authed(
Ok((username, permissioned_as, email))
}
/// Convert the triggerables from the old format to the new format.
fn empty_triggerables(mut policy: Policy) -> Policy {
use std::mem::take;
if let Some(triggerables) = take(&mut policy.triggerables) {
let mut triggerables_v2 = take(&mut policy.triggerables_v2).unwrap_or_default();
for (k, static_inputs) in triggerables.into_iter() {
triggerables_v2.insert(
k,
PolicyTriggerableInputs { static_inputs, ..Default::default() },
);
}
policy.triggerables_v2 = Some(triggerables_v2);
}
policy
}
async fn execute_component(
OptAuthed(opt_authed): OptAuthed,
Extension(db): Extension<DB>,
@@ -1228,99 +1283,129 @@ async fn execute_component(
};
let path = path.to_path();
let (arc_policy, policy): (Arc<Policy>, Policy);
let policy_triggerables_default = Default::default();
let policy = match payload.clone() {
// Two cases here:
// 1. The component is executed from the editor (i.e. in "preview" mode), then:
// - The policy is set to default (in `Viewer` execution mode).
// - The policy triggerables are built by the frontend and retrieved from the request
// payload.
// - In case of inline script, the `RawCode` from the request is pushed as is to the
// job queue.
// 2. Otherwise (i.e. "run" mode):
// - The policy and triggerables are fetched from the database.
// - In case of inline script, if an entry exists in the `app_script` table, push
// an `AppScript` job payload, as in (.1) otherwise.
let (policy, policy_triggerables) = match payload {
// 1. "preview" mode.
ExecuteApp {
force_viewer_static_fields: Some(static_fields),
force_viewer_one_of_fields: Some(one_of_fields),
force_viewer_static_fields: Some(static_inputs),
force_viewer_one_of_fields: Some(one_of_inputs),
force_viewer_allow_user_resources: Some(allow_user_resources),
..
} => {
let mut hm = HashMap::new();
if let Some(path) = payload.path.clone() {
hm.insert(
format!("{}:{path}", payload.component),
PolicyTriggerableInputs {
static_inputs: static_fields,
one_of_inputs: one_of_fields,
allow_user_resources,
},
);
} else {
hm.insert(
format!(
"{}:{}",
payload.component,
digest(payload.raw_code.clone().unwrap().content.as_str())
),
PolicyTriggerableInputs {
static_inputs: static_fields,
one_of_inputs: one_of_fields,
allow_user_resources,
},
);
}
Policy {
execution_mode: ExecutionMode::Viewer,
triggerables: None,
triggerables_v2: Some(hm),
on_behalf_of: None,
on_behalf_of_email: None,
s3_inputs: None,
}
}
} => (
&Policy { execution_mode: ExecutionMode::Viewer, ..Default::default() },
&PolicyTriggerableInputs { static_inputs, one_of_inputs, allow_user_resources },
),
// 2. "run" mode.
_ => {
let policy_o = sqlx::query_scalar!(
"SELECT policy from app WHERE path = $1 AND workspace_id = $2",
// Policy is fetched from the database on app `path` and `workspace_id`.
let policy_fut = sqlx::query_scalar!(
"SELECT policy as \"policy: sqlx::types::Json<Box<RawValue>>\"
FROM app WHERE app.path = $1 AND app.workspace_id = $2 LIMIT 1",
path,
&w_id
&w_id,
)
.fetch_optional(&db)
.await?;
.map_err(Into::<Error>::into)
.map(|policy_o| Result::Ok(not_found_if_none(policy_o?, "App", path)?))
.map(|policy| Result::Ok(serde_json::from_str(policy?.get())?))
.map_ok(empty_triggerables);
let policy = not_found_if_none(policy_o, "App", path)?;
// 1. The app `version` is provided: cache the fetched policy.
// 2. Otherwise, always fetch the policy from the database.
let policy = if let Some(id) = payload.version {
let cache = cache::anon!({ u64 => Arc<Policy> } in "policy" <= 1000);
arc_policy = policy_fut
.map_ok(Arc::new)
.cached(cache, &(id as u64))
.await?;
&*arc_policy
} else {
policy = policy_fut.await?;
&policy
};
serde_json::from_value::<Policy>(policy).map_err(to_anyhow)?
// Compute the path for the triggerables map:
// - flow: `flow/<payload.path>`
// - script: `script/<payload.path>`
// - inline script: `rawscript/<sha256(raw_code.content)>`
let path = match &payload {
// flow or script: just use the `payload.path`.
ExecuteApp { path: Some(path), .. } => path,
// inline script: without entry in the `app_script` table.
ExecuteApp { raw_code: Some(raw_code), id: None, .. } => &digest(&raw_code.content),
// inline script: with an entry in the `app_script` table.
ExecuteApp { raw_code: Some(_), id: Some(id), .. } => {
let cache = cache::anon!({ u64 => Arc<String> } in "appscriptpath" <= 10000);
// `id` is unique, cache the result.
&*sqlx::query_scalar!(
"SELECT format('rawscript/%s', code_sha256) as \"path!: String\"
FROM app_script WHERE id = $1 LIMIT 1",
id
)
.fetch_one(&db)
.map_err(Into::<Error>::into)
.map_ok(Arc::new)
.cached(cache, &(*id as u64))
.await?
}
_ => unreachable!(),
};
// Retrieve the triggerables from the policy on `path` or `<component>:<path>`.
let triggerables_v2 = policy
.triggerables_v2
.as_ref()
.ok_or_else(|| Error::BadRequest(format!("Policy is missing triggerables")))?;
let policy_triggerables = triggerables_v2
.get(path) // start with `path` in case we can avoid the next` format!`.
.or_else(|| triggerables_v2.get(&format!("{}:{}", payload.component, &path)))
.or(match policy.execution_mode {
ExecutionMode::Viewer => Some(&policy_triggerables_default),
_ => None,
})
.ok_or_else(|| Error::BadRequest(format!("Path {path} forbidden by policy")))?;
(policy, policy_triggerables)
}
};
let (username, permissioned_as, email) =
get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed).await?;
let (job_payload, (args, job_id), tag) = match payload {
ExecuteApp { args, component, raw_code: Some(raw_code), path: None, .. } => {
let content = &raw_code.content;
let payload = JobPayload::Code(raw_code.clone());
let path = digest(content);
let args = build_args(
policy,
&component,
path,
args,
opt_authed.as_ref(),
&user_db,
&db,
&w_id,
)
.await?;
(payload, args, None)
}
ExecuteApp { args, component, raw_code: None, path: Some(path), .. } => {
let (payload, tag) = get_payload_tag_from_prefixed_path(&path, &db, &w_id).await?;
let args = build_args(
policy,
&component,
path.to_string(),
args,
opt_authed.as_ref(),
&user_db,
&db,
&w_id,
)
.await?;
(payload, args, tag)
}
let (args, job_id) = build_args(
policy,
policy_triggerables,
payload.args,
opt_authed.as_ref(),
&user_db,
&db,
&w_id,
)
.await?;
let (job_payload, tag) = match (payload.path, payload.raw_code, payload.id) {
// flow or script:
(Some(path), None, None) => get_payload_tag_from_prefixed_path(&path, &db, &w_id).await?,
// inline script: in "preview" mode or without entry in the `app_script` table.
(None, Some(raw_code), None) => (JobPayload::Code(raw_code), None),
// inline script: in "run" mode and with an entry in the `app_script` table.
(None, Some(RawCode { language, path, cache_ttl, .. }), Some(id)) => (
JobPayload::AppScript { id: AppScriptId(id), cache_ttl, language, path },
None,
),
_ => unreachable!(),
};
let tx = windmill_queue::PushIsolationLevel::IsolatedRoot(db.clone());
@@ -1655,9 +1740,12 @@ async fn exists_app(
}
async fn build_args(
policy: Policy,
component: &str,
path: String,
policy: &Policy,
PolicyTriggerableInputs {
static_inputs,
one_of_inputs,
allow_user_resources,
}: &PolicyTriggerableInputs,
mut args: HashMap<String, Box<RawValue>>,
authed: Option<&ApiAuthed>,
user_db: &UserDB,
@@ -1665,54 +1753,6 @@ async fn build_args(
w_id: &str,
) -> Result<(PushArgsOwned, Option<Uuid>)> {
let mut job_id: Option<Uuid> = None;
let key = format!("{}:{}", component, &path);
let (static_inputs, one_of_inputs, allow_user_resources) = match policy {
Policy { triggerables_v2: Some(t), .. } => {
let PolicyTriggerableInputs { static_inputs, one_of_inputs, allow_user_resources } = t
.get(&key)
.or_else(|| t.get(&path))
.map(|x| x.clone())
.or_else(|| {
if matches!(policy.execution_mode, ExecutionMode::Viewer) {
Some(PolicyTriggerableInputs {
static_inputs: HashMap::new(),
one_of_inputs: HashMap::new(),
allow_user_resources: Vec::new(),
})
} else {
None
}
})
.ok_or_else(|| {
Error::BadRequest(format!("path {} is not allowed in the app policy", path))
})?;
(static_inputs, one_of_inputs, allow_user_resources)
}
Policy { triggerables: Some(t), .. } => {
let static_inputs = t
.get(&key)
.or_else(|| t.get(&path))
.map(|x| x.clone())
.or_else(|| {
if matches!(policy.execution_mode, ExecutionMode::Viewer) {
Some(HashMap::new())
} else {
None
}
})
.ok_or_else(|| {
Error::BadRequest(format!("path {} is not allowed in the app policy", path))
})?;
(static_inputs, HashMap::new(), Vec::new())
}
_ => Err(Error::BadRequest(format!(
"Policy is missing triggerables for {}",
key
)))?,
};
let mut safe_args = HashMap::<String, Box<RawValue>>::new();
// tracing::error!("{:?}", allow_user_resources);
@@ -1761,16 +1801,16 @@ async fn build_args(
}
for (k, v) in one_of_inputs {
if safe_args.contains_key(&k) {
if safe_args.contains_key(k) {
continue;
}
if let Some(arg_val) = args.get(&k) {
if let Some(arg_val) = args.get(k) {
let arg_str = arg_val.get();
let options_str_vec = v.iter().map(|x| x.get()).collect::<Vec<&str>>();
if options_str_vec.contains(&arg_str) {
safe_args.insert(k.to_string(), arg_val.clone());
args.remove(&k);
args.remove(k);
continue;
}
@@ -1781,7 +1821,7 @@ async fn build_args(
.all(|x| options_str_vec.contains(&x.get()))
{
safe_args.insert(k.to_string(), arg_val.clone());
args.remove(&k);
args.remove(k);
continue;
}
}
backend/windmill-api/src/tracing_init.rs
+8 -6
View File
@@ -29,11 +29,13 @@ impl<B> OnResponse<B> for MyOnResponse {
_span: &tracing::Span,
) {
if *LOG_REQUESTS {
tracing::info!(
latency = latency.as_millis(),
status = response.status().as_u16(),
"response"
)
let latency = latency.as_millis();
let status = response.status().as_u16();
if response.status().is_success() {
tracing::info!(latency = latency, status = status, "response")
} else {
tracing::error!(latency = latency, status = status, "response")
}
}
}
}
@@ -67,7 +69,7 @@ impl<B> MakeSpan<B> for MyMakeSpan {
uri = %request.uri(),
username = field::Empty,
workspace_id = field::Empty,
trace_id = tracing_id,
traceId = tracing_id,
email = field::Empty,
)
}
backend/windmill-api/src/users.rs
+21 -10
View File
@@ -847,6 +847,7 @@ pub struct UserInfo {
pub folders_read: Vec<String>,
pub folders: Vec<String>,
pub folders_owners: Vec<String>,
pub name: Option<String>,
}
#[derive(FromRow, Serialize)]
@@ -1218,6 +1219,7 @@ async fn whoami(
workspace_id: w_id,
email: email.clone(),
username: email,
name: None,
is_admin,
is_super_admin: is_admin,
created_at: chrono::Utc::now(),
@@ -1306,22 +1308,30 @@ async fn get_usage(
Ok(usage.to_string())
}
#[derive(FromRow, Serialize)]
pub struct User2 {
pub workspace_id: String,
pub email: String,
pub username: String,
pub is_admin: bool,
pub created_at: chrono::DateTime<chrono::Utc>,
pub operator: bool,
pub disabled: bool,
pub role: Option<String>,
pub super_admin: bool,
pub name: Option<String>,
}
async fn get_user(w_id: &str, username: &str, db: &DB) -> Result<Option<UserInfo>> {
let user = sqlx::query_as!(
User,
"SELECT * FROM usr where username = $1 AND workspace_id = $2",
User2,
"SELECT usr.*, password.super_admin, password.name FROM usr LEFT JOIN password ON usr.email = password.email Where usr.username = $1 AND workspace_id = $2
",
username,
w_id
)
.fetch_optional(db)
.await?;
let is_super_admin = sqlx::query_scalar!(
"SELECT super_admin FROM password WHERE email = $1",
user.as_ref().map(|x| &x.email)
)
.fetch_optional(db)
.await?
.unwrap_or(false);
let groups = get_groups_for_user(
&w_id,
username,
@@ -1339,8 +1349,9 @@ async fn get_user(w_id: &str, username: &str, db: &DB) -> Result<Option<UserInfo
workspace_id: usr.workspace_id,
email: usr.email,
username: usr.username,
name: usr.name,
is_admin: usr.is_admin,
is_super_admin,
is_super_admin: usr.super_admin,
created_at: usr.created_at,
operator: usr.operator,
disabled: usr.disabled,
backend/windmill-common/Cargo.toml
+10
View File
@@ -13,6 +13,8 @@ flamegraph = ["dep:tracing-flame"]
loki = ["dep:tracing-loki"]
benchmark = []
parquet = ["dep:object_store", "dep:aws-config", "dep:aws-sdk-sts"]
otel = ["dep:opentelemetry-semantic-conventions", "dep:opentelemetry-otlp", "dep:opentelemetry_sdk",
"dep:opentelemetry", "dep:tracing-opentelemetry", "dep:opentelemetry-appender-tracing", "dep:tonic"]
[lib]
name = "windmill_common"
@@ -64,5 +66,13 @@ croner = "2.0.6"
quick_cache.workspace = true
pin-project-lite.workspace = true
opentelemetry-semantic-conventions = { workspace = true, optional = true }
opentelemetry-otlp = { workspace = true, optional = true }
opentelemetry_sdk = { workspace = true, optional = true }
opentelemetry = { workspace = true, optional = true }
tracing-opentelemetry = { workspace = true, optional = true }
opentelemetry-appender-tracing = { workspace = true, optional = true }
tonic = { workspace = true, optional = true }
[target.'cfg(not(target_env = "msvc"))'.dependencies]
tikv-jemalloc-ctl = { optional = true, workspace = true }
backend/windmill-common/src/apps.rs
+12 -1
View File
@@ -6,7 +6,18 @@
* LICENSE-AGPL for a copy of the license.
*/
use serde::Deserialize;
use serde::{Deserialize, Serialize};
/// Id in the `app_script` table.
#[derive(Serialize, Deserialize, Debug, Copy, Clone, Hash, Eq, PartialEq)]
#[serde(transparent)]
pub struct AppScriptId(pub i64);
impl Into<u64> for AppScriptId {
fn into(self) -> u64 {
self.0 as u64
}
}
#[derive(Deserialize)]
pub struct ListAppQuery {
backend/windmill-common/src/cache.rs
+291 -64
View File
@@ -1,25 +1,149 @@
use crate::error;
use std::future::Future;
use std::hash::Hash;
use std::path::{Path, PathBuf};
use quick_cache::sync::Cache;
use quick_cache::Equivalent;
use serde::{Deserialize, Serialize};
use sqlx::PgExecutor;
pub use const_format::concatcp;
pub use lazy_static::lazy_static;
pub use quick_cache::sync::Cache;
/// Cache directory for windmill server/worker(s).
pub const CACHE_DIR: &str = "/tmp/windmill/cache/";
/// A file-system backed concurrent cache.
pub struct FsBackedCache<Key, Val> {
cache: Cache<Key, Val>,
root: &'static str,
}
impl<Key: Eq + Hash, Val: Clone + fs::Bundle> FsBackedCache<Key, Val> {
/// Create a new file-system backed cache with `items_capacity` capacity.
/// The cache will be stored in the `root` directory.
pub fn new(root: &'static str, items_capacity: usize) -> Self {
Self { cache: Cache::new(items_capacity), root }
}
/// Gets or inserts an item in the cache with key `key`.
pub async fn get_or_insert_async<'a, Q, F>(&'a self, key: &Q, with: F) -> error::Result<Val>
where
Q: Hash + Equivalent<Key> + ToOwned<Owned = Key> + Copy + Into<u64>,
F: Future<Output = error::Result<Val>>,
{
self.cache
.get_or_insert_async(
key,
fs::import_or_insert_with(self.root, (*key).into(), with),
)
.await
}
}
/// Like [`lazy_static`]`, but for file-system backed caches.
///
/// # Example
/// ```rust
/// use windmill_common::make_static;
///
/// make_static! {
/// /// String cache with a maximum capacity of 1000 items stored in the
/// /// "subdirectory" directory.
/// static ref CACHE: { u64 => String } in "subdirectory" <= 1000;
/// /// Another cache.
/// static ref ANOTHER_CACHE: { u64 => Vec<String> } in "another" <= 100;
/// }
/// ```
#[macro_export]
macro_rules! make_static {
{ $( $(#[$attr:meta])* static ref $name:ident: { $Key:ty => $Val:ty } in $root:literal <= $cap:literal; )+ } => {
$crate::cache::lazy_static! {
$(
$(#[$attr])*
static ref $name: $crate::cache::FsBackedCache<$Key, $Val> =
$crate::cache::FsBackedCache::new(
$crate::cache::concatcp!($crate::cache::CACHE_DIR, $root),
$cap
);
)+
}
};
}
// re-export:
pub use make_static;
/// Create an anonymous file-system backed cache for one-time use.
///
/// # Example
/// ```rust
/// use windmill_common::anon;
/// let cache = anon!({ u64 => String } in "subdirectory" <= 1000);
/// ```
#[macro_export]
macro_rules! anon {
({ $Key:ty => $Val:ty } in $root:literal <= $cap:literal) => {{
$crate::cache::make_static! {
static ref __ANON__: { $Key => $Val } in $root <= $cap;
}
&__ANON__
}};
}
// re-export:
pub use anon;
pub mod future {
use super::*;
/// Extension trait for futures that can be cached.
pub trait FutureCachedExt<Val: Clone + fs::Bundle>:
Future<Output = error::Result<Val>> + Sized
{
/// Get or insert the future result in the cache.
///
/// # Example
/// ```rust
/// use windmill_common::cache::{self, future::FutureCachedExt};
///
/// async {
/// let result = std::future::ready(Ok(42))
/// .cached(cache::anon!({ u64 => u64 } in "test" <= 1), &42)
/// .await;
///
/// assert_eq!(result.unwrap(), 42);
/// };
/// ```
fn cached<Key: Eq + Hash, Q>(
self,
cache: &FsBackedCache<Key, Val>,
key: &Q,
) -> impl Future<Output = error::Result<Val>>
where
Q: Hash + Equivalent<Key> + ToOwned<Owned = Key> + Copy + Into<u64>,
{
cache.get_or_insert_async(key, self)
}
}
impl<Val: Clone + fs::Bundle, F: Future<Output = error::Result<Val>> + Sized>
FutureCachedExt<Val> for F
{
}
}
pub mod flow {
use super::*;
use crate::flows::{FlowNodeId, FlowValue};
/// Cache directory for windmill server/worker(s) flow nodes.
pub const CACHE_DIR: &str = const_format::concatcp!(super::CACHE_DIR, "flow");
lazy_static::lazy_static! {
make_static! {
/// Flow node cache.
/// FIXME: This should be a static but [`Cache`] does not have a const constructor.
/// FIXME: Use `Arc<Val>` for cheap cloning.
static ref CACHE: Cache<FlowNodeId, Val> = Cache::new(1000);
static ref CACHE: { FlowNodeId => Val } in "flow" <= 1000;
}
/// Flow node cache value.
@@ -75,38 +199,35 @@ pub mod flow {
// so only one thread will be able to fetch the data from the database and write it to
// the file system and cache, hence no race on the file system.
CACHE
.get_or_insert_async(
&node,
fs::import_or_insert_with(CACHE_DIR, node.0 as u64, async {
sqlx::query!(
"SELECT \
.get_or_insert_async(&node, async {
sqlx::query!(
"SELECT \
lock AS \"lock: String\", \
code AS \"code: String\", \
flow::text AS \"flow: Box<str>\" \
FROM flow_node WHERE id = $1 LIMIT 1",
node.0,
)
.fetch_one(e)
.await
.map_err(Into::into)
.and_then(|r| {
Ok(Val {
lock: r
.lock
.and_then(|x| if x.is_empty() { None } else { Some(x) }),
code: r.code,
flow: match r.flow {
None => None,
Some(flow) => serde_json::from_str(&flow).map_err(|err| {
error::Error::InternalErr(format!(
"Unable to parse flow value: {err:?}"
))
})?,
},
})
node.0,
)
.fetch_one(e)
.await
.map_err(Into::into)
.and_then(|r| {
Ok(Val {
lock: r
.lock
.and_then(|x| if x.is_empty() { None } else { Some(x) }),
code: r.code,
flow: match r.flow {
None => None,
Some(flow) => serde_json::from_str(&flow).map_err(|err| {
error::Error::InternalErr(format!(
"Unable to parse flow value: {err:?}"
))
})?,
},
})
}),
)
})
})
.await
}
@@ -164,14 +285,10 @@ pub mod script {
use super::*;
use crate::scripts::{ScriptHash, ScriptLang};
/// Cache directory for windmill server/worker(s) scripts.
pub const CACHE_DIR: &str = const_format::concatcp!(super::CACHE_DIR, "script");
lazy_static::lazy_static! {
make_static! {
/// Scripts cache.
/// FIXME: This should be a static but [`Cache`] does not have a const constructor.
/// FIXME: Use `Arc<Val>` for cheap cloning.
static ref CACHE: Cache<ScriptHash, Val> = Cache::new(1000);
static ref CACHE: { ScriptHash => Val } in "script" <= 1000;
}
/// Script cache value.
@@ -197,34 +314,31 @@ pub mod script {
// so only one thread will be able to fetch the data from the database and write it to
// the file system and cache, hence no race on the file system.
CACHE
.get_or_insert_async(
&hash,
fs::import_or_insert_with(CACHE_DIR, hash.0 as u64, async {
sqlx::query!(
"SELECT \
.get_or_insert_async(&hash, async {
sqlx::query!(
"SELECT \
lock AS \"lock: String\", \
content AS \"code!: String\",
language AS \"language: Option<ScriptLang>\", \
envs AS \"envs: Vec<String>\", \
codebase AS \"codebase: String\" \
FROM script WHERE hash = $1 AND workspace_id = $2 LIMIT 1",
hash.0,
workspace_id,
)
.fetch_one(e)
.await
.map_err(Into::into)
.map(|r| Val {
lock: r
.lock
.and_then(|x| if x.is_empty() { None } else { Some(x) }),
code: r.code,
language: r.language,
envs: r.envs,
codebase: r.codebase,
})
}),
)
hash.0,
workspace_id,
)
.fetch_one(e)
.await
.map_err(Into::into)
.map(|r| Val {
lock: r
.lock
.and_then(|x| if x.is_empty() { None } else { Some(x) }),
code: r.code,
language: r.language,
envs: r.envs,
codebase: r.codebase,
})
})
.await
}
@@ -280,11 +394,99 @@ pub mod script {
}
}
pub mod app {
use super::*;
use crate::apps::AppScriptId;
make_static! {
/// App scripts cache.
/// FIXME: Use `Arc<Val>` for cheap cloning.
static ref CACHE: { AppScriptId => Val } in "app" <= 1000;
}
/// App app script cache value.
#[derive(Debug, Clone, Default)]
pub struct Val {
pub lock: Option<String>,
pub code: String,
}
/// Fetch the app script referenced by `id` from the cache.
/// If not present, import from the file-system cache or fetch it from the database and write
/// it to the file system and cache.
/// This should be preferred over fetching the database directly.
pub async fn fetch_script(
e: impl PgExecutor<'_>,
id: AppScriptId,
) -> error::Result<(Option<String>, String)> {
// If not present, `get_or_insert_async` will lock the key until the future completes,
// so only one thread will be able to fetch the data from the database and write it to
// the file system and cache, hence no race on the file system.
CACHE
.get_or_insert_async(&id, async {
sqlx::query!(
"SELECT lock, code FROM app_script WHERE id = $1 LIMIT 1",
id.0,
)
.fetch_one(e)
.await
.map_err(Into::into)
.map(|r| Val {
lock: r
.lock
.and_then(|x| if x.is_empty() { None } else { Some(x) }),
code: r.code,
})
})
.await
.map(|Val { lock, code }| (lock, code))
}
// ----------------------------------------------------------------------------------------------
// impl `fs::Bundle` for `Val`.
#[derive(Copy, Clone)]
pub enum Item {
Lock,
Code,
}
impl fs::Item for Item {
fn path(&self, root: &Path) -> PathBuf {
match self {
Item::Lock => root.join("lock.txt"),
Item::Code => root.join("code.txt"),
}
}
}
impl fs::Bundle for Val {
type Item = Item;
fn items() -> &'static [Self::Item] {
&[Item::Lock, Item::Code]
}
fn import(&mut self, item: Self::Item, data: Vec<u8>) -> error::Result<()> {
match item {
Item::Lock => self.lock = Some(String::from_utf8(data)?),
Item::Code => self.code = String::from_utf8(data)?,
}
Ok(())
}
fn export(&self, item: Self::Item) -> error::Result<Option<Vec<u8>>> {
match item {
Item::Lock => Ok(self.lock.as_ref().map(|s| s.as_bytes().to_vec())),
Item::Code => Ok(Some(self.code.as_bytes().to_vec())),
}
}
}
}
mod fs {
use super::*;
use std::future::Future;
use std::fs::{self, OpenOptions};
use std::io::{Read, Write};
@@ -363,4 +565,29 @@ mod fs {
}
Ok(data)
}
// Auto-implement `Bundle` for all `serde` serializable types.
impl Item for () {
fn path(&self, root: &Path) -> PathBuf {
root.join("self.json")
}
}
impl<T: for<'de> Deserialize<'de> + Serialize + Default> Bundle for T {
type Item = ();
fn items() -> &'static [Self::Item] {
&[()]
}
fn import(&mut self, _: Self::Item, data: Vec<u8>) -> error::Result<()> {
*self = serde_json::from_slice(&data)?;
Ok(())
}
fn export(&self, _: Self::Item) -> error::Result<Option<Vec<u8>>> {
Ok(Some(serde_json::to_vec(self)?))
}
}
}
backend/windmill-common/src/db.rs
+8 -8
View File
@@ -80,14 +80,14 @@ impl UserDB {
let mut folders_read = folders_read.clone();
folders_read.extend(folders_write.clone());
tracing::debug!(
"Setting role to {} {:?} {:?} {:?} {:?}",
user,
authed.username(),
authed.groups(),
folders_read,
folders_write
);
// tracing::debug!(
// "Setting role to {} {:?} {:?} {:?} {:?}",
// user,
// authed.username(),
// authed.groups(),
// folders_read,
// folders_write
// );
let mut tx = self.db.begin().await?;
backend/windmill-common/src/flow_status.rs
+2 -2
View File
@@ -17,7 +17,7 @@ use crate::flows::FlowValue;
const MINUTES: Duration = Duration::from_secs(60);
const HOURS: Duration = MINUTES.saturating_mul(60);
pub const MAX_RETRY_ATTEMPTS: u16 = 1000;
pub const MAX_RETRY_ATTEMPTS: u32 = u32::MAX;
pub const MAX_RETRY_INTERVAL: Duration = HOURS.saturating_mul(6);
pub fn is_retry_default(v: &RetryStatus) -> bool {
@@ -48,7 +48,7 @@ pub struct FlowStatus {
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
#[serde(default)]
pub struct RetryStatus {
pub fail_count: u16,
pub fail_count: u32,
pub failed_jobs: Vec<Uuid>,
}
backend/windmill-common/src/flows.rs
+75 -23
View File
@@ -147,7 +147,7 @@ impl Retry {
/// Takes the number of previous retries and returns the interval until the next retry if any.
///
/// May return [`Duration::ZERO`] to retry immediately.
pub fn interval(&self, previous_attempts: u16, silent: bool) -> Option<Duration> {
pub fn interval(&self, previous_attempts: u32, silent: bool) -> Option<Duration> {
let Self { constant, exponential } = self;
if previous_attempts < constant.attempts {
@@ -178,7 +178,7 @@ impl Retry {
self.constant.attempts != 0 || self.exponential.attempts != 0
}
pub fn max_attempts(&self) -> u16 {
pub fn max_attempts(&self) -> u32 {
self.constant
.attempts
.saturating_add(self.exponential.attempts)
@@ -194,7 +194,7 @@ impl Retry {
#[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq)]
#[serde(default)]
pub struct ConstantDelay {
pub attempts: u16,
pub attempts: u32,
pub seconds: u16,
}
@@ -202,7 +202,7 @@ pub struct ConstantDelay {
#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
#[serde(default)]
pub struct ExponentialDelay {
pub attempts: u16,
pub attempts: u32,
pub multiplier: u16,
pub seconds: u16,
pub random_factor: Option<i8>, // percentage, defaults to 0 for no jitter
@@ -407,6 +407,12 @@ pub enum InputTransform {
#[serde(transparent)]
pub struct FlowNodeId(pub i64);
impl Into<u64> for FlowNodeId {
fn into(self) -> u64 {
self.0 as u64
}
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct Branch {
#[serde(skip_serializing_if = "Option::is_none")]
@@ -713,10 +719,14 @@ pub async fn resolve_maybe_value<T>(
workspace_id: &str,
with_code: bool,
maybe: Option<T>,
value_mut: impl FnOnce(&mut T) -> Option<&mut Json<Box<JsonRawValue>>>
value_mut: impl FnOnce(&mut T) -> Option<&mut Json<Box<JsonRawValue>>>,
) -> Result<Option<T>, Error> {
let Some(mut container) = maybe else { return Ok(None); };
let Some(value) = value_mut(&mut container) else { return Ok(Some(container)); };
let Some(mut container) = maybe else {
return Ok(None);
};
let Some(value) = value_mut(&mut container) else {
return Ok(Some(container));
};
resolve_value(e, workspace_id, &mut value.0, with_code).await?;
Ok(Some(container))
}
@@ -728,8 +738,9 @@ pub async fn resolve_value(
value: &mut Box<JsonRawValue>,
with_code: bool,
) -> Result<(), Error> {
let mut val = serde_json::from_str::<FlowValue>(value.get())
.map_err(|err| Error::InternalErr(format!("resolve: Failed to parse flow value: {}", err)))?;
let mut val = serde_json::from_str::<FlowValue>(value.get()).map_err(|err| {
Error::InternalErr(format!("resolve: Failed to parse flow value: {}", err))
})?;
for module in &mut val.modules {
resolve_module(e, workspace_id, &mut module.value, with_code).await?;
}
@@ -746,16 +757,29 @@ pub async fn resolve_module(
) -> Result<(), Error> {
use FlowModuleValue::*;
let mut val = serde_json::from_str::<FlowModuleValue>(value.get())
.map_err(|err| Error::InternalErr(format!("resolve: Failed to parse flow module value: {}", err)))?;
let mut val = serde_json::from_str::<FlowModuleValue>(value.get()).map_err(|err| {
Error::InternalErr(format!(
"resolve: Failed to parse flow module value: {}",
err
))
})?;
match &mut val {
FlowScript { .. } => {
// In order to avoid an unnecessary `.clone()` of `val`, take ownership of it's content
// using `std::mem::replace`.
let FlowScript {
input_transforms, id, tag, language,
custom_concurrency_key, concurrent_limit, concurrency_time_window_s, is_trigger
} = std::mem::replace(&mut val, Identity) else { unreachable!() };
input_transforms,
id,
tag,
language,
custom_concurrency_key,
concurrent_limit,
concurrency_time_window_s,
is_trigger,
} = std::mem::replace(&mut val, Identity)
else {
unreachable!()
};
// Load script lock file and code content.
let (lock, content) = if !with_code {
(Some("...".to_string()), "...".to_string())
@@ -763,22 +787,44 @@ pub async fn resolve_module(
cache::flow::fetch_script(e, id).await?
};
val = RawScript {
input_transforms, content, lock, path: None, tag, language, custom_concurrency_key,
concurrent_limit, concurrency_time_window_s, is_trigger
input_transforms,
content,
lock,
path: None,
tag,
language,
custom_concurrency_key,
concurrent_limit,
concurrency_time_window_s,
is_trigger,
};
},
ForloopFlow { modules, modules_node, .. } | WhileloopFlow { modules, modules_node, .. } => {
}
ForloopFlow { modules, modules_node, .. } | WhileloopFlow { modules, modules_node, .. } => {
resolve_modules(e, workspace_id, modules, modules_node.take(), with_code).await?;
},
}
BranchOne { branches, default, default_node } => {
resolve_modules(e, workspace_id, default, default_node.take(), with_code).await?;
for branch in branches {
resolve_modules(e, workspace_id, &mut branch.modules, branch.modules_node.take(), with_code).await?;
resolve_modules(
e,
workspace_id,
&mut branch.modules,
branch.modules_node.take(),
with_code,
)
.await?;
}
},
}
BranchAll { branches, .. } => {
for branch in branches {
resolve_modules(e, workspace_id, &mut branch.modules, branch.modules_node.take(), with_code).await?;
resolve_modules(
e,
workspace_id,
&mut branch.modules,
branch.modules_node.take(),
with_code,
)
.await?;
}
}
_ => {}
@@ -801,7 +847,13 @@ pub async fn resolve_modules(
.map(|flow| flow.modules)?;
}
for module in modules.iter_mut() {
Box::pin(resolve_module(e, workspace_id, &mut module.value, with_code)).await?;
Box::pin(resolve_module(
e,
workspace_id,
&mut module.value,
with_code,
))
.await?;
}
Ok(())
}
backend/windmill-common/src/global_settings.rs
+5 -1
View File
@@ -35,8 +35,9 @@ pub const CRITICAL_ALERT_MUTE_UI_SETTING: &str = "critical_alert_mute_ui";
pub const DEV_INSTANCE_SETTING: &str = "dev_instance";
pub const JWT_SECRET_SETTING: &str = "jwt_secret";
pub const EMAIL_DOMAIN_SETTING: &str = "email_domain";
pub const OTEL_SETTING: &str = "otel";
pub const ENV_SETTINGS: [&str; 51] = [
pub const ENV_SETTINGS: [&str; 54] = [
"DISABLE_NSJAIL",
"MODE",
"NUM_WORKERS",
@@ -88,4 +89,7 @@ pub const ENV_SETTINGS: [&str; 51] = [
"WORKER_GROUP",
"SAML_METADATA",
"INSTANCE_IS_DEV",
"OTEL_METRICS",
"OTEL_TRACING",
"OTEL_LOGS",
];
backend/windmill-common/src/jobs.rs
+8
View File
@@ -14,6 +14,7 @@ pub const ENTRYPOINT_OVERRIDE: &str = "_ENTRYPOINT_OVERRIDE";
pub const PREPROCESSOR_FAKE_ENTRYPOINT: &str = "__WM_PREPROCESSOR";
use crate::{
apps::AppScriptId,
error::{self, to_anyhow, Error},
flow_status::{FlowStatus, RestartedFrom},
flows::{FlowNodeId, FlowValue, Retry},
@@ -41,6 +42,7 @@ pub enum JobKind {
DeploymentCallback,
FlowScript,
FlowNode,
AppScript,
}
#[derive(sqlx::FromRow, Debug, Serialize, Clone)]
@@ -278,6 +280,12 @@ pub enum JobPayload {
id: FlowNodeId, // flow_node(id).
path: String, // flow node inner path (e.g. `outer/branchall-42`).
},
AppScript {
id: AppScriptId, // app_script(id).
path: Option<String>,
language: ScriptLang,
cache_ttl: Option<i32>,
},
Code(RawCode),
Dependencies {
path: String,
backend/windmill-common/src/lib.rs
+8 -2
View File
@@ -36,20 +36,20 @@ pub mod job_s3_helpers_ee;
pub mod jobs;
pub mod more_serde;
pub mod oauth2;
pub mod otel_ee;
pub mod queue;
pub mod s3_helpers;
pub mod schedule;
pub mod scripts;
pub mod server;
pub mod stats_ee;
pub mod tracing_init;
pub mod users;
pub mod utils;
pub mod variables;
pub mod worker;
pub mod workspaces;
pub mod tracing_init;
pub const DEFAULT_MAX_CONNECTIONS_SERVER: u32 = 50;
pub const DEFAULT_MAX_CONNECTIONS_WORKER: u32 = 5;
pub const DEFAULT_MAX_CONNECTIONS_INDEXER: u32 = 5;
@@ -87,6 +87,12 @@ lazy_static::lazy_static! {
.unwrap_or_else(|| SocketAddr::from(([0, 0, 0, 0], *METRICS_PORT)));
pub static ref METRICS_ENABLED: AtomicBool = AtomicBool::new(std::env::var("METRICS_PORT").is_ok() || std::env::var("METRICS_ADDR").is_ok());
pub static ref OTEL_METRICS_ENABLED: AtomicBool = AtomicBool::new(std::env::var("OTEL_METRICS").is_ok());
pub static ref OTEL_TRACING_ENABLED: AtomicBool = AtomicBool::new(std::env::var("OTEL_TRACING").is_ok());
pub static ref OTEL_LOGS_ENABLED: AtomicBool = AtomicBool::new(std::env::var("OTEL_LOGS").is_ok());
pub static ref METRICS_DEBUG_ENABLED: AtomicBool = AtomicBool::new(false);
pub static ref CRITICAL_ALERT_MUTE_UI_ENABLED: AtomicBool = AtomicBool::new(false);
backend/windmill-common/src/otel_ee.rs
+58
View File
@@ -0,0 +1,58 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use crate::{jobs::QueuedJob, utils::Mode};
use uuid::Uuid;
pub fn set_span_parent(_span: &tracing::Span, _rj: &Uuid) {}
#[cfg(not(all(feature = "otel", feature = "enterprise")))]
pub(crate) type OtelProvider = Option<()>;
#[cfg(all(feature = "otel", feature = "enterprise"))]
pub(crate) type OtelProvider = Option<opentelemetry_sdk::metrics::SdkMeterProvider>;
#[cfg(not(feature = "otel"))]
pub fn otel_ctx() -> () {}
#[cfg(feature = "otel")]
#[inline(always)]
pub fn otel_ctx() -> opentelemetry::Context {
opentelemetry::Context::current()
}
#[cfg(not(feature = "otel"))]
impl<T: Sized> FutureExt for T {}
#[cfg(not(feature = "otel"))]
pub trait FutureExt: Sized {
fn with_context(self, _otel_cx: ()) -> Self {
self
}
}
use tracing_subscriber::EnvFilter;
pub(crate) fn init_logs_bridge(_mode: &Mode, _hostname: &str, _env: &str) -> Option<EnvFilter> {
None
}
#[cfg(all(feature = "otel", feature = "enterprise"))]
pub(crate) fn init_otlp_tracer(
_mode: &Mode,
_hostname: &str,
_env: &str,
) -> Option<opentelemetry_sdk::trace::Tracer> {
None
}
pub(crate) fn init_meter_provider(_mode: &Mode, _hostname: &str, _env: &str) -> OtelProvider {
None
}
pub fn add_root_flow_job_to_otlp(_queued_job: &QueuedJob, _success: bool) {}
backend/windmill-common/src/scripts.rs
+6
View File
@@ -75,6 +75,12 @@ impl ScriptLang {
#[sqlx(transparent)]
pub struct ScriptHash(pub i64);
impl Into<u64> for ScriptHash {
fn into(self) -> u64 {
self.0 as u64
}
}
#[derive(PartialEq, sqlx::Type)]
#[sqlx(transparent, no_pg_array)]
pub struct ScriptHashes(pub Vec<i64>);
backend/windmill-common/src/tracing_init.rs
+69 -39
View File
@@ -7,13 +7,23 @@
*/
use const_format::concatcp;
use std::{
collections::HashMap,
sync::{Arc, RwLock},
};
use tracing::{level_filters::LevelFilter, Event};
use tracing_appender::non_blocking::{NonBlockingBuilder, WorkerGuard};
use tracing_subscriber::layer::Context;
use tracing_subscriber::{
filter::Targets,
fmt::{format, Layer},
prelude::*,
EnvFilter,
};
use crate::utils::Mode;
fn json_layer<S>() -> Layer<S, format::JsonFields, format::Format<format::Json>> {
tracing_subscriber::fmt::layer()
.json()
@@ -34,17 +44,40 @@ pub const LOGS_SERVICE: &str = "logs/services/";
pub const TMP_WINDMILL_LOGS_SERVICE: &str = concatcp!("/tmp/windmill/", LOGS_SERVICE);
pub fn initialize_tracing(hostname: &str) -> WorkerGuard {
pub fn initialize_tracing(
hostname: &str,
mode: &Mode,
environment: &str,
) -> (WorkerGuard, crate::otel_ee::OtelProvider) {
let style = std::env::var("RUST_LOG_STYLE").unwrap_or_else(|_| "auto".into());
if std::env::var("RUST_LOG").is_ok_and(|x| x == "debug" || x == "info") {
let rust_log_env = std::env::var("RUST_LOG");
if rust_log_env
.as_ref()
.is_ok_and(|x| x == "debug" || x == "info")
{
std::env::set_var(
"RUST_LOG",
&format!("windmill={}", std::env::var("RUST_LOG").unwrap()),
&format!("windmill={}", rust_log_env.as_ref().unwrap()),
)
}
let default_env_filter = if rust_log_env.is_ok_and(|x| x == "debug") {
LevelFilter::DEBUG
} else {
LevelFilter::INFO
};
let meter_provider = crate::otel_ee::init_meter_provider(mode, hostname, environment);
#[cfg(all(feature = "otel", feature = "enterprise"))]
let opentelemetry = crate::otel_ee::init_otlp_tracer(mode, hostname, environment)
.map(|x| tracing_opentelemetry::layer().with_tracer(x));
#[cfg(not(all(feature = "otel", feature = "enterprise")))]
let opentelemetry: Option<EnvFilter> = None;
let logs_bridge = crate::otel_ee::init_logs_bridge(&mode, hostname, environment);
let env_filter = EnvFilter::from_default_env();
use tracing_appender::rolling::{RollingFileAppender, Rotation};
let log_dir = format!("{}/{}/", TMP_WINDMILL_LOGS_SERVICE, hostname);
@@ -61,39 +94,59 @@ pub fn initialize_tracing(hostname: &str) -> WorkerGuard {
.finish(file_appender);
let stdout_and_log_file_writer = std::io::stdout.and(log_file_writer);
let ts_base = tracing_subscriber::registry().with(env_filter);
// let job_logs_filter = tracing_subscriber::filter::Targets::new()
// .with_target("windmill:job_log", tracing::Level::TRACE);
#[cfg(feature = "loki")]
let ts_base = {
let (layer, task) = tracing_loki::builder()
.build_url(reqwest::Url::parse("http://127.0.0.1:3100").unwrap())
.expect("build loki url");
tokio::spawn(task);
ts_base.with(layer)
};
let env_filter = EnvFilter::builder()
.with_default_directive(tracing::level_filters::LevelFilter::ERROR.into())
.from_env_lossy();
let ts_base = tracing_subscriber::registry().with(env_filter);
match *JSON_FMT {
true => ts_base
.with(logs_bridge)
.with(opentelemetry)
// .with(env_filter2.add_directive("windmill:job_log=off".parse().unwrap()))
.with(
json_layer()
.with_writer(stdout_and_log_file_writer)
.flatten_event(true),
.flatten_event(true)
.with_filter(
Targets::new()
.with_target(
"windmill:job_log",
tracing::level_filters::LevelFilter::OFF,
)
.with_default(default_env_filter),
),
)
.with(CountingLayer::new())
.init(),
false => ts_base
.with(logs_bridge)
.with(opentelemetry)
// .with(env_filter2.add_directive("windmill:job_log=off".parse().unwrap()))
.with(
compact_layer()
.with_writer(stdout_and_log_file_writer)
.with_ansi(style.to_lowercase() != "never")
.with_file(true)
.with_line_number(true)
.with_target(false),
.with_target(false)
.with_filter(
Targets::new()
.with_target(
"windmill:job_log",
tracing::level_filters::LevelFilter::OFF,
)
.with_default(default_env_filter),
),
)
.with(CountingLayer::new())
.init(),
}
_guard
(_guard, meter_provider)
}
#[cfg(feature = "flamegraph")]
@@ -112,13 +165,6 @@ pub fn setup_flamegraph() -> impl Drop {
_guard
}
use std::{
collections::HashMap,
sync::{Arc, RwLock},
};
use tracing::Event;
use tracing_subscriber::layer::Context;
lazy_static::lazy_static! {
pub static ref LOG_COUNTING_BY_MIN: Arc<RwLock<HashMap<String, LogCounter>>> = Arc::new(RwLock::new(HashMap::new()));
}
@@ -144,22 +190,6 @@ impl CountingLayer {
}
}
// impl CountingLayer {
// pub fn new() -> Self {
// CountingLayer { counter: Arc::new(Mutex::new(LogCounter::new())) }
// }
// pub fn get_counts(&self) -> (usize, usize) {
// let counter = self.counter.lock().unwrap();
// (counter.non_error_count, counter.error_count)
// }
// pub fn reset_counts(&self) {
// let mut counter = self.counter.lock().unwrap();
// counter.reset();
// }
// }
pub const LOG_TIMESTAMP_FMT: &str = "%Y-%m-%d-%H-%M";
impl<S> tracing_subscriber::Layer<S> for CountingLayer
backend/windmill-common/src/worker.rs
+12 -3
View File
@@ -17,7 +17,9 @@ use std::{
use tokio::sync::RwLock;
use windmill_macros::annotations;
use crate::{error, global_settings::CUSTOM_TAGS_SETTING, indexer::TantivyIndexerSettings, server::Smtp, DB};
use crate::{
error, global_settings::CUSTOM_TAGS_SETTING, indexer::TantivyIndexerSettings, server::Smtp, DB,
};
lazy_static::lazy_static! {
pub static ref WORKER_GROUP: String = std::env::var("WORKER_GROUP").unwrap_or_else(|_| "default".to_string());
@@ -352,6 +354,11 @@ pub struct SqlAnnotations {
pub return_last_result: bool,
}
#[annotations("#")]
pub struct BashAnnotations {
pub docker: bool,
}
pub async fn load_cache(bin_path: &str, _remote_path: &str) -> (bool, String) {
if tokio::fs::metadata(&bin_path).await.is_ok() {
(true, format!("loaded from local cache: {}\n", bin_path))
@@ -577,8 +584,10 @@ pub fn get_windmill_memory_usage() -> Option<i64> {
}
}
pub async fn update_min_version<'c, E: sqlx::Executor<'c, Database = sqlx::Postgres>>(executor: E) -> bool {
use crate::utils::{GIT_VERSION, GIT_SEM_VERSION};
pub async fn update_min_version<'c, E: sqlx::Executor<'c, Database = sqlx::Postgres>>(
executor: E,
) -> bool {
use crate::utils::{GIT_SEM_VERSION, GIT_VERSION};
// fetch all pings with a different version than self from the last 5 minutes.
let pings = sqlx::query_scalar!(
backend/windmill-queue/Cargo.toml
+2 -1
View File
@@ -43,4 +43,5 @@ bigdecimal.workspace = true
axum.workspace = true
serde_urlencoded.workspace = true
regex.workspace = true
backon.workspace = true
backon.workspace = true
opentelemetry.workspace = true
backend/windmill-queue/src/jobs.rs
+21 -8
View File
@@ -32,7 +32,6 @@ use sqlx::{types::Json, FromRow, Pool, Postgres, Transaction};
#[cfg(feature = "benchmark")]
use std::time::Instant;
use tokio::{sync::RwLock, time::sleep};
use tracing::{instrument, Instrument};
use ulid::Ulid;
use uuid::Uuid;
use windmill_audit::audit_ee::{audit_log, AuditAuthor};
@@ -454,7 +453,6 @@ where
}
}
#[instrument(level = "trace", skip_all)]
pub async fn add_completed_job_error(
db: &Pool<Postgres>,
queued_job: &QueuedJob,
@@ -510,7 +508,6 @@ lazy_static::lazy_static! {
pub static ref GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE: Option<String> = std::env::var("GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE").ok();
}
#[instrument(level = "trace", skip_all, name = "add_completed_job")]
pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
db: &Pool<Postgres>,
queued_job: &QueuedJob,
@@ -643,9 +640,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
.fetch_one(&mut *tx)
.await
.map_err(|e| Error::InternalErr(format!("Could not add completed job {job_id}: {e:#}")))?;
// tracing::error!("2 {:?}", start.elapsed());
// add_time!(bench, "add_completed_job query END");
if !queued_job.is_flow_step {
if _duration > 500
@@ -1259,7 +1254,6 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync>
Ok(())
}
#[instrument(level = "trace", skip_all)]
pub async fn handle_maybe_scheduled_job<'c>(
db: &Pool<Postgres>,
job: &QueuedJob,
@@ -2429,7 +2423,6 @@ async fn extract_result_from_job_result(
}
}
#[instrument(level = "trace", skip_all)]
pub async fn delete_job<'c>(
mut tx: Transaction<'c, Postgres>,
w_id: &str,
@@ -3229,6 +3222,26 @@ pub async fn push<'c, 'd>(
None,
)
}
JobPayload::AppScript {
id, // app_script(id).
path,
language,
cache_ttl,
} => (
Some(id.0),
path,
None,
JobKind::AppScript,
None,
None,
Some(language),
None,
None,
None,
cache_ttl,
None,
None,
),
JobPayload::ScriptHub { path } => {
if path == "hub/7771/slack" || path == "hub/7836/slack" {
permissioned_as = SUPERADMIN_NOTIFICATION_EMAIL.to_string();
@@ -3989,6 +4002,7 @@ pub async fn push<'c, 'd>(
JobKind::DeploymentCallback => "jobs.run.deployment_callback",
JobKind::FlowScript => "jobs.run.flow_script",
JobKind::FlowNode => "jobs.run.flow_node",
JobKind::AppScript => "jobs.run.app_script",
};
let audit_author = if format!("u/{user}") != permissioned_as && user != permissioned_as {
@@ -4018,7 +4032,6 @@ pub async fn push<'c, 'd>(
script_path.as_ref().map(|x| x.as_str()),
Some(hm),
)
.instrument(tracing::info_span!("job_run", email = &email))
.await?;
}
backend/windmill-worker/Cargo.toml
+11 -1
View File
@@ -18,7 +18,10 @@ parquet = ["windmill-common/parquet", "dep:object_store"]
flow_testing = []
cloud = []
sqlx = []
deno_core = ["dep:deno_fetch", "dep:deno_webidl", "dep:deno_web", "dep:deno_net", "dep:deno_console", "dep:deno_url", "dep:deno_core", "dep:deno_ast", "dep:deno_tls"]
deno_core = ["dep:deno_fetch", "dep:deno_webidl", "dep:deno_web", "dep:deno_net", "dep:deno_console", "dep:deno_url", "dep:deno_core",
"dep:deno_ast", "dep:deno_tls", "dep:deno_permissions"]
otel = ["windmill-common/otel", "dep:opentelemetry"]
dind = ["dep:bollard"]
[dependencies]
windmill-queue.workspace = true
@@ -68,6 +71,8 @@ deno_url = { workspace = true, optional = true }
deno_core = { workspace = true, optional = true }
deno_ast = { workspace = true, optional = true }
deno_tls = { workspace = true, optional = true }
deno_permissions = { workspace = true, optional = true }
postgres-native-tls.workspace = true
native-tls.workspace = true
mysql_async.workspace = true
@@ -92,6 +97,9 @@ yaml-rust.workspace = true
swc_ecma_parser.workspace = true
backon.workspace = true
opentelemetry = { workspace = true, optional = true }
bollard = { workspace = true, optional = true }
[build-dependencies]
deno_fetch = { workspace = true, optional = true }
deno_webidl = { workspace = true, optional = true }
@@ -102,4 +110,6 @@ deno_url = { workspace = true, optional = true }
deno_core = { workspace = true, optional = true }
deno_ast = { workspace = true, optional = true }
deno_tls = { workspace = true, optional = true }
deno_permissions = { workspace = true, optional = true }
zstd.workspace = true
backend/windmill-worker/build.rs
+8 -7
View File
@@ -25,7 +25,7 @@ impl FetchPermissions for PermissionsContainer {
&mut self,
_url: &deno_core::url::Url,
_api_name: &str,
) -> Result<(), deno_core::error::AnyError> {
) -> Result<(), deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
@@ -34,7 +34,7 @@ impl FetchPermissions for PermissionsContainer {
&mut self,
_p: &'a std::path::Path,
_api_name: &str,
) -> Result<Cow<'a, Path>, deno_core::error::AnyError> {
) -> Result<Cow<'a, Path>, deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
}
@@ -53,7 +53,7 @@ impl NetPermissions for PermissionsContainer {
&mut self,
_p: &'a str,
_api_name: &str,
) -> Result<PathBuf, deno_core::error::AnyError> {
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
@@ -61,7 +61,7 @@ impl NetPermissions for PermissionsContainer {
&mut self,
_p: &'a str,
_api_name: &str,
) -> Result<PathBuf, deno_core::error::AnyError> {
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
@@ -69,7 +69,7 @@ impl NetPermissions for PermissionsContainer {
&mut self,
_host: &(T, Option<u16>),
_api_name: &str,
) -> Result<(), deno_core::error::AnyError> {
) -> Result<(), deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
@@ -77,7 +77,7 @@ impl NetPermissions for PermissionsContainer {
&mut self,
_: &'a Path,
_: &str,
) -> Result<Cow<'a, Path>, deno_core::anyhow::Error> {
) -> Result<Cow<'a, Path>, deno_permissions::PermissionCheckError> {
todo!()
}
}
@@ -95,6 +95,7 @@ fn main() {
println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap());
let exts = vec![
// deno_telemetry::deno_telemetry::init_ops_and_esm(),
deno_webidl::deno_webidl::init_ops_and_esm(),
deno_url::deno_url::init_ops_and_esm(),
deno_console::deno_console::init_ops_and_esm(),
@@ -116,10 +117,10 @@ fn main() {
deno_core::snapshot::CreateSnapshotOptions {
cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"),
startup_snapshot: None,
extension_transpiler: None,
extensions: exts,
with_runtime_cb: None,
skip_op_registration: false,
extension_transpiler: None,
},
None,
)
backend/windmill-worker/src/bash_executor.rs
+227 -1
View File
@@ -1,14 +1,30 @@
use std::{collections::HashMap, fs, process::Stdio};
#[cfg(feature = "dind")]
use bollard::container::{
KillContainerOptions, RemoveContainerOptions, StatsOptions, StopContainerOptions,
};
#[cfg(feature = "dind")]
use futures::{stream, StreamExt, TryStreamExt};
use regex::Regex;
use serde_json::{json, value::RawValue};
use sqlx::types::Json;
use tokio::process::Command;
#[cfg(feature = "dind")]
use uuid::Uuid;
use windmill_common::{
error::Error,
jobs::QueuedJob,
worker::{to_raw_value, write_file},
};
#[cfg(feature = "dind")]
use windmill_common::DB;
#[cfg(feature = "dind")]
use windmill_common::error::to_anyhow;
use windmill_queue::{append_logs, CanceledBy};
lazy_static::lazy_static! {
@@ -22,6 +38,9 @@ lazy_static::lazy_static! {
static ref RE_POWERSHELL_IMPORTS: Regex = Regex::new(r#"^Import-Module\s+(?:-Name\s+)?"?([^-\s"]+)"?"#).unwrap();
}
#[cfg(feature = "dind")]
use crate::handle_child::run_future_with_polling_update_job_poller;
use crate::{
common::{
build_args_map, get_reserved_variables, read_file, read_file_content, start_child_process,
@@ -54,8 +73,14 @@ pub async fn handle_bash_job(
worker_name: &str,
envs: HashMap<String, String>,
occupancy_metrics: &mut OccupancyMetrics,
_killpill_rx: &mut tokio::sync::broadcast::Receiver<()>,
) -> Result<Box<RawValue>, Error> {
let logs1 = "\n\n--- BASH CODE EXECUTION ---\n".to_string();
let annotation = windmill_common::worker::BashAnnotations::parse(&content);
let mut logs1 = "\n\n--- BASH CODE EXECUTION ---\n".to_string();
if annotation.docker {
logs1.push_str("docker mode\n");
}
append_logs(&job.id, &job.workspace_id, logs1, db).await;
write_file(job_dir, "main.sh", &format!("set -e\n{content}"))?;
@@ -193,6 +218,22 @@ exit $exit_status
)
.await?;
#[cfg(feature = "dind")]
if annotation.docker {
return handle_docker_job(
job.id,
&job.workspace_id,
db,
job.timeout,
mem_peak,
canceled_by,
worker_name,
occupancy_metrics,
_killpill_rx,
)
.await;
}
let result_json_path = format!("{job_dir}/result.json");
if let Ok(metadata) = tokio::fs::metadata(&result_json_path).await {
if metadata.len() > 0 {
@@ -214,6 +255,7 @@ exit $exit_status
.await?
.trim()
.to_string();
return Ok(to_raw_value(&json!(result)));
}
@@ -222,6 +264,190 @@ exit $exit_status
)))
}
#[cfg(feature = "dind")]
async fn handle_docker_job(
job_id: Uuid,
workspace_id: &str,
db: &DB,
job_timeout: Option<i32>,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
worker_name: &str,
occupancy_metrics: &mut OccupancyMetrics,
killpill_rx: &mut tokio::sync::broadcast::Receiver<()>,
) -> Result<Box<RawValue>, Error> {
let client = bollard::Docker::connect_with_unix_defaults().map_err(to_anyhow)?;
let container_id = job_id.to_string();
let inspected = client.inspect_container(&container_id, None).await;
if inspected.is_err() || inspected.unwrap().state.is_none() {
return Ok(to_raw_value(&format!(
"Container not found at {job_id}, you must use --name and not --rm it"
)));
}
let wait_f = async {
let wait = client
.wait_container::<String>(&container_id, None)
.try_collect::<Vec<_>>()
.await
.map_err(|e| {
tracing::error!("Error waiting for container: {:?}", e);
anyhow::anyhow!("Error waiting for container")
})?;
let waited = wait.first().map(|x| x.status_code);
Ok(waited)
};
let ncontainer_id = container_id.to_string();
let w_id = workspace_id.to_string();
let j_id = job_id.clone();
let db2 = db.clone();
let (tx, mut rx) = tokio::sync::broadcast::channel::<()>(1);
let mut killpill_rx = killpill_rx.resubscribe();
let logs = tokio::spawn(async move {
let client = bollard::Docker::connect_with_unix_defaults().map_err(to_anyhow);
if let Ok(client) = client {
let mut log_stream = client.logs(
&ncontainer_id,
Some(bollard::container::LogsOptions {
follow: true,
stdout: true,
stderr: true,
tail: "all",
..Default::default()
}),
);
loop {
tokio::select! {
log = log_stream.next() => {
match log {
Some(Ok(log)) => {
append_logs(&j_id, w_id.clone(), log.to_string(), db2.clone()).await;
}
Some(Err(e)) => {
tracing::error!("Error getting logs: {:?}", e);
}
_ => {
tracing::error!("End of stream");
return
}
};
},
_ = killpill_rx.recv() => {
tracing::error!("killing container after receving killpill");
if let Err(e) = client
.stop_container(&ncontainer_id, Some(StopContainerOptions { t: 3 }))
.await
{
tracing::error!("Error stopping container: {:?}", e);
}
return
},
_ = rx.recv() => {
return
}
}
}
}
});
let mem_client = bollard::Docker::connect_with_unix_defaults().map_err(to_anyhow);
let ncontainer_id = container_id.clone();
let result = run_future_with_polling_update_job_poller(
job_id,
job_timeout,
db,
mem_peak,
canceled_by,
wait_f,
worker_name,
workspace_id,
&mut Some(occupancy_metrics),
Box::pin(match mem_client {
Ok(client) => client
.stats(
&ncontainer_id,
Some(StatsOptions { stream: true, one_shot: false }),
)
.map(|x| {
x.map(|x| x.memory_stats.usage.map(|m| m / 1024).unwrap_or_default() as i32)
.unwrap_or_default()
})
.boxed(),
_ => stream::once(async { 0 }).boxed(),
}),
)
.await;
if let Err(e) = result {
if !logs.is_finished() {
let _ = tx.send(());
let _ = logs.await;
}
if container_is_alive(&client, &container_id).await {
kill_container(&client, &container_id, "SIGINT").await;
if container_is_alive(&client, &container_id).await {
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
if let Err(e) = client
.stop_container(&container_id, Some(StopContainerOptions { t: 3 }))
.await
{
tracing::error!("Error stopping container: {:?}", e);
}
}
}
return Err(e);
}
if let Err(e) = client
.remove_container(
&container_id,
Some(RemoveContainerOptions { force: true, ..Default::default() }),
)
.await
{
tracing::error!("Error removing container: {:?}", e);
}
let result = result.unwrap();
return Ok(to_raw_value(&json!(format!(
"Docker exit status: {}",
result
.map(|x| x.to_string())
.unwrap_or_else(|| "none".to_string())
))));
}
#[cfg(feature = "dind")]
async fn kill_container(client: &bollard::Docker, container_id: &str, signal: &str) {
if let Err(e) = client
.kill_container(&container_id, Some(KillContainerOptions { signal }))
.await
{
tracing::error!("Error killing container with signal {signal}: {:?}", e);
}
}
#[cfg(feature = "dind")]
async fn container_is_alive(client: &bollard::Docker, container_id: &str) -> bool {
let inspect = client.inspect_container(container_id, None).await;
if let Ok(inspect) = inspect {
let r = inspect
.state
.map(|x| x.running.unwrap_or_default())
.unwrap_or_default();
tracing::error!("Container {container_id} is alive: {r}");
r
} else {
false
}
}
fn raw_to_string(x: &str) -> String {
match serde_json::from_str::<serde_json::Value>(x) {
Ok(serde_json::Value::String(x)) => x,
backend/windmill-worker/src/bigquery_executor.rs
+1
View File
@@ -366,6 +366,7 @@ pub async fn do_bigquery(
worker_name,
&job.workspace_id,
&mut Some(occupancy_metrics),
Box::pin(futures::stream::once(async { 0 })),
)
.await?;
backend/windmill-worker/src/common.rs
+5 -5
View File
@@ -866,11 +866,11 @@ fn tentatively_improve_error(err: Error, executable: &str) -> Error {
#[cfg(windows)]
let err_msg = "program not found";
if err
.to_string()
.contains(&err_msg)
{
return Error::InternalErr(format!("Executable {executable} not found on worker. PATH: {}", *PATH_ENV));
if err.to_string().contains(&err_msg) {
return Error::InternalErr(format!(
"Executable {executable} not found on worker. PATH: {}",
*PATH_ENV
));
}
return err;
}
backend/windmill-worker/src/dedicated_worker.rs
+7 -7
View File
@@ -16,8 +16,7 @@ use windmill_common::error::Error;
use windmill_common::flows::FlowValue;
use windmill_common::worker::WORKER_CONFIG;
use windmill_common::{
cache,
error,
cache, error,
flows::{FlowModule, FlowModuleValue},
jobs::QueuedJob,
scripts::{ScriptHash, ScriptLang},
@@ -416,17 +415,18 @@ async fn spawn_dedicated_workers_for_flow(
job_completed_tx,
Some(module.id.clone()),
)
.await
.await
{
workers.push(dedi_w);
}
},
}
Err(err) => tracing::error!(
"failed to get script for module: {:?}, err: {:?}",
module, err
)
module,
err
),
}
},
}
FlowModuleValue::Flow { .. } => (),
FlowModuleValue::Identity => (),
}
backend/windmill-worker/src/graphql_executor.rs
+2 -1
View File
@@ -1,7 +1,7 @@
use std::collections::HashMap;
use anyhow::anyhow;
use futures::TryStreamExt;
use futures::{stream, TryStreamExt};
use serde_json::{json, value::RawValue};
use sqlx::types::Json;
use windmill_common::jobs::QueuedJob;
@@ -154,6 +154,7 @@ pub async fn do_graphql(
worker_name,
&job.workspace_id,
&mut Some(occupation_metrics),
Box::pin(stream::once(async { 0 })),
)
.await?;
backend/windmill-worker/src/handle_child.rs
+20 -18
View File
@@ -87,7 +87,7 @@ async fn kill_process_tree(pid: Option<u32>) -> Result<(), String> {
/// - update the `last_line` and `logs` strings with the program output
/// - update "queue"."last_ping" every five seconds
/// - kill process if we exceed timeout or "queue"."canceled" is set
#[tracing::instrument(level = "trace", skip_all)]
#[tracing::instrument(name="run_subprocess", level = "info", skip_all, fields(otel.name = %child_name))]
pub async fn handle_child(
job_id: &Uuid,
db: &Pool<Postgres>,
@@ -126,7 +126,7 @@ pub async fn handle_child(
let (tx, rx) = broadcast::channel::<()>(3);
let mut rx2 = tx.subscribe();
let output = child_joined_output_stream(&mut child);
let output = child_joined_output_stream(&mut child, job_id.clone());
let job_id = job_id.clone();
@@ -137,7 +137,9 @@ pub async fn handle_child(
db,
mem_peak,
canceled_by_ref,
|| get_mem_peak(pid, nsjail),
Box::pin(stream::unfold((), move |_| async move {
Some((get_mem_peak(pid, nsjail).await, ()))
})),
worker,
w_id,
rx,
@@ -435,8 +437,6 @@ pub async fn handle_child(
}
}
async fn get_mem_peak(pid: Option<u32>, nsjail: bool) -> i32 {
if pid.is_none() {
return -1;
@@ -488,7 +488,7 @@ async fn get_mem_peak(pid: Option<u32>, nsjail: bool) -> i32 {
}
}
pub async fn run_future_with_polling_update_job_poller<Fut, T>(
pub async fn run_future_with_polling_update_job_poller<Fut, T, S>(
job_id: Uuid,
timeout: Option<i32>,
db: &DB,
@@ -498,9 +498,11 @@ pub async fn run_future_with_polling_update_job_poller<Fut, T>(
worker_name: &str,
w_id: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
get_mem: S,
) -> error::Result<T>
where
Fut: Future<Output = anyhow::Result<T>>,
S: stream::Stream<Item = i32> + Unpin,
{
let (tx, rx) = broadcast::channel::<()>(3);
@@ -509,7 +511,7 @@ where
db,
mem_peak,
canceled_by_ref,
|| async { 0 },
get_mem,
worker_name,
w_id,
rx,
@@ -550,20 +552,19 @@ pub enum UpdateJobPollingExit {
AlreadyCompleted,
}
pub async fn update_job_poller<F, Fut>(
pub async fn update_job_poller<S>(
job_id: Uuid,
db: &DB,
mem_peak: &mut i32,
canceled_by_ref: &mut Option<CanceledBy>,
get_mem: F,
mut get_mem: S,
worker_name: &str,
w_id: &str,
mut rx: broadcast::Receiver<()>,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> UpdateJobPollingExit
where
F: Fn() -> Fut,
Fut: Future<Output = i32>,
S: stream::Stream<Item = i32> + Unpin,
{
let update_job_interval = Duration::from_millis(500);
@@ -608,7 +609,7 @@ where
.expect("update worker ping");
}
}
let current_mem = get_mem().await;
let current_mem = get_mem.next().await.unwrap_or(0);
if current_mem > *mem_peak {
*mem_peak = current_mem
}
@@ -679,6 +680,7 @@ where
/// builds a stream joining both stdout and stderr each read line by line
fn child_joined_output_stream(
child: &mut Child,
job_id: Uuid,
) -> impl stream::FusedStream<Item = io::Result<String>> {
let stderr = child
.stderr
@@ -692,24 +694,24 @@ fn child_joined_output_stream(
let stdout = BufReader::new(stdout).lines();
let stderr = BufReader::new(stderr).lines();
stream::select(lines_to_stream(stderr, true), lines_to_stream(stdout, false))
stream::select(
lines_to_stream(stderr, true, job_id.clone()),
lines_to_stream(stdout, false, job_id),
)
}
pub fn lines_to_stream<R: tokio::io::AsyncBufRead + Unpin>(
mut lines: tokio::io::Lines<R>,
stderr: bool,
job_id: Uuid,
) -> impl futures::Stream<Item = io::Result<String>> {
stream::poll_fn(move |cx| {
std::pin::Pin::new(&mut lines)
.poll_next_line(cx)
.map(|result| {
process_streaming_log_lines(result, stderr)
})
.map(|result| process_streaming_log_lines(result, stderr, &job_id))
})
}
pub fn process_status(status: ExitStatus) -> error::Result<()> {
if status.success() {
Ok(())
backend/windmill-worker/src/job_logger_ee.rs
+1
View File
@@ -34,6 +34,7 @@ pub(crate) async fn default_disk_log_storage(
pub(crate) fn process_streaming_log_lines(
r: Result<Option<String>, io::Error>,
_stderr: bool,
_job_id: &Uuid,
) -> Option<Result<String, io::Error>> {
r.transpose()
}
backend/windmill-worker/src/js_eval.rs
+40 -51
View File
@@ -7,14 +7,7 @@
*/
#[cfg(feature = "deno_core")]
use std::{
borrow::Cow,
cell::RefCell,
env,
io::{self, BufReader},
path::PathBuf,
rc::Rc,
};
use std::{borrow::Cow, cell::RefCell, env, path::PathBuf, rc::Rc};
use std::{collections::HashMap, sync::Arc};
@@ -22,7 +15,6 @@ use std::{collections::HashMap, sync::Arc};
use deno_ast::ParseParams;
#[cfg(feature = "deno_core")]
use deno_core::{
error::AnyError,
op2, serde_v8, url,
v8::{self, IsolateHandle},
Extension, JsRuntime, OpState, PollEventLoopOptions, RuntimeOptions,
@@ -31,8 +23,7 @@ use deno_core::{
use deno_fetch::FetchPermissions;
#[cfg(feature = "deno_core")]
use deno_net::NetPermissions;
#[cfg(feature = "deno_core")]
use deno_tls::{rustls::RootCertStore, rustls_pemfile};
#[cfg(feature = "deno_core")]
use deno_web::{BlobStore, TimersPermission};
#[cfg(feature = "deno_core")]
@@ -68,35 +59,35 @@ pub struct IdContext {
pub previous_id: String,
}
#[cfg(feature = "deno_core")]
pub struct ContainerRootCertStoreProvider {
root_cert_store: RootCertStore,
}
// #[cfg(feature = "deno_core")]
// pub struct ContainerRootCertStoreProvider {
// root_cert_store: RootCertStore,
// }
#[cfg(feature = "deno_core")]
impl ContainerRootCertStoreProvider {
fn new() -> ContainerRootCertStoreProvider {
return ContainerRootCertStoreProvider {
root_cert_store: deno_tls::create_default_root_cert_store(),
};
}
// #[cfg(feature = "deno_core")]
// impl ContainerRootCertStoreProvider {
// fn new() -> ContainerRootCertStoreProvider {
// return ContainerRootCertStoreProvider {
// root_cert_store: deno_tls::create_default_root_cert_store(),
// };
// }
fn add_certificate(&mut self, cert_path: String) -> io::Result<()> {
let cert_file = std::fs::File::open(cert_path)?;
let mut reader = BufReader::new(cert_file);
let pem_file = rustls_pemfile::certs(&mut reader).collect::<Result<Vec<_>, _>>()?;
// fn add_certificate(&mut self, cert_path: String) -> io::Result<()> {
// let cert_file = std::fs::File::open(cert_path)?;
// let mut reader = BufReader::new(cert_file);
// let pem_file = rustls_pemfile::certs(&mut reader).collect::<Result<Vec<_>, _>>()?;
self.root_cert_store.add_parsable_certificates(pem_file);
Ok(())
}
}
// self.root_cert_store.add_parsable_certificates(pem_file);
// Ok(())
// }
// }
#[cfg(feature = "deno_core")]
impl deno_tls::RootCertStoreProvider for ContainerRootCertStoreProvider {
fn get_or_try_init(&self) -> Result<&RootCertStore, AnyError> {
Ok(&self.root_cert_store)
}
}
// #[cfg(feature = "deno_core")]
// impl deno_tls::RootCertStoreProvider for ContainerRootCertStoreProvider {
// fn get_or_try_init(&self) -> Result<&RootCertStore, AnyError> {
// Ok(&self.root_cert_store)
// }
// }
#[cfg(feature = "deno_core")]
pub struct PermissionsContainer;
@@ -108,7 +99,7 @@ impl FetchPermissions for PermissionsContainer {
&mut self,
_url: &deno_core::url::Url,
_api_name: &str,
) -> Result<(), deno_core::error::AnyError> {
) -> Result<(), deno_permissions::PermissionCheckError> {
Ok(())
}
@@ -117,7 +108,7 @@ impl FetchPermissions for PermissionsContainer {
&mut self,
p: &'a std::path::Path,
_api_name: &str,
) -> Result<Cow<'a, std::path::Path>, anyhow::Error> {
) -> Result<Cow<'a, std::path::Path>, deno_permissions::PermissionCheckError> {
Ok(Cow::Borrowed(p))
}
}
@@ -136,7 +127,7 @@ impl NetPermissions for PermissionsContainer {
&mut self,
p: &'a str,
_api_name: &str,
) -> Result<PathBuf, deno_core::error::AnyError> {
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
Ok(PathBuf::from(p))
}
@@ -144,7 +135,7 @@ impl NetPermissions for PermissionsContainer {
&mut self,
p: &'a str,
_api_name: &str,
) -> Result<PathBuf, deno_core::error::AnyError> {
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
Ok(PathBuf::from(p))
}
@@ -152,7 +143,7 @@ impl NetPermissions for PermissionsContainer {
&mut self,
_host: &(T, Option<u16>),
_api_name: &str,
) -> Result<(), deno_core::error::AnyError> {
) -> Result<(), deno_permissions::PermissionCheckError> {
Ok(())
}
@@ -160,7 +151,7 @@ impl NetPermissions for PermissionsContainer {
&mut self,
p: &'a std::path::Path,
_api_name: &str,
) -> Result<std::borrow::Cow<'a, std::path::Path>, AnyError> {
) -> Result<std::borrow::Cow<'a, std::path::Path>, deno_permissions::PermissionCheckError> {
Ok(Cow::Borrowed(p))
}
}
@@ -669,9 +660,12 @@ pub fn transpile_ts(expr: String) -> anyhow::Result<String> {
text: deno_core::ModuleCodeString::from(expr).into(),
})?;
Ok(parsed
.transpile(&Default::default(), &Default::default())?
.transpile(
&Default::default(),
&Default::default(),
&Default::default(),
)?
.into_source()
.into_string()?
.text)
}
@@ -814,13 +808,7 @@ pub async fn eval_fetch_timeout(
let ext = Extension { name: "windmill", ops: ops.into(), ..Default::default() };
let fetch_options = deno_fetch::Options {
root_cert_store_provider: if let Some(cert_path) = env::var("DENO_CERT").ok() {
let mut cert_store_provider = ContainerRootCertStoreProvider::new();
cert_store_provider.add_certificate(cert_path)?;
Some(Arc::new(cert_store_provider))
} else {
None
},
root_cert_store_provider: None,
user_agent: ann.useragent.unwrap_or_else(|| "windmill/beta".to_string()),
proxy: ann.proxy.map(|x| deno_tls::Proxy {
url: x.0,
@@ -930,6 +918,7 @@ pub async fn eval_fetch_timeout(
worker_name,
w_id,
&mut Some(occupation_metrics),
Box::pin(futures::stream::once(async { 0 })),
)
.await
.map_err(|e| {
backend/windmill-worker/src/mssql_executor.rs
+1
View File
@@ -160,6 +160,7 @@ pub async fn do_mssql(
worker_name,
&job.workspace_id,
&mut Some(occupancy_metrics),
Box::pin(futures::stream::once(async { 0 })),
)
.await?;
backend/windmill-worker/src/mysql_executor.rs
+1
View File
@@ -298,6 +298,7 @@ pub async fn do_mysql(
worker_name,
&job.workspace_id,
&mut Some(occupancy_metrics),
Box::pin(futures::stream::once(async { 0 })),
)
.await?;
backend/windmill-worker/src/pg_executor.rs
+1
View File
@@ -355,6 +355,7 @@ pub async fn do_postgresql(
worker_name,
&job.workspace_id,
&mut Some(occupancy_metrics),
Box::pin(futures::stream::once(async { 0 })),
)
.await?;
backend/windmill-worker/src/python_executor.rs
+80 -57
View File
@@ -3,7 +3,7 @@ use std::{
fs,
path::Path,
process::Stdio,
sync::Arc
sync::Arc,
};
use anyhow::anyhow;
@@ -318,7 +318,7 @@ pub async fn uv_pip_compile(
if let Some(cert_path) = PIP_INDEX_CERT.as_ref() {
args.extend(["--cert", cert_path]);
}
tracing::debug!("uv args: {:?}", args);
tracing::error!("uv args: {:?}", args);
#[cfg(windows)]
let uv_cmd = "uv";
@@ -329,7 +329,8 @@ pub async fn uv_pip_compile(
let mut child_cmd = Command::new(uv_cmd);
child_cmd
.current_dir(job_dir)
.args(args)
.env("HOME", HOME_ENV.to_string())
.args(&args)
.stdout(Stdio::piped())
.stderr(Stdio::piped());
let child_process = start_child_process(child_cmd, uv_cmd).await?;
@@ -350,7 +351,12 @@ pub async fn uv_pip_compile(
occupancy_metrics,
)
.await
.map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?;
.map_err(|e| {
Error::ExecutionErr(format!(
"Lock file generation failed.\n\ncommand: {uv_cmd} {}\n\n{e:?}",
args.join(" ")
))
})?;
}
let path_lock = format!("{job_dir}/requirements.txt");
@@ -423,12 +429,12 @@ async fn postinstall(
.ok_or(anyhow::anyhow!("Cannot convert OsString to String"))?
.to_owned();
if name == "bin" || name.contains("dist-info") {
if name == "bin" || name == "__pycache__" || name.contains("dist-info") {
continue;
}
if let Some(existing_paths) = lookup_table.get_mut(&name) {
tracing::info!(
tracing::debug!(
"Found existing package name: {:?} in {}",
entry.file_name(),
path
@@ -479,18 +485,22 @@ fn copy_dir_recursively(src: &Path, dst: &Path) -> windmill_common::error::Resul
fs::create_dir_all(dst)?;
}
tracing::debug!("Copying recursively from {:?} to {:?}", src, dst);
for entry in fs::read_dir(src)? {
let entry = entry?;
let src_path = entry.path();
let dst_path = dst.join(entry.file_name());
if src_path.is_dir() {
if src_path.is_dir() && !src_path.is_symlink() {
copy_dir_recursively(&src_path, &dst_path)?;
} else {
fs::copy(&src_path, &dst_path)?;
}
}
tracing::debug!("Finished copying recursively from {:?} to {:?}", src, dst);
Ok(())
}
@@ -529,10 +539,13 @@ pub async fn handle_python_job(
)
.await?;
tracing::debug!("Finished handling python dependencies");
if !PythonAnnotations::parse(inner_content).no_postinstall {
if let Err(e) = postinstall(&mut additional_python_paths, job_dir, job, db).await {
tracing::error!("Postinstall stage has failed. Reason: {e}");
}
tracing::debug!("Finished deps postinstall stage");
}
append_logs(
@@ -563,9 +576,12 @@ pub async fn handle_python_job(
)
.await?;
tracing::debug!("Finished preparing wrapper");
let apply_preprocessor = pre_spread.is_some();
create_args_and_out_file(&client, job, job_dir, db).await?;
tracing::debug!("Finished preparing wrapper");
let preprocessor = if let Some(pre_spread) = pre_spread {
format!(
@@ -664,6 +680,8 @@ except BaseException as e:
);
write_file(job_dir, "wrapper.py", &wrapper_content)?;
tracing::debug!("Finished writing wrapper");
let client = client.get_authed().await;
let mut reserved_variables = get_reserved_variables(job, &client.token, db).await?;
let additional_python_paths_folders = additional_python_paths.iter().join(":");
@@ -749,7 +767,8 @@ mount {{
.stdout(Stdio::piped())
.stderr(Stdio::piped());
#[cfg(windows)] {
#[cfg(windows)]
{
python_cmd.env("SystemRoot", SYSTEM_ROOT.as_str());
python_cmd.env("USERPROFILE", crate::USERPROFILE_ENV.as_str());
}
@@ -1093,7 +1112,7 @@ async fn handle_python_deps(
let mut venv_path = handle_python_reqs(
requirements
.split("\n")
.filter(|x| !x.starts_with("--"))
.filter(|x| !x.starts_with("--") && !x.trim().is_empty())
.collect(),
job_id,
w_id,
@@ -1239,29 +1258,28 @@ async fn spawn_uv_install(
#[cfg(unix)]
{
if no_uv_install {
let mut flock_cmd = Command::new(FLOCK_PATH.as_str());
flock_cmd
.env_clear()
.envs(PROXY_ENVS.clone())
.envs(envs)
.args([
"-x",
&format!(
"{}/{}-{}.lock",
LOCK_CACHE_DIR,
if no_uv_install { "pip" } else { "py311" },
fssafe_req
),
"--command",
&command_args.join(" "),
])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
start_child_process(flock_cmd, FLOCK_PATH.as_str()).await
let mut flock_cmd = Command::new(FLOCK_PATH.as_str());
flock_cmd
.env_clear()
.envs(PROXY_ENVS.clone())
.envs(envs)
.args([
"-x",
&format!(
"{}/{}-{}.lock",
LOCK_CACHE_DIR,
if no_uv_install { "pip" } else { "py311" },
fssafe_req
),
"--command",
&command_args.join(" "),
])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
start_child_process(flock_cmd, FLOCK_PATH.as_str()).await
} else {
let mut cmd = Command::new(command_args[0]);
cmd
.env_clear()
cmd.env_clear()
.envs(PROXY_ENVS.clone())
.envs(envs)
.args(&command_args[1..])
@@ -1321,7 +1339,6 @@ pub async fn handle_python_reqs(
mut no_uv_install: bool,
is_ansible: bool,
) -> error::Result<Vec<String>> {
let counter_arc = Arc::new(tokio::sync::Mutex::new(0));
// Append logs with line like this:
// [9/21] + requests==2.32.3 << (S3) | in 57ms
@@ -1405,7 +1422,6 @@ pub async fn handle_python_reqs(
.map(handle_ephemeral_token),
);
// Prepare NSJAIL
if !*DISABLE_NSJAIL {
let _ = write_file(
@@ -1438,7 +1454,7 @@ pub async fn handle_python_reqs(
let mut in_cache = vec![];
for req in requirements {
// Ignore python version annotation backed into lockfile
if req.starts_with('#') {
if req.starts_with('#') || req.starts_with('-') || req.trim().is_empty() {
continue;
}
// TODO: Remove
@@ -1461,12 +1477,19 @@ pub async fn handle_python_reqs(
}
}
if in_cache.len() > 0 {
append_logs(&job_id, w_id, format!("\nenv deps from local cache: {}\n", in_cache.join(", ")), db).await;
append_logs(
&job_id,
w_id,
format!("\nenv deps from local cache: {}\n", in_cache.join(", ")),
db,
)
.await;
}
let (kill_tx, ..) = tokio::sync::broadcast::channel::<()>(1);
let kill_rxs: Vec<tokio::sync::broadcast::Receiver<()>> =
(0..req_with_penv.len()).map(|_| kill_tx.subscribe()).collect();
let kill_rxs: Vec<tokio::sync::broadcast::Receiver<()>> = (0..req_with_penv.len())
.map(|_| kill_tx.subscribe())
.collect();
// ________ Read comments at the end of the function to get more context
let (_done_tx, mut done_rx) = tokio::sync::mpsc::channel::<()>(1);
@@ -1506,21 +1529,21 @@ pub async fn handle_python_reqs(
if canceled {
tracing::info!(
// If there is listener on other side,
// If there is listener on other side,
workspace_id = %w_id_2,
"cancelling installations",
);
if let Err(ref e) = kill_tx.send(()){
tracing::error!(
// If there is listener on other side,
// If there is listener on other side,
workspace_id = %w_id_2,
"failed to send done: Probably receiving end closed too early or have not opened yet\n{}",
// If there is no listener, it will be dropped safely
e
);
}
}
}
}
// Once done_tx is dropped, this will be fired
_ = done_rx.recv() => break
@@ -1555,9 +1578,15 @@ pub async fn handle_python_reqs(
// Do we use Nsjail?
if !*DISABLE_NSJAIL {
logs.push_str(&format!("\nStarting isolated installation... ({} tasks in parallel) \n", parallel_limit));
logs.push_str(&format!(
"\nStarting isolated installation... ({} tasks in parallel) \n",
parallel_limit
));
} else {
logs.push_str(&format!("\nStarting installation... ({} tasks in parallel) \n", parallel_limit));
logs.push_str(&format!(
"\nStarting installation... ({} tasks in parallel) \n",
parallel_limit
));
}
append_logs(&job_id, w_id, logs, db).await;
}
@@ -1617,13 +1646,12 @@ pub async fn handle_python_reqs(
tokio::select! {
// Cancel was called on the job
_ = kill_rx.recv() => return Err(anyhow::anyhow!("S3 pull was canceled")),
pull = pull_from_tar(os, venv_p.clone(), no_uv_install) => {
if let Err(e) = pull {
tracing::info!(
workspace_id = %w_id,
"No tarball was found on S3 or different problem occured {job_id}:\n{e}",
);
);
} else {
print_success(
true,
@@ -1653,12 +1681,12 @@ pub async fn handle_python_reqs(
no_uv_install,
).await {
Ok(r) => r,
Err(e) => {
Err(e) => {
append_logs(
&job_id,
w_id,
format!(
"\nError while spawning proccess:\n{e}",
"\nError while spawning proccess:\n{e}",
),
db,
)
@@ -1698,7 +1726,7 @@ pub async fn handle_python_reqs(
&job_id,
w_id,
format!(
"\nError while installing {}:\n{buf}",
"\nError while installing {}:\n{buf}",
&req
),
db,
@@ -1757,7 +1785,10 @@ pub async fn handle_python_reqs(
let mut failed = false;
for (handle, (_, venv_p)) in handles.into_iter().zip(req_with_penv.into_iter()) {
if let Err(e) = handle.await.unwrap_or(Err(anyhow!("Problem by joining handle"))) {
if let Err(e) = handle
.await
.unwrap_or(Err(anyhow!("Problem by joining handle")))
{
failed = true;
tracing::warn!(
workspace_id = %w_id,
@@ -1778,19 +1809,11 @@ pub async fn handle_python_reqs(
if has_work {
let total_time = total_time.elapsed().as_millis();
append_logs(
&job_id,
w_id,
format!(
"\nenv set in {}ms",
total_time
),
db,
).await;
append_logs(&job_id, w_id, format!("\nenv set in {}ms", total_time), db).await;
}
// Usually done_tx will drop after this return
// If there is listener on other side,
// If there is listener on other side,
// it will be triggered
// If there is no listener, it will be dropped safely
return if failed {
backend/windmill-worker/src/result_processor.rs
+91 -27
View File
@@ -1,3 +1,6 @@
#[cfg(feature = "otel")]
use opentelemetry::trace::FutureExt;
use serde::Serialize;
use sqlx::{types::Json, Pool, Postgres};
use std::{
@@ -7,6 +10,9 @@ use std::{
Arc,
},
};
use tracing::{field, Instrument};
#[cfg(not(feature = "otel"))]
use windmill_common::otel_ee::FutureExt;
use uuid::Uuid;
@@ -85,7 +91,49 @@ pub fn start_background_processor(
JobKind::Dependencies | JobKind::FlowDependencies
);
handle_receive_completed_job(
let success = jc.success;
let span = tracing::span!(
tracing::Level::INFO,
"job_postprocessing",
job_id = %jc.job.id, root_job = field::Empty, workspace_id = %jc.job.workspace_id, worker = %worker_name,tag = %jc.job.tag,
// hostname = %hostname,
language = field::Empty,
script_path = field::Empty,
flow_step_id = field::Empty,
parent_job = field::Empty,
otel.name = field::Empty
);
let rj = if let Some(root_job) = jc.job.root_job {
root_job
} else {
jc.job.id
};
windmill_common::otel_ee::set_span_parent(&span, &rj);
if let Some(lg) = jc.job.language.as_ref() {
span.record("language", lg.as_str());
}
if let Some(step_id) = jc.job.flow_step_id.as_ref() {
span.record(
"otel.name",
format!("job_postprocessing {}", step_id).as_str(),
);
span.record("flow_step_id", step_id.as_str());
} else {
span.record("otel.name", "job postprocessing");
}
if let Some(parent_job) = jc.job.parent_job.as_ref() {
span.record("parent_job", parent_job.to_string().as_str());
}
if let Some(script_path) = jc.job.script_path.as_ref() {
span.record("script_path", script_path.as_str());
}
if let Some(root_job) = jc.job.root_job.as_ref() {
span.record("root_job", root_job.to_string().as_str());
}
let root_job = handle_receive_completed_job(
jc,
&base_internal_url,
&db,
@@ -96,8 +144,13 @@ pub fn start_background_processor(
#[cfg(feature = "benchmark")]
&mut bench,
)
.instrument(span)
.await;
if let Some(root_job) = root_job {
windmill_common::otel_ee::add_root_flow_job_to_otlp(&root_job, success);
}
if is_init_script_and_failure {
tracing::error!("init script errored, exiting");
killpill_tx.send(()).unwrap_or_default();
@@ -199,7 +252,11 @@ async fn send_job_completed(
token,
duration,
};
job_completed_tx.send(jc).await.expect("send job completed")
job_completed_tx
.send(jc)
.with_context(windmill_common::otel_ee::otel_ctx())
.await
.expect("send job completed")
}
pub async fn process_result(
@@ -271,6 +328,7 @@ pub async fn process_result(
token,
duration,
)
.with_context(windmill_common::otel_ee::otel_ctx())
.await;
Ok(true)
}
@@ -315,6 +373,7 @@ pub async fn process_result(
token,
duration,
)
.with_context(windmill_common::otel_ee::otel_ctx())
.await;
Ok(false)
}
@@ -330,7 +389,7 @@ pub async fn handle_receive_completed_job(
worker_name: &str,
job_completed_tx: Sender<SendResult>,
#[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) {
) -> Option<Arc<QueuedJob>> {
let token = jc.token.clone();
let workspace = jc.job.workspace_id.clone();
let client = AuthedClient {
@@ -342,7 +401,7 @@ pub async fn handle_receive_completed_job(
let job = jc.job.clone();
let mem_peak = jc.mem_peak.clone();
let canceled_by = jc.canceled_by.clone();
if let Err(err) = process_completed_job(
match process_completed_job(
jc,
&client,
db,
@@ -355,26 +414,29 @@ pub async fn handle_receive_completed_job(
)
.await
{
handle_job_error(
db,
&client,
job.as_ref(),
mem_peak,
canceled_by,
err,
false,
same_worker_tx.clone(),
&worker_dir,
worker_name,
job_completed_tx,
#[cfg(feature = "benchmark")]
bench,
)
.await;
Err(err) => {
handle_job_error(
db,
&client,
job.as_ref(),
mem_peak,
canceled_by,
err,
false,
same_worker_tx.clone(),
&worker_dir,
worker_name,
job_completed_tx,
#[cfg(feature = "benchmark")]
bench,
)
.await;
None
}
Ok(r) => r,
}
}
#[tracing::instrument(name = "completed_job", level = "info", skip_all, fields(job_id = %job.id))]
pub async fn process_completed_job(
JobCompleted { job, result, mem_peak, success, cached_res_path, canceled_by, duration, .. }: JobCompleted,
client: &AuthedClient,
@@ -384,7 +446,7 @@ pub async fn process_completed_job(
worker_name: &str,
job_completed_tx: Sender<SendResult>,
#[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) -> windmill_common::error::Result<()> {
) -> windmill_common::error::Result<Option<Arc<QueuedJob>>> {
if success {
// println!("bef completed job{:?}", SystemTime::now());
if let Some(cached_path) = cached_res_path {
@@ -414,8 +476,8 @@ pub async fn process_completed_job(
if is_flow_step {
if let Some(parent_job) = parent_job {
tracing::info!(parent_flow = %parent_job, subflow = %job_id, "updating flow status (2)");
update_flow_status_after_job_completion(
// tracing::info!(parent_flow = %parent_job, subflow = %job_id, "updating flow status (2)");
let r = update_flow_status_after_job_completion(
db,
client,
parent_job,
@@ -434,9 +496,10 @@ pub async fn process_completed_job(
)
.warn_after_seconds(10)
.await?;
add_time!(bench, "updated flow status END");
return Ok(r);
}
}
add_time!(bench, "updated flow status END");
} else {
let result = add_completed_job_error(
db,
@@ -454,7 +517,7 @@ pub async fn process_completed_job(
if job.is_flow_step {
if let Some(parent_job) = job.parent_job {
tracing::error!(parent_flow = %parent_job, subflow = %job.id, "process completed job error, updating flow status");
update_flow_status_after_job_completion(
let r = update_flow_status_after_job_completion(
db,
client,
parent_job,
@@ -473,10 +536,11 @@ pub async fn process_completed_job(
)
.warn_after_seconds(10)
.await?;
return Ok(r);
}
}
}
Ok(())
return Ok(None);
}
#[tracing::instrument(name = "job_error", level = "info", skip_all, fields(job_id = %job.id))]
backend/windmill-worker/src/snowflake_executor.rs
+1
View File
@@ -419,6 +419,7 @@ pub async fn do_snowflake(
worker_name,
&job.workspace_id,
&mut Some(occupancy_metrics),
Box::pin(futures::stream::once(async { 0 })),
)
.await?;
*mem_peak = (r.get().len() / 1000) as i32;
backend/windmill-worker/src/windmill-client.js
+6
View File
@@ -558,6 +558,9 @@ var $QueuedJob = {
"identity",
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode",
"appscript",
],
},
schedule_path: {
@@ -697,6 +700,9 @@ var $CompletedJob = {
"identity",
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode",
"appscript",
],
},
schedule_path: {
backend/windmill-worker/src/worker.rs
+113 -27
View File
@@ -6,7 +6,11 @@
* LICENSE-AGPL for a copy of the license.
*/
// #[cfg(feature = "otel")]
// use opentelemetry::{global, KeyValue};
use windmill_common::{
apps::AppScriptId,
auth::{fetch_authed_from_permissioned_as, JWTAuthClaims, JobPerms, JWT_SECRET},
scripts::PREVIEW_IS_TAR_CODEBASE_HASH,
utils::WarnAfterExt,
@@ -24,7 +28,7 @@ use const_format::concatcp;
#[cfg(feature = "prometheus")]
use prometheus::IntCounter;
use tracing::Instrument;
use tracing::{field, Instrument};
#[cfg(feature = "prometheus")]
use windmill_common::METRICS_DEBUG_ENABLED;
#[cfg(feature = "prometheus")]
@@ -719,7 +723,10 @@ fn add_outstanding_wait_time(
}.in_current_span());
}
#[tracing::instrument(name = "worker", level = "info", skip_all, fields(worker = %worker_name, hostname = %hostname))]
// struct WorkerMtrics {
// job_
// }
pub async fn run_worker(
db: &Pool<Postgres>,
hostname: &str,
@@ -735,6 +742,7 @@ pub async fn run_worker(
#[cfg(not(feature = "enterprise"))]
if !*DISABLE_NSJAIL {
tracing::warn!(
worker = %worker_name, hostname = %hostname,
"NSJAIL to sandbox process in untrusted environments is an enterprise feature but allowed to be used for testing purposes"
);
}
@@ -742,10 +750,10 @@ pub async fn run_worker(
let start_time = Instant::now();
let worker_dir = format!("{TMP_DIR}/{worker_name}");
tracing::debug!(worker_dir = %worker_dir, "Creating worker dir");
tracing::debug!(worker = %worker_name, hostname = %hostname, worker_dir = %worker_dir, "Creating worker dir");
if let Some(ref netrc) = *NETRC {
tracing::info!("Writing netrc at {}/.netrc", HOME_ENV.as_str());
tracing::info!(worker = %worker_name, hostname = %hostname, "Writing netrc at {}/.netrc", HOME_ENV.as_str());
write_file(&HOME_ENV, ".netrc", netrc).expect("could not write netrc");
}
@@ -960,6 +968,14 @@ pub async fn run_worker(
None
};
// let worker_resource = &[
// KeyValue::new("hostname", hostname.to_string()),
// KeyValue::new("worker", worker_name.to_string()),
// ];
// // Create a meter from the above MeterProvider.
// let meter = global::meter("windmill");
// let counter = meter.u64_counter("jobs.execution").build();
let mut occupancy_metrics = OccupancyMetrics::new(start_time);
let mut jobs_executed = 0;
@@ -1015,6 +1031,7 @@ pub async fn run_worker(
IS_READY.store(true, Ordering::Relaxed);
tracing::info!(
worker = %worker_name, hostname = %hostname,
"listening for jobs, WORKER_GROUP: {}, config: {:?}",
*WORKER_GROUP,
WORKER_CONFIG.read().await
@@ -1050,7 +1067,7 @@ pub async fn run_worker(
if i_worker == 1 {
if let Err(e) = queue_init_bash_maybe(db, same_worker_tx.clone(), &worker_name).await {
killpill_tx.send(()).unwrap_or_default();
tracing::error!("Error queuing init bash script for worker {worker_name}: {e:#}");
tracing::error!(worker = %worker_name, hostname = %hostname, "Error queuing init bash script for worker {worker_name}: {e:#}");
return;
}
}
@@ -1079,11 +1096,12 @@ pub async fn run_worker(
let mut last_suspend_first = Instant::now();
let mut killed_but_draining_same_worker_jobs = false;
let mut killpill_rx2 = killpill_rx.resubscribe();
loop {
#[cfg(feature = "enterprise")]
{
if let Ok(_) = killpill_rx.try_recv() {
tracing::info!("killpill received on worker waiting for valid key");
tracing::info!(worker = %worker_name, hostname = %hostname, "killpill received on worker waiting for valid key");
job_completed_tx
.0
.send(SendResult::Kill)
@@ -1095,6 +1113,7 @@ pub async fn run_worker(
if !valid_key {
tracing::error!(
worker = %worker_name, hostname = %hostname,
"Invalid license key, workers require a valid license key, sleeping for 30s waiting for valid key to be set"
);
tokio::time::sleep(Duration::from_secs(10)).await;
@@ -1108,7 +1127,7 @@ pub async fn run_worker(
#[cfg(feature = "prometheus")]
if let Some(wk) = worker_busy.as_ref() {
wk.set(0);
tracing::debug!("set worker busy to 0");
tracing::debug!(worker = %worker_name, hostname = %hostname, "set worker busy to 0");
}
occupancy_metrics.running_job_started_at = None;
@@ -1120,7 +1139,7 @@ pub async fn run_worker(
.try_into()
.unwrap(),
);
tracing::debug!("set uptime metric");
tracing::debug!(worker = %worker_name, hostname = %hostname, "set uptime metric");
}
if last_ping.elapsed().as_secs() > NUM_SECS_PING {
@@ -1164,15 +1183,19 @@ pub async fn run_worker(
)
.notify(|err, dur| {
tracing::error!(
worker = %worker_name, hostname = %hostname,
"retrying updating worker ping in {dur:#?}, err: {err:#?}"
);
})
.sleep(tokio::time::sleep)
.await {
tracing::error!("failed to update worker ping, exiting: {}", e);
tracing::error!(
worker = %worker_name, hostname = %hostname,
"failed to update worker ping, exiting: {}", e);
killpill_tx.send(()).unwrap_or_default();
}
tracing::info!(
worker = %worker_name, hostname = %hostname,
"ping update, memory: container={}MB, windmill={}MB",
memory_usage.unwrap_or_default() / (1024 * 1024),
wm_memory_usage.unwrap_or_default() / (1024 * 1024)
@@ -1184,16 +1207,18 @@ pub async fn run_worker(
if (jobs_executed as u32 + vacuum_shift) % VACUUM_PERIOD == 0 {
let db2 = db.clone();
let current_span = tracing::Span::current();
let worker_name = worker_name.clone();
let hostname = hostname.to_string();
tokio::task::spawn(
(async move {
tracing::info!("vacuuming queue and completed_job");
tracing::info!(worker = %worker_name, hostname = %hostname, "vacuuming queue");
if let Err(e) = sqlx::query!("VACUUM (skip_locked) queue")
.execute(&db2)
.await
{
tracing::error!("failed to vacuum queue: {}", e);
tracing::error!(worker = %worker_name, hostname = %hostname, "failed to vacuum queue: {}", e);
}
tracing::info!("vacuumed queue and completed_job");
tracing::info!(worker = %worker_name, hostname = %hostname, "vacuumed queue");
})
.instrument(current_span),
);
@@ -1229,6 +1254,7 @@ pub async fn run_worker(
if let Ok(same_worker_job) = same_worker_rx.try_recv() {
same_worker_queue_size.fetch_sub(1, Ordering::SeqCst);
tracing::debug!(
worker = %worker_name, hostname = %hostname,
"received {} from same worker channel",
same_worker_job.job_id
);
@@ -1241,6 +1267,7 @@ pub async fn run_worker(
.map_err(|_| Error::InternalErr("Impossible to fetch same_worker job".to_string()));
if r.is_err() && !same_worker_job.recoverable {
tracing::error!(
worker = %worker_name, hostname = %hostname,
"failed to fetch same_worker job on a non recoverable job, exiting"
);
job_completed_tx
@@ -1254,7 +1281,7 @@ pub async fn run_worker(
}
} else if let Ok(_) = killpill_rx.try_recv() {
if !killed_but_draining_same_worker_jobs {
tracing::info!("received killpill for worker {}, jobs are not pulled anymore except same_worker jobs", i_worker);
tracing::info!(worker = %worker_name, hostname = %hostname, "received killpill for worker {}, jobs are not pulled anymore except same_worker jobs", i_worker);
killed_but_draining_same_worker_jobs = true;
job_completed_tx
.0
@@ -1265,10 +1292,10 @@ pub async fn run_worker(
continue;
} else if killed_but_draining_same_worker_jobs {
if job_completed_processor_is_done.load(Ordering::SeqCst) {
tracing::info!("all running jobs have completed and all completed jobs have been fully processed, exiting");
tracing::info!(worker = %worker_name, hostname = %hostname, "all running jobs have completed and all completed jobs have been fully processed, exiting");
break;
} else {
tracing::info!("there may be same_worker jobs to process later, waiting for job_completed_processor to finish progressing all remaining flows before exiting");
tracing::info!(worker = %worker_name, hostname = %hostname, "there may be same_worker jobs to process later, waiting for job_completed_processor to finish progressing all remaining flows before exiting");
tokio::time::sleep(Duration::from_millis(200)).await;
continue;
}
@@ -1293,7 +1320,7 @@ pub async fn run_worker(
if !agent_mode && duration_pull_s > 0.5 {
let empty = job.as_ref().is_ok_and(|x| x.0.is_none());
tracing::warn!("pull took more than 0.5s ({duration_pull_s}), this is a sign that the database is VERY undersized for this load. empty: {empty}, err: {err_pull}");
tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.5s ({duration_pull_s}), this is a sign that the database is VERY undersized for this load. empty: {empty}, err: {err_pull}");
#[cfg(feature = "prometheus")]
if empty {
if let Some(wp) = worker_pull_over_500_counter_empty.as_ref() {
@@ -1304,7 +1331,7 @@ pub async fn run_worker(
}
} else if !agent_mode && duration_pull_s > 0.1 {
let empty = job.as_ref().is_ok_and(|x| x.0.is_none());
tracing::warn!("pull took more than 0.1s ({duration_pull_s}) this is a sign that the database is undersized for this load. empty: {empty}, err: {err_pull}");
tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.1s ({duration_pull_s}) this is a sign that the database is undersized for this load. empty: {empty}, err: {err_pull}");
#[cfg(feature = "prometheus")]
if empty {
if let Some(wp) = worker_pull_over_100_counter_empty.as_ref() {
@@ -1358,7 +1385,7 @@ pub async fn run_worker(
last_executed_job = None;
jobs_executed += 1;
tracing::debug!("started handling of job {}", job.id);
tracing::debug!(worker = %worker_name, hostname = %hostname, "started handling of job {}", job.id);
if matches!(job.job_kind, JobKind::Script | JobKind::Preview) {
if !dedicated_workers.is_empty() {
@@ -1423,6 +1450,11 @@ pub async fn run_worker(
)
.await;
// counter.add(
// 1,
// worker_resource
// );
#[cfg(feature = "prometheus")]
let _timer = register_metric(
&WORKER_EXECUTION_DURATION,
@@ -1510,6 +1542,39 @@ pub async fn run_worker(
let PulledJob { job, raw_code, raw_lock, raw_flow } = job;
let arc_job = Arc::new(job);
add_time!(bench, "handle_queued_job START");
let span = tracing::span!(tracing::Level::INFO, "job",
job_id = %arc_job.id, root_job = field::Empty, workspace_id = %arc_job.workspace_id, worker = %worker_name, hostname = %hostname, tag = %arc_job.tag,
language = field::Empty,
script_path = field::Empty, flow_step_id = field::Empty, parent_job = field::Empty,
otel.name = field::Empty);
let rj = if let Some(root_job) = arc_job.root_job {
root_job
} else {
arc_job.id
};
if let Some(lg) = arc_job.language.as_ref() {
span.record("language", lg.as_str());
}
if let Some(step_id) = arc_job.flow_step_id.as_ref() {
span.record("otel.name", format!("job {}", step_id).as_str());
span.record("flow_step_id", step_id.as_str());
} else {
span.record("otel.name", "job");
}
if let Some(parent_job) = arc_job.parent_job.as_ref() {
span.record("parent_job", parent_job.to_string().as_str());
}
if let Some(script_path) = arc_job.script_path.as_ref() {
span.record("script_path", script_path.as_str());
}
if let Some(root_job) = arc_job.root_job.as_ref() {
span.record("root_job", root_job.to_string().as_str());
}
windmill_common::otel_ee::set_span_parent(&span, &rj);
// span.context().span().add_event_with_timestamp("job created".to_string(), arc_job.created_at.into(), vec![]);
match handle_queued_job(
arc_job.clone(),
raw_code,
@@ -1525,9 +1590,11 @@ pub async fn run_worker(
base_internal_url,
job_completed_tx.clone(),
&mut occupancy_metrics,
&mut killpill_rx2,
#[cfg(feature = "benchmark")]
&mut bench,
)
.instrument(span)
.await
{
Err(err) => {
@@ -1606,7 +1673,7 @@ pub async fn run_worker(
if let Some(secs) = *EXIT_AFTER_NO_JOB_FOR_SECS {
if let Some(lj) = last_executed_job {
if lj.elapsed().as_secs() > secs {
tracing::info!("no job for {} seconds, exiting", secs);
tracing::info!(worker = %worker_name, hostname = %hostname, "no job for {} seconds, exiting", secs);
break;
}
} else {
@@ -1637,12 +1704,12 @@ pub async fn run_worker(
});
}
Err(err) => {
tracing::error!("Failed to pull jobs: {}", err);
tracing::error!(worker = %worker_name, hostname = %hostname, "Failed to pull jobs: {}", err);
}
};
}
tracing::info!("worker {} exiting", worker_name);
tracing::info!(worker = %worker_name, hostname = %hostname, "worker {} exiting", worker_name);
#[cfg(feature = "benchmark")]
{
@@ -1657,20 +1724,20 @@ pub async fn run_worker(
if has_dedicated_workers {
for handle in dedicated_handles {
if let Err(e) = handle.await {
tracing::error!("error in dedicated worker waiting for it to end: {:?}", e)
tracing::error!(worker = %worker_name, hostname = %hostname, "error in dedicated worker waiting for it to end: {:?}", e)
}
}
tracing::info!("all dedicated workers have exited");
tracing::info!(worker = %worker_name, hostname = %hostname, "all dedicated workers have exited");
}
drop(job_completed_tx);
tracing::info!("waiting for job_completed_processor to finish processing remaining jobs");
tracing::info!(worker = %worker_name, hostname = %hostname, "waiting for job_completed_processor to finish processing remaining jobs");
if let Err(e) = send_result.await {
tracing::error!("error in awaiting send_result process: {e:?}")
}
tracing::info!("worker {} exited", worker_name);
tracing::info!("number of jobs executed: {}", jobs_executed);
tracing::info!(worker = %worker_name, hostname = %hostname, "worker {} exited", worker_name);
tracing::info!(worker = %worker_name, hostname = %hostname, "number of jobs executed: {}", jobs_executed);
}
async fn queue_init_bash_maybe<'c>(
@@ -1797,7 +1864,6 @@ pub struct PreviousResult<'a> {
pub previous_result: Option<&'a RawValue>,
}
#[tracing::instrument(name = "job", level = "info", skip_all, fields(job_id = %job.id))]
async fn handle_queued_job(
job: Arc<QueuedJob>,
raw_code: Option<String>,
@@ -1813,8 +1879,11 @@ async fn handle_queued_job(
base_internal_url: &str,
job_completed_tx: JobCompletedSender,
occupancy_metrics: &mut OccupancyMetrics,
killpill_rx: &mut tokio::sync::broadcast::Receiver<()>,
#[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) -> windmill_common::error::Result<bool> {
// Extract the active span from the context
if job.canceled {
return Err(Error::JsonErr(canceled_job_to_result(&job)));
}
@@ -2120,6 +2189,7 @@ async fn handle_queued_job(
&mut column_order,
&mut new_args,
occupancy_metrics,
killpill_rx,
)
.await;
occupancy_metrics.total_duration_of_running_jobs +=
@@ -2263,6 +2333,7 @@ async fn handle_code_execution_job(
column_order: &mut Option<Vec<String>>,
new_args: &mut Option<HashMap<String, Box<RawValue>>>,
occupancy_metrics: &mut OccupancyMetrics,
killpill_rx: &mut tokio::sync::broadcast::Receiver<()>,
) -> error::Result<Box<RawValue>> {
let ContentReqLangEnvs {
content: inner_content,
@@ -2311,6 +2382,20 @@ async fn handle_code_execution_job(
codebase: None,
}
}
JobKind::AppScript => {
let (lockfile, content) = cache::app::fetch_script(
db,
AppScriptId(job.script_hash.unwrap_or(ScriptHash(0)).0),
)
.await?;
ContentReqLangEnvs {
content,
lockfile,
language: job.language.to_owned(),
envs: None,
codebase: None,
}
}
JobKind::DeploymentCallback => {
get_script_content_by_path(job.script_path.clone(), &job.workspace_id, db).await?
}
@@ -2592,6 +2677,7 @@ mount {{
worker_name,
envs,
occupancy_metrics,
killpill_rx,
)
.await
}
backend/windmill-worker/src/worker_flow.rs
+270 -209
View File
@@ -77,45 +77,34 @@ pub async fn update_flow_status_after_job_completion(
worker_name: &str,
job_completed_tx: Sender<SendResult>,
#[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) -> error::Result<()> {
) -> error::Result<Option<Arc<QueuedJob>>> {
// this is manual tailrecursion because async_recursion blows up the stack
// todo!();
potentially_crash_for_testing();
let mut rec = update_flow_status_after_job_completion_internal(
db,
client,
let mut rec = RecUpdateFlowStatusAfterJobCompletion {
flow,
job_id_for_status,
w_id,
job_id_for_status: job_id_for_status.clone(),
success,
result,
unrecoverable,
same_worker_tx.clone(),
worker_dir,
stop_early_override,
false,
worker_name,
job_completed_tx.clone(),
#[cfg(feature = "benchmark")]
bench,
)
.await?;
while let Some(nrec) = rec {
skip_error_handler: false,
};
let mut unrecoverable = unrecoverable;
loop {
potentially_crash_for_testing();
rec = match update_flow_status_after_job_completion_internal(
let nrec = match update_flow_status_after_job_completion_internal(
db,
client,
nrec.flow,
&nrec.job_id_for_status,
rec.flow,
&rec.job_id_for_status,
w_id,
nrec.success,
nrec.result,
false,
rec.success,
rec.result,
unrecoverable,
same_worker_tx.clone(),
worker_dir,
nrec.stop_early_override,
nrec.skip_error_handler,
rec.stop_early_override,
rec.skip_error_handler,
worker_name,
job_completed_tx.clone(),
#[cfg(feature = "benchmark")]
@@ -125,12 +114,12 @@ pub async fn update_flow_status_after_job_completion(
{
Ok(j) => j,
Err(e) => {
tracing::error!("Error while updating flow status of {} after completion of {}, updating flow status again with error: {e:#}", nrec.flow,&nrec.job_id_for_status);
tracing::error!("Error while updating flow status of {} after completion of {}, updating flow status again with error: {e:#}", rec.flow, &rec.job_id_for_status);
update_flow_status_after_job_completion_internal(
db,
client,
nrec.flow,
&nrec.job_id_for_status,
rec.flow,
&rec.job_id_for_status,
w_id,
false,
Arc::new(to_raw_value(&Json(&WrappedError {
@@ -139,8 +128,8 @@ pub async fn update_flow_status_after_job_completion(
true,
same_worker_tx.clone(),
worker_dir,
nrec.stop_early_override,
nrec.skip_error_handler,
rec.stop_early_override,
rec.skip_error_handler,
worker_name,
job_completed_tx.clone(),
#[cfg(feature = "benchmark")]
@@ -148,9 +137,33 @@ pub async fn update_flow_status_after_job_completion(
)
.await?
}
};
unrecoverable = false;
match nrec {
UpdateFlowStatusAfterJobCompletion::Done(job) => {
add_time!(bench, "update flow status internal END");
return Ok(Some(job));
}
UpdateFlowStatusAfterJobCompletion::Rec(nrec) => {
rec = nrec;
}
UpdateFlowStatusAfterJobCompletion::NonLastParallelBranch => {
add_time!(bench, "update flow status internal END");
return Ok(None);
}
UpdateFlowStatusAfterJobCompletion::NotDone => {
add_time!(bench, "update flow status internal END");
return Ok(None);
}
}
}
Ok(())
}
pub enum UpdateFlowStatusAfterJobCompletion {
Rec(RecUpdateFlowStatusAfterJobCompletion),
Done(Arc<QueuedJob>),
NotDone,
NonLastParallelBranch,
}
pub struct RecUpdateFlowStatusAfterJobCompletion {
flow: uuid::Uuid,
@@ -188,7 +201,7 @@ pub async fn update_flow_status_after_job_completion_internal(
worker_name: &str,
job_completed_tx: Sender<SendResult>,
#[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) -> error::Result<Option<RecUpdateFlowStatusAfterJobCompletion>> {
) -> error::Result<UpdateFlowStatusAfterJobCompletion> {
add_time!(bench, "update flow status internal START");
let (
should_continue_flow,
@@ -272,93 +285,96 @@ pub async fn update_flow_status_after_job_completion_internal(
let is_failure_step =
old_status.step >= old_status.modules.len() as i32 && old_status.modules.len() > 0;
let (mut stop_early, mut skip_if_stop_early, continue_on_error) = if let Some(se) =
stop_early_override
{
//do not stop early if module is a flow step
let step = match module_step {
Step::PreprocessorStep => None,
Step::FailureStep => None,
Step::Step(i) => Some(i),
};
let (mut stop_early, mut skip_if_stop_early, continue_on_error) =
if let Some(se) = stop_early_override {
//do not stop early if module is a flow step
let step = match module_step {
Step::PreprocessorStep => None,
Step::FailureStep => None,
Step::Step(i) => Some(i),
};
let is_flow = if let Some(_) = step {
#[derive(Deserialize)]
struct GetType<'j> { r#type: &'j str }
let is_flow = if let Some(_) = step {
#[derive(Deserialize)]
struct GetType<'j> {
r#type: &'j str,
}
current_module
.as_ref()
.map(|module| serde_json::from_str::<GetType>(module.value.get()).map(|v| v.r#type == "flow"))
.unwrap_or(Ok(false))
.unwrap_or(false)
} else {
false
};
if is_flow {
(false, false, false)
} else {
(true, se, false)
}
} else if is_failure_step || matches!(module_step, Step::PreprocessorStep) {
(false, false, false)
} else if let Some(current_module) = current_module.as_ref() {
let stop_early = success
&& !is_branch_all
&& if let Some(ref expr) = current_module
.stop_after_if
.as_ref()
.map(|x| x.expr.clone())
{
let all_iters = match &module_status {
FlowStatusModule::InProgress { flow_jobs: Some(flow_jobs), .. }
if expr.contains("all_iters") =>
{
Some(Arc::new(
retrieve_flow_jobs_results(db, w_id, flow_jobs).await?,
))
}
_ => None,
};
let args = sqlx::query_as::<_, RowArgs>(
"SELECT
args
FROM queue
WHERE id = $2",
)
.bind(old_status.step)
.bind(flow)
.fetch_one(db)
.await
.map_err(|e| {
Error::InternalErr(format!("retrieval of args from state: {e:#}"))
})?;
compute_bool_from_expr(
&expr,
Marc::new(args.args.unwrap_or_default().0),
result.clone(),
all_iters,
None,
Some(client),
None,
None,
)
.await?
current_module
.as_ref()
.map(|module| {
serde_json::from_str::<GetType>(module.value.get())
.map(|v| v.r#type == "flow")
})
.unwrap_or(Ok(false))
.unwrap_or(false)
} else {
false
};
(
stop_early,
current_module
.stop_after_if
.as_ref()
.map(|x| x.skip_if_stopped)
.unwrap_or(false),
current_module.continue_on_error.unwrap_or(false),
)
} else {
(false, false, false)
};
if is_flow {
(false, false, false)
} else {
(true, se, false)
}
} else if is_failure_step || matches!(module_step, Step::PreprocessorStep) {
(false, false, false)
} else if let Some(current_module) = current_module.as_ref() {
let stop_early = success
&& !is_branch_all
&& if let Some(ref expr) = current_module
.stop_after_if
.as_ref()
.map(|x| x.expr.clone())
{
let all_iters =
match &module_status {
FlowStatusModule::InProgress {
flow_jobs: Some(flow_jobs), ..
} if expr.contains("all_iters") => Some(Arc::new(
retrieve_flow_jobs_results(db, w_id, flow_jobs).await?,
)),
_ => None,
};
let args = sqlx::query_as::<_, RowArgs>(
"SELECT
args
FROM queue
WHERE id = $2",
)
.bind(old_status.step)
.bind(flow)
.fetch_one(db)
.await
.map_err(|e| {
Error::InternalErr(format!("retrieval of args from state: {e:#}"))
})?;
compute_bool_from_expr(
&expr,
Marc::new(args.args.unwrap_or_default().0),
result.clone(),
all_iters,
None,
Some(client),
None,
None,
)
.await?
} else {
false
};
(
stop_early,
current_module
.stop_after_if
.as_ref()
.map(|x| x.skip_if_stopped)
.unwrap_or(false),
current_module.continue_on_error.unwrap_or(false),
)
} else {
(false, false, false)
};
let skip_branch_failure = match module_status {
FlowStatusModule::InProgress {
@@ -420,7 +436,7 @@ pub async fn update_flow_status_after_job_completion_internal(
None
};
let nindex = if let Some(position) = position {
let nindex = if let Some(position) = position {
sqlx::query_scalar!(
"UPDATE queue
SET flow_status = JSONB_SET(
@@ -467,7 +483,7 @@ pub async fn update_flow_status_after_job_completion_internal(
None
};
let nindex = if let Some(position) = position {
let nindex = if let Some(position) = position {
sqlx::query_scalar!(
"UPDATE queue
SET flow_status = JSONB_SET(
@@ -603,7 +619,7 @@ pub async fn update_flow_status_after_job_completion_internal(
);
}
add_time!(bench, "non final parallel flow finished");
return Ok(None);
return Ok(UpdateFlowStatusAfterJobCompletion::NonLastParallelBranch);
}
}
FlowStatusModule::InProgress {
@@ -677,16 +693,15 @@ pub async fn update_flow_status_after_job_completion_internal(
}
}
if success || (flow_jobs.is_some() && (skip_loop_failures || skip_branch_failure)) {
let is_skipped = if current_module.as_ref().is_some_and(|m| m.skip_if.is_some()) {
let is_skipped = if current_module.as_ref().is_some_and(|m| m.skip_if.is_some())
{
sqlx::query_scalar!(
"SELECT job_kind = 'identity' FROM completed_job WHERE id = $1",
job_id_for_status
)
.fetch_one(db)
.await
.map_err(|e| {
Error::InternalErr(format!("error during skip check: {e:#}"))
})?
.map_err(|e| Error::InternalErr(format!("error during skip check: {e:#}")))?
.unwrap_or(false)
} else {
false
@@ -905,13 +920,15 @@ pub async fn update_flow_status_after_job_completion_internal(
.context("remove flow status retry")?;
}
let flow_job = sqlx::query_as::<_, PulledJob>("SELECT * FROM queue WHERE id = $1 AND workspace_id = $2")
.bind(flow)
.bind(w_id)
.fetch_optional(&mut *tx)
.await
.map_err(Into::<Error>::into)?
.ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?;
let flow_job = sqlx::query_as::<_, PulledJob>(
"SELECT * FROM queue WHERE id = $1 AND workspace_id = $2",
)
.bind(flow)
.bind(w_id)
.fetch_optional(&mut *tx)
.await
.map_err(Into::<Error>::into)?
.ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?;
tx.commit().await?;
let job_root = flow_job
@@ -927,7 +944,8 @@ pub async fn update_flow_status_after_job_completion_internal(
sqlx::query_scalar!(
"SELECT raw_flow AS \"raw_flow!: Json<Box<sqlx::types::JsonRawValue>>\"
FROM job WHERE id = $1 AND workspace_id = $2 LIMIT 1",
&flow_job.id, w_id
&flow_job.id,
w_id
)
.fetch_one(db)
.await
@@ -950,9 +968,16 @@ pub async fn update_flow_status_after_job_completion_internal(
flow_value
.as_ref()
.and_then(|value| match module_step {
Step::PreprocessorStep => value.preprocessor_module.as_ref().and_then(|m| m.retry.as_ref()),
Step::Step(i) => value.modules.get(i).as_ref().and_then(|m| m.retry.as_ref()),
Step::FailureStep => value.failure_module.as_ref().and_then(|m| m.retry.as_ref()),
Step::PreprocessorStep => value
.preprocessor_module
.as_ref()
.and_then(|m| m.retry.as_ref()),
Step::Step(i) => {
value.modules.get(i).as_ref().and_then(|m| m.retry.as_ref())
}
Step::FailureStep => {
value.failure_module.as_ref().and_then(|m| m.retry.as_ref())
}
})
.unwrap_or(&Retry::default()),
&old_status.retry,
@@ -964,7 +989,10 @@ pub async fn update_flow_status_after_job_completion_internal(
false
if !is_failure_step
&& !skip_error_handler
&& flow_value.as_ref().map(|v| v.failure_module.is_some()).unwrap_or(false) =>
&& flow_value
.as_ref()
.map(|v| v.failure_module.is_some())
.unwrap_or(false) =>
{
true
}
@@ -1032,7 +1060,7 @@ pub async fn update_flow_status_after_job_completion_internal(
reason: flow_job.canceled_reason.clone(),
}),
canceled_job_to_result(&flow_job),
worker_name,
worker_name,
true,
None,
)
@@ -1108,7 +1136,7 @@ pub async fn update_flow_status_after_job_completion_internal(
Some(nresult.clone()),
same_worker_tx.clone(),
worker_dir,
job_completed_tx,
job_completed_tx,
)
.warn_after_seconds(10)
.await
@@ -1122,17 +1150,8 @@ pub async fn update_flow_status_after_job_completion_internal(
db,
)
.await;
let _ = add_completed_job_error(
db,
&flow_job,
0,
None,
e,
worker_name,
true,
None,
)
.await;
let _ = add_completed_job_error(db, &flow_job, 0, None, e, worker_name, true, None)
.await;
true
}
Ok(_) => false,
@@ -1148,23 +1167,25 @@ pub async fn update_flow_status_after_job_completion_internal(
if let Some(parent_job) = flow_job.parent_job {
tracing::info!(subflow_id = %flow_job.id, parent_id = %parent_job, "subflow is finished, updating parent flow status");
return Ok(Some(RecUpdateFlowStatusAfterJobCompletion {
flow: parent_job,
job_id_for_status: flow,
success: success && !is_failure_step,
result: nresult.clone(),
stop_early_override: if stop_early {
Some(skip_if_stop_early)
} else {
None
return Ok(UpdateFlowStatusAfterJobCompletion::Rec(
RecUpdateFlowStatusAfterJobCompletion {
flow: parent_job,
job_id_for_status: flow,
success: success && !is_failure_step,
result: nresult.clone(),
stop_early_override: if stop_early {
Some(skip_if_stop_early)
} else {
None
},
skip_error_handler: skip_error_handler || is_failure_step,
},
skip_error_handler: skip_error_handler || is_failure_step,
}));
));
}
}
Ok(None)
Ok(UpdateFlowStatusAfterJobCompletion::Done(flow_job))
} else {
Ok(None)
Ok(UpdateFlowStatusAfterJobCompletion::NotDone)
}
}
@@ -1292,7 +1313,7 @@ async fn compute_skip_branchall_failure<'c>(
// )))
// }
fn next_retry(retry: &Retry, status: &RetryStatus) -> Option<(u16, Duration)> {
fn next_retry(retry: &Retry, status: &RetryStatus) -> Option<(u32, Duration)> {
(status.fail_count <= MAX_RETRY_ATTEMPTS)
.then(|| &retry)
.and_then(|retry| retry.interval(status.fail_count, false))
@@ -1495,8 +1516,7 @@ pub async fn handle_flow(
worker_dir: &str,
job_completed_tx: Sender<SendResult>,
) -> anyhow::Result<()> {
let flow = flow_value
.with_context(|| "Unable to parse flow definition")?;
let flow = flow_value.with_context(|| "Unable to parse flow definition")?;
let status = flow_job
.parse_flow_status()
.with_context(|| "Unable to parse flow status")?;
@@ -1511,14 +1531,15 @@ pub async fn handle_flow(
let schedule_path = flow_job.schedule_path.as_ref().unwrap();
let schedule =
get_schedule_opt(&mut tx, &flow_job.workspace_id, schedule_path).warn_after_seconds(5).await?;
let schedule = get_schedule_opt(&mut tx, &flow_job.workspace_id, schedule_path)
.warn_after_seconds(5)
.await?;
tx.commit().await?;
if let Some(schedule) = schedule {
if let Err(err) = handle_maybe_scheduled_job(
db,
db,
&flow_job,
&schedule,
flow_job.script_path.as_ref().unwrap(),
@@ -2247,14 +2268,16 @@ async fn push_next_flow_job(
} else {
let value = module.get_value();
match &value {
Ok(_) if matches!(value, Ok(FlowModuleValue::Identity)) || is_skipped => serde_json::from_str(
&serde_json::to_string(&PreviousResult {
previous_result: Some(&arc_last_job_result),
})
.unwrap(),
)
.map(Marc::new)
.map_err(|e| error::Error::InternalErr(format!("identity: {e:#}"))),
Ok(_) if matches!(value, Ok(FlowModuleValue::Identity)) || is_skipped => {
serde_json::from_str(
&serde_json::to_string(&PreviousResult {
previous_result: Some(&arc_last_job_result),
})
.unwrap(),
)
.map(Marc::new)
.map_err(|e| error::Error::InternalErr(format!("identity: {e:#}")))
}
Ok(
FlowModuleValue::Script { input_transforms, .. }
| FlowModuleValue::RawScript { input_transforms, .. }
@@ -2276,7 +2299,7 @@ async fn push_next_flow_job(
)
.await
.map(Marc::new)
},
}
Ok(_) => Ok(arc_flow_job_args.clone()),
Err(e) => {
return Err(error::Error::InternalErr(format!(
@@ -2533,7 +2556,9 @@ async fn push_next_flow_job(
};
tracing::debug!(id = %flow_job.id, root_id = %job_root, "computed perms for job {i} of {len}");
let tag = if flow_job.tag == "flow" || flow_job.tag == format!("flow-{}", flow_job.workspace_id) {
let tag = if flow_job.tag == "flow"
|| flow_job.tag == format!("flow-{}", flow_job.workspace_id)
{
payload_tag.tag.clone()
} else {
Some(flow_job.tag.clone())
@@ -2950,7 +2975,7 @@ fn payload_from_modules<'a>(
if let Some(id) = modules_node {
return Some(JobPayload::FlowNode { id, path: path() });
}
add_virtual_items_if_necessary(&mut modules);
let mut failure_module = failure_module.cloned();
@@ -3094,7 +3119,7 @@ async fn compute_next_flow_transform(
ContinuePayload::SingleJob(payload),
NextStatus::NextStep,
))
},
}
FlowModuleValue::WhileloopFlow { modules, modules_node, .. } => {
// if it's a simple single step flow, we will collapse it as an optimization and need to pass flow_input as an arg
let is_simple = is_simple_modules(&modules, flow.failure_module.as_ref());
@@ -3196,15 +3221,21 @@ async fn compute_next_flow_transform(
.into_iter()
.filter_map(|i| {
let Some(payload) = payload_from_modules(
modules.clone(), modules_node, flow.failure_module.as_ref(), flow.same_worker,
modules.clone(),
modules_node,
flow.failure_module.as_ref(),
flow.same_worker,
|| format!("{}-{i}", status.step),
|| format!("{}/forloop-{i}", flow_job.script_path()),
true
true,
) else {
return None;
};
Some(JobPayloadWithTag {
payload, tag: None, delete_after_use, timeout: None
payload,
tag: None,
delete_after_use,
timeout: None,
})
})
.collect::<Vec<_>>();
@@ -3268,28 +3299,37 @@ async fn compute_next_flow_transform(
)))?,
};
let (modules, modules_node) = match branch {
BranchChosen::Default => (default, default_node),
BranchChosen::Branch { branch } => branches
.into_iter()
.nth(branch)
.map(|Branch { modules, modules_node, .. }| (modules, modules_node))
.ok_or_else(|| Error::BadRequest(format!("Unrecognized branch for BranchOne {status_module:?}")))?
.ok_or_else(|| {
Error::BadRequest(format!(
"Unrecognized branch for BranchOne {status_module:?}"
))
})?,
};
let Some(payload) = payload_from_modules(
modules, modules_node, flow.failure_module.as_ref(), flow.same_worker,
modules,
modules_node,
flow.failure_module.as_ref(),
flow.same_worker,
|| status.step.to_string(),
|| format!("{}/branchone-{}", flow_job.script_path(), status.step),
true
true,
) else {
return Ok(NextFlowTransform::EmptyInnerFlows);
};
Ok(NextFlowTransform::Continue(
ContinuePayload::SingleJob(JobPayloadWithTag {
payload, tag: None, delete_after_use, timeout: None
payload,
tag: None,
delete_after_use,
timeout: None,
}),
NextStatus::BranchChosen(branch),
))
@@ -3308,15 +3348,21 @@ async fn compute_next_flow_transform(
.enumerate()
.filter_map(|(i, Branch { modules, modules_node, .. })| {
let Some(payload) = payload_from_modules(
modules, modules_node, flow.failure_module.as_ref(), flow.same_worker,
modules,
modules_node,
flow.failure_module.as_ref(),
flow.same_worker,
|| format!("{}-{i}", status.step),
|| format!("{}/branchall-{}", flow_job.script_path(), i),
false
false,
) else {
return None;
};
Some(JobPayloadWithTag {
payload, tag: None, delete_after_use, timeout: None
payload,
tag: None,
delete_after_use,
timeout: None,
})
})
.collect::<Vec<_>>();
@@ -3358,20 +3404,36 @@ async fn compute_next_flow_transform(
let Branch { modules, modules_node, .. } = branches
.into_iter()
.nth(branch_status.branch)
.ok_or_else(|| Error::BadRequest(format!("Unrecognized branch for BranchAll {status_module:?}")))?;
.ok_or_else(|| {
Error::BadRequest(format!(
"Unrecognized branch for BranchAll {status_module:?}"
))
})?;
let Some(payload) = payload_from_modules(
modules, modules_node, flow.failure_module.as_ref(), flow.same_worker,
modules,
modules_node,
flow.failure_module.as_ref(),
flow.same_worker,
|| format!("{}-{}", status.step, branch_status.branch),
|| format!("{}/branchall-{}", flow_job.script_path(), branch_status.branch),
false
|| {
format!(
"{}/branchall-{}",
flow_job.script_path(),
branch_status.branch
)
},
false,
) else {
return Ok(NextFlowTransform::EmptyInnerFlows);
};
Ok(NextFlowTransform::Continue(
ContinuePayload::SingleJob(JobPayloadWithTag {
payload, tag: None, delete_after_use, timeout: None
payload,
tag: None,
delete_after_use,
timeout: None,
}),
NextStatus::NextBranchStep(NextBranch {
status: branch_status,
@@ -3404,25 +3466,25 @@ async fn next_loop_iteration(
| FlowModuleValue::FlowScript { input_transforms, .. }
| FlowModuleValue::Flow { input_transforms, .. } => {
Some(std::mem::take(input_transforms))
},
}
_ => None,
};
return Ok(NextFlowTransform::Continue(
ContinuePayload::SingleJob(
payload_from_simple_module(value, db, flow_job, module, Some(inner_path())).await?
payload_from_simple_module(value, db, flow_job, module, Some(inner_path())).await?,
),
NextStatus::NextLoopIteration {
next: ns,
simple_input_transforms,
},
NextStatus::NextLoopIteration { next: ns, simple_input_transforms },
));
}
let Some(payload) = payload_from_modules(
modules, modules_node, flow.failure_module.as_ref(), flow.same_worker,
modules,
modules_node,
flow.failure_module.as_ref(),
flow.same_worker,
|| format!("{}-{}", status.step, ns.index),
inner_path,
true
true,
) else {
return Ok(NextFlowTransform::EmptyInnerFlows);
};
@@ -3438,7 +3500,10 @@ async fn next_loop_iteration(
))
}
pub(super) fn is_simple_modules(modules: &Vec<FlowModule>, failure_module: Option<&Box<FlowModule>>) -> bool {
pub(super) fn is_simple_modules(
modules: &Vec<FlowModule>,
failure_module: Option<&Box<FlowModule>>,
) -> bool {
let is_simple = modules.len() == 1
&& modules[0].is_simple()
&& modules[0].sleep.is_none()
@@ -3685,11 +3750,7 @@ fn raw_script_to_payload(
}
fn flow_to_payload(path: String, delete_after_use: bool) -> JobPayloadWithTag {
let payload = JobPayload::Flow {
path,
dedicated_worker: None,
apply_preprocessor: false,
};
let payload = JobPayload::Flow { path, dedicated_worker: None, apply_preprocessor: false };
JobPayloadWithTag { payload, tag: None, delete_after_use, timeout: None }
}
backend/windmill-worker/src/worker_lockfiles.rs
+102 -10
View File
@@ -15,6 +15,7 @@ use windmill_common::jobs::JobPayload;
use windmill_common::scripts::ScriptHash;
use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file};
use windmill_common::{
apps::AppScriptId,
error::{self, to_anyhow},
flows::{add_virtual_items_if_necessary, FlowValue},
jobs::QueuedJob,
@@ -648,7 +649,7 @@ pub async fn handle_flow_dependency_job(
// Compute a lite version of the flow value (`RawScript` => `FlowScript`).
let mut value_lite = flow.clone();
tx = reduce(
tx = reduce_flow(
tx,
&mut value_lite.modules,
&job_path,
@@ -1027,8 +1028,6 @@ async fn insert_flow_node<'c>(
) -> Result<(sqlx::Transaction<'c, sqlx::Postgres>, FlowNodeId)> {
let hash = {
let mut hasher = sha2::Sha256::new();
hasher.update(path);
hasher.update(workspace_id);
hasher.update(code.unwrap_or(&Default::default()));
hasher.update(lock.unwrap_or(&Default::default()));
hasher.update(flow.unwrap_or(&Default::default()).get());
@@ -1040,7 +1039,7 @@ async fn insert_flow_node<'c>(
r#"
INSERT INTO flow_node (path, workspace_id, hash_v2, lock, code, flow)
VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (hash_v2) DO UPDATE SET path = EXCLUDED.path -- trivial update to return the id
ON CONFLICT (path, workspace_id, hash_v2) DO UPDATE SET path = EXCLUDED.path -- trivial update to return the id
RETURNING id
"#,
path,
@@ -1055,6 +1054,41 @@ async fn insert_flow_node<'c>(
Ok((tx, FlowNodeId(id)))
}
async fn insert_app_script(
db: &sqlx::Pool<sqlx::Postgres>,
app: i64,
code: String,
lock: Option<String>,
) -> Result<AppScriptId> {
let code_sha256 = format!("{:x}", sha2::Sha256::digest(&code));
let hash = {
let mut hasher = sha2::Sha256::new();
hasher.update(app.to_le_bytes());
hasher.update(&code_sha256);
hasher.update(lock.as_ref().unwrap_or(&Default::default()));
format!("{:x}", hasher.finalize())
};
// Insert the app script if it doesn't exist.
sqlx::query_scalar!(
r#"
INSERT INTO app_script (app, hash, lock, code, code_sha256)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (hash) DO UPDATE SET app = EXCLUDED.app -- trivial update to return the id
RETURNING id
"#,
app,
hash,
lock,
code,
code_sha256
)
.fetch_one(db)
.await
.map(AppScriptId)
.map_err(Into::into)
}
async fn insert_flow_modules<'c>(
mut tx: sqlx::Transaction<'c, sqlx::Postgres>,
path: &str,
@@ -1064,7 +1098,7 @@ async fn insert_flow_modules<'c>(
modules: &mut Vec<FlowModule>,
modules_node: &mut Option<FlowNodeId>,
) -> Result<sqlx::Transaction<'c, sqlx::Postgres>> {
tx = Box::pin(reduce(
tx = Box::pin(reduce_flow(
tx,
modules,
path,
@@ -1096,7 +1130,7 @@ async fn insert_flow_modules<'c>(
Ok(tx)
}
async fn reduce<'c>(
async fn reduce_flow<'c>(
mut tx: sqlx::Transaction<'c, sqlx::Postgres>,
modules: &mut Vec<FlowModule>,
path: &str,
@@ -1109,7 +1143,7 @@ async fn reduce<'c>(
let mut val =
serde_json::from_str::<FlowModuleValue>(module.value.get()).map_err(|err| {
Error::InternalErr(format!(
"reduce: Failed to parse flow module value: {}",
"reduce_flow: Failed to parse flow module value: {}",
err
))
})?;
@@ -1205,6 +1239,48 @@ async fn reduce<'c>(
Ok(tx)
}
async fn reduce_app(db: &sqlx::Pool<sqlx::Postgres>, value: &mut Value, app: i64) -> Result<()> {
match value {
Value::Object(object) => {
if let Some(Value::Object(script)) = object.get_mut("inlineScript") {
if script
.get("language")
.and_then(|x| x.as_str())
.is_some_and(|x| x == "frontend")
{
return Ok(());
}
// replace `content` with an empty string:
let Some(Value::String(code)) = script.get_mut("content").map(std::mem::take)
else {
return Err(error::Error::InternalErr(
"Missing `content` in inlineScript".to_string(),
));
};
// remove `lock`:
let lock = script.remove("lock").and_then(|x| match x {
Value::String(s) => Some(s),
_ => None,
});
let id = insert_app_script(db, app, code, lock).await?;
// insert the `id` into the `script` object:
script.insert("id".to_string(), json!(id.0));
} else {
for (_, value) in object {
Box::pin(reduce_app(db, value, app)).await?;
}
}
}
Value::Array(array) => {
for value in array {
Box::pin(reduce_app(db, value, app)).await?;
}
}
_ => {}
}
Ok(())
}
fn skip_creating_new_lock(language: &ScriptLang, content: &str) -> bool {
if language == &ScriptLang::Bun || language == &ScriptLang::Bunnative {
let anns = windmill_common::worker::TypeScriptAnnotations::parse(&content);
@@ -1390,11 +1466,12 @@ pub async fn handle_app_dependency_job(
.clone()
.ok_or_else(|| Error::InternalErr("App Dependency requires script hash".to_owned()))?
.0;
let value = sqlx::query_scalar!("SELECT value FROM app_version WHERE id = $1", id)
let record = sqlx::query!("SELECT app_id, value FROM app_version WHERE id = $1", id)
.fetch_optional(db)
.await?;
.await?
.map(|record| (record.app_id, record.value));
if let Some(value) = value {
if let Some((app_id, value)) = record {
let value = lock_modules_app(
value,
job,
@@ -1411,6 +1488,21 @@ pub async fn handle_app_dependency_job(
)
.await?;
// Compute a lite version of the app value (w/ `inlineScript.{lock,code}`).
let mut value_lite = value.clone();
reduce_app(db, &mut value_lite, app_id).await?;
if let Value::Object(object) = &mut value_lite {
object.insert("version".to_string(), json!(id));
}
sqlx::query!(
"INSERT INTO app_version_lite (id, value) VALUES ($1, $2)
ON CONFLICT (id) DO UPDATE SET value = EXCLUDED.value",
id,
sqlx::types::Json(to_raw_value(&value_lite)) as sqlx::types::Json<Box<RawValue>>,
)
.execute(db)
.await?;
// Re-check cancelation to ensure we don't accidentially override an app.
if sqlx::query_scalar!("SELECT canceled FROM queue WHERE id = $1", job.id)
.fetch_optional(db)
benchmarks/lib.ts
+1 -1
View File
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
export const VERSION = "v1.435.2";
export const VERSION = "v1.437.1";
export async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({
cli/main.ts
+1 -1
View File
@@ -60,7 +60,7 @@ export {
// }
// });
export const VERSION = "1.435.2";
export const VERSION = "1.437.1";
const command = new Command()
.name("wmill")
docker/DockerfileSlim
+1 -1
View File
@@ -20,7 +20,7 @@ RUN /usr/local/bin/python3 -m pip install pip-tools
# Install UV
RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv
COPY --from=oven/bun:1.1.34 /usr/local/bin/bun /usr/bin/bun
COPY --from=oven/bun:1.1.38 /usr/local/bin/bun /usr/bin/bun
# add the docker client to call docker from a worker if enabled
COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/
docker/DockerfileSlimEe
+1 -1
View File
@@ -19,7 +19,7 @@ RUN /usr/local/bin/python3 -m pip install pip-tools
# Install UV
RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv
COPY --from=oven/bun:1.1.34 /usr/local/bin/bun /usr/bin/bun
COPY --from=oven/bun:1.1.38 /usr/local/bin/bun /usr/bin/bun
# add the docker client to call docker from a worker if enabled
COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/
flake.lock
Generated
+6 -6
View File
@@ -20,11 +20,11 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1732014248,
"narHash": "sha256-y/MEyuJ5oBWrWAic/14LaIr/u5E0wRVzyYsouYY3W6w=",
"lastModified": 1733759999,
"narHash": "sha256-463SNPWmz46iLzJKRzO3Q2b0Aurff3U1n0nYItxq7jU=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "23e89b7da85c3640bbc2173fe04f4bd114342367",
"rev": "a73246e2eef4c6ed172979932bc80e1404ba2d56",
"type": "github"
},
"original": {
@@ -61,11 +61,11 @@
"nixpkgs": "nixpkgs_2"
},
"locked": {
"lastModified": 1732156292,
"narHash": "sha256-XuTCME5ZausokOJ28AsIoayBVD1soscdoiKweT4VY50=",
"lastModified": 1733884434,
"narHash": "sha256-8GXR9kC07dyOIshAyfZhG11xfvBRSZzYghnZ2weOKJU=",
"owner": "oxalica",
"repo": "rust-overlay",
"rev": "2d484c7a0db32f2700e253160bcd2aaa6cdca3ba",
"rev": "d0483df44ddf0fd1985f564abccbe568e020ddf2",
"type": "github"
},
"original": {
flake.nix
+1 -1
View File
@@ -14,7 +14,7 @@
};
buildInputs = with pkgs; [
openssl openssl.dev libxml2.dev xmlsec.dev libxslt.dev
rust-bin.stable.latest.default nodejs
rust-bin.stable.latest.default nodejs cmake
postgresql
pkg-config
];
frontend/package-lock.json
Generated
+6 -6
View File
@@ -1,12 +1,12 @@
{
"name": "windmill-components",
"version": "1.435.2",
"version": "1.437.1",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "windmill-components",
"version": "1.435.2",
"version": "1.437.1",
"license": "AGPL-3.0",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.1",
@@ -72,7 +72,7 @@
"windmill-parser-wasm-go": "^1.429.0",
"windmill-parser-wasm-php": "^1.429.0",
"windmill-parser-wasm-py": "^1.429.0",
"windmill-parser-wasm-regex": "^1.429.0",
"windmill-parser-wasm-regex": "^1.435.2",
"windmill-parser-wasm-rust": "^1.429.0",
"windmill-parser-wasm-ts": "^1.429.0",
"windmill-parser-wasm-yaml": "^1.429.0",
@@ -13660,9 +13660,9 @@
"integrity": "sha512-cqc+tblQVHVrc8wNA4esVYD1Dv59XQJ4mHXFFPa8Lx5UjXv7FO/0VQxyQuRyXaP/V6J6DDy0oeN107eFNLxrBg=="
},
"node_modules/windmill-parser-wasm-regex": {
"version": "1.429.0",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.429.0.tgz",
"integrity": "sha512-rjgrgSYjEBU2+RB6ffFR+nmvEJdiqVKYiwugzHQE+VgVTYMJ0jLBE370n5dBjDj5KPsF+Sfmj6DIUK69uu9wSA=="
"version": "1.435.2",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.435.2.tgz",
"integrity": "sha512-kQShKPNjXok8jV7b7llfvKhypHfqePwiKgTHm8MlzBt4BZC87xrQ7TH0eS9h7OSK9Q98H9fIiUr2AbN8zTCraA=="
},
"node_modules/windmill-parser-wasm-rust": {
"version": "1.429.0",
frontend/package.json
+3 -3
View File
@@ -1,6 +1,6 @@
{
"name": "windmill-components",
"version": "1.435.2",
"version": "1.437.1",
"scripts": {
"dev": "vite dev",
"build": "vite build",
@@ -128,8 +128,8 @@
"ol": "^7.4.0",
"openai": "^4.57.2",
"p-limit": "^6.1.0",
"pdfjs-dist": "4.8.69",
"panzoom": "^9.4.3",
"pdfjs-dist": "4.8.69",
"quill": "^1.3.7",
"rfc4648": "^1.5.3",
"svelte-carousel": "^1.0.25",
@@ -145,7 +145,7 @@
"windmill-parser-wasm-go": "^1.429.0",
"windmill-parser-wasm-php": "^1.429.0",
"windmill-parser-wasm-py": "^1.429.0",
"windmill-parser-wasm-regex": "^1.429.0",
"windmill-parser-wasm-regex": "^1.435.2",
"windmill-parser-wasm-rust": "^1.429.0",
"windmill-parser-wasm-ts": "^1.429.0",
"windmill-parser-wasm-yaml": "^1.429.0",
frontend/src/lib/components/AuthSettings.svelte
+256
View File
@@ -0,0 +1,256 @@
<script lang="typescript">
import { enterpriseLicense } from '$lib/stores'
import { Alert, Button, Tab, Tabs } from './common'
import OAuthSetting from '$lib/components/OAuthSetting.svelte'
import OktaSetting from './OktaSetting.svelte'
import CloseButton from './common/CloseButton.svelte'
import KeycloakSetting from './KeycloakSetting.svelte'
import CustomSso from './CustomSso.svelte'
import AuthentikSetting from '$lib/components/AuthentikSetting.svelte'
import AutheliaSetting from '$lib/components/AutheliaSetting.svelte'
import KanidmSetting from '$lib/components/KanidmSetting.svelte'
import ZitadelSetting from '$lib/components/ZitadelSetting.svelte'
import CustomOauth from './CustomOauth.svelte'
import { capitalize } from '$lib/utils'
import Toggle from './Toggle.svelte'
import { ExternalLink, Plus } from 'lucide-svelte'
export let snowflakeAccountIdentifier = ''
export let oauths: Record<string, any> = {}
const windmillBuiltins = [
'github',
'gitlab',
'bitbucket',
'slack',
'gsheets',
'gdrive',
'gmail',
'gcal',
'gforms',
'gcloud',
'gworkspace',
'basecamp',
'linkedin',
'quickbooks',
'visma',
'spotify',
'snowflake_oauth',
'xero'
]
let oauth_name = undefined
let clientName = ''
let resourceName = ''
let tab: 'sso' | 'oauth' | 'scim' = 'sso'
let requirePreexistingUserForOauth: boolean = false
</script>
<div>
<Tabs bind:selected={tab} class="mt-2 mb-4">
<Tab value="sso">SSO</Tab>
<Tab value="oauth">OAuth</Tab>
<Tab value="scim">SCIM/SAML</Tab>
</Tabs>
</div>
<div class="mb-6">
{#if tab === 'sso'}
{#if !$enterpriseLicense || $enterpriseLicense.endsWith('_pro')}
<Alert type="warning" title="Limited to 10 SSO users">
Without EE, the number of SSO users is limited to 10. SCIM/SAML is available on EE
</Alert>
{/if}
<div class="py-1" />
<div class="mb-2">
<span class="text-primary text-sm"
>When at least one of the below options is set, users will be able to login to Windmill via
their third-party account.
<br /> To test SSO, the recommended workflow is to to save the settings and try to login in
an incognito window.
<a target="_blank" href="https://www.windmill.dev/docs/misc/setup_oauth#sso">Learn more</a
></span
>
</div>
<div class="flex flex-col gap-3 py-4">
<OAuthSetting name="google" bind:value={oauths['google']} />
<OAuthSetting name="microsoft" bind:value={oauths['microsoft']} />
<OktaSetting bind:value={oauths['okta']} />
<OAuthSetting name="github" bind:value={oauths['github']} />
<OAuthSetting name="gitlab" bind:value={oauths['gitlab']} />
<OAuthSetting name="jumpcloud" bind:value={oauths['jumpcloud']} />
<KeycloakSetting bind:value={oauths['keycloak']} />
<AuthentikSetting bind:value={oauths['authentik']} />
<AutheliaSetting bind:value={oauths['authelia']} />
<KanidmSetting bind:value={oauths['kanidm']} />
<ZitadelSetting bind:value={oauths['zitadel']} />
{#each Object.keys(oauths) as k}
{#if !['authelia', 'authentik', 'google', 'microsoft', 'github', 'gitlab', 'jumpcloud', 'okta', 'keycloak', 'slack', 'kanidm', 'zitadel'].includes(k) && 'login_config' in oauths[k]}
{#if oauths[k]}
<div class="flex flex-col gap-2 pb-4">
<div class="flex flex-row items-center gap-2">
<!-- svelte-ignore a11y-label-has-associated-control -->
<label class="text-md font-medium text-primary">{k}</label>
<CloseButton
on:close={() => {
delete oauths[k]
oauths = { ...oauths }
}}
/>
</div>
<div class="p-2 border rounded">
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Custom Name</span>
<input
type="text"
placeholder="Custom Name"
bind:value={oauths[k]['display_name']}
/>
</label>
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Client Id</span>
<input type="text" placeholder="Client Id" bind:value={oauths[k]['id']} />
</label>
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Client Secret</span>
<input type="text" placeholder="Client Secret" bind:value={oauths[k]['secret']} />
</label>
{#if !windmillBuiltins.includes(k) && k != 'slack'}
<CustomSso bind:login_config={oauths[k]['login_config']} />
{/if}
</div>
</div>
{/if}
{/if}
{/each}
</div>
<div class="flex gap-2 py-4">
<input type="text" placeholder="client_id" bind:value={clientName} />
<Button
variant="border"
color="blue"
hover="yo"
size="sm"
endIcon={{ icon: Plus }}
disabled={clientName == ''}
on:click={() => {
oauths[clientName] = { id: '', secret: '', login_config: {} }
clientName = ''
}}
>
Add custom SSO client {!$enterpriseLicense ? '(requires ee)' : ''}
</Button>
</div>
<div class="flex gap-2 py-4">
<Toggle
options={{
right: 'Require users to have been added manually to Windmill to sign in through OAuth'
}}
bind:checked={requirePreexistingUserForOauth}
/>
</div>
{:else if tab === 'oauth'}
<div class="mb-2">
<span class="text-primary text-sm"
>When one of the below options is set, you will be able to create a specific resource
containing a token automatically generated by the third-party provider.
<br />
To test it after setting an oauth client, go to the Resources menu and create a new one of the
type of your oauth client (i.e. a 'github' resource if you set Github OAuth).
<br /><a target="_blank" href="https://www.windmill.dev/docs/misc/setup_oauth#oauth"
>Learn more</a
></span
>
</div>
<div class="py-1" />
<OAuthSetting login={false} name="slack" bind:value={oauths['slack']} />
<div class="py-1" />
{#each Object.keys(oauths) as k}
{#if oauths[k] && !('login_config' in oauths[k])}
{#if !['slack'].includes(k) && oauths[k]}
<div class="flex flex-col gap-2 pb-4">
<div class="flex flex-row items-center gap-2">
<!-- svelte-ignore a11y-label-has-associated-control -->
<label class="text-md font-medium text-primary">{k}</label>
<CloseButton
on:close={() => {
delete oauths[k]
oauths = { ...oauths }
}}
/>
</div>
<div class="p-2 border rounded">
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Client Id</span>
<input type="text" placeholder="Client Id" bind:value={oauths[k]['id']} />
</label>
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Client Secret</span>
<input type="text" placeholder="Client Secret" bind:value={oauths[k]['secret']} />
</label>
{#if !windmillBuiltins.includes(k) && k != 'slack'}
<CustomOauth bind:connect_config={oauths[k]['connect_config']} />
{/if}
{#if k == 'snowflake_oauth'}
<label class="block pb-2">
<span class="text-primary font-semibold text-sm flex gap-2 items-center"
><a
href="https://docs.snowflake.com/en/user-guide/admin-account-identifier#using-an-account-name-as-an-identifier"
target="_blank">Snowflake Account Identifier</a
><ExternalLink size={12} /></span
>
<input
type="text"
placeholder="<orgname>-<account_name>"
required={true}
bind:value={snowflakeAccountIdentifier}
/>
</label>
{/if}
</div>
</div>
{/if}
{/if}
{/each}
<div class="flex gap-2">
<select name="oauth_name" id="oauth_name" bind:value={oauth_name}>
<option value={undefined}>Select an OAuth client</option>
<option value="custom">Fully Custom (requires ee)</option>
{#each windmillBuiltins as name}
<option value={name}>{capitalize(name)}</option>
{/each}
</select>
{#if oauth_name == 'custom'}
<input type="text" placeholder="client_id" bind:value={resourceName} />
{:else}
<input type="text" value={oauth_name ?? ''} disabled />
{/if}
<Button
variant="border"
color="blue"
hover="yo"
size="sm"
endIcon={{ icon: Plus }}
disabled={!oauth_name ||
(oauth_name == 'custom' && resourceName == '') ||
(oauth_name == 'custom' && !$enterpriseLicense)}
on:click={() => {
let name = oauth_name == 'custom' ? resourceName : oauth_name
oauths[name ?? ''] = { id: '', secret: '' }
resourceName = ''
}}
>
Add OAuth client {oauth_name == 'custom' && !$enterpriseLicense ? '(requires ee)' : ''}
</Button>
</div>
{:else if tab == 'scim'}
<slot name="scim" />
{/if}
</div>
frontend/src/lib/components/FlowStatusViewerInner.svelte
+12 -5
View File
@@ -52,6 +52,7 @@
flowJobs: string[]
flowJobsSuccess: (boolean | undefined)[]
length: number
branchall?: boolean
}
| undefined = undefined
@@ -466,9 +467,7 @@
let modId = flowJobIds?.moduleId ?? ''
let common = {
iteration_from:
// $localDurationStatuses?.[modId]?.iteration_from ??
Math.max(flowJobIds.flowJobs.length - 20, 0),
iteration_from: flowJobIds?.branchall ? 0 : Math.max(flowJobIds.flowJobs.length - 20, 0),
iteration_total: $localDurationStatuses?.[modId]?.iteration_total ?? flowJobIds?.length
}
$localDurationStatuses[modId] = {
@@ -818,6 +817,8 @@
return rec(ids, undefined)
}
let subflowsSize = 500
</script>
{#if notAnonynmous}
@@ -947,7 +948,12 @@
</h3>
<div class="overflow-auto max-h-1/2">
{#each flowJobIds?.flowJobs ?? [] as loopJobId, j (loopJobId)}
{#if render}
{#if render && j + subflowsSize + 1 == (flowJobIds?.flowJobs.length ?? 0)}
<Button variant="border" color="light" on:click={() => (subflowsSize += 500)}
>Load 500 more...</Button
>
{/if}
{#if render && j + subflowsSize + 1 > (flowJobIds?.flowJobs.length ?? 0)}
<Button
variant={forloop_selected === loopJobId ? 'contained' : 'border'}
color={flowJobIds?.flowJobsSuccess?.[j] === false
@@ -1141,7 +1147,8 @@
moduleId: mod.id,
flowJobs: mod.flow_jobs,
flowJobsSuccess: mod.flow_jobs_success,
length: mod.iterator?.itered?.length ?? mod.flow_jobs.length
length: mod.iterator?.itered?.length ?? mod.flow_jobs.length,
branchall: job?.raw_flow?.modules?.[i]?.value?.type == 'branchall'
}
: undefined}
on:jobsLoaded={(e) => {
frontend/src/lib/components/InstanceSetting.svelte
+789
View File
@@ -0,0 +1,789 @@
<script lang="typescript">
import { isCloudHosted } from '$lib/cloud'
import { enterpriseLicense, isCriticalAlertsUIOpen } from '$lib/stores'
import {
AlertCircle,
AlertTriangle,
BadgeCheck,
BadgeX,
Info,
Plus,
Slack,
X
} from 'lucide-svelte'
import type { Setting } from './instanceSettings'
import Tooltip from './Tooltip.svelte'
import ObjectStoreConfigSettings from './ObjectStoreConfigSettings.svelte'
import { sendUserToast } from '$lib/toast'
import ConfirmButton from './ConfirmButton.svelte'
import { IndexSearchService, SettingService } from '$lib/gen'
import { Button, SecondsInput, Skeleton } from './common'
import Password from './Password.svelte'
import { classNames } from '$lib/utils'
import Popover from './Popover.svelte'
import Toggle from './Toggle.svelte'
import type { Writable } from 'svelte/store'
import { createEventDispatcher } from 'svelte'
import { fade } from 'svelte/transition'
import { base } from '$lib/base'
export let setting: Setting
export let version: string
export let values: Writable<Record<string, any>>
export let loading = true
const dispatch = createEventDispatcher()
let latestKeyRenewalAttempt: {
result: string
attempted_at: string
} | null
function showSetting(setting: string, values: Record<string, any>) {
if (setting == 'dev_instance') {
if (values['license_key'] == undefined) {
return false
}
}
return true
}
let licenseKeyChanged = false
let renewing = false
let opening = false
async function reloadKeyrenewalAttemptInfo() {
latestKeyRenewalAttempt = await SettingService.getLatestKeyRenewalAttempt()
}
if (setting.key == 'license_key') {
reloadKeyrenewalAttemptInfo()
}
export async function renewLicenseKey() {
renewing = true
try {
await SettingService.renewLicenseKey({
licenseKey: $values['license_key'] || undefined
})
sendUserToast('Key renewal successful')
reloadKeyrenewalAttemptInfo()
} catch (err) {
latestKeyRenewalAttempt = await SettingService.getLatestKeyRenewalAttempt()
throw err
} finally {
renewing = false
}
}
export async function openCustomerPortal() {
opening = true
try {
const url = await SettingService.createCustomerPortalSession({
licenseKey: $values['license_key'] || undefined
})
window.open(url, '_blank')
} finally {
opening = false
}
}
function parseLicenseKey(key: string): {
valid: boolean
expiration?: string
} {
let splitted = key.split('.')
if (splitted.length >= 3) {
try {
let i = parseInt(splitted[1])
let date = new Date(i * 1000)
const stringDate = date.toLocaleDateString()
if (stringDate !== 'Invalid Date') {
return {
valid: date.getTime() > Date.now(),
expiration: date.toLocaleDateString()
}
}
} catch {}
}
return {
valid: false
}
}
</script>
{#if (!setting.cloudonly || isCloudHosted()) && showSetting(setting.key, $values) && !(setting.hiddenIfNull && $values[setting.key] == null)}
{#if setting.ee_only != undefined && !$enterpriseLicense}
<div class="flex text-xs items-center gap-1 text-yellow-500 whitespace-nowrap">
<AlertTriangle size={16} />
EE only {#if setting.ee_only != ''}<Tooltip>{setting.ee_only}</Tooltip>{/if}
</div>
{/if}
<!-- svelte-ignore a11y-label-has-associated-control -->
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">{setting.label}</span>
{#if setting.description}
<span class="text-secondary text-xs">
{@html setting.description}
</span>
{/if}
{#if setting.tooltip}
<Tooltip>{setting.tooltip}</Tooltip>
{/if}
{#if $values}
{@const hasError = setting.isValid && !setting.isValid($values[setting.key])}
{#if loading}
<Skeleton layout={[[2.5]]} />
{:else if setting.fieldType == 'text'}
<input
disabled={setting.ee_only != undefined && !$enterpriseLicense}
type="text"
placeholder={setting.placeholder}
class={hasError
? 'border !border-red-700 !border-opacity-30 !focus:border-red-700 !focus:border-opacity-30'
: ''}
bind:value={$values[setting.key]}
/>
{#if setting.advancedToggle}
<div class="mt-1">
<Toggle
size="xs"
options={{ right: setting.advancedToggle.label }}
checked={setting.advancedToggle.checked($values)}
on:change={() => {
if (setting.advancedToggle) {
$values = setting.advancedToggle.onChange($values)
}
}}
/>
</div>
{/if}
{:else if setting.fieldType == 'textarea'}
<textarea
disabled={!$enterpriseLicense}
rows="2"
placeholder={setting.placeholder}
bind:value={$values[setting.key]}
/>
{#if setting.key == 'saml_metadata'}
<div class="flex mt-2">
<Button
disabled={!$enterpriseLicense}
on:click={async (e) => {
const res = await SettingService.testMetadata({
requestBody: $values[setting.key]
})
sendUserToast(`Metadata valid, see console for full content`)
console.log(`Metadata content:`, res)
}}>Test content/url</Button
>
</div>
{/if}
{:else if setting.fieldType == 'license_key'}
{@const { valid, expiration } = parseLicenseKey($values[setting.key] ?? '')}
<div class="flex gap-2">
<Password
small
placeholder={setting.placeholder}
on:keydown={() => {
licenseKeyChanged = true
}}
bind:password={$values[setting.key]}
/>
<Button
variant={$values[setting.key] ? 'contained' : 'border'}
size="xs"
on:click={async () => {
await SettingService.testLicenseKey({
requestBody: { license_key: $values[setting.key] }
})
sendUserToast('Valid key')
}}
>
Test Key
</Button>
</div>
<div class="mt-1 flex flex-col gap-1 items-start">
{#if $values[setting.key]?.length > 0}
{#if valid}
<div class="flex flex-row gap-1 items-center">
<Info size={12} class="text-tertiary" />
<span class="text-tertiary text-xs">License key expires on {expiration ?? ''}</span>
</div>
{:else if expiration}
<div class="flex flex-row gap-1 items-center">
<AlertCircle size={12} class="text-red-600" />
<span class="text-red-600 dark:text-red-400 text-xs"
>License key expired on {expiration}</span
>
</div>
{:else}
<div class="flex flex-row gap-1 items-center">
<AlertCircle size={12} class="text-red-600" />
<span class="text-red-600 dark:text-red-400 text-xs"
>Invalid license key format</span
>
</div>
{/if}
{/if}
{#if latestKeyRenewalAttempt}
{@const attemptedAt = new Date(latestKeyRenewalAttempt.attempted_at).toLocaleString()}
{@const isTrial = latestKeyRenewalAttempt.result.startsWith('error: trial:')}
<div class="relative">
<Popover notClickable>
<div class="flex flex-row items-center gap-1">
{#if latestKeyRenewalAttempt.result === 'success'}
<BadgeCheck class="text-green-600" size={12} />
{:else}
<BadgeX class={isTrial ? 'text-yellow-600' : 'text-red-600'} size={12} />
{/if}
<span
class={classNames(
'text-xs',
latestKeyRenewalAttempt.result === 'success'
? 'text-green-600'
: isTrial
? 'text-yellow-600'
: 'text-red-600'
)}
>
{latestKeyRenewalAttempt.result === 'success'
? 'Latest key renewal succeeded'
: isTrial
? 'Latest key renewal ignored because in trial'
: 'Latest key renewal failed'}
on {attemptedAt}
</span>
</div>
<div slot="text">
{#if latestKeyRenewalAttempt.result === 'success'}
<span class="text-green-300">
Latest key renewal succeeded on {attemptedAt}
</span>
{:else if isTrial}
<span class="text-yellow-300">
License key cannot be renewed during trial ({attemptedAt})
</span>
{:else}
<span class="text-red-300">
Latest key renewal failed on {attemptedAt}: {latestKeyRenewalAttempt.result.replace(
'error: ',
''
)}
</span>
{/if}
<br />
As long as invoices are paid and usage corresponds to the subscription, the key is
renewed daily with a validity of 35 days (grace period).
</div>
</Popover>
</div>
{/if}
{#if licenseKeyChanged && !$enterpriseLicense}
{#if version.startsWith('CE')}
<div class="text-red-400"
>License key is set but image used is the Community Edition {version}. Switch image
to EE.</div
>
{/if}
{/if}
{#if valid || expiration}
<div class="flex flex-row gap-2 mt-1">
<Button on:click={renewLicenseKey} loading={renewing} size="xs" color="dark"
>Renew key
</Button>
<Button color="dark" size="xs" loading={opening} on:click={openCustomerPortal}>
Open customer portal
</Button>
</div>
{/if}
</div>
{:else if setting.fieldType == 'email'}
<input type="email" placeholder={setting.placeholder} bind:value={$values[setting.key]} />
{:else if setting.key == 'critical_alert_mute_ui'}
<div class="flex flex-col gap-y-2 my-2 py-2">
<Toggle
disabled={!$enterpriseLicense}
bind:checked={$values[setting.key]}
options={{ right: setting.description }}
/>
<div class="flex flex-row">
<Button
variant="border"
color="light"
disabled={!$enterpriseLicense}
size="xs"
on:click={() => {
isCriticalAlertsUIOpen.set(true)
dispatch('closeDrawer')
}}
>
Show Critical Alerts
</Button>
</div>
</div>
{:else if setting.fieldType == 'critical_error_channels'}
<div class="w-full flex gap-x-16 flex-wrap">
<div class="w-full max-w-lg">
{#if $enterpriseLicense && Array.isArray($values[setting.key])}
{#each $values[setting.key] ?? [] as v, i}
<div class="flex w-full max-w-lg mt-1 gap-2 items-center">
<select
class="w-20"
on:change={(e) => {
if (e.target?.['value']) {
$values[setting.key][i] = {
[e.target['value']]: ''
}
}
}}
value={v && 'slack_channel' in v ? 'slack_channel' : 'email'}
>
<option value="email">Email</option>
<option value="slack_channel">Slack</option>
</select>
{#if v && 'slack_channel' in v}
<input
type="text"
placeholder="Slack channel"
on:input={(e) => {
if (e.target?.['value']) {
$values[setting.key][i] = {
slack_channel: e.target['value']
}
}
}}
value={v?.slack_channel ?? ''}
/>
{:else}
<input
type="email"
placeholder="Email address"
on:input={(e) => {
if (e.target?.['value']) {
$values[setting.key][i] = {
email: e.target['value']
}
}
}}
value={v?.email ?? ''}
/>
{/if}
<button
transition:fade|local={{ duration: 100 }}
class="rounded-full p-1 bg-surface-secondary duration-200 hover:bg-surface-hover"
aria-label="Clear"
on:click={() => {
$values[setting.key] = $values[setting.key].filter((_, index) => index !== i)
}}
>
<X size={14} />
</button>
</div>
{/each}
{/if}
</div>
</div>
<div class="flex mt-2 gap-20 items-center">
<Button
variant="border"
color="light"
size="md"
btnClasses="mt-1"
on:click={() => {
if ($values[setting.key] == undefined || !Array.isArray($values[setting.key])) {
$values[setting.key] = []
}
$values[setting.key] = $values[setting.key].concat('')
}}
id="arg-input-add-item"
startIcon={{ icon: Plus }}
disabled={!$enterpriseLicense}
>
Add channel
</Button>
<div class="flex mt-1">
<Button
disabled={!$enterpriseLicense}
variant="border"
color="light"
size="md"
on:click={async () => {
try {
await SettingService.testCriticalChannels({
requestBody: $values[setting.key]
})
sendUserToast('Test message sent successfully to critical channels', false)
} catch (error) {
sendUserToast('Failed to send test message: ' + error.message, true)
}
}}
>
Test Channels
</Button>
</div>
</div>
{:else if setting.fieldType == 'slack_connect'}
<div class="flex flex-col items-start self-start">
{#if $values[setting.key] && 'team_name' in $values[setting.key]}
<div class="text-sm">
Connected to <code>{$values[setting.key]['team_name']}</code>
</div>
<Button
size="sm"
endIcon={{ icon: Slack }}
btnClasses="mt-2"
variant="border"
on:click={async () => {
$values[setting.key] = undefined
}}
>
Disconnect Slack
</Button>
{:else}
<Button
size="xs"
color="dark"
href="{base}/api/oauth/connect_slack?instance=true"
startIcon={{ icon: Slack }}
disabled={!$enterpriseLicense}
>
Connect to Slack
</Button>
{/if}
</div>
{:else if setting.fieldType == 'indexer_rates'}
<div class="flex flex-col gap-4 mt-4">
{#if $values[setting.key]}
<div>
<label for="writer_memory_budget" class="block text-sm font-medium">
Index writer memory budget (MB)
<Tooltip>
The allocated memory arena for the indexer. A bigger value means less writing to
disk and potentially higher indexing throughput
</Tooltip>
</label>
<input
disabled={!$enterpriseLicense}
type="number"
id="writer_memory_budget"
placeholder="300"
on:input={(e) => {
if (e.target instanceof HTMLInputElement) {
if (e.target.valueAsNumber) {
$values[setting.key].writer_memory_budget =
e.target.valueAsNumber * (1024 * 1024)
}
}
}}
value={$values[setting.key].writer_memory_budget / (1024 * 1024)}
/>
</div>
<h3>Completed Job Index</h3>
<div>
<label for="commit_job_max_batch_size" class="block text-sm font-medium">
Commit max batch size <Tooltip>
The max amount of documents (here jobs) per commit. To optimize indexing
throughput, it is best to keep this as high as possible. However, especially when
reindexing the whole instance, it can be useful to have a limit on how many jobs
can be written without being commited. A commit will make the jobs available for
search, constitute a "checkpoint" state in the indexing and will be logged.
</Tooltip>
</label>
<input
disabled={!$enterpriseLicense}
type="number"
id="commit_job_max_batch_size"
placeholder="100000"
bind:value={$values[setting.key].commit_job_max_batch_size}
/>
</div>
<div>
<label for="refresh_index_period" class="block text-sm font-medium">
Refresh index period (s) <Tooltip>
The index will query new jobs peridically and write them on the index. This
setting sets that period.
</Tooltip></label
>
<input
disabled={!$enterpriseLicense}
type="number"
id="refresh_index_period"
placeholder="300"
bind:value={$values[setting.key].refresh_index_period}
/>
</div>
<div>
<label for="max_indexed_job_log_size" class="block text-sm font-medium">
Max indexed job log size (KB) <Tooltip>
Job logs are included when indexing, but to avoid the index size growing
artificially, the logs will be truncated after a size has been reached.
</Tooltip>
</label>
<input
disabled={!$enterpriseLicense}
type="number"
id="max_indexed_job_log_size"
placeholder="1024"
on:input={(e) => {
if (e.target instanceof HTMLInputElement) {
if (e.target.valueAsNumber) {
$values[setting.key].max_indexed_job_log_size = e.target.valueAsNumber * 1024
}
}
}}
value={$values[setting.key].max_indexed_job_log_size / 1024}
/>
</div>
<h3>Service Logs Index</h3>
<div>
<label for="commit_log_max_batch_size" class="block text-sm font-medium"
>Commit max batch size Commit max batch size <Tooltip>
The max amount of documents per commit. In this case 1 document is one log file
representing all logs during 1 minute for a specific host. To optimize indexing
throughput, it is best to keep this as high as possible. However, especially when
reindexing the whole instance, it can be useful to have a limit on how many logs
can be written without being commited. A commit will make the logs available for
search, appear as a log line, and be a "checkpoint" of the indexing progress.
</Tooltip>
</label>
<input
disabled={!$enterpriseLicense}
type="number"
id="commit_log_max_batch_size"
placeholder="10000"
bind:value={$values[setting.key].commit_log_max_batch_size}
/>
</div>
<div>
<label for="refresh_log_index_period" class="block text-sm font-medium">
Refresh index period (s) <Tooltip>
The index will query new service logs peridically and write them on the index.
This setting sets that period.
</Tooltip></label
>
<input
disabled={!$enterpriseLicense}
type="number"
id="refresh_log_index_period"
placeholder="300"
bind:value={$values[setting.key].refresh_log_index_period}
/>
</div>
<h3>Reset Index</h3>
This buttons will clear the whole index, and the service will start reindexing from scratch.
Full text search might be down during this time.
<div>
<ConfirmButton
on:click={async () => {
let r = await IndexSearchService.clearIndex({
idxName: 'JobIndex'
})
console.log('asasd')
sendUserToast(r)
}}>Clear <b>Jobs</b> Index</ConfirmButton
>
<ConfirmButton
on:click={async () => {
let r = await IndexSearchService.clearIndex({
idxName: 'ServiceLogIndex'
})
console.log('asasd')
sendUserToast(r)
}}>Clear <b>Service Logs</b> Index</ConfirmButton
>
</div>
{/if}
</div>
{:else if setting.fieldType == 'smtp_connect'}
<div class="flex flex-col gap-4 border rounded p-4">
{#if $values[setting.key]}
<div>
<label for="smtp_host" class="block text-sm font-medium">Host</label>
<input
type="text"
id="smtp_host"
placeholder="smtp.gmail.com"
bind:value={$values[setting.key].smtp_host}
/>
</div>
<div>
<label for="smtp_port" class="block text-sm font-medium">Port</label>
<input
type="number"
id="smtp_port"
placeholder="587"
bind:value={$values[setting.key].smtp_port}
/>
</div>
<div>
<label for="smtp_username" class="block text-sm font-medium">Username</label>
<input
type="text"
id="smtp_username"
placeholder="ruben@windmill.dev"
bind:value={$values[setting.key].smtp_username}
/>
</div>
<div>
<label for="smtp_password" class="block text-sm font-medium">Password</label>
<Password bind:password={$values[setting.key].smtp_password} />
</div>
<div>
<label for="smtp_from" class="block text-sm font-medium">From Address</label>
<input
type="email"
id="smtp_from"
placeholder="noreply@windmill.dev"
bind:value={$values[setting.key].smtp_from}
/>
</div>
<div>
<Toggle
disabled={$values[setting.key].smtp_disable_tls == true || !$enterpriseLicense}
id="smtp_tls_implicit"
bind:checked={$values[setting.key].smtp_tls_implicit}
options={{ right: 'Implicit TLS' }}
label="Implicit TLS"
/>
</div>
<div>
<Toggle
id="smtp_disable_tls"
disabled={!$enterpriseLicense}
bind:checked={$values[setting.key].smtp_disable_tls}
on:change={() => {
if ($values[setting.key].smtp_disable_tls) {
$values[setting.key].smtp_tls_implicit = false
}
}}
options={{ right: 'Disable TLS' }}
label="Disable TLS"
/>
</div>
{/if}
</div>
{:else if setting.fieldType == 'otel'}
<div class="flex flex-col gap-4 border rounded p-4">
{#if $values[setting.key]}
<div class="flex gap-8">
<Toggle
disabled={!$enterpriseLicense}
id="tracing_enabled"
bind:checked={$values[setting.key].tracing_enabled}
options={{ right: 'Tracing' }}
label="Tracing"
/>
<Toggle
disabled={!$enterpriseLicense}
id="logs_enabled"
bind:checked={$values[setting.key].logs_enabled}
options={{ right: 'Logs' }}
label="logs"
/>
<Toggle
disabled
id="metrics_enabled"
bind:checked={$values[setting.key].logs_enabled}
options={{ right: 'Metrics (coming soon)' }}
label="metrics"
/>
</div>
<div>
<label for="OTEL_EXPORTER_OTLP_ENDPOINT" class="block text-sm font-medium"
>Endpoint</label
>
<input
disabled={!$enterpriseLicense}
type="text"
id="OTEL_EXPORTER_OTLP_ENDPOINT"
placeholder="http://otel-collector.example.com:4317"
bind:value={$values[setting.key].otel_exporter_otlp_endpoint}
/>
</div>
<div>
<label for="OTEL_EXPORTER_OTLP_HEADERS" class="block text-sm font-medium"
>Headers</label
>
<input
disabled={!$enterpriseLicense}
type="text"
id="OTEL_EXPORTER_OTLP_HEADERS"
placeholder="Authorization=Bearer my-secret-token,Env=production"
bind:value={$values[setting.key].otel_exporter_otlp_headers}
/>
</div>
<div>
<label for="OTEL_EXPORTER_OTLP_PROTOCOL" class="block text-sm font-medium"
>Protocol</label
>
gRPC
</div>
<!-- <div>
<label for="OTEL_EXPORTER_OTLP_PROTOCOL" class="block text-sm font-medium"
>Protocol<span class="text-2xs text-tertiary ml-4"
>grpc, http/protobuf, http/json</span
></label
>
<input
type="text"
id="OTEL_EXPORTER_OTLP_PROTOCOL"
placeholder="grpc"
bind:value={$values[setting.key].otel_exporter_otlp_protocol}
/>
</div>
<div>
<label for="OTEL_EXPORTER_OTLP_COMPRESSION" class="block text-sm font-medium"
>Compression <span class="text-2xs text-tertiary ml-4">none, gzip</span></label
>
<input
type="text"
id="OTEL_EXPORTER_OTLP_COMPRESSION"
placeholder="none"
bind:value={$values[setting.key].otel_exporter_otlp_compression}
/>
</div> -->
{/if}
</div>
{:else if setting.fieldType == 'object_store_config'}
<ObjectStoreConfigSettings bind:bucket_config={$values[setting.key]} />
<div class="mb-6" />
{:else if setting.fieldType == 'number'}
<input type="number" placeholder={setting.placeholder} bind:value={$values[setting.key]} />
{:else if setting.fieldType == 'password'}
<input
autocomplete="new-password"
type="password"
placeholder={setting.placeholder}
bind:value={$values[setting.key]}
/>
{:else if setting.fieldType == 'boolean'}
<div class="mt-0.5">
<Toggle
disabled={setting.ee_only != undefined && !$enterpriseLicense}
bind:checked={$values[setting.key]}
/>
</div>
{:else if setting.fieldType == 'seconds'}
<div>
<SecondsInput
max={setting.ee_only != undefined && !$enterpriseLicense
? 60 * 60 * 24 * 30
: undefined}
bind:seconds={$values[setting.key]}
/>
</div>
{/if}
{#if hasError}
<span class="text-red-500 dark:text-red-400 text-sm">
{setting.error ?? ''}
</span>
{/if}
{:else}
<input disabled placeholder="Loading..." />
{/if}
</label>
{/if}
frontend/src/lib/components/InstanceSettings.svelte
+74 -1033
View File
File diff suppressed because it is too large Load Diff
frontend/src/lib/components/OAuthSetting.svelte
+4 -3
View File
@@ -50,9 +50,10 @@
}
</script>
<div class="flex flex-col gap-1">
<div class="flex flex-col">
<!-- svelte-ignore a11y-label-has-associated-control -->
<label class="text-sm flex gap-4 items-center font-medium text-primary"
<label
class="text-sm flex gap-4 items-center font-medium text-primary {enabled ? 'rounded py-2' : ''}"
><div class="w-[120px]"><IconedResourceType {name} after={true} /></div><Toggle
checked={enabled}
on:change={(e) => {
@@ -65,7 +66,7 @@
/></label
>
{#if enabled}
<div class="p-2 rounded border">
<div class="p-2 rounded border mb-4">
{#if name != 'slack'}
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Custom Name</span>
frontend/src/lib/components/OauthExtraParams.svelte
+1 -1
View File
@@ -32,7 +32,7 @@
<div class="flex items-center mt-1">
<Button
variant="border"
color="blue"
color="light"
hover="yo"
size="sm"
endIcon={{ icon: Plus }}
frontend/src/lib/components/OauthScopes.svelte
+2 -2
View File
@@ -11,7 +11,7 @@
<input type="text" bind:value={v} />
<Button
variant="border"
color="red"
color="light"
size="xs"
btnClasses="mx-6"
on:click={() => {
@@ -27,7 +27,7 @@
<div class="flex items-center mt-1">
<Button
variant="border"
color="blue"
color="light"
hover="yo"
size="sm"
endIcon={{ icon: Plus }}
frontend/src/lib/components/ObjectStoreConfigSettings.svelte
+141 -128
View File
@@ -5,6 +5,7 @@
import { SettingService } from '$lib/gen'
import { sendUserToast } from '$lib/toast'
import TestConnection from './TestConnection.svelte'
import { enterpriseLicense } from '$lib/stores'
type S3Config = {
type: 'S3'
@@ -51,9 +52,10 @@
}
</script>
<div>
<div class="my-0.5">
<Toggle
options={{ right: 'Enable' }}
disabled={!$enterpriseLicense}
options={{ right: bucket_config ? '' : 'set object store' }}
checked={Boolean(bucket_config)}
on:change={(e) => {
if (e.detail) {
@@ -72,134 +74,145 @@
/>
</div>
{#if bucket_config}
<div class="flex gap-2 py-1">
<Button
spacingSize="sm"
size="xs"
btnClasses="h-8"
color="light"
variant="border"
on:click={testConnection}
<div class="p-2">
<div class="flex gap-2 py-1">
<Button
spacingSize="sm"
size="xs"
btnClasses="h-8"
color="light"
variant="border"
on:click={testConnection}
>
{#if loading}
<Loader2 class="animate-spin mr-2 !h-4 !w-4" />
{:else}
<Database class="mr-2 !h-4 !w-4" />
{/if}
Test from a server
</Button>
<TestConnection
args={bucket_config}
resourceType="s3_bucket"
workspaceOverride="admins"
buttonTextOverride="Test from a worker"
/>
</div>
<Tabs
bind:selected={bucket_config.type}
on:selected={(e) => {
if (e.detail === 'S3') {
bucket_config = {
type: 'S3',
bucket: '',
region: '',
access_key: '',
secret_key: '',
endpoint: ''
}
} else if (e.detail === 'Azure') {
bucket_config = {
type: 'Azure',
accountName: '',
containerName: '',
useSSL: false,
tenantId: '',
clientId: '',
accessKey: ''
}
}
}}
>
{#if loading}
<Loader2 class="animate-spin mr-2 !h-4 !w-4" />
{:else}
<Database class="mr-2 !h-4 !w-4" />
{/if}
Test from a server
</Button>
<TestConnection
args={bucket_config}
resourceType="s3_bucket"
workspaceOverride="admins"
buttonTextOverride="Test from a worker"
/>
</div>
<Tabs
bind:selected={bucket_config.type}
on:selected={(e) => {
if (e.detail === 'S3') {
bucket_config = {
type: 'S3',
bucket: '',
region: '',
access_key: '',
secret_key: '',
endpoint: ''
}
} else if (e.detail === 'Azure') {
bucket_config = {
type: 'Azure',
accountName: '',
containerName: '',
useSSL: false,
tenantId: '',
clientId: '',
accessKey: ''
}
}
}}
>
<Tab size="sm" value="S3">S3</Tab>
<Tab size="sm" value="Azure">Azure Blob</Tab>
</Tabs>
<div class="flex flex-col gap-2 mt-2 p-2 border rounded-md">
{#if bucket_config.type === 'S3'}
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Bucket</span>
<input type="text" placeholder="bucket-name" bind:value={bucket_config.bucket} />
</label>
<Tab size="sm" value="S3">S3</Tab>
<Tab size="sm" value="Azure">Azure Blob</Tab>
</Tabs>
<div class="flex flex-col gap-2 mt-2 p-2 border rounded-md">
{#if bucket_config.type === 'S3'}
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Bucket</span>
<input type="text" placeholder="bucket-name" bind:value={bucket_config.bucket} />
</label>
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Region</span>
<span class="text-tertiary text-2xs"
>If left empty, will be derived automatically from $AWS_REGION</span
>
<input type="text" bind:value={bucket_config.region} />
</label>
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Access key ID</span>
<span class="text-tertiary text-2xs"
>If left empty, will be derived automatically from $AWS_ACCESS_KEY_ID, pod or ec2 profile</span
>
<input type="text" bind:value={bucket_config.access_key} />
</label>
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Secret key</span>
<span class="text-tertiary text-2xs"
>If left empty, will be derived automatically from $AWS_SECRET_KEY, pod or ec2 profile</span
>
<input type="password" autocomplete="new-password" bind:value={bucket_config.secret_key} />
</label>
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Endpoint</span>
<span class="text-tertiary text-2xs"
>Only needed for non AWS S3 providers like R2 or MinIo</span
>
<input type="text" bind:value={bucket_config.endpoint} />
</label>
<div class="block pb-2">
<span class="text-tertiary text-2xs">Disable if using https only policy</span>
<div>
<Toggle bind:checked={bucket_config.allow_http} options={{ right: 'Allow http' }} />
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Region</span>
<span class="text-tertiary text-2xs"
>If left empty, will be derived automatically from $AWS_REGION</span
>
<input type="text" bind:value={bucket_config.region} />
</label>
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Access key ID</span>
<span class="text-tertiary text-2xs"
>If left empty, will be derived automatically from $AWS_ACCESS_KEY_ID, pod or ec2
profile</span
>
<input type="text" bind:value={bucket_config.access_key} />
</label>
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Secret key</span>
<span class="text-tertiary text-2xs"
>If left empty, will be derived automatically from $AWS_SECRET_KEY, pod or ec2 profile</span
>
<input
type="password"
autocomplete="new-password"
bind:value={bucket_config.secret_key}
/>
</label>
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Endpoint</span>
<span class="text-tertiary text-2xs"
>Only needed for non AWS S3 providers like R2 or MinIo</span
>
<input type="text" bind:value={bucket_config.endpoint} />
</label>
<div class="block pb-2">
<span class="text-tertiary text-2xs">Disable if using https only policy</span>
<div>
<Toggle bind:checked={bucket_config.allow_http} options={{ right: 'Allow http' }} />
</div>
</div>
</div>
{:else if bucket_config.type === 'Azure'}
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Account name</span>
<input type="text" placeholder="account-name" bind:value={bucket_config.accountName} />
</label>
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Container name</span>
<input type="text" placeholder="container-name" bind:value={bucket_config.containerName} />
</label>
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Access key</span>
<input type="password" autocomplete="new-password" bind:value={bucket_config.accessKey} />
</label>
<label class="block pb-2">
<span class="text-primary font-semibold text-sm"
>Tenant ID <span class="text-2xs text-tertiary">(optional)</span></span
>
<input type="text" bind:value={bucket_config.tenantId} />
</label>
<label class="block pb-2">
<span class="text-primary font-semibold text-sm"
>Client ID <span class="text-2xs text-tertiary">(optional)</span></span
>
<input type="text" bind:value={bucket_config.clientId} />
</label>
<label class="block pb-2">
<span class="text-primary font-semibold text-sm"
>Endpoint <span class="text-2xs text-tertiary">(optional)</span></span
>
<span class="text-tertiary text-2xs"
>Only needed for non Azure Blob providers like Azurite</span
>
<input type="text" bind:value={bucket_config.endpoint} />
</label>
{:else}
<div>Unknown bucket type {bucket_config['type']}</div>
{/if}
{:else if bucket_config.type === 'Azure'}
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Account name</span>
<input type="text" placeholder="account-name" bind:value={bucket_config.accountName} />
</label>
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Container name</span>
<input
type="text"
placeholder="container-name"
bind:value={bucket_config.containerName}
/>
</label>
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Access key</span>
<input type="password" autocomplete="new-password" bind:value={bucket_config.accessKey} />
</label>
<label class="block pb-2">
<span class="text-primary font-semibold text-sm"
>Tenant ID <span class="text-2xs text-tertiary">(optional)</span></span
>
<input type="text" bind:value={bucket_config.tenantId} />
</label>
<label class="block pb-2">
<span class="text-primary font-semibold text-sm"
>Client ID <span class="text-2xs text-tertiary">(optional)</span></span
>
<input type="text" bind:value={bucket_config.clientId} />
</label>
<label class="block pb-2">
<span class="text-primary font-semibold text-sm"
>Endpoint <span class="text-2xs text-tertiary">(optional)</span></span
>
<span class="text-tertiary text-2xs"
>Only needed for non Azure Blob providers like Azurite</span
>
<input type="text" bind:value={bucket_config.endpoint} />
</label>
{:else}
<div>Unknown bucket type {bucket_config['type']}</div>
{/if}
</div>
</div>
{/if}
frontend/src/lib/components/ServiceLogsInner.svelte
+1 -1
View File
@@ -587,7 +587,7 @@
<div
class="text-sm pt-2 pl-0.5 whitespace-nowrap"
title={hn}
style="width: 90px;">{truncateRev(hn, 8)}</div
style="width: 90px;">{truncateRev(hn, countsPerHost || loadingLogs ? 40 : 8)}</div
>
{#if loadingLogCounts}
<Loader2 size={15} class="animate-spin" />
frontend/src/lib/components/SuperadminSettings.svelte
+24 -14
View File
@@ -23,7 +23,7 @@
import { isCloudHosted } from '$lib/cloud'
import InstanceNameEditor from './InstanceNameEditor.svelte'
import Toggle from './Toggle.svelte'
import { instanceSettingsSelectedTab } from '$lib/stores';
import { instanceSettingsSelectedTab } from '$lib/stores'
let drawer: Drawer
let filter = ''
@@ -57,7 +57,7 @@
let tab: 'users' | string = 'users'
$: $instanceSettingsSelectedTab, tab = $instanceSettingsSelectedTab
$: $instanceSettingsSelectedTab, (tab = $instanceSettingsSelectedTab)
$: tab, instanceSettingsSelectedTab.set(tab)
let nbDisplayed = 50
@@ -133,7 +133,7 @@
</div>
<div class="pt-4 h-full">
<Tabs bind:selected={tab}>
<Tab value="users">Global users</Tab>
<Tab value="users">Users</Tab>
{#each settingsKeys as category}
<Tab value={category}>{category}</Tab>
{/each}
@@ -252,7 +252,7 @@
{/if}
<td>
<ToggleButtonGroup
selected={super_admin ? "super_admin" : devops ? "devops" : "user"}
selected={super_admin ? 'super_admin' : devops ? 'devops' : 'user'}
on:selected={async (e) => {
if (email == $userStore?.email) {
sendUserToast('You cannot demote yourself', true)
@@ -260,9 +260,9 @@
return
}
let role = e.detail;
let role = e.detail
if (role === "super_admin") {
if (role === 'super_admin') {
await UserService.globalUserUpdate({
email,
requestBody: {
@@ -271,7 +271,7 @@
}
})
}
if (role === "devops") {
if (role === 'devops') {
await UserService.globalUserUpdate({
email,
requestBody: {
@@ -279,9 +279,8 @@
is_devops: true
}
})
}
if (role === "user") {
if (role === 'user') {
await UserService.globalUserUpdate({
email,
requestBody: {
@@ -294,15 +293,20 @@
listUsers(activeOnly)
}}
>
<ToggleButton value={"user"} size="xs" label="User" />
<ToggleButton value={"devops"} size="xs" label="Devops" tooltip="Devops is a role that grants visibilty similar to that of a super admin, but without giving all rights. For example devops users can see service logs and crtical alerts. You can think of it as a 'readonly' super admin" />
<ToggleButton value={"super_admin"} size="xs" label="Superadmin" />
<ToggleButton value={'user'} size="xs" label="User" />
<ToggleButton
value={'devops'}
size="xs"
label="Devops"
tooltip="Devops is a role that grants visibilty similar to that of a super admin, but without giving all rights. For example devops users can see service logs and crtical alerts. You can think of it as a 'readonly' super admin"
/>
<ToggleButton value={'super_admin'} size="xs" label="Superadmin" />
</ToggleButtonGroup>
</td>
<td>
<div class="flex flex-row gap-x-1 justify-end">
<InstanceNameEditor
login_type={login_type}
{login_type}
value={name}
{username}
{email}
@@ -351,7 +355,13 @@
</div>
</TabContent>
<TabContent value="" values={settingsKeys}>
<InstanceSettings bind:this={instanceSettings} hideTabs hideSave {tab} {closeDrawer}/>
<InstanceSettings
bind:this={instanceSettings}
hideTabs
hideSave
{tab}
{closeDrawer}
/>
</TabContent>
</svelte:fragment>
</Tabs>
frontend/src/lib/components/apps/components/helpers/RunnableComponent.svelte
+9 -2
View File
@@ -377,11 +377,14 @@
: runnable
if (inlineScript) {
if (inlineScript.id !== undefined) {
requestBody['id'] = inlineScript.id
}
requestBody['raw_code'] = {
content: inlineScript.content,
content: inlineScript.id === undefined ? inlineScript.content : '',
language: inlineScript.language ?? '',
path: inlineScript.path,
lock: inlineScript.lock,
lock: inlineScript.id === undefined ? inlineScript.lock : undefined,
cache_ttl: inlineScript.cache_ttl
}
}
@@ -390,6 +393,10 @@
requestBody['path'] = runType !== 'hubscript' ? `${runType}/${path}` : `script/${path}`
}
if ($app.version !== undefined) {
requestBody['version'] = $app.version
}
const uuid = await AppService.executeComponent({
workspace,
path: defaultIfEmptyString($appPath, `u/${$userStore?.username ?? 'unknown'}/newapp`),
frontend/src/lib/components/apps/editor/AppEditor.svelte
+1
View File
@@ -125,6 +125,7 @@
email: $userStore?.email,
groups: $userStore?.groups,
username: $userStore?.username,
name: $userStore?.name,
query: Object.fromEntries(new URL(window.location.href).searchParams.entries()),
hash: window.location.hash.substring(1),
workspace: $workspaceStore,
frontend/src/lib/components/apps/editor/AppEditorHeader.svelte
+1 -1
View File
@@ -482,7 +482,7 @@
policy,
path: npath,
deployment_message: deploymentMsg,
custom_path: $userStore?.is_admin || $userStore?.is_super_admin ? customPath : undefined
custom_path: $userStore?.is_admin || $userStore?.is_super_admin ? (customPath ?? '') : undefined
}
})
savedApp = {
frontend/src/lib/components/apps/types.ts
+2
View File
@@ -117,6 +117,7 @@ export type InlineScript = {
cache_ttl?: number
refreshOn?: { id: string; key: string }[]
suggestedRefreshOn?: { id: string; key: string }[]
id?: number
}
export type AppCssItemName = 'viewer' | 'grid' | AppComponent['type']
@@ -163,6 +164,7 @@ export type App = {
theme: AppTheme | undefined
hideLegacyTopBar?: boolean | undefined
mobileViewOnSmallerScreens?: boolean | undefined
version?: number
}
export type ConnectingInput = {
frontend/src/lib/components/flows/content/FlowModuleEarlyStop.svelte
+2 -2
View File
@@ -85,7 +85,7 @@
/>
<div
class="w-full border p-2 flex flex-col {flowModule.stop_after_if
class="w-full mt-1 border p-2 flex flex-col {flowModule.stop_after_if
? ''
: 'bg-surface-secondary'}"
>
@@ -179,7 +179,7 @@
/>
<div
class="w-full border p-2 flex flex-col {flowModule.stop_after_all_iters_if
class="w-full border mt-1 p-2 flex flex-col {flowModule.stop_after_all_iters_if
? ''
: 'bg-surface-secondary'}"
>
frontend/src/lib/components/flows/content/FlowRetries.svelte
+33 -5
View File
@@ -50,6 +50,8 @@
$: flowModuleRetry === undefined && resetDelayType()
$: !loaded && initialLoad()
const u32Max = 4294967295
</script>
<div class="h-full flex flex-col {$$props.class ?? ''}">
@@ -76,14 +78,34 @@
{#if delayType === 'constant'}
{#if flowModuleRetry?.constant}
<div class="text-xs font-bold !mt-2">Attempts</div>
<input bind:value={flowModuleRetry.constant.attempts} type="number" />
<div class="flex gap-1">
<input
max={u32Max.toString()}
bind:value={flowModuleRetry.constant.attempts}
type="number"
/>
<button
class="text-xs"
on:click={() =>
flowModuleRetry?.constant && (flowModuleRetry.constant.attempts = u32Max)}
>max</button
>
</div>
<div class="text-xs font-bold !mt-2">Delay</div>
<SecondsInput bind:seconds={flowModuleRetry.constant.seconds} />
{/if}
{:else if delayType === 'exponential'}
{#if flowModuleRetry?.exponential}
<div class="text-xs font-bold !mt-2">Attempts</div>
<input bind:value={flowModuleRetry.exponential.attempts} type="number" />
<div class="flex gap-1">
<input max="100" bind:value={flowModuleRetry.exponential.attempts} type="number" />
<button
class="text-xs"
on:click={() =>
flowModuleRetry?.exponential && (flowModuleRetry.exponential.attempts = 100)}
>max</button
>
</div>
<div class="text-xs font-bold !mt-2">Multiplier</div>
<span class="text-xs text-tertiary">delay = multiplier * base ^ (number of attempt)</span>
<input bind:value={flowModuleRetry.exponential.multiplier} type="number" />
@@ -127,9 +149,9 @@
multiplier,
random_factor
} = flowModuleRetry?.exponential || {}}
{@const cArray = Array.from({ length: cAttempts || 0 }, () => cSeconds)}
{@const cArray = Array.from({ length: Math.min(cAttempts || 0, 100) }, () => cSeconds)}
{@const eArray = Array.from(
{ length: eAttempts || 0 },
{ length: Math.min(eAttempts || 0, 100) },
(_, i) => (multiplier || 0) * (eSeconds || 0) ** (i + cArray.length + 1)
)}
{@const array = [...cArray, ...eArray]}
@@ -146,7 +168,7 @@
seconds){/if}</td
>
</tr>
{#each array.slice(1) as delay, i}
{#each array.slice(1, 100) as delay, i}
{@const index = i + 2}
<tr>
<td class="font-semibold pr-1 align-top">{index}:</td>
@@ -163,6 +185,12 @@
</td>
</tr>
{/each}
{#if (cAttempts ?? 0) > 100 || (eAttempts ?? 0) > 100}
<tr>
<td class="font-semibold pr-1 align-top">...</td>
<td class="pb-1">...</td>
</tr>
{/if}
</table>
{:else}
<div class="text-xs">No retries</div>
frontend/src/lib/components/icons/GitIcon.svelte
+5 -2
View File
@@ -3,6 +3,9 @@
export let width = '24px'
</script>
<svg {width} {height} viewBox="0 0 168 168" fill="none" xmlns="http://www.w3.org/2000/svg">
<path fill="#F05133" d="M92.71 44.408 52.591 4.291c-2.31-2.311-6.057-2.311-8.369 0l-8.33 8.332L46.459 23.19c2.456-.83 5.272-.273 7.229 1.685 1.969 1.97 2.521 4.81 1.67 7.275l10.186 10.185c2.465-.85 5.307-.3 7.275 1.671 2.75 2.75 2.75 7.206 0 9.958-2.752 2.751-7.208 2.751-9.961 0-2.068-2.07-2.58-5.11-1.531-7.658l-9.5-9.499v24.997c.67.332 1.303.774 1.861 1.332 2.75 2.75 2.75 7.206 0 9.959-2.75 2.749-7.209 2.749-9.957 0-2.75-2.754-2.75-7.21 0-9.959.68-.679 1.467-1.193 2.307-1.537v-25.23c-.84-.344-1.625-.853-2.307-1.537-2.083-2.082-2.584-5.14-1.516-7.698L31.798 16.715 4.288 44.222c-2.311 2.313-2.311 6.06 0 8.371l40.121 40.118c2.31 2.311 6.056 2.311 8.369 0L92.71 52.779c2.311-2.311 2.311-6.06 0-8.371z"/>
<svg {width} {height} viewBox="0 0 100 100" fill="none" xmlns="http://www.w3.org/2000/svg">
<path
fill="#F05133"
d="M92.71 44.408 52.591 4.291c-2.31-2.311-6.057-2.311-8.369 0l-8.33 8.332L46.459 23.19c2.456-.83 5.272-.273 7.229 1.685 1.969 1.97 2.521 4.81 1.67 7.275l10.186 10.185c2.465-.85 5.307-.3 7.275 1.671 2.75 2.75 2.75 7.206 0 9.958-2.752 2.751-7.208 2.751-9.961 0-2.068-2.07-2.58-5.11-1.531-7.658l-9.5-9.499v24.997c.67.332 1.303.774 1.861 1.332 2.75 2.75 2.75 7.206 0 9.959-2.75 2.749-7.209 2.749-9.957 0-2.75-2.754-2.75-7.21 0-9.959.68-.679 1.467-1.193 2.307-1.537v-25.23c-.84-.344-1.625-.853-2.307-1.537-2.083-2.082-2.584-5.14-1.516-7.698L31.798 16.715 4.288 44.222c-2.311 2.313-2.311 6.06 0 8.371l40.121 40.118c2.31 2.311 6.056 2.311 8.369 0L92.71 52.779c2.311-2.311 2.311-6.06 0-8.371z"
/>
</svg>
frontend/src/lib/components/icons/XeroIcon.svelte
+25
View File
@@ -0,0 +1,25 @@
<script lang="ts">
export let height = '24px'
export let width = '24px'
</script>
<svg
{width}
{height}
viewBox="0 0 256 256"
version="1.1"
xmlns="http://www.w3.org/2000/svg"
xmlns:xlink="http://www.w3.org/1999/xlink"
preserveAspectRatio="xMidYMid"
>
<g>
<path
d="M128.00285,256 C198.693118,256 256,198.688775 256,128 C256,57.2998444 198.698818,0 128.00285,0 C57.3068822,0 0,57.2998444 0,128 C0,198.688775 57.3068822,256 128.00285,256"
fill="#1FC0E7"
/>
<path
d="M62.3672889,127.967763 L84.1671111,106.065541 C84.8896,105.337363 85.2935111,104.358874 85.2935111,103.323496 C85.2935111,101.161719 83.5413333,99.4095407 81.3852444,99.4095407 C80.3328,99.4095407 79.3486222,99.8191407 78.6090667,100.575763 L56.8206222,122.381274 L34.9468444,100.535941 C34.2072889,99.807763 33.2288,99.4095407 32.1877333,99.4095407 C30.0259556,99.4095407 28.2794667,101.161719 28.2794667,103.317807 C28.2794667,104.370252 28.7004444,105.365807 29.4513778,106.105363 L51.2512,127.93363 L29.4627556,149.778963 C28.7004444,150.541274 28.2794667,151.531141 28.2794667,152.583585 C28.2794667,154.745363 30.0316444,156.497541 32.1877333,156.497541 C33.2288,156.497541 34.2072889,156.099319 34.9468444,155.359763 L56.7921778,133.503052 L78.5521778,155.27443 C79.3258667,156.070874 80.3157333,156.497541 81.3852444,156.497541 C83.5356444,156.497541 85.2878222,154.745363 85.2878222,152.583585 C85.2878222,151.542519 84.8896,150.56403 84.1500444,149.824474 L62.3616,127.945007 L62.3672889,127.967763 Z M191.965867,127.962074 C191.965867,131.887407 195.151644,135.073185 199.076978,135.073185 C202.979556,135.073185 206.165333,131.887407 206.165333,127.962074 C206.165333,124.036741 202.979556,120.850963 199.071289,120.850963 C195.163022,120.850963 191.982933,124.036741 191.982933,127.962074 L191.965867,127.962074 Z M178.511644,127.962074 C178.511644,116.612741 187.727644,107.368296 199.071289,107.368296 C210.392178,107.368296 219.625244,116.612741 219.625244,127.962074 C219.625244,139.311407 210.397867,148.555852 199.071289,148.555852 C187.733333,148.555852 178.511644,139.322785 178.511644,127.962074 L178.511644,127.962074 Z M170.422044,127.962074 C170.422044,143.777185 183.278933,156.65683 199.071289,156.65683 C214.863644,156.65683 227.720533,143.788563 227.720533,127.967763 C227.720533,112.152652 214.863644,99.2730074 199.071289,99.2730074 C183.273244,99.2730074 170.422044,112.146963 170.422044,127.967763 L170.422044,127.962074 Z M168.391111,99.7622519 L167.196444,99.7622519 C163.584,99.7622519 160.1024,100.90003 157.195378,103.147141 C156.797156,101.406341 155.227022,100.075141 153.361067,100.075141 C151.216356,100.075141 149.492622,101.810252 149.492622,103.966341 L149.504,152.293452 C149.504,154.438163 151.267556,156.184652 153.400889,156.184652 C155.551289,156.184652 157.303467,154.438163 157.309156,152.282074 L157.309156,122.563319 C157.309156,112.664652 158.219378,108.659674 166.684444,107.601541 C167.480889,107.50483 168.322844,107.521896 168.334222,107.521896 C170.643911,107.436563 172.293689,105.837985 172.293689,103.681896 C172.293689,101.520119 170.530133,99.7679407 168.368356,99.7679407 L168.391111,99.7622519 Z M93.3774222,123.245985 C93.3774222,123.132207 93.3888,123.01843 93.3944889,122.91603 C95.6586667,113.927585 103.7824,107.288652 113.447822,107.288652 C123.232711,107.288652 131.424711,114.081185 133.586489,123.228919 L93.3717333,123.228919 L93.3774222,123.245985 Z M141.579378,122.50643 C139.895467,114.524919 135.532089,107.96563 128.893156,103.755852 C119.176533,97.5777185 106.353778,97.9190519 96.9784889,104.609185 C89.3212444,110.053452 84.9009778,118.97363 84.9009778,128.161185 C84.9009778,130.465185 85.1854222,132.791941 85.7543111,135.084563 C88.6442667,146.462341 98.4177778,155.086696 110.068622,156.514607 C113.527467,156.935585 116.895289,156.742163 120.365511,155.831941 C123.369244,155.092385 126.264889,153.886341 128.932978,152.168296 C131.703467,150.387674 134.018844,148.032474 136.271644,145.216474 C136.305778,145.159585 136.351289,145.119763 136.3968,145.062874 C137.955556,143.128652 137.665422,140.363852 135.958756,139.055407 C134.513778,137.946074 132.090311,137.496652 130.190222,139.942874 C129.780622,140.534519 129.319822,141.137541 128.824889,141.734874 C127.305956,143.413096 125.422933,145.03443 123.170133,146.297363 C120.291556,147.833363 117.026133,148.703763 113.555911,148.726519 C102.189511,148.595674 96.1137778,140.648296 93.9463111,134.982163 C93.5708444,133.918341 93.2807111,132.820385 93.0759111,131.682607 L93.0190222,131.085274 L133.808356,131.085274 C139.394844,130.960119 142.398578,127.006341 141.568,122.495052 L141.579378,122.50643 Z"
fill="#FFFFFF"
/>
</g>
</svg>
frontend/src/lib/components/icons/index.ts
+5 -3
View File
@@ -93,7 +93,7 @@ import JumpCloudIcon from './JumpCloudIcon.svelte'
import KeycloakIcon from './KeycloakIcon.svelte'
import ZitadelIcon from './ZitadelIcon.svelte'
import SpotifyIcon from './SpotifyIcon.svelte'
import XeroIcon from './XeroIcon.svelte'
export const APP_TO_ICON_COMPONENT = {
postgresql: PostgresIcon,
mysql: Mysql,
@@ -191,7 +191,8 @@ export const APP_TO_ICON_COMPONENT = {
jumpcloud: JumpCloudIcon,
keycloak: KeycloakIcon,
zitadel: ZitadelIcon,
spotify: SpotifyIcon
spotify: SpotifyIcon,
xero: XeroIcon
} as const
export {
@@ -283,5 +284,6 @@ export {
VismaIcon,
JumpCloudIcon,
KeycloakIcon,
ZitadelIcon
ZitadelIcon,
XeroIcon
}
frontend/src/lib/components/instanceSettings.ts
+100 -94
View File
@@ -7,20 +7,21 @@ export interface Setting {
tooltip?: string
key: string
fieldType:
| 'text'
| 'number'
| 'boolean'
| 'password'
| 'select'
| 'textarea'
| 'seconds'
| 'email'
| 'license_key'
| 'object_store_config'
| 'critical_error_channels'
| 'slack_connect'
| 'smtp_connect'
| 'indexer_rates'
| 'text'
| 'number'
| 'boolean'
| 'password'
| 'select'
| 'textarea'
| 'seconds'
| 'email'
| 'license_key'
| 'object_store_config'
| 'critical_error_channels'
| 'slack_connect'
| 'smtp_connect'
| 'indexer_rates'
| 'otel'
storage: SettingStorage
advancedToggle?: {
label: string
@@ -36,6 +37,27 @@ export interface Setting {
export type SettingStorage = 'setting'
export const scimSamlSetting: Setting[] = [
{
label: 'SCIM token',
description: 'Token used to authenticate requests from the IdP',
key: 'scim_token',
fieldType: 'text',
placeholder: 'mytoken',
storage: 'setting',
ee_only: ''
},
{
label: 'SAML metadata',
description: 'XML metadata url OR content for the SAML IdP',
key: 'saml_metadata',
fieldType: 'textarea',
placeholder: 'https://dev-2578259.okta.com/app/exkaell8gidiiUWrg5d7/sso/saml/metadata ',
storage: 'setting',
ee_only: ''
}
]
export const settings: Record<string, Setting[]> = {
Core: [
{
@@ -50,9 +72,9 @@ export const settings: Record<string, Setting[]> = {
isValid: (value: string | undefined) =>
value
? value?.startsWith('http') &&
value.includes('://') &&
!value?.endsWith('/') &&
!value?.endsWith(' ')
value.includes('://') &&
!value?.endsWith('/') &&
!value?.endsWith(' ')
: false
},
{
@@ -82,6 +104,13 @@ export const settings: Record<string, Setting[]> = {
storage: 'setting',
cloudonly: false
},
{
label: 'Keep job directories for debug',
key: 'keep_job_dir',
fieldType: 'boolean',
description: 'Keep Job directories after execution at /tmp/windmill/WORKER/JOB_ID',
storage: 'setting'
},
{
label: 'Max timeout for sync endpoints',
description:
@@ -107,7 +136,8 @@ export const settings: Record<string, Setting[]> = {
'Whether we should consider the reported usage of this instance as non-prod. <a href="https://www.windmill.dev/docs/advanced/instance_settings#non-prod-instance">Learn more</a>',
key: 'dev_instance',
fieldType: 'boolean',
storage: 'setting'
storage: 'setting',
ee_only: ''
},
{
label: 'Retention period in secs',
@@ -122,21 +152,14 @@ export const settings: Record<string, Setting[]> = {
},
{
label: 'Delete logs from s3 periodically',
description: 'Job and service logs are periodically deleted from disk. When this setting is on, they will also be deleted from the object storage.',
description:
'Job and service logs are periodically deleted from disk. When this setting is on, they will also be deleted from the object storage.',
key: 'monitor_logs_on_s3',
fieldType: 'boolean',
storage: 'setting',
ee_only: ''
},
{
label: 'Expose metrics',
description:
'Expose Prometheus metrics for workers and servers on port 8001 at /metrics. <a href="https://www.windmill.dev/docs/advanced/instance_settings#expose-metrics">Learn more</a>',
key: 'expose_metrics',
fieldType: 'boolean',
storage: 'setting',
ee_only: ''
},
{
label: 'Instance object storage',
description:
@@ -146,24 +169,7 @@ export const settings: Record<string, Setting[]> = {
storage: 'setting',
ee_only: ''
},
{
label: 'Critical alert channels',
description:
'Channels to send critical alerts to. SMTP must be configured for the email channel. A Slack workspace must be connected to the instance for the Slack channel. <a href="https://www.windmill.dev/docs/core_concepts/critical_alerts">Learn more</a>',
key: 'critical_error_channels',
fieldType: 'critical_error_channels',
storage: 'setting',
ee_only: 'Channels other than tracing are only available in the EE version'
},
{
label: 'Mute critical alerts in UI',
description: 'Enable to mute critical alerts in the UI',
key: 'critical_alert_mute_ui',
fieldType: 'boolean',
storage: 'setting',
requiresReloadOnChange: true,
ee_only: 'Critical alerts in UI are only available in the EE version'
},
{
label: 'Azure OpenAI base path',
description:
@@ -209,7 +215,7 @@ export const settings: Record<string, Setting[]> = {
requiresReloadOnChange: true
}
],
'SSO/OAuth': [],
'Auth/OAuth': [],
Registries: [
{
label: 'Pip Index Url',
@@ -249,7 +255,32 @@ export const settings: Record<string, Setting[]> = {
ee_only: ''
}
],
SMTP: [
Alerts: [
{
label: 'Critical alert channels',
description:
'Channels to send critical alerts to. SMTP and Slack must be configured below. <a href="https://www.windmill.dev/docs/core_concepts/critical_alerts">Learn more</a>',
key: 'critical_error_channels',
fieldType: 'critical_error_channels',
storage: 'setting',
ee_only: 'Channels other than tracing are only available in the EE version'
},
{
label: 'Mute critical alerts in UI',
description: 'Enable to mute critical alerts in the UI',
key: 'critical_alert_mute_ui',
fieldType: 'boolean',
storage: 'setting',
requiresReloadOnChange: true,
ee_only: 'Critical alerts in UI are only available in the EE version'
},
{
label: 'Slack',
key: 'slack',
fieldType: 'slack_connect',
storage: 'setting',
ee_only: ''
},
{
label: 'SMTP',
key: 'smtp_settings',
@@ -258,7 +289,26 @@ export const settings: Record<string, Setting[]> = {
ee_only: ''
}
],
'Indexer/Search': [
'OTEL/Prom': [
{
label: 'OpenTelemetry',
key: 'otel',
fieldType: 'otel',
storage: 'setting',
ee_only: ''
},
{
label: 'Prometheus',
description:
'Expose Prometheus metrics for workers and servers on port 8001 at /metrics. <a href="https://www.windmill.dev/docs/advanced/instance_settings#expose-metrics">Learn more</a>',
key: 'expose_metrics',
fieldType: 'boolean',
storage: 'setting',
ee_only: ''
}
],
Indexer: [
{
label: '',
key: 'indexer_settings',
@@ -267,51 +317,7 @@ export const settings: Record<string, Setting[]> = {
ee_only: 'Full text search across jobs and service logs is an EE feature'
}
],
Slack: [
{
label: 'Slack',
key: 'slack',
fieldType: 'slack_connect',
storage: 'setting',
ee_only: ''
}
],
'SCIM/SAML': [
{
label: 'SCIM token',
description: 'Token used to authenticate requests from the IdP',
key: 'scim_token',
fieldType: 'text',
placeholder: 'mytoken',
storage: 'setting',
ee_only: ''
},
{
label: 'SAML metadata',
description: 'XML metadata url OR content for the SAML IdP',
key: 'saml_metadata',
fieldType: 'textarea',
placeholder: 'https://dev-2578259.okta.com/app/exkaell8gidiiUWrg5d7/sso/saml/metadata ',
storage: 'setting',
ee_only: ''
}
],
Debug: [
{
label: 'Keep job directories',
key: 'keep_job_dir',
fieldType: 'boolean',
description: 'Keep Job directories after execution at /tmp/windmill/WORKER/JOB_ID',
storage: 'setting'
},
{
label: 'Expose debug metrics',
key: 'expose_debug_metrics',
fieldType: 'boolean',
description: 'Expose additional metrics (require metrics to be enabled)',
storage: 'setting'
}
],
Telemetry: [
{
label: 'Disable telemetry',

Some files were not shown because too many files have changed in this diff Show More