Merge branch 'main' into fr/flow-triggers-v0

CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## [1.399.0](https://github.com/windmill-labs/windmill/compare/v1.398.1...v1.399.0) (2024-09-25)
+
+
+### Features
+
+* add tag filtering to external JWT authentication ([#4425](https://github.com/windmill-labs/windmill/issues/4425)) ([590321f](https://github.com/windmill-labs/windmill/commit/590321fd3c88046a657e0a751b62a19424102060))
+
+
+### Bug Fixes
+
+* **cli:** version the whole client for deno compatibility ([81c2bb0](https://github.com/windmill-labs/windmill/commit/81c2bb069176a95b8fb9c52b31e5e03c1cc78afc))
+* correct AI generation for CRON and regex ([#4437](https://github.com/windmill-labs/windmill/issues/4437)) ([aeb5b5b](https://github.com/windmill-labs/windmill/commit/aeb5b5bcd163e2e7d1413d7e4aed8bd769996f24))
+
 ## [1.398.1](https://github.com/windmill-labs/windmill/compare/v1.398.0...v1.398.1) (2024-09-23)
 
 

backend/.sqlx/query-0e52a588f3edeb8fb58d6d62247b8590e51171e2811c62737bdb81fb0ac8f182.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE worker_ping SET \n        ping_at = now(), \n        jobs_executed = 1, \n        current_job_id = $1, \n        current_job_workspace_id = 'admins' \n        WHERE worker = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "0e52a588f3edeb8fb58d6d62247b8590e51171e2811c62737bdb81fb0ac8f182"
+}

backend/Cargo.lock

@@ -696,9 +696,9 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de"
 
 [[package]]
 name = "async-trait"
-version = "0.1.82"
+version = "0.1.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1"
+checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -4670,9 +4670,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.158"
+version = "0.2.159"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439"
+checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5"
 
 [[package]]
 name = "libgit2-sys"
@@ -4710,7 +4710,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d"
 dependencies = [
  "bitflags 2.6.0",
  "libc",
- "redox_syscall 0.5.4",
+ "redox_syscall 0.5.5",
 ]
 
 [[package]]
@@ -5672,7 +5672,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
 dependencies = [
  "cfg-if",
  "libc",
- "redox_syscall 0.5.4",
+ "redox_syscall 0.5.5",
  "smallvec",
  "windows-targets 0.52.6",
 ]
@@ -6120,7 +6120,7 @@ version = "3.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b"
 dependencies = [
- "toml_edit 0.22.21",
+ "toml_edit 0.22.22",
 ]
 
 [[package]]
@@ -6623,9 +6623,9 @@ dependencies = [
 
 [[package]]
 name = "redox_syscall"
-version = "0.5.4"
+version = "0.5.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853"
+checksum = "62871f2d65009c0256aed1b9cfeeb8ac272833c404e13d53d400cd0dad7a2ac0"
 dependencies = [
  "bitflags 2.6.0",
 ]
@@ -9019,9 +9019,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
 
 [[package]]
 name = "tar"
-version = "0.4.41"
+version = "0.4.42"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb797dad5fb5b76fcf519e702f4a589483b5ef06567f160c392832c1f5e44909"
+checksum = "4ff6c40d3aedb5e06b57c6f669ad17ab063dd1e63d977c6a88e7f4dfa4f04020"
 dependencies = [
  "filetime",
  "libc",
@@ -9433,7 +9433,7 @@ dependencies = [
  "serde",
  "serde_spanned",
  "toml_datetime",
- "toml_edit 0.22.21",
+ "toml_edit 0.22.22",
 ]
 
 [[package]]
@@ -9460,15 +9460,15 @@ dependencies = [
 
 [[package]]
 name = "toml_edit"
-version = "0.22.21"
+version = "0.22.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b072cee73c449a636ffd6f32bd8de3a9f7119139aff882f44943ce2986dc5cf"
+checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5"
 dependencies = [
  "indexmap 2.5.0",
  "serde",
  "serde_spanned",
  "toml_datetime",
- "winnow 0.6.18",
+ "winnow 0.6.19",
 ]
 
 [[package]]
@@ -10382,7 +10382,7 @@ version = "1.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d"
 dependencies = [
- "redox_syscall 0.5.4",
+ "redox_syscall 0.5.5",
  "wasite",
  "web-sys",
 ]
@@ -10426,7 +10426,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
 [[package]]
 name = "windmill"
-version = "1.398.1"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "axum",
@@ -10468,7 +10468,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api"
-version = "1.398.1"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "argon2",
@@ -10553,7 +10553,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-client"
-version = "1.398.1"
+version = "1.399.0"
 dependencies = [
  "base64 0.21.7",
  "chrono",
@@ -10571,7 +10571,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-audit"
-version = "1.398.1"
+version = "1.399.0"
 dependencies = [
  "chrono",
  "serde",
@@ -10584,7 +10584,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-common"
-version = "1.398.1"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "async-stream",
@@ -10628,7 +10628,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-git-sync"
-version = "1.398.1"
+version = "1.399.0"
 dependencies = [
  "regex",
  "rsmq_async",
@@ -10643,7 +10643,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-indexer"
-version = "1.398.1"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "bytes",
@@ -10664,7 +10664,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser"
-version = "1.398.1"
+version = "1.399.0"
 dependencies = [
  "convert_case 0.6.0",
  "serde",
@@ -10673,7 +10673,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-bash"
-version = "1.398.1"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -10685,7 +10685,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-go"
-version = "1.398.1"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "gosyn",
@@ -10697,7 +10697,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-graphql"
-version = "1.398.1"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -10709,7 +10709,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-php"
-version = "1.398.1"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "itertools 0.13.0",
@@ -10720,7 +10720,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py"
-version = "1.398.1"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "itertools 0.13.0",
@@ -10731,7 +10731,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py-imports"
-version = "1.398.1"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "async-recursion",
@@ -10749,7 +10749,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-rust"
-version = "1.398.1"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "convert_case 0.6.0",
@@ -10766,7 +10766,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-sql"
-version = "1.398.1"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -10778,7 +10778,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-ts"
-version = "1.398.1"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -10796,7 +10796,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-wasm"
-version = "1.398.1"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "getrandom 0.2.15",
@@ -10817,7 +10817,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-yaml"
-version = "1.398.1"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "serde_json",
@@ -10827,7 +10827,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-queue"
-version = "1.398.1"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "async-recursion",
@@ -10860,7 +10860,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-sql-datatype-parser-wasm"
-version = "1.398.1"
+version = "1.399.0"
 dependencies = [
  "wasm-bindgen",
  "wasm-bindgen-test",
@@ -10870,7 +10870,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-worker"
-version = "1.398.1"
+version = "1.399.0"
 dependencies = [
  "anyhow",
  "async-recursion",
@@ -11142,9 +11142,9 @@ dependencies = [
 
 [[package]]
 name = "winnow"
-version = "0.6.18"
+version = "0.6.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f"
+checksum = "c52ac009d615e79296318c1bcce2d422aaca15ad08515e344feeda07df67a587"
 dependencies = [
  "memchr",
 ]

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.398.1"
+version = "1.399.0"
 authors.workspace = true
 edition.workspace = true
 
@@ -27,7 +27,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.398.1"
+version = "1.399.0"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 

backend/ee-repo-ref.txt

@@ -1 +1 @@
-e093d51a219ce4ff0562a02db24ec402554a1f05 
+3d37b6c31155265d8d026ae9d6ced0b433078f87

backend/parsers/windmill-parser-wasm/publish-pkgs.sh

@@ -21,3 +21,6 @@ popd
 
 pushd "pkg-rust" && npm publish ${args}
 popd
+
+pushd "pkg-yaml" && npm publish ${args}
+popd

backend/parsers/windmill-parser-yaml/src/lib.rs

@@ -25,14 +25,19 @@ pub fn parse_ansible_sig(inner_content: &str) -> anyhow::Result<MainArgSignature
                     if let Yaml::Hash(v) = &value {
                         for (key, arg) in v {
                             if let Yaml::String(arg_name) = key {
-                                args.push(Arg {
-                                    name: arg_name.to_string(),
-                                    otyp: None,
-                                    typ: parse_ansible_typ(&arg),
-                                    default: None,
-                                    has_default: false,
-                                    oidx: None,
-                                })
+                                // if not then it is a static resource or var, so skip
+                                // it will be fetched by the worker
+                                if let ArgTyp::Typ(typ) = parse_ansible_arg_typ(arg) {
+                                    let default = get_default_for_typ(&arg);
+                                    args.push(Arg {
+                                        name: arg_name.to_string(),
+                                        otyp: None,
+                                        typ,
+                                        has_default: default.is_some(),
+                                        default,
+                                        oidx: None,
+                                    })
+                                }
                             }
                         }
                     }
@@ -65,6 +70,50 @@ pub fn parse_ansible_sig(inner_content: &str) -> anyhow::Result<MainArgSignature
     })
 }
 
+fn get_default_for_typ(arg: &Yaml) -> Option<serde_json::Value> {
+    if let Yaml::Hash(arg) = arg {
+        if let Some(def) = arg.get(&Yaml::String("default".to_string())) {
+            if let Some(Yaml::String(typ)) = arg.get(&Yaml::String("type".to_string())) {
+                if let Yaml::String(path) = def {
+                    if typ.as_str() == "windmill_resource" {
+                        return Some(serde_json::Value::String(format!("$res:{}", path)));
+                    }
+                }
+            }
+            return Some(yaml_to_json(def));
+        }
+    }
+    None
+}
+
+enum ArgTyp {
+    Typ(Typ),
+    StaticVar(String),
+    StaticResource(String),
+}
+
+fn parse_ansible_arg_typ(arg: &Yaml) -> ArgTyp {
+    if let Yaml::Hash(a) = arg {
+        if let Some(Yaml::String(typ)) = a.get(&Yaml::String("type".to_string())) {
+            if typ.as_str() == "windmill_variable" {
+                if let Some(Yaml::String(variable_path)) =
+                    a.get(&Yaml::String("variable".to_string()))
+                {
+                    return ArgTyp::StaticVar(variable_path.to_string());
+                }
+            }
+            if typ.as_str() == "windmill_resource" {
+                if let Some(Yaml::String(resource_path)) =
+                    a.get(&Yaml::String("resource".to_string()))
+                {
+                    return ArgTyp::StaticResource(resource_path.to_string());
+                }
+            }
+        }
+    }
+    ArgTyp::Typ(parse_ansible_typ(arg))
+}
+
 fn parse_ansible_typ(arg: &Yaml) -> Typ {
     if let Yaml::Hash(arg) = arg {
         if let Some(Yaml::String(typ)) = arg.get(&Yaml::String("type".to_string())) {
@@ -127,7 +176,19 @@ fn parse_ansible_typ(arg: &Yaml) -> Typ {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
+pub struct AnsiblePlaybookOptions {
+    // There are a lot more options
+    // TODO: Add the options as customers require them
+    // Add it here and then on the executors cmd_options
+    pub verbosity: Option<String>,
+    pub forks: Option<i64>,
+    pub timeout: Option<i64>,
+    pub flush_cache: Option<()>,
+    pub force_handlers: Option<()>,
+}
+
+#[derive(Debug, Clone)]
 pub struct FileResource {
     pub resource_path: String,
     pub target_path: String,
@@ -140,12 +201,15 @@ pub struct AnsibleInventory {
     resource_type: Option<String>,
     pub pinned_resource: Option<String>,
 }
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct AnsibleRequirements {
     pub python_reqs: Vec<String>,
     pub collections: Option<String>,
     pub file_resources: Vec<FileResource>,
     pub inventories: Vec<AnsibleInventory>,
+    pub vars: Vec<(String, String)>,
+    pub resources: Vec<(String, String)>,
+    pub options: AnsiblePlaybookOptions,
 }
 
 fn parse_inventories(inventory_yaml: &Yaml) -> anyhow::Result<Vec<AnsibleInventory>> {
@@ -204,11 +268,21 @@ pub fn parse_ansible_reqs(
         return Ok((logs, None, inner_content.to_string()));
     }
 
+    let opts = AnsiblePlaybookOptions {
+        verbosity: None,
+        forks: None,
+        timeout: None,
+        flush_cache: None,
+        force_handlers: None,
+    };
     let mut ret = AnsibleRequirements {
         python_reqs: vec![],
         collections: None,
         file_resources: vec![],
         inventories: vec![],
+        vars: vec![],
+        resources: vec![],
+        options: opts,
     };
 
     if let Yaml::Hash(doc) = &docs[0] {
@@ -242,10 +316,33 @@ pub fn parse_ansible_reqs(
                         ret.file_resources.append(&mut resources?);
                     }
                 }
-                Yaml::String(key) if key == "extra_vars" => {}
+                Yaml::String(key) if key == "extra_vars" => {
+                    if let Yaml::Hash(v) = &value {
+                        for (key, arg) in v {
+                            if let Yaml::String(arg_name) = key {
+                                match parse_ansible_arg_typ(arg) {
+                                    ArgTyp::StaticVar(p) => {
+                                        ret.vars
+                                            .push((arg_name.to_string(), format!("$var:{}", p)));
+                                    }
+                                    ArgTyp::StaticResource(p) => {
+                                        ret.resources
+                                            .push((arg_name.to_string(), format!("$res:{}", p)));
+                                    }
+                                    ArgTyp::Typ(_) => (),
+                                }
+                            }
+                        }
+                    }
+                }
                 Yaml::String(key) if key == "inventory" => {
                     ret.inventories = parse_inventories(value)?;
                 }
+                Yaml::String(key) if key == "options" => {
+                    if let Yaml::Array(opts) = &value {
+                        ret.options = parse_ansible_options(opts);
+                    }
+                }
                 Yaml::String(key) => logs.push_str(&format!("\nUnknown field `{}`. Ignoring", key)),
                 _ => (),
             }
@@ -260,6 +357,82 @@ pub fn parse_ansible_reqs(
     Ok((logs, Some(ret), out_str))
 }
 
+fn parse_ansible_options(opts: &Vec<Yaml>) -> AnsiblePlaybookOptions {
+    let mut ret = AnsiblePlaybookOptions {
+        verbosity: None,
+        forks: None,
+        timeout: None,
+        flush_cache: None,
+        force_handlers: None,
+    };
+    for opt in opts {
+        if let Yaml::String(o) = opt {
+            match o.as_str() {
+                "flush_cache" => {
+                    ret.flush_cache = Some(());
+                }
+                "force_handlers" => {
+                    ret.force_handlers = Some(());
+                }
+                s if count_consecutive_vs(s) > 0 => {
+                    ret.verbosity = Some("v".repeat(count_consecutive_vs(s).min(6)));
+                }
+                _ => (),
+            }
+        }
+        if let Yaml::Hash(m) = opt {
+            if let Some((Yaml::String(name), value)) = m.iter().last() {
+                match name.as_str() {
+                    "forks" => {
+                        if let Yaml::Integer(count) = value {
+                            ret.forks = Some(*count);
+                        }
+                    }
+                    "timeout" => {
+                        if let Yaml::Integer(timeout) = value {
+                            ret.timeout = Some(*timeout);
+                        }
+                    }
+                    "verbosity" => {
+                        if let Yaml::String(verbosity) = value {
+                            let c = count_consecutive_vs(verbosity);
+                            if c > 0 && c <= 6 {
+                                ret.verbosity = Some("v".repeat(c.min(6)));
+                            }
+
+                        }
+                    }
+                    _ => ()
+
+                }
+            }
+
+
+        }
+    }
+
+    ret
+}
+
+fn count_consecutive_vs(s: &str) -> usize {
+    let mut max_count = 0;
+    let mut current_count = 0;
+
+    for c in s.chars() {
+        if c == 'v' {
+            current_count += 1;
+            if current_count == 6 {
+                return 6;  // Stop early if we reach 6
+            }
+        } else {
+            current_count = 0;  // Reset count if the character is not 'v'
+        }
+        max_count = max_count.max(current_count);
+    }
+
+    max_count
+}
+
 fn parse_file_resource(yaml: &Yaml) -> anyhow::Result<FileResource> {
     if let Yaml::Hash(f) = yaml {
         if let Some(Yaml::String(resource_path)) = f.get(&Yaml::String("resource".to_string())) {
@@ -279,3 +452,31 @@ fn parse_file_resource(yaml: &Yaml) -> anyhow::Result<FileResource> {
     }
     return Err(anyhow!("Invalid file resource: Should be a dictionary."));
 }
+
+fn yaml_to_json(yaml: &Yaml) -> serde_json::Value {
+    match yaml {
+        Yaml::Array(arr) => {
+            let json_array: Vec<serde_json::Value> = arr.into_iter().map(yaml_to_json).collect();
+            serde_json::Value::Array(json_array)
+        }
+        Yaml::Hash(hash) => {
+            let json_object = hash
+                .into_iter()
+                .map(|(k, v)| {
+                    let key = match k {
+                        Yaml::String(s) => s.clone(),
+                        _ => k.as_str().unwrap_or("").to_string(),
+                    };
+                    (key, yaml_to_json(v))
+                })
+                .collect();
+            serde_json::Value::Object(json_object)
+        }
+        Yaml::String(s) => serde_json::Value::String(s.to_string()),
+        Yaml::Integer(i) => serde_json::Value::Number(i.clone().into()),
+        Yaml::Real(r) => serde_json::Value::Number(r.parse().unwrap_or(0.into())),
+        Yaml::Boolean(b) => serde_json::Value::Bool(*b),
+        Yaml::Null => serde_json::Value::Null,
+        _ => serde_json::Value::Null,
+    }
+}

backend/tests/worker.rs

@@ -1912,6 +1912,109 @@ def main():
     assert_eq!(result, serde_json::json!("test-workspace"));
 }
 
+#[sqlx::test(fixtures("base"))]
+async fn test_bun_job_datetime(db: Pool<Postgres>) {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await;
+    let port = server.addr.port();
+
+    let content = r#"
+export async function main(a: Date) {
+    return typeof a;
+}
+        "#
+    .to_owned();
+
+    let result = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content,
+        path: None,
+        lock: None,
+        language: ScriptLang::Bun,
+        custom_concurrency_key: None,
+        concurrent_limit: None,
+        concurrency_time_window_s: None,
+        cache_ttl: None,
+        dedicated_worker: None,
+    }))
+    .arg("a", json!("2024-09-24T10:00:00.000Z"))
+    .run_until_complete(&db, port)
+    .await
+    .json_result()
+    .unwrap();
+
+    assert_eq!(result, serde_json::json!("object"));
+}
+
+#[sqlx::test(fixtures("base"))]
+async fn test_deno_job_datetime(db: Pool<Postgres>) {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await;
+    let port = server.addr.port();
+
+    let content = r#"
+export async function main(a: Date) {
+    return typeof a;
+}
+        "#
+    .to_owned();
+
+    let result = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content,
+        path: None,
+        lock: None,
+        language: ScriptLang::Deno,
+        custom_concurrency_key: None,
+        concurrent_limit: None,
+        concurrency_time_window_s: None,
+        cache_ttl: None,
+        dedicated_worker: None,
+    }))
+    .arg("a", json!("2024-09-24T10:00:00.000Z"))
+    .run_until_complete(&db, port)
+    .await
+    .json_result()
+    .unwrap();
+
+    assert_eq!(result, serde_json::json!("object"));
+}
+
+#[sqlx::test(fixtures("base"))]
+async fn test_python_job_datetime_and_bytes(db: Pool<Postgres>) {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await;
+    let port = server.addr.port();
+
+    let content = r#"
+from datetime import datetime
+def main(a: datetime, b: bytes):
+    return (isinstance(a, datetime), isinstance(b, bytes))
+        "#
+    .to_owned();
+
+    let result = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content,
+        path: None,
+        lock: None,
+        language: ScriptLang::Python3,
+        custom_concurrency_key: None,
+        concurrent_limit: None,
+        concurrency_time_window_s: None,
+        cache_ttl: None,
+        dedicated_worker: None,
+    }))
+    .arg("a", json!("2024-09-24T10:00:00.000Z"))
+    .arg("b", json!("dGVzdA=="))
+    .run_until_complete(&db, port)
+    .await
+    .json_result()
+    .unwrap();
+
+    assert_eq!(result, serde_json::json!([true, true]));
+}
+
 #[sqlx::test(fixtures("base"))]
 async fn test_empty_loop_1(db: Pool<Postgres>) {
     initialize_tracing().await;

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.398.1
+  version: 1.399.0
   title: Windmill API
 
   contact:

backend/windmill-api/src/concurrency_groups.rs

@@ -144,7 +144,7 @@ async fn get_concurrent_intervals(
     Query(iq): Query<ExtendedJobsParams>,
     Query(lq): Query<ListCompletedQuery>,
 ) -> JsonResult<ExtendedJobs> {
-    check_scopes(&authed, || format!("listjobs"))?;
+    check_scopes(&authed, || format!("jobs:listjobs"))?;
 
     if lq.success.is_some() && lq.running.is_some_and(|x| x) {
         return Err(error::Error::BadRequest(

backend/windmill-api/src/jobs.rs

@@ -33,6 +33,7 @@ use crate::add_webhook_allowed_origin;
 use crate::concurrency_groups::join_concurrency_key;
 use crate::db::ApiAuthed;
 
+use crate::users::get_scope_tags;
 use crate::utils::content_plain;
 use crate::{
     db::DB,
@@ -1248,13 +1249,18 @@ pub fn list_queue_jobs_query(
     lq: &ListQueueQuery,
     fields: &[&str],
     join_outstanding_wait_times: bool,
+    tags: Option<Vec<&str>>,
 ) -> SqlBuilder {
-    let sqlb = SqlBuilder::select_from("queue")
+    let mut sqlb = SqlBuilder::select_from("queue")
         .fields(fields)
         .order_by("created_at", lq.order_desc.unwrap_or(true))
         .limit(1000)
         .clone();
 
+    if let Some(tags) = tags {
+        sqlb.and_where_in("tag", &tags.iter().map(|x| quote(x)).collect::<Vec<_>>());
+    }
+
     filter_list_queue_query(sqlb, lq, w_id, join_outstanding_wait_times)
 }
 
@@ -1310,6 +1316,7 @@ async fn list_queue_jobs(
             "workspace_id",
         ],
         false,
+        get_scope_tags(&authed),
     )
     .sql()?;
     let mut tx = user_db.begin(&authed).await?;
@@ -1498,6 +1505,10 @@ async fn list_filtered_uuids(
 
     sqlb.and_where_is_null("schedule_path");
 
+    if let Some(tags) = get_scope_tags(&authed) {
+        sqlb.and_where_in("tag", &tags.iter().map(|x| quote(x)).collect::<Vec<_>>());
+    }
+
     sqlb = filter_list_queue_query(sqlb, &lq, w_id.as_str(), false);
 
     let sql = sqlb.query()?;
@@ -1557,7 +1568,7 @@ async fn list_jobs(
     Query(lq): Query<ListCompletedQuery>,
     Extension(_api_list_jobs_query_duration): Extension<Option<Histo>>,
 ) -> error::JsonResult<Vec<Job>> {
-    check_scopes(&authed, || format!("listjobs"))?;
+    check_scopes(&authed, || format!("jobs:listjobs"))?;
 
     let (per_page, offset) = paginate(pagination);
     let lqc = lq.clone();
@@ -1575,6 +1586,7 @@ async fn list_jobs(
             &ListCompletedQuery { order_desc: Some(true), ..lqc },
             UnifiedJob::completed_job_fields(),
             true,
+            get_scope_tags(&authed),
         ))
     } else {
         None
@@ -1590,6 +1602,7 @@ async fn list_jobs(
             &ListQueueQuery { order_desc: Some(true), ..lq.into() },
             UnifiedJob::queued_job_fields(),
             true,
+            get_scope_tags(&authed),
         );
 
         if let Some(sqlc) = sqlc {
@@ -1640,7 +1653,7 @@ pub async fn resume_suspended_flow_as_owner(
     Path((_w_id, flow_id)): Path<(String, Uuid)>,
     QueryOrBody(value): QueryOrBody<serde_json::Value>,
 ) -> error::Result<StatusCode> {
-    check_scopes(&authed, || format!("resumeflow"))?;
+    check_scopes(&authed, || format!("jobs:resumeflow"))?;
     let value = value.unwrap_or(serde_json::Value::Null);
     let mut tx = db.begin().await?;
 
@@ -3847,7 +3860,7 @@ async fn run_preview_script(
     #[cfg(feature = "enterprise")]
     check_license_key_valid().await?;
 
-    check_scopes(&authed, || format!("runscript"))?;
+    check_scopes(&authed, || format!("jobs:runscript"))?;
     if authed.is_operator {
         return Err(error::Error::NotAuthorized(
             "Operators cannot run preview jobs for security reasons".to_string(),
@@ -3917,7 +3930,7 @@ async fn run_bundle_preview_script(
 
     check_license_key_valid().await?;
 
-    check_scopes(&authed, || format!("runscript"))?;
+    check_scopes(&authed, || format!("jobs:runscript"))?;
     if authed.is_operator {
         return Err(error::Error::NotAuthorized(
             "Operators cannot run preview jobs for security reasons".to_string(),
@@ -4403,7 +4416,7 @@ async fn run_preview_flow_job(
     Query(run_query): Query<RunJobQuery>,
     Json(raw_flow): Json<PreviewFlow>,
 ) -> error::Result<(StatusCode, String)> {
-    check_scopes(&authed, || format!("runflow"))?;
+    check_scopes(&authed, || format!("jobs:runflow"))?;
     if authed.is_operator {
         return Err(error::Error::NotAuthorized(
             "Operators cannot run preview jobs for security reasons".to_string(),
@@ -4843,14 +4856,19 @@ pub fn list_completed_jobs_query(
     lq: &ListCompletedQuery,
     fields: &[&str],
     join_outstanding_wait_times: bool,
+    tags: Option<Vec<&str>>,
 ) -> SqlBuilder {
-    let sqlb = SqlBuilder::select_from("completed_job")
+    let mut sqlb = SqlBuilder::select_from("completed_job")
         .fields(fields)
         .order_by("created_at", lq.order_desc.unwrap_or(true))
         .offset(offset)
         .limit(per_page)
         .clone();
 
+    if let Some(tags) = tags {
+        sqlb.and_where_in("tag", &tags.iter().map(|x| quote(x)).collect::<Vec<_>>());
+    }
+
     filter_list_completed_query(sqlb, lq, w_id, join_outstanding_wait_times)
 }
 #[derive(Deserialize, Clone)]
@@ -4895,7 +4913,7 @@ async fn list_completed_jobs(
     Query(pagination): Query<Pagination>,
     Query(lq): Query<ListCompletedQuery>,
 ) -> error::JsonResult<Vec<ListableCompletedJob>> {
-    check_scopes(&authed, || format!("listjobs"))?;
+    check_scopes(&authed, || format!("jobs:listjobs"))?;
 
     let (per_page, offset) = paginate(pagination);
 
@@ -4937,6 +4955,7 @@ async fn list_completed_jobs(
             "'CompletedJob' as type",
         ],
         false,
+        get_scope_tags(&authed),
     )
     .sql()?;
     let mut tx = user_db.begin(&authed).await?;
@@ -5161,7 +5180,7 @@ async fn delete_completed_job<'a>(
     Extension(user_db): Extension<UserDB>,
     Path((w_id, id)): Path<(String, Uuid)>,
 ) -> error::Result<Response> {
-    check_scopes(&authed, || format!("deletejob"))?;
+    check_scopes(&authed, || format!("jobs:deletejob"))?;
 
     let mut tx = user_db.begin(&authed).await?;
 

backend/windmill-api/src/users.rs

@@ -651,9 +651,12 @@ where
                 {
                     if let Some(authed) = cache.get_authed(workspace_id.clone(), &token).await {
                         parts.extensions.insert(authed.clone());
-                        if authed.scopes.is_some()
-                            && (path_vec.len() < 3
-                                || (path_vec[4] != "jobs" && path_vec[4] != "jobs_u"))
+                        if authed.scopes.as_ref().is_some_and(|scopes| {
+                            scopes
+                                .iter()
+                                .any(|s| s.starts_with("jobs:") || s.starts_with("run:"))
+                        }) && (path_vec.len() < 3
+                            || (path_vec[4] != "jobs" && path_vec[4] != "jobs_u"))
                         {
                             BRUTE_FORCE_COUNTER.increment().await;
                             return Err((
@@ -681,15 +684,30 @@ pub fn check_scopes<F>(authed: &ApiAuthed, required: F) -> error::Result<()>
 where
     F: FnOnce() -> String,
 {
-    if let Some(scopes) = &authed.scopes {
+    if authed.scopes.as_ref().is_some_and(|scopes| {
+        scopes
+            .iter()
+            .any(|s| s.starts_with("jobs:") || s.starts_with("run:"))
+    }) {
         let req = &required();
-        if !scopes.contains(req) {
+        if !authed.scopes.as_ref().unwrap().contains(req) {
             return Err(Error::BadRequest(format!("missing required scope: {req}")));
         }
     }
     Ok(())
 }
 
+pub fn get_scope_tags(authed: &ApiAuthed) -> Option<Vec<&str>> {
+    authed
+        .scopes
+        .as_ref()?
+        .iter()
+        .find_map(|s| match s.split(":").collect::<Vec<_>>().as_slice() {
+            ["if_jobs", "filter_tags", tags] => Some(tags.split(",").collect::<Vec<_>>()),
+            _ => None,
+        })
+}
+
 #[derive(Clone, Debug)]
 pub struct OptAuthed(pub Option<ApiAuthed>);
 

backend/windmill-api/src/workspaces.rs

@@ -2577,7 +2577,7 @@ async fn tarball_workspace(
                 }
                 ScriptLang::Php => "php",
                 ScriptLang::Rust => "rs",
-                ScriptLang::Ansible => "yaml",
+                ScriptLang::Ansible => "playbook.yml",
             };
             archive
                 .write_to_archive(&script.content, &format!("{}.{}", script.path, ext))

backend/windmill-common/src/auth.rs

@@ -25,6 +25,7 @@ pub struct JWTAuthClaims {
     pub workspace_id: String,
     pub exp: usize,
     pub job_id: Option<String>,
+    pub scopes: Option<Vec<String>>,
 }
 
 #[derive(Deserialize)]

backend/windmill-common/src/ee.rs

@@ -26,7 +26,10 @@ pub async fn get_license_plan() -> LicensePlan {
 
 #[derive(Deserialize)]
 #[serde(untagged)]
-pub enum CriticalErrorChannel {}
+pub enum CriticalErrorChannel {
+    Email { email: String },
+    Slack { slack_channel: String },
+}
 
 pub enum CriticalAlertKind {
     #[cfg(feature = "enterprise")]

backend/windmill-queue/src/jobs.rs

@@ -475,6 +475,11 @@ where
     }
 }
 
+#[derive(Deserialize)]
+struct RawFlowFailureModule {
+    failure_module: Option<Box<RawValue>>,
+}
+
 #[instrument(level = "trace", skip_all)]
 pub async fn add_completed_job_error<R: rsmq_async::RsmqConnection + Clone + Send>(
     db: &Pool<Postgres>,
@@ -922,7 +927,17 @@ pub async fn add_completed_job<
             )
             .await;
         } else if !skip_downstream_error_handlers
-            && matches!(queued_job.job_kind, JobKind::Flow | JobKind::Script)
+            && (matches!(queued_job.job_kind, JobKind::Script)
+                || matches!(queued_job.job_kind, JobKind::Flow)
+                    && queued_job
+                        .raw_flow
+                        .as_ref()
+                        .and_then(|v| {
+                            serde_json::from_str::<RawFlowFailureModule>((**v).get())
+                                .ok()
+                                .and_then(|v| v.failure_module)
+                        })
+                        .is_none())
             && queued_job.parent_job.is_none()
         {
             let result = serde_json::from_str(

backend/windmill-worker/src/ansible_executor.rs

@@ -1,5 +1,8 @@
 use std::{
-    collections::HashMap, os::unix::fs::PermissionsExt, path::{Path, PathBuf}, process::Stdio
+    collections::HashMap,
+    os::unix::fs::PermissionsExt,
+    path::{Path, PathBuf},
+    process::Stdio,
 };
 
 use anyhow::anyhow;
@@ -10,16 +13,20 @@ use uuid::Uuid;
 use windmill_common::{
     error,
     jobs::QueuedJob,
-    worker::{write_file, write_file_at_user_defined_location, WORKER_CONFIG},
+    worker::{to_raw_value, write_file, write_file_at_user_defined_location, WORKER_CONFIG},
 };
 use windmill_parser_yaml::AnsibleRequirements;
 use windmill_queue::{append_logs, CanceledBy};
 
 use crate::{
-    bash_executor::BIN_BASH, common::{
+    bash_executor::BIN_BASH,
+    common::{
         get_reserved_variables, handle_child, read_and_check_result, start_child_process,
         transform_json,
-    }, python_executor::{create_dependencies_dir, handle_python_reqs, pip_compile}, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, TZ_ENV
+    },
+    python_executor::{create_dependencies_dir, handle_python_reqs, pip_compile},
+    AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
+    TZ_ENV,
 };
 
 lazy_static::lazy_static! {
@@ -198,7 +205,6 @@ pub async fn handle_ansible_job(
     append_logs(&job.id, &job.workspace_id, logs, db).await;
     write_file(job_dir, "main.yml", &playbook)?;
 
-
     let additional_python_paths = handle_ansible_python_deps(
         job_dir,
         requirements_o,
@@ -215,7 +221,13 @@ pub async fn handle_ansible_job(
 
     let interpolated_args;
     if let Some(args) = &job.args {
-        if let Some(x) = transform_json(client, &job.workspace_id, &args.0, job, db).await? {
+        let mut args = args.0.clone();
+        if let Some(reqs) = reqs.clone() {
+            for (name, path) in reqs.resources.iter().chain(reqs.vars.iter()) {
+                args.insert(name.clone(), to_raw_value(path));
+            }
+        }
+        if let Some(x) = transform_json(client, &job.workspace_id, &args, job, db).await? {
             write_file(
                 job_dir,
                 "args.json",
@@ -228,7 +240,7 @@ pub async fn handle_ansible_job(
                 "args.json",
                 &serde_json::to_string(&args).unwrap_or_else(|_| "{}".to_string()),
             )?;
-            interpolated_args = Some(args.clone().0);
+            interpolated_args = Some(args);
         }
     } else {
         interpolated_args = None;
@@ -236,6 +248,12 @@ pub async fn handle_ansible_job(
     };
     write_file(job_dir, "result.json", "")?;
 
+    let cmd_options: Vec<String> = reqs
+        .as_ref()
+        .map(|r| r.options.clone())
+        .map(|r| get_cmd_options(r))
+        .unwrap_or_default();
+
     let inventories: Vec<String> = reqs
         .as_ref()
         .map(|x| {
@@ -282,14 +300,16 @@ pub async fn handle_ansible_job(
         db,
     )
     .await;
-    let ansible_cfg_content = format!(r#"
+    let ansible_cfg_content = format!(
+        r#"
 [defaults]
 collections_path = ./
 roles_path = ./roles
 home={job_dir}/.ansible
 local_tmp={job_dir}/.ansible/tmp
 remote_tmp={job_dir}/.ansible/tmp
-"#);
+"#
+    );
     write_file(job_dir, "ansible.cfg", &ansible_cfg_content)?;
 
     let mut reserved_variables = get_reserved_variables(job, &authed_client.token, db).await?;
@@ -331,10 +351,11 @@ mount {{
 
     let mut cmd_args = vec!["main.yml", "--extra-vars", "@args.json"];
     cmd_args.extend(inventories.iter().map(|s| s.as_str()));
+    cmd_args.extend(cmd_options.iter().map(|s| s.as_str()));
 
     let child = if !*DISABLE_NSJAIL {
-
-        let wrapper = format!(r#"set -eou pipefail
+        let wrapper = format!(
+            r#"set -eou pipefail
 {0} "$@"
 if [ -f "result" ]; then
     cat result > result_nsjail_mount.json
@@ -342,7 +363,9 @@ fi
 if [ -f "result.json" ]; then
     cat result.json > result_nsjail_mount.json
 fi
-"#, ANSIBLE_PLAYBOOK_PATH.as_str());
+"#,
+            ANSIBLE_PLAYBOOK_PATH.as_str()
+        );
 
         let file = write_file(job_dir, "wrapper.sh", &wrapper)?;
 
@@ -406,6 +429,36 @@ fi
     read_and_check_result(job_dir).await
 }
 
+fn get_cmd_options(r: windmill_parser_yaml::AnsiblePlaybookOptions) -> Vec<String> {
+    let mut ret = vec![];
+
+    if let Some(v) = r.verbosity {
+        if v.chars().all(|c| c == 'v') && v.len() <= 6 {
+            ret.push(format!("-{}", v));
+        }
+    }
+
+    if let Some(o) = r.timeout {
+        ret.push("--timeout".to_string());
+        ret.push(o.to_string());
+    }
+
+    if let Some(o) = r.forks {
+        ret.push("--forks".to_string());
+        ret.push(o.to_string());
+    }
+
+    if r.flush_cache.is_some() {
+        ret.push("--flush-cache".to_string());
+    }
+
+    if r.force_handlers.is_some() {
+        ret.push("--force-handlers".to_string());
+    }
+
+    ret
+}
+
 fn define_nsjail_mount(job_dir: &str, path: &PathBuf) -> anyhow::Result<String> {
     Ok(format!(
         r#"

backend/windmill-worker/src/common.rs

@@ -573,6 +573,28 @@ pub enum UpdateJobPollingExit {
     AlreadyCompleted,
 }
 
+pub async fn update_worker_ping_for_failed_init_script(
+    db: &DB,
+    worker_name: &str,
+    last_job_id: Uuid,
+) {
+    if let Err(e) = sqlx::query!(
+        "UPDATE worker_ping SET 
+        ping_at = now(), 
+        jobs_executed = 1, 
+        current_job_id = $1, 
+        current_job_workspace_id = 'admins' 
+        WHERE worker = $2",
+        last_job_id,
+        worker_name
+    )
+    .execute(db)
+    .await
+    {
+        tracing::error!("Error updating worker ping for failed init script: {e:?}");
+    }
+}
+
 pub async fn update_job_poller<F, Fut>(
     job_id: Uuid,
     db: &DB,

backend/windmill-worker/src/result_processor.rs

@@ -55,7 +55,7 @@ pub async fn process_result(
     column_order: Option<Vec<String>>,
     new_args: Option<HashMap<String, Box<RawValue>>>,
     db: &DB,
-) -> error::Result<()> {
+) -> error::Result<bool> {
     match result {
         Ok(r) => {
             let job = if column_order.is_some() || new_args.is_some() {
@@ -111,6 +111,7 @@ pub async fn process_result(
                 token,
             )
             .await;
+            Ok(true)
         }
         Err(e) => {
             let error_value = match e {
@@ -152,9 +153,9 @@ pub async fn process_result(
                 token,
             )
             .await;
+            Ok(false)
         }
-    };
-    Ok(())
+    }
 }
 
 pub async fn handle_receive_completed_job<

backend/windmill-worker/src/worker.rs

@@ -84,8 +84,7 @@ use rand::Rng;
 
 use crate::{
     ansible_executor::handle_ansible_job, bash_executor::{handle_bash_job, handle_powershell_job}, bun_executor::handle_bun_job, common::{
-        build_args_map, get_cached_resource_value_if_valid, get_reserved_variables, hash_args,
-        NO_LOGS_AT_ALL, SLOW_LOGS,
+        build_args_map, get_cached_resource_value_if_valid, get_reserved_variables, hash_args, update_worker_ping_for_failed_init_script, NO_LOGS_AT_ALL, SLOW_LOGS
     }, deno_executor::handle_deno_job, go_executor::handle_go_job, graphql_executor::do_graphql, handle_job_error, js_eval::{eval_fetch_timeout, transpile_ts}, mysql_executor::do_mysql, pg_executor::do_postgresql, php_executor::handle_php_job, python_executor::handle_python_job, result_processor::{handle_receive_completed_job, process_result}, rust_executor::handle_rust_job, worker_flow::{
         handle_flow, update_flow_status_after_job_completion, update_flow_status_in_progress, Step,
     }, worker_lockfiles::{
@@ -195,6 +194,7 @@ pub async fn create_token_for_owner(
         exp: (chrono::Utc::now() + chrono::Duration::seconds(expires_in as i64)).timestamp()
             as usize,
         job_id: Some(job_id.to_string()),
+        scopes: None,
     };
 
     let token = jsonwebtoken::encode(
@@ -1143,6 +1143,8 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
 
     let job_completed_processor_is_done2 = job_completed_processor_is_done.clone();
     let same_worker_queue_size2 = same_worker_queue_size.clone();
+
+
     let send_result = tokio::spawn(
         (async move {
             let mut has_been_killed = false;
@@ -1178,6 +1180,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
                         if is_init_script_and_failure {
                             tracing::error!("init script errored, exiting");
                             killpill_tx2.send(()).unwrap_or_default();
+                            break;
                         }
                         if is_dependency_job && is_dedicated_worker {
                             tracing::error!("Dedicated worker executed a dependency job, a new script has been deployed. Exiting expecting to be restarted.");
@@ -1695,8 +1698,10 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
                     #[cfg(feature = "prometheus")]
                     let tag = job.tag.clone();
 
+                    let is_init_script: bool = job.tag.as_str() == INIT_SCRIPT_TAG;
                     let arc_job = Arc::new(job);
-                    if let Err(err) = handle_queued_job(
+                    
+                    match handle_queued_job(
                         arc_job.clone(),
                         db,
                         &authed_client,
@@ -1712,10 +1717,9 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
                         worker_flow_initial_transition_duration.clone(),
                         worker_code_execution_duration.clone(),
                     )
-                    .await
-                    {
-                        let is_init_script = arc_job.tag.as_str() == INIT_SCRIPT_TAG;
-                        handle_job_error(
+                    .await {
+                        Err(err) => {
+                            handle_job_error(
                             db,
                             &authed_client.get_authed().await,
                             arc_job.as_ref(),
@@ -1730,12 +1734,20 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
                             (&job_completed_tx.0).clone(),
                         )
                         .await;
-                        if is_init_script {
-                            tracing::error!("failed to execute init_script, exiting");
-                            killpill_tx.send(()).unwrap();
+                        if is_init_script  {
+                            tracing::error!("init script job failed (in handler), exiting");
+                            update_worker_ping_for_failed_init_script(db, &worker_name, arc_job.id).await;
                             break;
                         }
-                    };
+                    },
+                    Ok(false) if is_init_script => {
+                            tracing::error!("init script job failed, exiting");
+                            update_worker_ping_for_failed_init_script(db, &worker_name, arc_job.id).await;
+                            break;
+                        
+                    }
+                    _ => {}
+                }
 
                     #[cfg(feature = "prometheus")]
                     if let Some(duration) = _timer.map(|x| x.stop_and_record()) {
@@ -1810,6 +1822,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
         };
     }
 
+    tracing::info!("worker {} exiting", worker_name);
     // #[cfg(feature = "benchmark")]
     // {
     //     println!("Writing benchmark file");
@@ -1824,10 +1837,14 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
 
     drop(dedicated_workers);
 
-    for handle in dedicated_handles {
-        if let Err(e) = handle.await {
-            tracing::error!("error in dedicated worker waiting for it to end: {:?}", e)
+    let has_dedicated_workers = !dedicated_handles.is_empty();
+    if has_dedicated_workers {
+        for handle in dedicated_handles {
+            if let Err(e) = handle.await {
+                tracing::error!("error in dedicated worker waiting for it to end: {:?}", e)
+            }
         }
+        tracing::info!("all dedicated workers have exited");
     }
 
     drop(job_completed_tx);
@@ -1843,7 +1860,7 @@ async fn queue_init_bash_maybe<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
     same_worker_tx: SameWorkerSender,
     worker_name: &str,
     rsmq: Option<R>,
-) -> error::Result<()> {
+) -> error::Result<bool> {
     if let Some(content) = WORKER_CONFIG.read().await.init_bash.clone() {
         let tx = PushIsolationLevel::IsolatedRoot(db.clone(), rsmq);
         let ehm = HashMap::new();
@@ -1889,8 +1906,10 @@ async fn queue_init_bash_maybe<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
             .await
             .map_err(to_anyhow)?;
         tracing::info!("Creating initial job {uuid} from initial script script: {content}");
+        Ok(true)
+    } else {
+        Ok(false)
     }
-    Ok(())
 }
 
 // async fn process_result<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
@@ -2041,7 +2060,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
     worker_code_execution_metric: &mut f32,
     _worker_flow_initial_transition_duration: Option<Histo>,
     _worker_code_execution_duration: Option<Histo>,
-) -> windmill_common::error::Result<()> {
+) -> windmill_common::error::Result<bool> {
     if job.canceled {
         return Err(Error::JsonErr(canceled_job_to_result(&job)));
     }
@@ -2137,7 +2156,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
             )
             .fetch_one(db)
             .await
-            .map_err(|e| Error::InternalErr(format!("fetching step flow status: {e:#}")))?
+            .map_err(|e| Error::InternalErr(format!("Fetching script path from queue for caching purposes: {e:#}")))?
             .ok_or_else(|| Error::InternalErr(format!("Expected script_path")))?;
             let step = match step.unwrap() {
                 Step::Step(i) => i.to_string(),
@@ -2186,12 +2205,10 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
                 .await
                 .expect("send job completed");
 
-            return Ok(());
+            return Ok(true);
         }
     };
     if job.is_flow() {
-        #[cfg(feature = "prometheus")]
-        let timer = _worker_flow_initial_transition_duration.map(|x| x.start_timer());
         handle_flow(
             job,
             db,
@@ -2203,8 +2220,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
             job_completed_tx.0.clone(),
         )
         .await?;
-        #[cfg(feature = "prometheus")]
-        timer.map(|x| x.stop_and_record());
+        Ok(true)
     } else {
         let mut logs = "".to_string();
         let mut mem_peak: i32 = 0;
@@ -2323,14 +2339,14 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
 
         //it's a test job, no need to update the db
         if job.as_ref().workspace_id == "" {
-            return Ok(());
+            return Ok(true);
         }
 
         if result
             .as_ref()
             .is_err_and(|err| matches!(err, &Error::AlreadyCompleted(_)))
         {
-            return Ok(());
+            return Ok(false);
         }
 
         process_result(
@@ -2346,9 +2362,8 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
             new_args,
             db,
         )
-        .await?;
-    };
-    Ok(())
+        .await
+    }
 }
 
 

benchmarks/lib.ts

@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
 import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
 import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
 
-export const VERSION = "v1.398.1";
+export const VERSION = "v1.399.0";
 
 export async function login(email: string, password: string): Promise<string> {
   return await windmill.UserService.login({

cli/.gitignore

@@ -1,2 +1 @@
 npm/
-gen/

cli/build.sh

@@ -1,2 +1,3 @@
+#!/bin/bash
 ./gen_wm_client.sh
-deno run -A dnt.ts
+deno run -A dnt.ts

cli/gen/core/ApiError.ts

@@ -0,0 +1,21 @@
+import type { ApiRequestOptions } from './ApiRequestOptions.ts';
+import type { ApiResult } from './ApiResult.ts';
+
+export class ApiError extends Error {
+	public readonly url: string;
+	public readonly status: number;
+	public readonly statusText: string;
+	public readonly body: unknown;
+	public readonly request: ApiRequestOptions;
+
+	constructor(request: ApiRequestOptions, response: ApiResult, message: string) {
+		super(message);
+
+		this.name = 'ApiError';
+		this.url = response.url;
+		this.status = response.status;
+		this.statusText = response.statusText;
+		this.body = response.body;
+		this.request = request;
+	}
+}

cli/gen/core/ApiRequestOptions.ts

@@ -0,0 +1,21 @@
+export type ApiRequestOptions<T = unknown> = {
+	readonly body?: any;
+	readonly cookies?: Record<string, unknown>;
+	readonly errors?: Record<number | string, string>;
+	readonly formData?: Record<string, unknown> | any[] | Blob | File;
+	readonly headers?: Record<string, unknown>;
+	readonly mediaType?: string;
+	readonly method:
+		| 'DELETE'
+		| 'GET'
+		| 'HEAD'
+		| 'OPTIONS'
+		| 'PATCH'
+		| 'POST'
+		| 'PUT';
+	readonly path?: Record<string, unknown>;
+	readonly query?: Record<string, unknown>;
+	readonly responseHeader?: string;
+	readonly responseTransformer?: (data: unknown) => Promise<T>;
+	readonly url: string;
+};

cli/gen/core/ApiResult.ts

@@ -0,0 +1,7 @@
+export type ApiResult<TData = any> = {
+	readonly body: TData;
+	readonly ok: boolean;
+	readonly status: number;
+	readonly statusText: string;
+	readonly url: string;
+};

cli/gen/core/CancelablePromise.ts

@@ -0,0 +1,126 @@
+export class CancelError extends Error {
+	constructor(message: string) {
+		super(message);
+		this.name = 'CancelError';
+	}
+
+	public get isCancelled(): boolean {
+		return true;
+	}
+}
+
+export interface OnCancel {
+	readonly isResolved: boolean;
+	readonly isRejected: boolean;
+	readonly isCancelled: boolean;
+
+	(cancelHandler: () => void): void;
+}
+
+export class CancelablePromise<T> implements Promise<T> {
+	private _isResolved: boolean;
+	private _isRejected: boolean;
+	private _isCancelled: boolean;
+	readonly cancelHandlers: (() => void)[];
+	readonly promise: Promise<T>;
+	private _resolve?: (value: T | PromiseLike<T>) => void;
+	private _reject?: (reason?: unknown) => void;
+
+	constructor(
+		executor: (
+			resolve: (value: T | PromiseLike<T>) => void,
+			reject: (reason?: unknown) => void,
+			onCancel: OnCancel
+		) => void
+	) {
+		this._isResolved = false;
+		this._isRejected = false;
+		this._isCancelled = false;
+		this.cancelHandlers = [];
+		this.promise = new Promise<T>((resolve, reject) => {
+			this._resolve = resolve;
+			this._reject = reject;
+
+			const onResolve = (value: T | PromiseLike<T>): void => {
+				if (this._isResolved || this._isRejected || this._isCancelled) {
+					return;
+				}
+				this._isResolved = true;
+				if (this._resolve) this._resolve(value);
+			};
+
+			const onReject = (reason?: unknown): void => {
+				if (this._isResolved || this._isRejected || this._isCancelled) {
+					return;
+				}
+				this._isRejected = true;
+				if (this._reject) this._reject(reason);
+			};
+
+			const onCancel = (cancelHandler: () => void): void => {
+				if (this._isResolved || this._isRejected || this._isCancelled) {
+					return;
+				}
+				this.cancelHandlers.push(cancelHandler);
+			};
+
+			Object.defineProperty(onCancel, 'isResolved', {
+				get: (): boolean => this._isResolved,
+			});
+
+			Object.defineProperty(onCancel, 'isRejected', {
+				get: (): boolean => this._isRejected,
+			});
+
+			Object.defineProperty(onCancel, 'isCancelled', {
+				get: (): boolean => this._isCancelled,
+			});
+
+			return executor(onResolve, onReject, onCancel as OnCancel);
+		});
+	}
+
+	get [Symbol.toStringTag]() {
+		return "Cancellable Promise";
+	}
+
+	public then<TResult1 = T, TResult2 = never>(
+		onFulfilled?: ((value: T) => TResult1 | PromiseLike<TResult1>) | null,
+		onRejected?: ((reason: unknown) => TResult2 | PromiseLike<TResult2>) | null
+	): Promise<TResult1 | TResult2> {
+		return this.promise.then(onFulfilled, onRejected);
+	}
+
+	public catch<TResult = never>(
+		onRejected?: ((reason: unknown) => TResult | PromiseLike<TResult>) | null
+	): Promise<T | TResult> {
+		return this.promise.catch(onRejected);
+	}
+
+	public finally(onFinally?: (() => void) | null): Promise<T> {
+		return this.promise.finally(onFinally);
+	}
+
+	public cancel(): void {
+		if (this._isResolved || this._isRejected || this._isCancelled) {
+			return;
+		}
+		this._isCancelled = true;
+		if (this.cancelHandlers.length) {
+			try {
+				for (const cancelHandler of this.cancelHandlers) {
+					cancelHandler();
+				}
+			} catch (error) {
+				console.warn('Cancellation threw an error', error);
+				return;
+			}
+		}
+		this.cancelHandlers.length = 0;
+		if (this._reject) this._reject(new CancelError('Request aborted'));
+	}
+
+	public get isCancelled(): boolean {
+		return this._isCancelled;
+	}
+}

cli/gen/core/OpenAPI.ts

@@ -0,0 +1,63 @@
+const getEnv = (key: string) => {
+  return Deno.env.get(key)
+};
+
+const baseUrl = getEnv("BASE_INTERNAL_URL") ?? getEnv("BASE_URL") ?? "http://localhost:8000";
+const baseUrlApi = (baseUrl ?? '') + "/api";
+
+import type { ApiRequestOptions } from './ApiRequestOptions.ts';
+
+type Headers = Record<string, string>;
+type Middleware<T> = (value: T) => T | Promise<T>;
+type Resolver<T> = (options: ApiRequestOptions<T>) => Promise<T>;
+
+export class Interceptors<T> {
+  _fns: Middleware<T>[];
+
+  constructor() {
+    this._fns = [];
+  }
+
+  eject(fn: Middleware<T>): void {
+    const index = this._fns.indexOf(fn);
+    if (index !== -1) {
+      this._fns = [...this._fns.slice(0, index), ...this._fns.slice(index + 1)];
+    }
+  }
+
+  use(fn: Middleware<T>): void {
+    this._fns = [...this._fns, fn];
+  }
+}
+
+export type OpenAPIConfig = {
+	BASE: string;
+	CREDENTIALS: 'include' | 'omit' | 'same-origin';
+	ENCODE_PATH?: ((path: string) => string) | undefined;
+	HEADERS?: Headers | Resolver<Headers> | undefined;
+	PASSWORD?: string | Resolver<string> | undefined;
+	TOKEN?: string | Resolver<string> | undefined;
+	USERNAME?: string | Resolver<string> | undefined;
+	VERSION: string;
+	WITH_CREDENTIALS: boolean;
+	interceptors: {
+		request: Interceptors<RequestInit>;
+		response: Interceptors<Response>;
+	};
+};
+
+export const OpenAPI: OpenAPIConfig = {
+	BASE: baseUrlApi,
+	CREDENTIALS: 'include',
+	ENCODE_PATH: undefined,
+	HEADERS: undefined,
+	PASSWORD: undefined,
+	TOKEN: getEnv("WM_TOKEN"),
+	USERNAME: undefined,
+	VERSION: '1.398.1',
+	WITH_CREDENTIALS: true,
+	interceptors: {
+		request: new Interceptors(),
+		response: new Interceptors(),
+	},
+};

cli/gen/core/request.ts

@@ -0,0 +1,350 @@
+import { ApiError } from './ApiError.ts';
+import type { ApiRequestOptions } from './ApiRequestOptions.ts';
+import type { ApiResult } from './ApiResult.ts';
+import { CancelablePromise } from './CancelablePromise.ts';
+import type { OnCancel } from './CancelablePromise.ts';
+import type { OpenAPIConfig } from './OpenAPI.ts';
+
+export const isString = (value: unknown): value is string => {
+	return typeof value === 'string';
+};
+
+export const isStringWithValue = (value: unknown): value is string => {
+	return isString(value) && value !== '';
+};
+
+export const isBlob = (value: any): value is Blob => {
+	return value instanceof Blob;
+};
+
+export const isFormData = (value: unknown): value is FormData => {
+	return value instanceof FormData;
+};
+
+export const base64 = (str: string): string => {
+	try {
+		return btoa(str);
+	} catch (err) {
+		// @ts-ignore
+		return Buffer.from(str).toString('base64');
+	}
+};
+
+export const getQueryString = (params: Record<string, unknown>): string => {
+	const qs: string[] = [];
+
+	const append = (key: string, value: unknown) => {
+		qs.push(`${encodeURIComponent(key)}=${encodeURIComponent(String(value))}`);
+	};
+
+	const encodePair = (key: string, value: unknown) => {
+		if (value === undefined || value === null) {
+			return;
+		}
+
+		if (value instanceof Date) {
+			append(key, value.toISOString());
+		} else if (Array.isArray(value)) {
+			value.forEach(v => encodePair(key, v));
+		} else if (typeof value === 'object') {
+			Object.entries(value).forEach(([k, v]) => encodePair(`${key}[${k}]`, v));
+		} else {
+			append(key, value);
+		}
+	};
+
+	Object.entries(params).forEach(([key, value]) => encodePair(key, value));
+
+	return qs.length ? `?${qs.join('&')}` : '';
+};
+
+const getUrl = (config: OpenAPIConfig, options: ApiRequestOptions): string => {
+	const encoder = config.ENCODE_PATH || encodeURI;
+
+	const path = options.url
+		.replace('{api-version}', config.VERSION)
+		.replace(/{(.*?)}/g, (substring: string, group: string) => {
+			if (options.path?.hasOwnProperty(group)) {
+				return encoder(String(options.path[group]));
+			}
+			return substring;
+		});
+
+	const url = config.BASE + path;
+	return options.query ? url + getQueryString(options.query) : url;
+};
+
+export const getFormData = (options: ApiRequestOptions): FormData | undefined => {
+	if (options.formData) {
+		const formData = new FormData();
+
+		const process = (key: string, value: unknown) => {
+			if (isString(value) || isBlob(value)) {
+				formData.append(key, value);
+			} else {
+				formData.append(key, JSON.stringify(value));
+			}
+		};
+
+		Object.entries(options.formData)
+			.filter(([, value]) => value !== undefined && value !== null)
+			.forEach(([key, value]) => {
+				if (Array.isArray(value)) {
+					value.forEach(v => process(key, v));
+				} else {
+					process(key, value);
+				}
+			});
+
+		return formData;
+	}
+	return undefined;
+};
+
+type Resolver<T> = (options: ApiRequestOptions<T>) => Promise<T>;
+
+export const resolve = async <T>(options: ApiRequestOptions<T>, resolver?: T | Resolver<T>): Promise<T | undefined> => {
+	if (typeof resolver === 'function') {
+		return (resolver as Resolver<T>)(options);
+	}
+	return resolver;
+};
+
+export const getHeaders = async <T>(config: OpenAPIConfig, options: ApiRequestOptions<T>): Promise<Headers> => {
+	const [token, username, password, additionalHeaders] = await Promise.all([
+		// @ts-ignore
+		resolve(options, config.TOKEN),
+		// @ts-ignore
+		resolve(options, config.USERNAME),
+		// @ts-ignore
+		resolve(options, config.PASSWORD),
+		// @ts-ignore
+		resolve(options, config.HEADERS),
+	]);
+
+	const headers = Object.entries({
+		Accept: 'application/json',
+		...additionalHeaders,
+		...options.headers,
+	})
+		.filter(([, value]) => value !== undefined && value !== null)
+		.reduce((headers, [key, value]) => ({
+			...headers,
+			[key]: String(value),
+		}), {} as Record<string, string>);
+
+	if (isStringWithValue(token)) {
+		headers['Authorization'] = `Bearer ${token}`;
+	}
+
+	if (isStringWithValue(username) && isStringWithValue(password)) {
+		const credentials = base64(`${username}:${password}`);
+		headers['Authorization'] = `Basic ${credentials}`;
+	}
+
+	if (options.body !== undefined) {
+		if (options.mediaType) {
+			headers['Content-Type'] = options.mediaType;
+		} else if (isBlob(options.body)) {
+			headers['Content-Type'] = options.body.type || 'application/octet-stream';
+		} else if (isString(options.body)) {
+			headers['Content-Type'] = 'text/plain';
+		} else if (!isFormData(options.body)) {
+			headers['Content-Type'] = 'application/json';
+		}
+	}
+
+	return new Headers(headers);
+};
+
+export const getRequestBody = (options: ApiRequestOptions): unknown => {
+	if (options.body !== undefined) {
+		if (options.mediaType?.includes('application/json') || options.mediaType?.includes('+json')) {
+			return JSON.stringify(options.body);
+		} else if (isString(options.body) || isBlob(options.body) || isFormData(options.body)) {
+			return options.body;
+		} else {
+			return JSON.stringify(options.body);
+		}
+	}
+	return undefined;
+};
+
+export const sendRequest = async (
+	config: OpenAPIConfig,
+	options: ApiRequestOptions,
+	url: string,
+	body: any,
+	formData: FormData | undefined,
+	headers: Headers,
+	onCancel: OnCancel
+): Promise<Response> => {
+	const controller = new AbortController();
+
+	let request: RequestInit = {
+		headers,
+		body: body ?? formData,
+		method: options.method,
+		signal: controller.signal,
+	};
+
+	if (config.WITH_CREDENTIALS) {
+		request.credentials = config.CREDENTIALS;
+	}
+
+	for (const fn of config.interceptors.request._fns) {
+		request = await fn(request);
+	}
+
+	onCancel(() => controller.abort());
+
+	return await fetch(url, request);
+};
+
+export const getResponseHeader = (response: Response, responseHeader?: string): string | undefined => {
+	if (responseHeader) {
+		const content = response.headers.get(responseHeader);
+		if (isString(content)) {
+			return content;
+		}
+	}
+	return undefined;
+};
+
+export const getResponseBody = async (response: Response): Promise<unknown> => {
+	if (response.status !== 204) {
+		try {
+			const contentType = response.headers.get('Content-Type');
+			if (contentType) {
+				const binaryTypes = ['application/octet-stream', 'application/pdf', 'application/zip', 'audio/', 'image/', 'video/'];
+				if (contentType.includes('application/json') || contentType.includes('+json')) {
+					return await response.json();
+				} else if (binaryTypes.some(type => contentType.includes(type))) {
+					return await response.blob();
+				} else if (contentType.includes('multipart/form-data')) {
+					return await response.formData();
+				} else if (contentType.includes('text/')) {
+					return await response.text();
+				}
+			}
+		} catch (error) {
+			console.error(error);
+		}
+	}
+	return undefined;
+};
+
+export const catchErrorCodes = (options: ApiRequestOptions, result: ApiResult): void => {
+	const errors: Record<number, string> = {
+		400: 'Bad Request',
+		401: 'Unauthorized',
+		402: 'Payment Required',
+		403: 'Forbidden',
+		404: 'Not Found',
+		405: 'Method Not Allowed',
+		406: 'Not Acceptable',
+		407: 'Proxy Authentication Required',
+		408: 'Request Timeout',
+		409: 'Conflict',
+		410: 'Gone',
+		411: 'Length Required',
+		412: 'Precondition Failed',
+		413: 'Payload Too Large',
+		414: 'URI Too Long',
+		415: 'Unsupported Media Type',
+		416: 'Range Not Satisfiable',
+		417: 'Expectation Failed',
+		418: 'Im a teapot',
+		421: 'Misdirected Request',
+		422: 'Unprocessable Content',
+		423: 'Locked',
+		424: 'Failed Dependency',
+		425: 'Too Early',
+		426: 'Upgrade Required',
+		428: 'Precondition Required',
+		429: 'Too Many Requests',
+		431: 'Request Header Fields Too Large',
+		451: 'Unavailable For Legal Reasons',
+		500: 'Internal Server Error',
+		501: 'Not Implemented',
+		502: 'Bad Gateway',
+		503: 'Service Unavailable',
+		504: 'Gateway Timeout',
+		505: 'HTTP Version Not Supported',
+		506: 'Variant Also Negotiates',
+		507: 'Insufficient Storage',
+		508: 'Loop Detected',
+		510: 'Not Extended',
+		511: 'Network Authentication Required',
+		...options.errors,
+	}
+
+	const error = errors[result.status];
+	if (error) {
+		throw new ApiError(options, result, error);
+	}
+
+	if (!result.ok) {
+		const errorStatus = result.status ?? 'unknown';
+		const errorStatusText = result.statusText ?? 'unknown';
+		const errorBody = (() => {
+			try {
+				return JSON.stringify(result.body, null, 2);
+			} catch (e) {
+				return undefined;
+			}
+		})();
+
+		throw new ApiError(options, result,
+			`Generic Error: status: ${errorStatus}; status text: ${errorStatusText}; body: ${errorBody}`
+		);
+	}
+};
+
+/**
+ * Request method
+ * @param config The OpenAPI configuration object
+ * @param options The request options from the service
+ * @returns CancelablePromise<T>
+ * @throws ApiError
+ */
+export const request = <T>(config: OpenAPIConfig, options: ApiRequestOptions<T>): CancelablePromise<T> => {
+	return new CancelablePromise(async (resolve, reject, onCancel) => {
+		try {
+			const url = getUrl(config, options);
+			const formData = getFormData(options);
+			const body = getRequestBody(options);
+			const headers = await getHeaders(config, options);
+
+			if (!onCancel.isCancelled) {
+				let response = await sendRequest(config, options, url, body, formData, headers, onCancel);
+
+				for (const fn of config.interceptors.response._fns) {
+					response = await fn(response);
+				}
+
+				const responseBody = await getResponseBody(response);
+				const responseHeader = getResponseHeader(response, options.responseHeader);
+
+				let transformedBody = responseBody;
+				if (options.responseTransformer && response.ok) {
+					transformedBody = await options.responseTransformer(responseBody)
+				}
+
+				const result: ApiResult = {
+					url,
+					ok: response.ok,
+					status: response.status,
+					statusText: response.statusText,
+					body: responseHeader ?? transformedBody,
+				};
+
+				catchErrorCodes(options, result);
+
+				resolve(result.body);
+			}
+		} catch (error) {
+			reject(error);
+		}
+	});
+};

cli/gen/index.ts

@@ -0,0 +1,6 @@
+// This file is auto-generated by @hey-api/openapi-ts
+export { ApiError } from './core/ApiError.ts';
+export { CancelablePromise, CancelError } from './core/CancelablePromise.ts';
+export { OpenAPI, type OpenAPIConfig } from './core/OpenAPI.ts';
+export * from './services.gen.ts';
+export * from './types.gen.ts';

cli/gen_wm_client.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 set -eou pipefail
 script_dirpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
@@ -19,4 +20,4 @@ sed -i "s/BASE: '\/api'/BASE: baseUrlApi/g" gen/core/OpenAPI.ts
 
 find gen/ -name "*.ts" -exec sed -i -E "s/(import.*from[[:space:]]*['\"][^'\"]+)(['\"])/\1.ts\2/g" {} \;
 
-find gen/ -name "*.ts" -exec sed -i -E "s/(export.*from[[:space:]]*['\"][^'\"]+)(['\"])/\1.ts\2/g" {} \;
+find gen/ -name "*.ts" -exec sed -i -E "s/(export.*from[[:space:]]*['\"][^'\"]+)(['\"])/\1.ts\2/g" {} \;

cli/main.ts

@@ -56,7 +56,7 @@ export {
 //   }
 // });
 
-export const VERSION = "1.398.1";
+export const VERSION = "1.399.0";
 
 const command = new Command()
   .name("wmill")

cli/metadata.ts

@@ -7,6 +7,7 @@ import {
 } from "./bootstrap/script_bootstrap.ts";
 import {
   instantiate as instantiateWasm,
+  parse_ansible,
   parse_bash,
   parse_bigquery,
   parse_deno,
@@ -14,8 +15,10 @@ import {
   parse_graphql,
   parse_mssql,
   parse_mysql,
+  parse_php,
   parse_powershell,
   parse_python,
+  parse_rust,
   parse_snowflake,
   parse_sql,
 } from "./wasm/windmill_parser_wasm.generated.js";
@@ -306,8 +309,10 @@ export async function updateScriptSchema(
     path
   );
   metadataContent.schema = result.schema;
-  metadataContent.has_preprocessor = result.has_preprocessor;
-  metadataContent.no_main_func = result.no_main_func;
+  if (result.has_preprocessor != null)
+    metadataContent.has_preprocessor = result.has_preprocessor;
+  if (result.no_main_func != null)
+    metadataContent.no_main_func = result.no_main_func;
 }
 
 async function updateScriptLock(
@@ -324,7 +329,9 @@ async function updateScriptLock(
       language == "python3" ||
       language == "go" ||
       language == "deno" ||
-      language == "php"
+      language == "php" ||
+      language == "rust" ||
+      language == "ansible"
     )
   ) {
     return;
@@ -482,6 +489,12 @@ export function inferSchema(
     inferedSchema = JSON.parse(parse_bash(content));
   } else if (language === "powershell") {
     inferedSchema = JSON.parse(parse_powershell(content));
+  } else if (language === "php") {
+    inferedSchema = JSON.parse(parse_php(content));
+  } else if (language === "rust") {
+    inferedSchema = JSON.parse(parse_rust(content));
+  } else if (language === "ansible") {
+    inferedSchema = JSON.parse(parse_ansible(content));
   } else {
     throw new Error("Invalid language: " + language);
   }

cli/script_common.ts

@@ -48,9 +48,9 @@ export function inferContentTypeFromFilePath(
     return "bash";
   } else if (contentPath.endsWith(".ps1")) {
     return "powershell";
-  } else if (contentPath.endsWith("php")) {
+  } else if (contentPath.endsWith(".php")) {
     return "php";
-  } else if (contentPath.endsWith("rs")) {
+  } else if (contentPath.endsWith(".rs")) {
     return "rust";
   } else if (contentPath.endsWith(".playbook.yml")) {
     return "ansible";

cli/sync.ts

@@ -504,18 +504,20 @@ function ZipFSElement(
 
       if (formatExtension) {
         const fileContent: string = parsed["value"]["content"];
-        r.push({
-          isDirectory: false,
-          path:
-            removeSuffix(finalPath, ".resource.json") +
-            ".resource.file." +
-            formatExtension,
-          async *getChildren() {},
-          // deno-lint-ignore require-await
-          async getContentText() {
-            return fileContent;
-          },
-        });
+        if (typeof(fileContent) === "string") {
+          r.push({
+            isDirectory: false,
+            path:
+              removeSuffix(finalPath, ".resource.json") +
+              ".resource.file." +
+              formatExtension,
+            async *getChildren() {},
+            // deno-lint-ignore require-await
+            async getContentText() {
+              return fileContent;
+            },
+          });
+        }
       }
     }
     return r;

cli/types.ts

@@ -200,7 +200,9 @@ export function getTypeStrFromPath(
     parsed.ext == ".gql" ||
     parsed.ext == ".ps1" ||
     parsed.ext == ".js" ||
-    parsed.ext == ".php"
+    parsed.ext == ".php" ||
+    parsed.ext == ".rs" ||
+    (parsed.ext == ".yml" && parsed.name.split(".").pop() == "playbook")
   ) {
     return "script";
   }

cli/wasm/windmill_parser_wasm.generated.js

@@ -15,20 +15,6 @@ function getObject(idx) {
   return heap[idx];
 }
 
-let heap_next = heap.length;
-
-function dropObject(idx) {
-  if (idx < 132) return;
-  heap[idx] = heap_next;
-  heap_next = idx;
-}
-
-function takeObject(idx) {
-  const ret = getObject(idx);
-  dropObject(idx);
-  return ret;
-}
-
 let WASM_VECTOR_LEN = 0;
 
 let cachedUint8Memory0 = null;
@@ -103,6 +89,38 @@ function getInt32Memory0() {
   return cachedInt32Memory0;
 }
 
+let heap_next = heap.length;
+
+function dropObject(idx) {
+  if (idx < 132) return;
+  heap[idx] = heap_next;
+  heap_next = idx;
+}
+
+function takeObject(idx) {
+  const ret = getObject(idx);
+  dropObject(idx);
+  return ret;
+}
+
+let cachedFloat64Memory0 = null;
+
+function getFloat64Memory0() {
+  if (cachedFloat64Memory0 === null || cachedFloat64Memory0.byteLength === 0) {
+    cachedFloat64Memory0 = new Float64Array(wasm.memory.buffer);
+  }
+  return cachedFloat64Memory0;
+}
+
+function addHeapObject(obj) {
+  if (heap_next === heap.length) heap.push(heap.length + 1);
+  const idx = heap_next;
+  heap_next = heap[idx];
+
+  heap[idx] = obj;
+  return idx;
+}
+
 const cachedTextDecoder = typeof TextDecoder !== "undefined"
   ? new TextDecoder("utf-8", { ignoreBOM: true, fatal: true })
   : {
@@ -118,24 +136,6 @@ function getStringFromWasm0(ptr, len) {
   return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len));
 }
 
-function addHeapObject(obj) {
-  if (heap_next === heap.length) heap.push(heap.length + 1);
-  const idx = heap_next;
-  heap_next = heap[idx];
-
-  heap[idx] = obj;
-  return idx;
-}
-
-let cachedFloat64Memory0 = null;
-
-function getFloat64Memory0() {
-  if (cachedFloat64Memory0 === null || cachedFloat64Memory0.byteLength === 0) {
-    cachedFloat64Memory0 = new Float64Array(wasm.memory.buffer);
-  }
-  return cachedFloat64Memory0;
-}
-
 let cachedBigInt64Memory0 = null;
 
 function getBigInt64Memory0() {
@@ -698,9 +698,6 @@ function handleError(f, args) {
 
 const imports = {
   __wbindgen_placeholder__: {
-    __wbindgen_object_drop_ref: function (arg0) {
-      takeObject(arg0);
-    },
     __wbindgen_string_get: function (arg0, arg1) {
       const obj = getObject(arg1);
       const ret = typeof obj === "string" ? obj : undefined;
@@ -715,9 +712,8 @@ const imports = {
       getInt32Memory0()[arg0 / 4 + 1] = len1;
       getInt32Memory0()[arg0 / 4 + 0] = ptr1;
     },
-    __wbindgen_error_new: function (arg0, arg1) {
-      const ret = new Error(getStringFromWasm0(arg0, arg1));
-      return addHeapObject(ret);
+    __wbindgen_object_drop_ref: function (arg0) {
+      takeObject(arg0);
     },
     __wbindgen_boolean_get: function (arg0) {
       const v = getObject(arg0);
@@ -755,7 +751,11 @@ const imports = {
       const ret = BigInt.asUintN(64, arg0);
       return addHeapObject(ret);
     },
-    __wbg_eval_b7405fdd08ddef3d: function (arg0, arg1) {
+    __wbindgen_error_new: function (arg0, arg1) {
+      const ret = new Error(getStringFromWasm0(arg0, arg1));
+      return addHeapObject(ret);
+    },
+    __wbg_eval_8243d86342a64a9b: function (arg0, arg1) {
       const ret = eval(getStringFromWasm0(arg0, arg1));
       return addHeapObject(ret);
     },

frontend/package-lock.json

@@ -1,12 +1,12 @@
 {
 	"name": "windmill-components",
-	"version": "1.398.1",
+	"version": "1.399.0",
 	"lockfileVersion": 3,
 	"requires": true,
 	"packages": {
 		"": {
 			"name": "windmill-components",
-			"version": "1.398.1",
+			"version": "1.399.0",
 			"license": "AGPL-3.0",
 			"dependencies": {
 				"@aws-crypto/sha256-js": "^4.0.0",
@@ -72,7 +72,7 @@
 				"windmill-parser-wasm-regex": "^1.397.2",
 				"windmill-parser-wasm-rust": "^1.397.2",
 				"windmill-parser-wasm-ts": "^1.397.2",
-				"windmill-parser-wasm-yaml": "^1.396.1",
+				"windmill-parser-wasm-yaml": "^1.398.1",
 				"windmill-sql-datatype-parser-wasm": "^1.318.0",
 				"y-monaco": "^0.1.4",
 				"y-websocket": "^1.5.4",
@@ -13525,9 +13525,9 @@
 			"integrity": "sha512-nspEC1zLfrvvh6ta8YSjKo99lDfFvsXh7rV2mmpTAmc2ZzzyuVDvW5Cm0Wmo8Ww3seO+oDi/v3jy+8QX8pIzwA=="
 		},
 		"node_modules/windmill-parser-wasm-yaml": {
-			"version": "1.396.1",
-			"resolved": "https://registry.npmjs.org/windmill-parser-wasm-yaml/-/windmill-parser-wasm-yaml-1.396.1.tgz",
-			"integrity": "sha512-ZRo5WIAwO21+ZweFPGe0Dx+PZbzJ9gd1qtAc9m0KeB7iNH1XY6lYKrmdQNaHJN5NqdwFiWTRgxIRgP62vvizBQ=="
+			"version": "1.398.1",
+			"resolved": "https://registry.npmjs.org/windmill-parser-wasm-yaml/-/windmill-parser-wasm-yaml-1.398.1.tgz",
+			"integrity": "sha512-f5MCJ9PVLrJQBcVuxnTlZNw561HIJkMqX/76HLgOqKehNpO7gboZyaHx0txUJP50MIDv/IMpn0SD6S7DZnynEg=="
 		},
 		"node_modules/windmill-sql-datatype-parser-wasm": {
 			"version": "1.318.0",

frontend/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "windmill-components",
-	"version": "1.398.1",
+	"version": "1.399.0",
 	"scripts": {
 		"dev": "vite dev",
 		"build": "vite build",
@@ -145,7 +145,7 @@
 		"windmill-parser-wasm-regex": "^1.397.2",
 		"windmill-parser-wasm-rust": "^1.397.2",
 		"windmill-parser-wasm-ts": "^1.397.2",
-		"windmill-parser-wasm-yaml": "^1.396.1",
+		"windmill-parser-wasm-yaml": "^1.398.1",
 		"windmill-sql-datatype-parser-wasm": "^1.318.0",
 		"y-monaco": "^0.1.4",
 		"y-websocket": "^1.5.4",
@@ -459,4 +459,4 @@
 	"optionalDependencies": {
 		"fsevents": "^2.3.3"
 	}
-}
+}

frontend/src/lib/components/ExecutionDuration.svelte

@@ -0,0 +1,52 @@
+<script lang="ts">
+	import { type Job } from '$lib/gen'
+	import { onDestroy } from 'svelte'
+
+	export let job: Job | undefined = undefined
+	/** Execution duration of current active job (in ms) */
+	export let executionDuration: number = 0
+	/** Is current job running more than specified value in `longDefinition` seconds */
+	export let longRunning: boolean = false
+	/** What do we count as "long" (in ms)*/
+	export let longDefinition: number = 30_000
+	/** How often component updates execution duration (in ms)
+	 *   Higher value -> more efficient component is, less accuracy it has
+	 *   Lower value -> less efficient component is, more accuracy it has
+	 */
+	export let updateResolution: number = 5_000
+
+	let startedAt: number | undefined = undefined
+	let busy: boolean = false
+	let interval: NodeJS.Timeout | undefined
+	// Detect when execution of job started
+	$: if (
+		!busy &&
+		job &&
+		'running' in job &&
+		(job.job_kind == 'script' || job?.job_kind == 'preview')
+	)
+		start(job)
+
+	function start(job: Job) {
+		busy = true
+		startedAt = new Date(job?.started_at ?? '').getTime()
+		interval = setInterval(updateDuration, updateResolution)
+	}
+
+	function updateDuration() {
+		if (job?.type == 'CompletedJob') {
+			clearInterval(interval)
+			return
+		}
+
+		if (startedAt) executionDuration = Date.now() - startedAt
+
+		// Detect long running
+		if (executionDuration >= longDefinition) longRunning = true
+	}
+
+	onDestroy(() => {
+		// Clear the interval when the component is destroyed
+		clearInterval(interval)
+	})
+</script>

frontend/src/lib/components/FlowGraphViewer.svelte

@@ -40,8 +40,14 @@
 				failureModule={flow?.value?.failure_module}
 				preprocessorModule={flow?.value?.preprocessor_module}
 				on:select={(e) => {
-					const mod = dfs(flow?.value?.modules ?? [], (m) => m).find((m) => m?.id === e?.detail)
-					stepDetail = mod ?? e.detail
+					if (e?.detail === 'failure') {
+						stepDetail = flow?.value?.failure_module
+					} else if (e?.detail === 'preprocessor') {
+						stepDetail = flow?.value?.preprocessor_module
+					} else {
+						stepDetail = dfs(flow?.value?.modules ?? [], (m) => m).find((m) => m?.id === e?.detail)
+					}
+					stepDetail = stepDetail ?? e?.detail
 					dispatch('select', stepDetail)
 				}}
 				on:triggerDetail

frontend/src/lib/components/FlowGraphViewerStep.svelte

@@ -103,12 +103,17 @@
 	{:else if typeof stepDetail != 'string' && stepDetail.value}
 		<div class="">
 			<div class="sticky top-0 bg-surface w-full flex items-center py-2">
-				{#if stepDetail.id}
+				{#if stepDetail.id && stepDetail.id != 'failure' && stepDetail.id != 'preprocessor'}
 					<Badge color="indigo">
 						{stepDetail.id}
 					</Badge>
 				{/if}
-				<span class="ml-2 font-medium text-lg">
+				<span
+					class={twMerge(
+						'font-medium text-lg',
+						stepDetail.id !== 'failure' && stepDetail.id !== 'preprocessor' ? 'ml-2' : ''
+					)}
+				>
 					{#if stepDetail.summary}
 						{stepDetail.summary}
 					{:else if stepDetail.value.type == 'identity'}
@@ -124,6 +129,10 @@
 						Inner flow
 					{:else if stepDetail.value.type == 'whileloopflow'}
 						While loop
+					{:else if stepDetail.id === 'failure'}
+						Error handler
+					{:else if stepDetail.id === 'preprocessor'}
+						Preprocessor
 					{:else}
 						Anonymous step
 					{/if}
@@ -147,10 +156,12 @@
 				An identity step returns its inputs as outputs
 			</p>
 		{:else if stepDetail.value.type == 'rawscript'}
-			<div class="text-xs">
-				<h3 class="mb-2 font-semibold mt-2">Step Inputs</h3>
-				<InputTransformsViewer inputTransforms={stepDetail?.value?.input_transforms ?? {}} />
-			</div>
+			{#if stepDetail.id !== 'preprocessor'}
+				<div class="text-xs">
+					<h3 class="mb-2 font-semibold mt-2">Step Inputs</h3>
+					<InputTransformsViewer inputTransforms={stepDetail?.value?.input_transforms ?? {}} />
+				</div>
+			{/if}
 
 			<div>
 				<div class="mb-2 mt-4 flex justify-between items-center">
@@ -180,10 +191,12 @@
 				</div>
 			</div>
 		{:else if stepDetail.value.type == 'script'}
-			<div class="text-2xs">
-				<h3 class="mb-2 font-semibold mt-2">Step Inputs</h3>
-				<InputTransformsViewer inputTransforms={stepDetail?.value?.input_transforms ?? {}} />
-			</div>
+			{#if stepDetail.id !== 'preprocessor'}
+				<div class="text-2xs">
+					<h3 class="mb-2 font-semibold mt-2">Step Inputs</h3>
+					<InputTransformsViewer inputTransforms={stepDetail?.value?.input_transforms ?? {}} />
+				</div>
+			{/if}
 			{#if stepDetail.value.path.startsWith('hub/')}
 				<div class="flex flex-col grow">
 					<div class="mb-2 flex justify-between items-center">

frontend/src/lib/components/FlowJobResult.svelte

@@ -40,7 +40,6 @@
 	let logOffset = 0
 	async function getLogs() {
 		if (jobId) {
-			console.log('getLogs')
 
 			const getUpdate = await JobService.getJobUpdates({
 				workspace: workspaceId ?? $workspaceStore!,

frontend/src/lib/components/TestJobLoader.svelte

@@ -15,16 +15,16 @@
 	export let lazyLogs = false
 	// Will be set to number if job is not a flow
 	// If you want to find out progress of subjobs of a flow, check job.flow_status.progress
-	export let scriptProgress: number | undefined = undefined;
+	export let scriptProgress: number | undefined = undefined
 
 	/// Last time asked for job progress
-	let lastTimeCheckedProgress: number | undefined = undefined; 
+	let lastTimeCheckedProgress: number | undefined = undefined
 
-	/// Will try to poll progress every 5s and if once progress returned was not undefined, will be ignored 
+	/// Will try to poll progress every 5s and if once progress returned was not undefined, will be ignored
 	/// and getProgressRate will be used instead
-	const getProgressRetryRate: number = 5000;
+	const getProgressRetryRate: number = 5000
 	/// How often loader poll progress
-	const getProgressRate: number = 1000;
+	const getProgressRate: number = 1000
 
 	const dispatch = createEventDispatcher()
 
@@ -105,13 +105,11 @@
 
 	export async function getLogs() {
 		if (job) {
-			console.log('getLogs')
-
 			const getUpdate = await JobService.getJobUpdates({
 				workspace: workspace!,
 				id: job.id,
 				running: `running` in job && job.running,
-				logOffset: job.logs?.length ?? 0,
+				logOffset: job.logs?.length ?? 0
 			})
 
 			if ((job.logs ?? '').length == 0) {
@@ -129,11 +127,10 @@
 		tag: string | undefined,
 		lock?: string
 	): Promise<string> {
-
 		// Reset in case we rerun job without reloading
-		scriptProgress = undefined;
-		lastTimeCheckedProgress = undefined;
-	
+		scriptProgress = undefined
+		lastTimeCheckedProgress = undefined
+
 		return abstractRun(() =>
 			JobService.runScriptPreview({
 				workspace: $workspaceStore!,
@@ -192,29 +189,30 @@
 		if (currentId === id) {
 			try {
 				if (job && `running` in job) {
-
-					let getProgress: boolean | undefined = undefined;
+					let getProgress: boolean | undefined = undefined
 					// We only pull individual job progress this way
 					// Flow's progress we are getting from FlowStatusModule of flow job
-					if (job.job_kind == "script" || job.job_kind == "preview"){
+					if (job.job_kind == 'script' || job.job_kind == 'preview') {
 						// First time, before running job, lastTimeCheckedProgress is always undefined
-						if (lastTimeCheckedProgress){
-							const lastTimeCheckedMs = Date.now() - lastTimeCheckedProgress;
+						if (lastTimeCheckedProgress) {
+							const lastTimeCheckedMs = Date.now() - lastTimeCheckedProgress
 							// Ask for progress if the last time we asked is >5s OR the progress was once not undefined
-							if (lastTimeCheckedMs > getProgressRetryRate || (scriptProgress != undefined && lastTimeCheckedMs > getProgressRate)){
-								lastTimeCheckedProgress = Date.now();	
-								getProgress = true;																								
+							if (
+								lastTimeCheckedMs > getProgressRetryRate ||
+								(scriptProgress != undefined && lastTimeCheckedMs > getProgressRate)
+							) {
+								lastTimeCheckedProgress = Date.now()
+								getProgress = true
 							}
 						} else {
 							// Make it think we asked for progress, but in reality we didnt. First 5s we want to wait without putting extra work on db
 							// 99.99% of the jobs won't have progress be set so we have to do a balance between having low-latency for jobs that use it and job that don't
 							// we would usually not care to have progress the first 5s and jobs that are less than 5s
-							lastTimeCheckedProgress = Date.now();
-						}					
+							lastTimeCheckedProgress = Date.now()
+						}
 					}
 
 					const offset = logOffset == 0 ? (job.logs?.length ? job.logs?.length + 1 : 0) : logOffset
-					console.log('getLogs')
 
 					let previewJobUpdates = await JobService.getJobUpdates({
 						workspace: workspace!,
@@ -225,14 +223,13 @@
 					})
 
 					// Clamp number between two values with the following line:
-					const clamp = (num, min, max) => Math.min(Math.max(num, min), max);
+					const clamp = (num, min, max) => Math.min(Math.max(num, min), max)
 
-					if (previewJobUpdates.progress){
+					if (previewJobUpdates.progress) {
 						// Progress cannot go back and cannot be set to 100
-						scriptProgress = clamp(previewJobUpdates.progress, scriptProgress ?? 0, 99);						
+						scriptProgress = clamp(previewJobUpdates.progress, scriptProgress ?? 0, 99)
 					}
 
-
 					if (previewJobUpdates.new_logs) {
 						if (offset == 0) {
 							job.logs = previewJobUpdates.new_logs ?? ''

frontend/src/lib/components/copilot/CronGen.svelte

@@ -15,7 +15,7 @@
 	let abortController = new AbortController()
 	$: instructionsField && setTimeout(() => instructionsField?.focus(), 100)
 	const SYSTEM =
-		"You are a helpful assitant for creating CRON schedules. The structure is 'second minute hour dayOfMonth month dayOfWeek'. Weekdays are Sunday (1), Monday (2), Tuesday (3), Wednesday (4), Thursday (5), Friday (6), Saturday (7). You only return the CRON string. If it is invalid, you will return an error message preceeded by 'ERROR:'."
+		"You are a helpful assitant for creating CRON schedules. The structure is 'second minute hour dayOfMonth month dayOfWeek'. Weekdays are Sunday (1), Monday (2), Tuesday (3), Wednesday (4), Thursday (5), Friday (6), Saturday (7). You only return the CRON string without any wrapping characters. If it is invalid, you will return an error message preceeded by 'ERROR:'."
 	const USER = 'CRON schedule instructions: {instructions}'
 	async function generateCron() {
 		genLoading = true

frontend/src/lib/components/copilot/RegexGen.svelte

@@ -31,7 +31,7 @@
 					{
 						role: 'system',
 						content:
-							'Generate a regex pattern that one can use in a javascript Regex object. Output only the regex itself. The regex should match the following:'
+							'Generate a regex pattern that one can use in a javascript Regex object. Output only the regex itself without any wrapping characters including the / characters. The regex should match the following:'
 					},
 					{
 						role: 'user',

frontend/src/lib/components/flows/FlowProgressBar.svelte

@@ -3,6 +3,7 @@
 	import ProgressBar from '../progressBar/ProgressBar.svelte'
 
 	export let job: Job | undefined = undefined
+	export let currentSubJobProgress: number | undefined = undefined
 
 	let error: number | undefined = undefined
 	let index = 0
@@ -59,6 +60,9 @@
 			//                  Jitter protection >^^^^^^^^
 			subStepLength = 100
 			subIndexIsPercent = true;
+			currentSubJobProgress = subStepIndex
+		} else {
+			currentSubJobProgress = undefined
 		}
 
 		error = newError

frontend/src/lib/components/graph/graphBuilder.ts

@@ -476,6 +476,10 @@ export default function graphBuilder(
 		addEdge(id, 'Input', { type: 'empty' })
 	}
 
+	if (failureModule && !extra.flowModuleStates) {
+		addNode(failureModule, 0, 'module')
+	}
+
 	Object.keys(parents).forEach((key) => {
 		const node = nodes.find((n) => n.id === key)
 

frontend/src/lib/components/sidebar/SidebarContent.svelte

@@ -239,7 +239,7 @@
 					<Menu>
 						<div
 							slot="trigger"
-							class="w-full text-gray-400 text-2xs flex flex-row justify-center gap-1 items-center"
+							class="w-full text-gray-400 text-2xs flex flex-row gap-1 py-1 items-center px-2 hover:bg-[#2A3648] dark:hover:bg-[#30404e] rounded"
 						>
 							<Plus size={14} />
 						</div>

frontend/src/routes/(root)/(logged)/routes/+page.svelte

@@ -160,7 +160,7 @@
 	<PageHeader
 		title="Custom HTTP routes"
 		tooltip="Every script and flow already has a canonical HTTP API endpoint/webhook attached to it, this is to create additional parametrizable ones."
-		documentationLink="https://www.windmill.dev/docs/core_concepts/http_api"
+		documentationLink="https://www.windmill.dev/docs/core_concepts/http_routing"
 	>
 		{#if $userStore?.is_admin || $userStore?.is_super_admin}
 			<Button size="md" startIcon={{ icon: Plus }} on:click={() => routeEditor.openNew(false)}>

frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte

@@ -92,11 +92,13 @@
 	import Popover from '$lib/components/Popover.svelte'
 	import HighlightTheme from '$lib/components/HighlightTheme.svelte'
 	import PreprocessedArgsDisplay from '$lib/components/runs/PreprocessedArgsDisplay.svelte'
+	import ExecutionDuration from '$lib/components/ExecutionDuration.svelte'
 
 	let job: Job | undefined
 	let jobUpdateLastFetch: Date | undefined
 
 	let scriptProgress: number | undefined = undefined;
+	let currentJobIsLongRunning: boolean = false
 
 	let viewTab: 'result' | 'logs' | 'code' | 'stats' = 'result'
 	let selectedJobStep: string | undefined = undefined
@@ -112,6 +114,8 @@
 	let persistentScriptDrawer: PersistentScriptDrawer
 	let getLogs: (() => Promise<void>) | undefined = undefined
 
+	let showExplicitProgressTip: boolean =
+		(localStorage.getItem('hideExplicitProgressTip') ?? 'false') == 'false'
 	$: job?.logs == undefined && job && viewTab == 'logs' && getLogs?.()
 
 	let lastJobId: string | undefined = undefined
@@ -747,7 +751,35 @@
 			</div>
 			<div>
 				<Skeleton loading={!job} layout={[[9.5]]} />
-				{#if job}<FlowMetadata {job} {scheduleEditor} />{/if}
+				{#if job}
+					<FlowMetadata {job} {scheduleEditor} />
+					{#if currentJobIsLongRunning && showExplicitProgressTip  && !scriptProgress && 'running' in job}
+						<Alert
+							class="mt-4 p-1 flex flex-row relative text-center"
+							size="xs"
+							type="info"
+							title="tip: Track progress of longer jobs"
+							tooltip="For better transparency and verbosity, you can try setting progress from within the script."
+							documentationLink="https://www.windmill.dev/docs/advanced/explicit_progress"
+						>
+							<button
+								type="button"
+								on:click={() => {
+									localStorage.setItem('hideExplicitProgressTip', 'true')
+									showExplicitProgressTip = false
+								}}
+								class="absolute m-2 top-0 right-0 inline-flex rounded-md bg-surface-secondary text-gray-400 hover:text-tertiary focus:outline-none"
+							>
+								<span class="sr-only">Close</span>
+								<svg class="h-5 w-5" viewBox="0 0 20 20" fill="currentColor" aria-hidden="true">
+									<path
+										d="M6.28 5.22a.75.75 0 00-1.06 1.06L8.94 10l-3.72 3.72a.75.75 0 101.06 1.06L10 11.06l3.72 3.72a.75.75 0 101.06-1.06L11.06 10l3.72-3.72a.75.75 0 00-1.06-1.06L10 8.94 6.28 5.22z"
+									/>
+								</svg>
+							</button>
+						</Alert>
+					{/if}
+				{/if}
 			</div>
 		</div>
 
@@ -757,6 +789,9 @@
 			</div>
 		{/if}
 		{#if job?.job_kind !== 'flow' && job?.job_kind !== 'flowpreview' && job?.job_kind !== 'singlescriptflow'}
+			{#if ['python3', 'bun', 'deno'].includes(job?.language ?? '') && (job?.job_kind == 'script' || job?.job_kind == 'preview')}
+				<ExecutionDuration bind:job bind:longRunning={currentJobIsLongRunning} />
+			{/if}
 			<div class="max-w-7xl mx-auto w-full px-4 mb-10">
 				{#if job?.flow_status && typeof job.flow_status == 'object' && !('_metadata' in job.flow_status)}
 					<div class="mt-10" />
@@ -823,7 +858,11 @@
 			</div>
 		{:else if !job?.['deleted']}
 			<div class="mt-10" />
-			<FlowProgressBar {job} class="py-4 max-w-7xl mx-auto px-4" />
+			<FlowProgressBar
+				{job}
+				bind:currentSubJobProgress={scriptProgress}
+				class="py-4 max-w-7xl mx-auto px-4"
+			/>
 			<div class="w-full mt-10">
 				<FlowStatusViewer
 					jobId={job.id}

lsp/Pipfile

@@ -4,8 +4,8 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-wmill = ">=1.398.1"
-wmill_pg = ">=1.398.1"
+wmill = ">=1.399.0"
+wmill_pg = ">=1.399.0"
 sendgrid = "*"
 mysql-connector-python = "*"
 pymongo = "*"

openflow.openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.398.1
+  version: 1.399.0
   title: OpenFlow Spec
   contact:
     name: Ruben Fiszel

powershell-client/WindmillClient/WindmillClient.psd1

@@ -12,7 +12,7 @@
 RootModule = 'WindmillClient.psm1'
 
 # Version number of this module.
-ModuleVersion = '1.398.1'
+ModuleVersion = '1.399.0'
 
 # Supported PSEditions
 # CompatiblePSEditions = @()

python-client/wmill/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "wmill"
-version = "1.398.1"
+version = "1.399.0"
 description = "A client library for accessing Windmill server wrapping the Windmill client API"
 license = "Apache-2.0"
 homepage = "https://windmill.dev"

python-client/wmill_pg/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "wmill-pg"
-version = "1.398.1"
+version = "1.399.0"
 description = "An extension client for the wmill client library focused on pg"
 license = "Apache-2.0"
 homepage = "https://windmill.dev"

typescript-client/jsr.json

@@ -1,6 +1,6 @@
 {
   "name": "@windmill/windmill",
-  "version": "1.398.1",
+  "version": "1.399.0",
   "exports": "./src/index.ts",
   "publish": {
     "exclude": ["!src", "./s3Types.ts", "./client.ts"]

typescript-client/package.json

@@ -1,7 +1,7 @@
 {
   "name": "windmill-client",
   "description": "Windmill SDK client for browsers and Node.js",
-  "version": "1.398.1",
+  "version": "1.399.0",
   "author": "Ruben Fiszel",
   "license": "Apache 2.0",
   "devDependencies": {

version.txt

@@ -1 +1 @@
-1.398.1
+1.399.0