feat: add java support (#5458)

* feat: add nu (nushell) support

* add worker tests

* deactivate tables and non-any types below top-level

full support will come in V1
for V0 it's better to keep things minimal and simple

* add syntax highlighting

used python's grammar, since nushell isn't supported by monaco nor svelte-highlights

for V1 nu will get it`s own grammar

* add logo

* partially implement plugin support

* change logo + ability to deploy + nsjail draft

* static variables + get_resource + get_variable

* lsp/dev.nu + initial nu lsp (not working yet)

* make it work with nsjail

* nullguard

* Much more flexible signature parsing and better error-messages

* add init script

* rename nulsp to nu

* install nu to dockerfile

* fix merge

* implement Default for MainArgSignature

* stage NU_CACHE_DIR

* improve dockerfiles

* dev.nu for parser-wasm + flake.nix

* update code for windows

* add nushell to flake

* upload Cargo.lock

* make build.sh work on nixos

* build wasm cli parsers

* add docs to README_DEV.md

* add helper script docker/dev.nu

* improve docker/dev.nu

* fix windows

* commit frontend/package(lock).json

* update cargo.lock

* correctly update cargo.lock

* remove lsp

* update flake.nix to include svelte server and nushell

* Revert base.sql to main

* remove PLUGIN_USE_RE

* make CARGO_PATH private

* add nu to cli

* Change flags to build wasm-nu-parser

* remove flake.nix from parser-wasm

* update wasm-build target

* remove unused import

* add cli support for nu

* update github workflows

* wasm-build 0.17 -> 0.19

* update build script

* update cargo.lock

* Fix typographical error

* start working on java

* do java boilerplate

* implement parser for java

* update Cargo.lock

* update ENV_SETTINGS

* use published nu parser

* update package.lock

* java is S3 + Caching enabled

* install nsjail backup

* commit v0

* fix nsjail

* v0.1

* rewrite parser in tree-sitter

* implement parser from scratch

* polishing

* change init script to match new parser

* fix imports

* fix cli build

* fix cli build

* refactor install phase

* implement .valid.windmill atomic verification

* implement java init functionality

* remove quick-xml

* fix windows not recognizing 'mvn'

* create empty settings.xml if there is no config provided

* clean up

* change default settings.xml

* change classpath format for windows

* docs to helper

* java copy bin cache instead of symlink

* remove comments

* merge

* fix package.json

* fix package.json 2

* minor fixing

* migrate to Coursier

* update misc

* Http(s) Proxy + CA certs

* remove unused .wasm

* make requirements insensitive to spaces

* update handle_child refs

* rework save_cache for directories

* fix s3 bug

* compile .wasm for cli

* remove uuid import

* fix compilation

* use reference

* fix zero-dep failure

* removing unsafe stuff

* remove unneeded imports

* revert: we still need winapi

* remove nix store from nsjail

* do not create cache_nomount

* add java to dnt

* remove duplicated dependency in init script

* fix typos

* fix CI

* use published parser
This commit is contained in:
pyranota
2025-03-27 15:12:55 +01:00
committed by GitHub
parent 1774d4955d
commit 5c32a4ae1b
81 changed files with 3144 additions and 188 deletions
+2 -2
View File
@@ -64,7 +64,7 @@ jobs:
platforms: linux/amd64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust,nu
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
secrets: |
rh_username=${{ secrets.RH_USERNAME }}
rh_password=${{ secrets.RH_PASSWORD }}
@@ -81,7 +81,7 @@ jobs:
platforms: linux/arm64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust,nu
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
secrets: |
rh_username=${{ secrets.RH_USERNAME }}
rh_password=${{ secrets.RH_PASSWORD }}
+1 -2
View File
@@ -51,8 +51,7 @@ jobs:
$env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static"
mkdir frontend/build && cd backend
New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,nats,sqs_trigger,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust,nu
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,nats,sqs_trigger,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
- name: Rename binary with corresponding architecture
run: |
Rename-Item -Path ".\backend\target\release\windmill.exe" -NewName "windmill-ee.exe"
+1 -1
View File
@@ -67,7 +67,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=embedding,parquet,openidconnect,deno_core,license,http_trigger,zip,oauth2,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust,nu
features=embedding,parquet,openidconnect,license,http_trigger,zip,oauth2,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
${{ steps.meta-public.outputs.tags }}
+2 -2
View File
@@ -95,7 +95,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=embedding,parquet,openidconnect,jemalloc,deno_core,license,http_trigger,zip,oauth2,dind,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust,nu
features=embedding,parquet,openidconnect,jemalloc,license,http_trigger,zip,oauth2,dind,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }}
${{ steps.meta-public.outputs.tags }}
@@ -157,7 +157,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,otel,dind,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust,nu
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,otel,dind,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }}
${{ steps.meta-ee-public.outputs.tags }}
+1 -2
View File
@@ -53,8 +53,7 @@ jobs:
$env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static"
mkdir frontend/build && cd backend
New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust,nu
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
- name: Rename binary with corresponding architecture
run: |
Rename-Item -Path ".\backend\target\release\windmill.exe" -NewName "windmill-ee.exe"
+1
View File
@@ -9,3 +9,4 @@ CaddyfileRemoteMalo
.direnv
.vscode
.dev-docker-wrapper*
backend/.minio-data
+27
View File
@@ -12792,6 +12792,16 @@ dependencies = [
"tree-sitter-language",
]
[[package]]
name = "tree-sitter-java"
version = "0.23.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0aa6cbcdc8c679b214e616fd3300da67da0e492e066df01bcf5a5921a71e90d6"
dependencies = [
"cc",
"tree-sitter-language",
]
[[package]]
name = "tree-sitter-language"
version = "0.1.5"
@@ -13878,6 +13888,7 @@ dependencies = [
"serde_json",
"sha2 0.10.8",
"sqlx",
"tar",
"tempfile",
"thiserror 2.0.12",
"tikv-jemalloc-ctl",
@@ -13998,6 +14009,18 @@ dependencies = [
"windmill-parser",
]
[[package]]
name = "windmill-parser-java"
version = "1.478.1"
dependencies = [
"anyhow",
"serde_json",
"tree-sitter",
"tree-sitter-java",
"wasm-bindgen",
"windmill-parser",
]
[[package]]
name = "windmill-parser-nu"
version = "1.478.1"
@@ -14112,6 +14135,7 @@ dependencies = [
"windmill-parser-csharp",
"windmill-parser-go",
"windmill-parser-graphql",
"windmill-parser-java",
"windmill-parser-nu",
"windmill-parser-php",
"windmill-parser-py",
@@ -14236,17 +14260,20 @@ dependencies = [
"tokio-postgres 0.7.13",
"tokio-util",
"tracing",
"url",
"urlencoding",
"uuid",
"winapi",
"windmill-audit",
"windmill-common",
"windmill-git-sync",
"windmill-macros",
"windmill-parser",
"windmill-parser-bash",
"windmill-parser-csharp",
"windmill-parser-go",
"windmill-parser-graphql",
"windmill-parser-java",
"windmill-parser-nu",
"windmill-parser-php",
"windmill-parser-py",
+16 -10
View File
@@ -23,6 +23,7 @@ members = [
"./parsers/windmill-parser-rust",
"./parsers/windmill-parser-csharp",
"./parsers/windmill-parser-nu",
"./parsers/windmill-parser-java",
"./parsers/windmill-parser-bash",
"./parsers/windmill-parser-py",
"./parsers/windmill-parser-py-imports",
@@ -62,31 +63,34 @@ cloud = ["windmill-queue/cloud", "windmill-worker/cloud", "windmill-common/cloud
jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"]
tantivy = ["dep:windmill-indexer", "windmill-api/tantivy", "windmill-indexer/enterprise", "windmill-indexer/parquet", "windmill-common/tantivy", "enterprise", "parquet"]
sqlx = ["windmill-worker/sqlx"]
deno_core = ["windmill-worker/deno_core", "dep:deno_core", "dep:v8"]
kafka = ["windmill-api/kafka"]
nats = ["windmill-api/nats"]
otel = ["windmill-common/otel", "windmill-worker/otel"]
dind = ["windmill-worker/dind"]
php = ["windmill-worker/php"]
rust = ["windmill-worker/rust"]
mysql = ["windmill-worker/mysql"]
oracledb = ["windmill-worker/oracledb"]
mssql = ["windmill-worker/mssql"]
bigquery = ["windmill-worker/bigquery"]
websocket = ["windmill-api/websocket"]
postgres_trigger = ["windmill-api/postgres_trigger"]
mqtt_trigger = ["windmill-api/mqtt_trigger"]
sqs_trigger = ["windmill-api/sqs_trigger"]
python = ["windmill-worker/python"]
smtp = ["windmill-api/smtp", "windmill-common/smtp"]
csharp = ["windmill-worker/csharp"]
nu = ["windmill-worker/nu"]
license = ["windmill-api/license"]
oauth2 = ["windmill-api/oauth2"]
http_trigger = ["windmill-api/http_trigger"]
zip = ["windmill-api/zip"]
static_frontend = ["windmill-api/static_frontend"]
scoped_cache = ["windmill-common/scoped_cache"]
# Languages
python = ["windmill-worker/python"]
deno_core = ["windmill-worker/deno_core", "dep:deno_core", "dep:v8"]
rust = ["windmill-worker/rust"]
mysql = ["windmill-worker/mysql"]
oracledb = ["windmill-worker/oracledb"]
mssql = ["windmill-worker/mssql"]
bigquery = ["windmill-worker/bigquery"]
php = ["windmill-worker/php"]
csharp = ["windmill-worker/csharp"]
nu = ["windmill-worker/nu"]
java = ["windmill-worker/java"]
all_languages = [ "python", "deno_core", "rust", "mysql", "oracledb", "mssql", "bigquery", "csharp", "nu", "php", "java"]
[dependencies]
@@ -158,6 +162,7 @@ windmill-parser-go = { path = "./parsers/windmill-parser-go" }
windmill-parser-rust = { path = "./parsers/windmill-parser-rust" }
windmill-parser-yaml = { path = "./parsers/windmill-parser-yaml" }
windmill-parser-csharp = { path = "./parsers/windmill-parser-csharp" }
windmill-parser-java = { path = "./parsers/windmill-parser-java" }
windmill-parser-nu = { path = "./parsers/windmill-parser-nu" }
windmill-parser-bash = { path = "./parsers/windmill-parser-bash" }
windmill-parser-sql = { path = "./parsers/windmill-parser-sql" }
@@ -363,5 +368,6 @@ yaml-rust = "0.4.5"
tokio-tungstenite = { version = "0.24.0", features = ["native-tls"] }
tree-sitter = { version = "0.23.0", features = [] }
tree-sitter-c-sharp = "0.23.0"
tree-sitter-java = "0.23.0"
oracle = { version = "0.6.3", features = ["chrono"] }
rumqttc = { version = "0.24.0", features = ["use-native-tls"]}
@@ -0,0 +1 @@
-- Add down migration script here
@@ -0,0 +1,3 @@
-- Add up migration script here
ALTER TYPE SCRIPT_LANG ADD VALUE IF NOT EXISTS 'java';
UPDATE config set config = jsonb_set(config, '{worker_tags}', config->'worker_tags' || '["java"]'::jsonb) where name = 'worker__default' and config @> '{"worker_tags": ["deno", "python3", "go", "bash", "powershell", "dependency", "flow", "hub", "other", "bun", "php", "rust", "ansible", "csharp", "nu"]}'::jsonb AND NOT config->'worker_tags' @> '"java"'::jsonb;
@@ -0,0 +1,21 @@
[package]
name = "windmill-parser-java"
version.workspace = true
edition.workspace = true
authors.workspace = true
[lib]
name = "windmill_parser_java"
path = "./src/lib.rs"
[dependencies]
windmill-parser.workspace = true
tree-sitter.workspace = true
tree-sitter-java.workspace = true
anyhow.workspace = true
wasm-bindgen.workspace = true
serde_json.workspace = true
# convert_case.workspace = true
# lazy_static.workspace = true
# regex.workspace = true
@@ -0,0 +1,446 @@
#![cfg_attr(target_arch = "wasm32", feature(c_variadic))]
#[cfg(target_arch = "wasm32")]
pub mod wasm_libc;
use anyhow::anyhow;
use anyhow::bail;
use serde_json::Value;
use tree_sitter::Node;
use windmill_parser::Arg;
use windmill_parser::MainArgSignature;
use windmill_parser::Typ;
#[derive(Debug)]
pub struct JavaMainSigMeta {
pub is_public: bool,
pub returns_void: bool,
pub class_name: Option<String>,
pub main_sig: MainArgSignature,
}
pub fn parse_java_sig_meta(code: &str) -> anyhow::Result<JavaMainSigMeta> {
let mut parser = tree_sitter::Parser::new();
let language = tree_sitter_java::LANGUAGE;
parser
.set_language(&language.into())
.map_err(|e| anyhow!("Error setting Java as language: {e}"))?;
// Parse code
let tree = parser
.parse(code, None)
.ok_or(anyhow!("Failed to parse code"))?;
let root_node = tree.root_node();
// Traverse the AST to find the Main method signature
let main_sig = find_main_signature(root_node, code);
let no_main_func = Some(main_sig.is_none());
let mut is_public = false;
let mut returns_void = false;
let mut class_name = None;
let mut args = vec![];
if let Some((sig, name)) = main_sig {
class_name = name;
for sig_node in sig.children(&mut sig.walk()) {
if sig_node.kind() == "modifier" && sig_node.utf8_text(code.as_bytes())? == "public" {
is_public = true;
}
}
if let Some(return_type) = sig.child_by_field_name("type") {
let return_type = return_type.utf8_text(code.as_bytes())?;
if return_type == "void" {
returns_void = true;
}
}
if let Some(param_list) = sig.child_by_field_name("parameters") {
for p_list_node in param_list.children(&mut param_list.walk()) {
if p_list_node.kind() == "formal_parameter" {
let (otyp, typ, name, default) = parse_java_typ(p_list_node, code)?;
args.push(Arg {
name,
otyp,
typ,
has_default: default.is_some(),
default,
oidx: None,
});
}
}
}
}
let main_sig = MainArgSignature {
star_args: false,
star_kwargs: false,
args,
has_preprocessor: None,
no_main_func,
};
Ok(JavaMainSigMeta { returns_void, class_name, main_sig, is_public })
}
pub fn parse_java_signature(code: &str) -> anyhow::Result<MainArgSignature> {
Ok(parse_java_sig_meta(code)?.main_sig)
}
fn find_typ<'a>(typ_node: Node<'a>, code: &str) -> anyhow::Result<(Typ, Option<Value>)> {
let null = Some(serde_json::Value::Null);
let res = match typ_node.kind() {
#[rustfmt::skip]
"type_identifier" => {
match typ_node.utf8_text(code.as_bytes()) {
Ok("String") => (Typ::Str(None), null),
Ok("Byte") => (Typ::Bytes, null),
Ok("Short") => (Typ::Int, null),
Ok("Integer") => (Typ::Int, null),
Ok("Long") => (Typ::Int, null),
Ok("Float") => (Typ::Float, null),
Ok("Double") => (Typ::Float, null),
Ok("Boolean") => (Typ::Bool, null),
Ok("Character") => (Typ::Str(None), null),
Ok("Object") => (Typ::Object(vec![]),null), // TODO: Complete the object type
Ok(s) => bail!("Unknown type `{s}`"),
Err(e) => bail!("Error getting type name: {}", e),
}
}
#[rustfmt::skip]
"integral_type" => {
match typ_node.utf8_text(code.as_bytes()) {
Ok("byte") => (Typ::Bytes, None),
Ok("short") => (Typ::Int, None),
Ok("int") => (Typ::Int, None),
Ok("long") => (Typ::Int, None),
Ok("char") => (Typ::Str(None), None),
Ok(s) => bail!("Unknown type `{s}`"),
Err(e) => bail!("Error getting type name: {}", e),
}
}
"floating_point_type" => {
match typ_node.utf8_text(code.as_bytes()) {
Ok("float") => (Typ::Float, None),
Ok("double") => (Typ::Float, None),
Ok(s) => bail!("Unknown type `{s}`"),
Err(e) => bail!("Error getting type name: {}", e),
}
}
"boolean_type" => {
match typ_node.utf8_text(code.as_bytes()) {
Ok("boolean") => (Typ::Bool, None),
Ok(s) => bail!("Unknown type `{s}`"),
Err(e) => bail!("Error getting type name: {}", e),
}
}
"array_type" => {
let new_typ_node = typ_node
.named_child(0)
.ok_or(anyhow!("Failed to find inner type of array type"))?;
(Typ::List(Box::new(find_typ(new_typ_node, code)?.0)), null)
}
wc => bail!(
"Unexpected Java type node kind: {} for '{}'. This type is not handled by Windmill, please open an issue if this seems to be an error",
wc,
typ_node.utf8_text(code.as_bytes())?
),
};
Ok(res)
}
fn parse_java_typ<'a>(
param_node: Node<'a>,
code: &str,
) -> anyhow::Result<(Option<String>, Typ, String, Option<Value>)> {
let name = param_node
.child_by_field_name("name")
.and_then(|n| n.utf8_text(code.as_bytes()).ok())
.unwrap_or("");
let otyp_node = param_node.child_by_field_name("type");
let otyp = otyp_node
.and_then(|n| n.utf8_text(code.as_bytes()).ok())
.map(|s| s.to_string());
let (typ, default) = find_typ(
otyp_node.ok_or(anyhow!(
"Internal error: Failed to get child by field name 'type'"
))?,
code,
)?;
Ok((otyp, typ, name.to_string(), default))
}
// Function to find the Main method's signature
fn find_main_signature<'a>(root_node: Node<'a>, code: &str) -> Option<(Node<'a>, Option<String>)> {
let mut cursor = root_node.walk();
for x in root_node.children(&mut cursor) {
if x.kind() == "class_declaration" {
let class_name = x
.child_by_field_name("name")
.and_then(|n| n.utf8_text(code.as_bytes()).ok().map(|s| s.to_string()));
for c in x.children(&mut x.walk()) {
if c.kind() == "class_body" {
for w in c.children(&mut c.walk()) {
if w.kind() == "method_declaration" {
for child in w.children(&mut w.walk()) {
if child
.utf8_text(code.as_bytes())
.map(|name| name == "main")
.unwrap_or(false)
{
return Some((w, class_name));
}
}
}
}
}
}
}
}
return None;
}
#[cfg(test)]
mod test {
use serde_json::json;
use super::*;
#[test]
fn test_parse_java_return_void() {
let code = r#"
class Main {
public static void main() {}
}"#;
let sig_meta = parse_java_sig_meta(code).unwrap();
assert_eq!(sig_meta.class_name, Some("Main".to_string()));
assert!(sig_meta.returns_void);
}
#[test]
fn test_parse_java_return_object() {
let code = r#"
class Main {
public static Object main() {}
}"#;
let sig_meta = parse_java_sig_meta(code).unwrap();
assert_eq!(sig_meta.class_name, Some("Main".to_string()));
assert!(!sig_meta.returns_void);
}
#[test]
fn test_parse_java_primitive_types() {
let code = r#"
class Main {
public static string main(byte a, short b, int c, long d, float e, double f, boolean g, char h) {}
}"#;
let sig_meta = parse_java_sig_meta(code).unwrap();
assert_eq!(sig_meta.class_name, Some("Main".to_string()));
let ret = sig_meta.main_sig;
assert_eq!(
ret.args,
vec![
Arg {
name: "a".into(),
otyp: Some("byte".into()),
typ: Typ::Bytes,
default: None,
has_default: false,
oidx: None
},
Arg {
name: "b".into(),
otyp: Some("short".into()),
typ: Typ::Int,
default: None,
has_default: false,
oidx: None
},
Arg {
name: "c".into(),
otyp: Some("int".into()),
typ: Typ::Int,
default: None,
has_default: false,
oidx: None
},
Arg {
name: "d".into(),
otyp: Some("long".into()),
typ: Typ::Int,
default: None,
has_default: false,
oidx: None
},
Arg {
name: "e".into(),
otyp: Some("float".into()),
typ: Typ::Float,
default: None,
has_default: false,
oidx: None
},
Arg {
name: "f".into(),
otyp: Some("double".into()),
typ: Typ::Float,
default: None,
has_default: false,
oidx: None
},
Arg {
name: "g".into(),
otyp: Some("boolean".into()),
typ: Typ::Bool,
default: None,
has_default: false,
oidx: None
},
Arg {
name: "h".into(),
otyp: Some("char".into()),
typ: Typ::Str(None),
default: None,
has_default: false,
oidx: None
},
]
);
}
#[test]
fn test_parse_java_objects() {
let code = r#"
class Main {
public static string main(Byte a, Short b, Integer c, Long d, Float e, Double f, Boolean g, Character h, Object i) {}
}"#;
let sig_meta = parse_java_sig_meta(code).unwrap();
assert_eq!(sig_meta.class_name, Some("Main".to_string()));
let ret = sig_meta.main_sig;
assert_eq!(
ret.args,
vec![
Arg {
name: "a".into(),
otyp: Some("Byte".into()),
typ: Typ::Bytes,
default: Some(json!(null)),
has_default: true,
oidx: None
},
Arg {
name: "b".into(),
otyp: Some("Short".into()),
typ: Typ::Int,
default: Some(json!(null)),
has_default: true,
oidx: None
},
Arg {
name: "c".into(),
otyp: Some("Integer".into()),
typ: Typ::Int,
default: Some(json!(null)),
has_default: true,
oidx: None
},
Arg {
name: "d".into(),
otyp: Some("Long".into()),
typ: Typ::Int,
default: Some(json!(null)),
has_default: true,
oidx: None
},
Arg {
name: "e".into(),
otyp: Some("Float".into()),
typ: Typ::Float,
default: Some(json!(null)),
has_default: true,
oidx: None
},
Arg {
name: "f".into(),
otyp: Some("Double".into()),
typ: Typ::Float,
default: Some(json!(null)),
has_default: true,
oidx: None
},
Arg {
name: "g".into(),
otyp: Some("Boolean".into()),
typ: Typ::Bool,
default: Some(json!(null)),
has_default: true,
oidx: None
},
Arg {
name: "h".into(),
otyp: Some("Character".into()),
typ: Typ::Str(None),
default: Some(json!(null)),
has_default: true,
oidx: None
},
Arg {
name: "i".into(),
otyp: Some("Object".into()),
typ: Typ::Object(vec![]),
default: Some(json!(null)),
has_default: true,
oidx: None
},
]
);
}
#[test]
fn test_parse_java_array() {
let code = r#"
class Main {
public static string main(int[] a, Object[] b, String[] c) {}
}"#;
let sig_meta = parse_java_sig_meta(code).unwrap();
assert_eq!(sig_meta.class_name, Some("Main".to_string()));
let ret = sig_meta.main_sig;
assert_eq!(
ret.args,
vec![
Arg {
name: "a".into(),
otyp: Some("int[]".into()),
typ: Typ::List(Box::new(Typ::Int)),
default: Some(json!(null)),
has_default: true,
oidx: None
},
Arg {
name: "b".into(),
otyp: Some("Object[]".into()),
typ: Typ::List(Box::new(Typ::Object(vec![]))),
default: Some(json!(null)),
has_default: true,
oidx: None
},
Arg {
name: "c".into(),
otyp: Some("String[]".into()),
typ: Typ::List(Box::new(Typ::Str(None))),
default: Some(json!(null)),
has_default: true,
oidx: None
},
]
);
}
}
@@ -0,0 +1,207 @@
use std::collections::BTreeMap;
use std::sync::{Mutex, OnceLock};
use std::{
alloc::{self, Layout},
ffi::{c_char, c_int, c_void},
mem::align_of,
ptr,
};
use wasm_bindgen::prelude::*;
/* -------------------------------- stdlib.h -------------------------------- */
#[no_mangle]
pub unsafe extern "C" fn abort() {
panic!("Aborted from C");
}
macro_rules! console_log {
($($t:tt)*) => (unsafe { log(&format_args!($($t)*).to_string()) })
}
#[wasm_bindgen]
extern "C" {
#[wasm_bindgen(js_namespace = console)]
fn log(a: &str);
}
#[no_mangle]
pub unsafe extern "C" fn malloc(size: usize) -> *mut c_void {
if size == 0 {
return ptr::null_mut();
}
let (layout, offset_to_data) = layout_for_size_prepended(size);
let buf = alloc::alloc(layout);
store_layout(buf, layout, offset_to_data)
}
#[no_mangle]
pub unsafe extern "C" fn calloc(count: usize, size: usize) -> *mut c_void {
if count == 0 || size == 0 {
return ptr::null_mut();
}
let (layout, offset_to_data) = layout_for_size_prepended(size * count);
let buf = alloc::alloc_zeroed(layout);
store_layout(buf, layout, offset_to_data)
}
#[no_mangle]
pub unsafe extern "C" fn realloc(buf: *mut c_void, new_size: usize) -> *mut c_void {
if buf.is_null() {
malloc(new_size)
} else if new_size == 0 {
free(buf);
ptr::null_mut()
} else {
let (old_buf, old_layout) = retrieve_layout(buf);
let (new_layout, offset_to_data) = layout_for_size_prepended(new_size);
let new_buf = alloc::realloc(old_buf, old_layout, new_layout.size());
store_layout(new_buf, new_layout, offset_to_data)
}
}
#[no_mangle]
pub unsafe extern "C" fn free(buf: *mut c_void) {
if buf.is_null() {
return;
}
let (buf, layout) = retrieve_layout(buf);
alloc::dealloc(buf, layout);
}
// In all these allocations, we store the layout before the data for later retrieval.
// This is because we need to know the layout when deallocating the memory.
// Here are some helper methods for that:
/// Given a pointer to the data, retrieve the layout and the pointer to the layout.
unsafe fn retrieve_layout(buf: *mut c_void) -> (*mut u8, Layout) {
let (_, layout_offset) = Layout::new::<Layout>()
.extend(Layout::from_size_align(0, align_of::<*const u8>() * 2).unwrap())
.unwrap();
let buf = (buf as *mut u8).offset(-(layout_offset as isize));
let layout = *(buf as *mut Layout);
(buf, layout)
}
/// Calculate a layout for a given size with space for storing a layout at the start.
/// Returns the layout and the offset to the data.
fn layout_for_size_prepended(size: usize) -> (Layout, usize) {
Layout::new::<Layout>()
.extend(Layout::from_size_align(size, align_of::<*const u8>() * 2).unwrap())
.unwrap()
}
/// Store a layout in the pointer, returning a pointer to where the data should be stored.
unsafe fn store_layout(buf: *mut u8, layout: Layout, offset_to_data: usize) -> *mut c_void {
*(buf as *mut Layout) = layout;
(buf as *mut u8).offset(offset_to_data as isize) as *mut c_void
}
/* -------------------------------- string.h -------------------------------- */
#[no_mangle]
pub unsafe extern "C" fn strncmp(ptr1: *const c_void, ptr2: *const c_void, n: usize) -> c_int {
let s1 = std::slice::from_raw_parts(ptr1 as *const u8, n);
let s2 = std::slice::from_raw_parts(ptr2 as *const u8, n);
for (a, b) in s1.iter().zip(s2.iter()) {
if *a != *b || *a == 0 {
return (*a as i32) - (*b as i32);
}
}
0
}
/* -------------------------------- wctype.h -------------------------------- */
#[no_mangle]
pub unsafe extern "C" fn iswspace(c: c_int) -> bool {
char::from_u32(c as u32).map_or(false, |c| c.is_whitespace())
}
#[no_mangle]
pub unsafe extern "C" fn iswalnum(c: c_int) -> bool {
char::from_u32(c as u32).map_or(false, |c| c.is_alphanumeric())
}
/* --------------------------------- time.h --------------------------------- */
#[no_mangle]
pub unsafe extern "C" fn clock() -> u64 {
panic!("clock is not supported");
}
/* --------------------------------- ctype.h -------------------------------- */
#[no_mangle]
pub unsafe extern "C" fn isprint(c: c_int) -> bool {
c >= 32 && c <= 126
}
/* --------------------------------- stdio.h -------------------------------- */
#[no_mangle]
pub unsafe extern "C" fn fprintf(_file: *mut c_void, _format: *const c_void, _args: ...) -> c_int {
panic!("fprintf is not supported");
}
#[no_mangle]
pub unsafe extern "C" fn fputs(_s: *const c_void, _file: *mut c_void) -> c_int {
panic!("fputs is not supported");
}
#[no_mangle]
pub unsafe extern "C" fn fputc(_c: c_int, _file: *mut c_void) -> c_int {
panic!("fputc is not supported");
}
#[no_mangle]
pub unsafe extern "C" fn fdopen(_fd: c_int, _mode: *const c_void) -> *mut c_void {
panic!("fdopen is not supported");
}
#[no_mangle]
pub unsafe extern "C" fn fclose(_file: *mut c_void) -> c_int {
panic!("fclose is not supported");
}
#[no_mangle]
pub unsafe extern "C" fn fwrite(
_ptr: *const c_void,
_size: usize,
_nmemb: usize,
_stream: *mut c_void,
) -> usize {
panic!("fwrite is not supported");
}
#[no_mangle]
pub unsafe extern "C" fn vsnprintf(
_buf: *mut c_char,
_size: usize,
_format: *const c_char,
_args: ...
) -> c_int {
panic!("vsnprintf is not supported");
}
#[no_mangle]
pub extern "C" fn clock_gettime(ptr: usize, new_size: usize) {
panic!("clock_gettime is not supported");
}
// int snprintf( char* restrict buffer, size_t bufsz, const char* restrict format, ... );
#[no_mangle]
pub extern "C" fn snprintf() {
panic!("snprintf is not supported");
}
#[no_mangle]
pub extern "C" fn __assert_fail(_: *const i32, _: *const i32, _: *const i32, _: *const i32) {
panic!("oh no");
}
@@ -28,6 +28,7 @@ graphql-parser = [ "dep:windmill-parser-graphql"]
ansible-parser = [ "dep:windmill-parser-yaml"]
csharp-parser = [ "dep:windmill-parser-csharp"]
nu-parser = [ "dep:windmill-parser-nu"]
java-parser = [ "dep:windmill-parser-java"]
[dependencies]
anyhow.workspace = true
@@ -43,6 +44,7 @@ windmill-parser-rust = { workspace = true, optional = true }
windmill-parser-yaml = { workspace = true, optional = true }
windmill-parser-csharp = { workspace = true, optional = true }
windmill-parser-nu = { workspace = true, optional = true }
windmill-parser-java = { workspace = true, optional = true }
wasm-bindgen.workspace = true
serde_json.workspace = true
getrandom = { workspace = true, features = ["js"] }
@@ -70,6 +70,10 @@ OUT_DIR="../../../cli/wasm/nu"
mkdir -p $OUT_DIR
wasm-pack build --release --target deno --out-dir $OUT_DIR --features "nu-parser" \
-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
# sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-nu"/' $OUT_DIR/package.json
rm $OUT_DIR/.gitignore
#-# Java
OUT_DIR="../../../cli/wasm/java"
mkdir -p $OUT_DIR
CFLAGS_wasm32_unknown_unknown="-I$(pwd)/wasm-sysroot -Wbad-function-cast -Wcast-function-type -fno-builtin" RUSTFLAGS="-Zwasm-c-abi=spec" wasm-pack build --release --target deno --out-dir $OUT_DIR --features "java-parser"
rm $OUT_DIR/.gitignore
@@ -61,3 +61,8 @@ OUT_DIR="pkg-nu"
wasm-pack build --release --target web --out-dir $OUT_DIR --features "nu-parser" \
-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-nu"/' $OUT_DIR/package.json
#-# Java (needs some more stuff to compile C tree sitter into wasm)
OUT_DIR="pkg-java"
CFLAGS_wasm32_unknown_unknown="-I$(pwd)/wasm-sysroot -Wbad-function-cast -Wcast-function-type -fno-builtin" RUSTFLAGS="-Zwasm-c-abi=spec" wasm-pack build --release --target web --out-dir $OUT_DIR --features "java-parser"
sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-java"/' $OUT_DIR/package.json
@@ -60,3 +60,7 @@ wasm-pack build --release --target web --out-dir $OUT_DIR --features "nu-parser"
-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-nu"/' $OUT_DIR/package.json
#-# Java (needs some more stuff to compile C tree sitter into wasm)
OUT_DIR="pkg-java"
CFLAGS_wasm32_unknown_unknown="-I$(pwd)/wasm-sysroot -Wbad-function-cast -Wcast-function-type -fno-builtin" RUSTFLAGS="-Zwasm-c-abi=spec" wasm-pack build --release --target web --out-dir $OUT_DIR --features "java-parser"
sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-java"/' $OUT_DIR/package.json
@@ -1,8 +0,0 @@
#!/usr/bin/env bash
set -eou pipefail
# Flags are needed to build correctly with C#
# NOTE: Currently does not work with nix nor nixos
CFLAGS_wasm32_unknown_unknown="-I$(pwd)/wasm-sysroot -Wbad-function-cast -Wcast-function-type -fno-builtin" RUSTFLAGS="-Zwasm-c-abi=spec" deno task wasmbuild --inline --out ../../../cli/wasm/ -p windmill-parser-wasm --all-features
@@ -155,3 +155,11 @@ pub fn parse_csharp(code: &str) -> String {
pub fn parse_nu(code: &str) -> String {
wrap_sig(windmill_parser_nu::parse_nu_signature(code))
}
#[cfg(feature = "java-parser")]
#[wasm_bindgen]
pub fn parse_java(code: &str) -> String {
wrap_sig(windmill_parser_java::parse_java_signature(code))
}
// KJQXZ
+19 -9
View File
@@ -9,7 +9,8 @@
use anyhow::Context;
use monitor::{
load_base_url, load_otel, reload_delete_logs_periodically_setting, reload_indexer_config,
reload_instance_python_version_setting, reload_nuget_config_setting,
reload_instance_python_version_setting, reload_maven_repos_setting,
reload_no_default_maven_setting, reload_nuget_config_setting,
reload_timeout_wait_result_setting, send_current_log_file_to_object_store,
send_logs_to_object_store,
};
@@ -37,11 +38,12 @@ use windmill_common::{
EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
HUB_BASE_URL_SETTING, INDEXER_SETTING, INSTANCE_PYTHON_VERSION_SETTING,
JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING,
LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING,
NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING,
REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING,
RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING,
TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING,
NO_DEFAULT_MAVEN_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING,
OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TEAMS_SETTING,
TIMEOUT_WAIT_RESULT_SETTING,
},
scripts::ScriptLang,
stats_ee::schedule_stats,
@@ -66,9 +68,9 @@ use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING;
use windmill_worker::{
get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, CSHARP_CACHE_DIR,
DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR,
NU_CACHE_DIR, POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, PY312_CACHE_DIR,
PY313_CACHE_DIR, RUST_CACHE_DIR, TAR_PY310_CACHE_DIR, TAR_PY311_CACHE_DIR, TAR_PY312_CACHE_DIR,
TAR_PY313_CACHE_DIR, UV_CACHE_DIR,
JAVA_CACHE_DIR, NU_CACHE_DIR, POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR,
PY312_CACHE_DIR, PY313_CACHE_DIR, RUST_CACHE_DIR, TAR_JAVA_CACHE_DIR, TAR_PY310_CACHE_DIR,
TAR_PY311_CACHE_DIR, TAR_PY312_CACHE_DIR, TAR_PY313_CACHE_DIR, UV_CACHE_DIR,
};
use crate::monitor::{
@@ -796,6 +798,12 @@ Windmill Community Edition {GIT_VERSION}
NUGET_CONFIG_SETTING => {
reload_nuget_config_setting(&db).await
},
MAVEN_REPOS_SETTING => {
reload_maven_repos_setting(&db).await
},
NO_DEFAULT_MAVEN_SETTING => {
reload_no_default_maven_setting(&db).await
},
KEEP_JOB_DIR_SETTING => {
load_keep_job_dir(&db).await;
},
@@ -1103,6 +1111,8 @@ pub async fn run_workers(
NU_CACHE_DIR,
HUB_CACHE_DIR,
POWERSHELL_CACHE_DIR,
JAVA_CACHE_DIR,
TAR_JAVA_CACHE_DIR, // KJQXZ
] {
DirBuilder::new()
.recursive(true)
+17 -3
View File
@@ -52,9 +52,7 @@ use windmill_common::{
};
use windmill_queue::{cancel_job, MiniPulledJob};
use windmill_worker::{
create_token_for_owner, handle_job_error, AuthedClient, SameWorkerPayload, SameWorkerSender,
SendResult, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR,
NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, SCRIPT_TOKEN_EXPIRY,
create_token_for_owner, handle_job_error, AuthedClient, SameWorkerPayload, SameWorkerSender, SendResult, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, SCRIPT_TOKEN_EXPIRY
};
#[cfg(feature = "parquet")]
@@ -186,6 +184,8 @@ pub async fn initial_load(
reload_bunfig_install_scopes_setting(&db).await;
reload_instance_python_version_setting(&db).await;
reload_nuget_config_setting(&db).await;
reload_maven_repos_setting(&db).await;
reload_no_default_maven_setting(&db).await;
}
}
@@ -969,6 +969,20 @@ pub async fn reload_nuget_config_setting(db: &DB) {
)
.await;
}
pub async fn reload_maven_repos_setting(db: &DB) {
reload_option_setting_with_tracing(db, windmill_common::global_settings::MAVEN_REPOS_SETTING, "MAVEN_REPOS", MAVEN_REPOS.clone())
.await;
}
pub async fn reload_no_default_maven_setting(db: &DB) {
let value = load_value_from_global_settings(db, windmill_common::global_settings::NO_DEFAULT_MAVEN_SETTING).await;
match value {
Ok(Some(serde_json::Value::Bool(t))) => NO_DEFAULT_MAVEN.store(t, Ordering::Relaxed),
Err(e) => {
tracing::error!("Error loading no default maven repository: {e:#}");
}
_ => (),
};
}
pub async fn reload_retention_period_setting(db: &DB) {
if let Err(e) = reload_setting(
+45
View File
@@ -1988,6 +1988,51 @@ def main [
assert_eq!(result, serde_json::json!(0));
}
#[cfg(feature = "java")]
#[sqlx::test(fixtures("base"))]
async fn test_java_job(db: Pool<Postgres>) {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await;
let port = server.addr.port();
let content = r#"
public class Main {
public static Object main(
// Primitive
int a,
float b,
// Objects
Integer age,
Float d
){
return "hello world";
}
}
"#
.to_owned();
let job = RunJob::from(JobPayload::Code(RawCode {
hash: None,
content,
path: None,
lock: None,
language: ScriptLang::Java,
custom_concurrency_key: None,
concurrent_limit: None,
concurrency_time_window_s: None,
cache_ttl: None,
dedicated_worker: None,
}))
.arg("a", json!(3))
.arg("b", json!(3.0))
.arg("age", json!(30))
.arg("d", json!(3.0))
.run_until_complete(&db, port)
.await;
assert_eq!(job.json_result(), Some(json!("hello world")));
}
#[sqlx::test(fixtures("base"))]
async fn test_python_job(db: Pool<Postgres>) {
initialize_tracing().await;
+3 -1
View File
@@ -13529,7 +13529,9 @@ components:
rust,
ansible,
csharp,
nu
nu,
java
# KJQXZ
]
Preview:
+14 -11
View File
@@ -624,17 +624,20 @@ async fn create_script_internal<'c>(
.unwrap_or(json!({}));
let lock = if ns.codebase.is_some() {
Some(String::new())
} else if !(ns.language == ScriptLang::Python3
|| ns.language == ScriptLang::Go
|| ns.language == ScriptLang::Bun
|| ns.language == ScriptLang::Bunnative
|| ns.language == ScriptLang::Deno
|| ns.language == ScriptLang::Rust
|| ns.language == ScriptLang::Ansible
|| ns.language == ScriptLang::CSharp
|| ns.language == ScriptLang::Nu
|| ns.language == ScriptLang::Php)
{
} else if !(
ns.language == ScriptLang::Python3
|| ns.language == ScriptLang::Go
|| ns.language == ScriptLang::Bun
|| ns.language == ScriptLang::Bunnative
|| ns.language == ScriptLang::Deno
|| ns.language == ScriptLang::Rust
|| ns.language == ScriptLang::Ansible
|| ns.language == ScriptLang::CSharp
|| ns.language == ScriptLang::Nu
|| ns.language == ScriptLang::Php
|| ns.language == ScriptLang::Java
// KJQXZ
) {
Some(String::new())
} else {
ns.lock
@@ -375,6 +375,8 @@ pub(crate) async fn tarball_workspace(
ScriptLang::CSharp => "cs",
ScriptLang::Nu => "nu",
ScriptLang::OracleDB => "odb.sql",
ScriptLang::Java => "java",
// KJQXZ
};
archive
.write_to_archive(&script.content, &format!("{}.{}", script.path, ext))
+1
View File
@@ -24,6 +24,7 @@ name = "windmill_common"
path = "src/lib.rs"
[dependencies]
tar.workspace = true
hmac.workspace = true
sha2.workspace = true
thiserror.workspace = true
@@ -11,6 +11,8 @@ pub const LICENSE_KEY_SETTING: &str = "license_key";
pub const NPM_CONFIG_REGISTRY_SETTING: &str = "npm_config_registry";
pub const BUNFIG_INSTALL_SCOPES_SETTING: &str = "bunfig_install_scopes";
pub const NUGET_CONFIG_SETTING: &str = "nuget_config";
pub const MAVEN_REPOS_SETTING: &str = "maven_repos";
pub const NO_DEFAULT_MAVEN_SETTING: &str = "no_default_maven";
pub const EXTRA_PIP_INDEX_URL_SETTING: &str = "pip_extra_index_url";
pub const PIP_INDEX_URL_SETTING: &str = "pip_index_url";
@@ -61,6 +63,8 @@ pub const ENV_SETTINGS: &[&str] = &[
"NU_PATH",
"DENO_PATH",
"GO_PATH",
"JAVA_PATH",
// KJQXZ
"GOPRIVATE",
"GOPROXY",
"NETRC",
+9 -21
View File
@@ -345,27 +345,15 @@ fn find_annotation(comm_lit: &str, annotation: &str, code: &str) -> bool {
pub fn should_validate_schema(code: &str, lang: &ScriptLang) -> bool {
let annotation = "schema_validation";
match lang {
ScriptLang::Nativets | ScriptLang::Bun | ScriptLang::Bunnative | ScriptLang::Deno => {
find_annotation("//", annotation, code)
}
ScriptLang::Python3 => find_annotation("#", annotation, code),
ScriptLang::Go => find_annotation("#", annotation, code),
ScriptLang::Bash => find_annotation("#", annotation, code),
ScriptLang::Powershell => find_annotation("#", annotation, code),
ScriptLang::Postgresql => find_annotation("--", annotation, code),
ScriptLang::Mysql => find_annotation("--", annotation, code),
ScriptLang::Bigquery => find_annotation("--", annotation, code),
ScriptLang::Snowflake => find_annotation("--", annotation, code),
ScriptLang::Graphql => find_annotation("#", annotation, code),
ScriptLang::Mssql => find_annotation("--", annotation, code),
ScriptLang::OracleDB => find_annotation("--", annotation, code),
ScriptLang::Php => find_annotation("//", annotation, code),
ScriptLang::Rust => find_annotation("//!", annotation, code),
ScriptLang::Ansible => find_annotation("#", annotation, code),
ScriptLang::CSharp => find_annotation("//", annotation, code),
ScriptLang::Nu => find_annotation("#", annotation, code),
}
use ScriptLang::*;
let comment = match lang {
Nativets | Bun | Bunnative | Deno | Php | CSharp | Java => "//",
Python3 | Go | Bash | Powershell | Graphql | Ansible | Nu => "#",
Postgresql | Mysql | Bigquery | Snowflake | Mssql | OracleDB => "--",
Rust => "//!",
// KJQXZ
};
find_annotation(comment, annotation, code)
}
#[derive(Serialize, Deserialize, Debug, Clone)]
+4
View File
@@ -51,6 +51,8 @@ pub enum ScriptLang {
Ansible,
CSharp,
Nu,
Java,
// KJQXZ
}
impl ScriptLang {
@@ -76,6 +78,8 @@ impl ScriptLang {
ScriptLang::Ansible => "ansible",
ScriptLang::CSharp => "csharp",
ScriptLang::Nu => "nu",
ScriptLang::Java => "java",
// KJQXZ
}
}
}
+113 -10
View File
@@ -9,7 +9,7 @@ use serde_json::value::RawValue;
use std::{
cmp::Reverse,
collections::{HashMap, HashSet},
fs::File,
fs::{self, File},
io::Write,
path::{Component, Path, PathBuf},
str::FromStr,
@@ -51,6 +51,8 @@ lazy_static::lazy_static! {
"ansible".to_string(),
"csharp".to_string(),
"nu".to_string(),
"java".to_string(),
// KJQXZ
"dependency".to_string(),
"flow".to_string(),
"other".to_string()
@@ -402,8 +404,43 @@ pub struct SqlAnnotations {
pub struct BashAnnotations {
pub docker: bool,
}
/// length = 5
/// value = "foo"
/// output = "foo "
/// 12345
pub fn pad_string(value: &str, total_length: usize) -> String {
if value.len() >= total_length {
value.to_string() // Return the original string if it's already long enough
} else {
let padding_needed = total_length - value.len();
format!("{value}{}", " ".repeat(padding_needed)) // Pad with spaces
}
}
pub fn copy_dir_recursively(src: &Path, dst: &Path) -> error::Result<()> {
if !dst.exists() {
fs::create_dir_all(dst)?;
}
pub async fn load_cache(bin_path: &str, _remote_path: &str) -> (bool, String) {
tracing::debug!("Copying recursively from {:?} to {:?}", src, dst);
for entry in fs::read_dir(src)? {
let entry = entry?;
let src_path = entry.path();
let dst_path = dst.join(entry.file_name());
if src_path.is_dir() && !src_path.is_symlink() {
copy_dir_recursively(&src_path, &dst_path)?;
} else {
fs::copy(&src_path, &dst_path)?;
}
}
tracing::debug!("Finished copying recursively from {:?} to {:?}", src, dst);
Ok(())
}
pub async fn load_cache(bin_path: &str, _remote_path: &str, is_dir: bool) -> (bool, String) {
if tokio::fs::metadata(&bin_path).await.is_ok() {
(true, format!("loaded from local cache: {}\n", bin_path))
} else {
@@ -417,12 +454,22 @@ pub async fn load_cache(bin_path: &str, _remote_path: &str) -> (bool, String) {
use crate::s3_helpers::attempt_fetch_bytes;
if let Ok(mut x) = attempt_fetch_bytes(os, _remote_path).await {
if let Err(e) = write_binary_file(bin_path, &mut x) {
tracing::error!("could not write bundle/bin file locally: {e:?}");
return (
false,
"error writing bundle/bin file from object store".to_string(),
);
if is_dir {
if let Err(e) = extract_tar(x, bin_path).await {
tracing::error!("could not write tar archive locally: {e:?}");
return (
false,
"error writing tar archive from object store".to_string(),
);
}
} else {
if let Err(e) = write_binary_file(bin_path, &mut x) {
tracing::error!("could not write bundle/bin file locally: {e:?}");
return (
false,
"error writing bundle/bin file from object store".to_string(),
);
}
}
tracing::info!("loaded from object store {}", bin_path);
return (
@@ -435,6 +482,7 @@ pub async fn load_cache(bin_path: &str, _remote_path: &str) -> (bool, String) {
);
}
}
let _ = is_dir;
(false, "".to_string())
}
}
@@ -462,6 +510,7 @@ pub async fn save_cache(
local_cache_path: &str,
_remote_cache_path: &str,
origin: &str,
is_dir: bool,
) -> crate::error::Result<String> {
let mut _cached_to_s3 = false;
#[cfg(all(feature = "enterprise", feature = "parquet"))]
@@ -471,11 +520,33 @@ pub async fn save_cache(
.clone()
{
use object_store::path::Path;
let file_to_cache = if is_dir {
let tar_path = format!(
"{ROOT_CACHE_DIR}/tar/{}_tar.tar",
local_cache_path
.split("/")
.last()
.unwrap_or(&uuid::Uuid::new_v4().to_string())
);
let tar_file = std::fs::File::create(&tar_path)?;
let mut tar = tar::Builder::new(tar_file);
tar.append_dir_all(".", &origin)?;
let tar_metadata = tokio::fs::metadata(&tar_path).await;
if tar_metadata.is_err() || tar_metadata.as_ref().unwrap().len() == 0 {
tracing::info!("Failed to tar cache: {origin}");
return Err(error::Error::ExecutionErr(format!(
"Failed to tar cache: {origin}"
)));
}
tar_path
} else {
origin.to_owned()
};
if let Err(e) = os
.put(
&Path::from(_remote_cache_path),
std::fs::read(origin)?.into(),
std::fs::read(&file_to_cache)?.into(),
)
.await
{
@@ -485,12 +556,19 @@ pub async fn save_cache(
);
} else {
_cached_to_s3 = true;
if is_dir {
tokio::fs::remove_dir_all(&file_to_cache).await?;
}
}
}
// if !*CLOUD_HOSTED {
if true {
std::fs::copy(origin, local_cache_path)?;
if is_dir {
copy_dir_recursively(&PathBuf::from(origin), &PathBuf::from(local_cache_path))?;
} else {
std::fs::copy(origin, local_cache_path)?;
}
Ok(format!(
"\nwrote cached binary: {} (backed by EE distributed object store: {_cached_to_s3})\n",
local_cache_path
@@ -505,6 +583,31 @@ pub async fn save_cache(
}
}
#[cfg(all(feature = "enterprise", feature = "parquet"))]
pub async fn extract_tar(tar: bytes::Bytes, folder: &str) -> error::Result<()> {
use std::time::Instant;
use bytes::Buf;
use tokio::fs::{self};
let start: Instant = Instant::now();
fs::create_dir_all(&folder).await?;
let mut ar = tar::Archive::new(tar.reader());
if let Err(e) = ar.unpack(folder) {
tracing::info!("Failed to untar to {folder}. Error: {:?}", e);
fs::remove_dir_all(&folder).await?;
return Err(error::Error::ExecutionErr(format!(
"Failed to untar tar {folder}"
)));
}
tracing::info!(
"Finished extracting tar to {folder}. Took {}ms",
start.elapsed().as_millis(),
);
Ok(())
}
#[cfg(all(feature = "enterprise", feature = "parquet"))]
fn write_binary_file(main_path: &str, byts: &mut bytes::Bytes) -> error::Result<()> {
use std::fs::{File, Permissions};
+5 -1
View File
@@ -30,17 +30,20 @@ python = ["dep:windmill-parser-py", "dep:windmill-parser-py-imports"]
csharp = ["dep:windmill-parser-csharp"]
rust = ["dep:windmill-parser-rust"]
nu = ["dep:windmill-parser-nu"]
java = ["dep:windmill-parser-java"]
[dependencies]
windmill-queue.workspace = true
windmill-audit.workspace = true # there isn't really a reason for audit-worth actions to happen in the worker.
windmill-common = { workspace = true, default-features = false }
windmill-macros.workspace = true
windmill-parser.workspace = true
windmill-parser-ts.workspace = true
windmill-parser-go.workspace = true
windmill-parser-rust = { workspace = true, optional = true }
windmill-parser-csharp = { workspace = true, optional = true }
windmill-parser-nu = { workspace = true, optional = true }
windmill-parser-java = { workspace = true, optional = true }
windmill-parser-py = { workspace = true, optional = true }
windmill-parser-yaml.workspace = true
windmill-parser-py-imports = { workspace = true, optional = true }
@@ -72,6 +75,7 @@ dyn-iter.workspace = true
once_cell.workspace = true
tokio-postgres.workspace = true
bit-vec.workspace = true
url.workspace = true
deno_telemetry = { workspace = true, optional = true }
deno_fetch = { workspace = true, optional = true }
deno_webidl = { workspace = true, optional = true }
@@ -127,4 +131,4 @@ deno_permissions = { workspace = true, optional = true }
deno_io = { workspace = true, optional = true }
deno_runtime = { workspace = true, optional = true }
deno_telemetry = { workspace = true, optional = true }
winapi = { workspace = true, optional = true }
winapi = { workspace = true, optional = true }
@@ -0,0 +1,107 @@
name: "java run script"
mode: ONCE
hostname: "java"
log_level: ERROR
disable_rl: true
cwd: "/tmp"
clone_newnet: false
clone_newuser: {CLONE_NEWUSER}
keep_caps: false
keep_env: true
mount_proc: true
mount {
src: "/bin"
dst: "/bin"
is_bind: true
}
mount {
src: "/lib"
dst: "/lib"
is_bind: true
}
mount {
src: "/lib64"
dst: "/lib64"
is_bind: true
mandatory: false
}
mount {
src: "/usr"
dst: "/usr"
is_bind: true
}
mount {
src: "/dev/null"
dst: "/dev/null"
is_bind: true
rw: true
}
mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size=500000000"
}
mount {
src: "{JOB_DIR}/target"
dst: "/tmp/target"
is_bind: true
mandatory: false
}
mount {
src: "{JOB_DIR}/args.json"
dst: "/tmp/args.json"
is_bind: true
}
mount {
src: "{JOB_DIR}/result.json"
dst: "/tmp/result.json"
rw: true
is_bind: true
}
mount {
src: "{CACHE_DIR}"
dst: "{CACHE_DIR}"
is_bind: true
mandatory: false
}
mount {
src: "/etc"
dst: "/etc"
is_bind: true
}
mount {
src: "/dev/random"
dst: "/dev/random"
is_bind: true
}
mount {
src: "/dev/urandom"
dst: "/dev/urandom"
is_bind: true
}
iface_no_lo: true
{SHARED_MOUNT}
@@ -170,6 +170,7 @@ async fn install_galaxy_collections(
None,
false,
&mut Some(occupancy_metrics),
None,
)
.await?;
@@ -432,6 +433,7 @@ fi
job.timeout,
false,
&mut Some(occupancy_metrics),
None,
)
.await?;
read_and_check_result(job_dir).await
@@ -226,6 +226,7 @@ exit $exit_status
job.timeout,
true,
&mut Some(occupancy_metrics),
None,
)
.await?;
@@ -569,6 +570,7 @@ pub async fn handle_powershell_job(
job.timeout,
false,
&mut Some(occupancy_metrics),
None,
)
.await?;
}
@@ -780,6 +782,7 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"",
job.timeout,
false,
&mut Some(occupancy_metrics),
None,
)
.await?;
+16 -4
View File
@@ -166,6 +166,7 @@ pub async fn gen_bun_lockfile(
None,
false,
occupancy_metrics,
None,
)
.await?;
} else {
@@ -349,6 +350,7 @@ pub async fn install_bun_lockfile(
None,
false,
occupancy_metrics,
None,
)
.await?
} else {
@@ -520,6 +522,7 @@ pub async fn generate_wrapper_mjs(
timeout,
false,
occupancy_metrics,
None,
)
.await?;
fs::rename(
@@ -570,6 +573,7 @@ pub async fn generate_bun_bundle(
timeout,
false,
occupancy_metrics,
None,
)
.await?;
} else {
@@ -594,7 +598,6 @@ pub async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result<()> {
if std::fs::metadata(&bun_cache_path).is_ok() {
tracing::info!("loading {bun_cache_path} from cache");
extract_saved_codebase(job_dir, &bun_cache_path, is_tar, &dst, false)?;
} else {
#[cfg(all(feature = "enterprise", feature = "parquet"))]
@@ -733,7 +736,7 @@ pub async fn prebundle_bun_script(
)
.await?;
save_cache(&local_path, &remote_path, &origin).await?;
save_cache(&local_path, &remote_path, &origin, false).await?;
Ok(())
}
@@ -852,7 +855,8 @@ pub async fn handle_bun_job(
)
.await;
let (cache, logs) = windmill_common::worker::load_cache(&local_path, &remote_path).await;
let (cache, logs) =
windmill_common::worker::load_cache(&local_path, &remote_path, false).await;
(cache, logs, local_path, remote_path)
} else {
(false, "".to_string(), "".to_string(), "".to_string())
@@ -1181,7 +1185,14 @@ try {{
)
.await?;
if !local_path.is_empty() {
match save_cache(&local_path, &remote_path, &format!("{job_dir}/main.js")).await {
match save_cache(
&local_path,
&remote_path,
&format!("{job_dir}/main.js"),
false,
)
.await
{
Err(e) => {
let em = format!("could not save {local_path} to bundle cache: {e:?}");
tracing::error!(em)
@@ -1421,6 +1432,7 @@ try {{
job.timeout,
false,
&mut Some(occupancy_metrics),
None,
)
.await?;
+578 -2
View File
@@ -11,6 +11,7 @@ use sha2::Digest;
use sqlx::types::Json;
use sqlx::{Pool, Postgres};
use tokio::process::Command;
use tokio::sync::{RwLock, Semaphore};
use tokio::{fs::File, io::AsyncReadExt};
#[cfg(feature = "parquet")]
@@ -28,7 +29,7 @@ use windmill_common::{
variables::ContextualVariable,
};
use anyhow::{anyhow, Result};
use anyhow::{anyhow, bail, Result};
use windmill_queue::MiniPulledJob;
use std::path::Path;
@@ -39,7 +40,10 @@ use windmill_common::{variables, DB};
use tokio::{io::AsyncWriteExt, process::Child, time::Instant};
use crate::{AuthedClient, JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, MAX_TIMEOUT_DURATION, PATH_ENV};
use crate::{
AuthedClient, DISABLE_NSJAIL, JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, MAX_TIMEOUT_DURATION,
PATH_ENV,
};
pub async fn build_args_map<'a>(
job: &'a MiniPulledJob,
@@ -946,3 +950,575 @@ pub fn build_http_client(timeout_duration: std::time::Duration) -> error::Result
.build()
.map_err(|e| Error::internal_err(format!("Error building http client: {e:#}")))
}
#[derive(Clone)]
pub struct RequiredDependency {
/// Expected directory of dependency in cache
/// For example:
/// /tmp/windmill/cache/python_311/rich==0.0.0
/// IMPORTANT!: path should not end with '/'
pub path: String,
/// Name to use for S3 tars
/// If not specified will use top level directory of path.
pub custom_name: Option<String>,
/// Display name
/// Name that will be used for console output and logging
/// If not specified will either use custom_name or top level directory of path.
pub short_name: Option<String>,
}
pub enum InstallStrategy {
/// Will invoke callback to install single dependency
Single(Arc<dyn Fn(RequiredDependency) -> Result<Command, error::Error> + Send + Sync>),
/// Will try to pull S3 first and will invoke closure to install the rest
AllAtOnce(Arc<dyn Fn(Vec<RequiredDependency>) -> Result<Command, error::Error> + Send + Sync>),
}
/// # General
///
/// Languages that compile usually include dependencies in final executable.
/// When dynamic languages do not and runtime dependencies provided separately.
///
/// This helper implies that the language is dynamic.
/// Python, Ruby, Java are dynamic and they can use this helper.
///
/// # Features
///
/// This helper will install all specified dependencies in parallel and if it is EE, cache to S3
/// It has atomic success file, allowing to distinguish failed installations from succesfull.
///
/// Besides that it provides console output and does logging.
///
/// # Usage
///
/// Most important arguments in this helper are `deps` and `install_fn`
///
/// In `deps` you specify all dependencies that are needed to be on worker in order to execute script.
/// You don't know which are actually installed and which are not.
///
/// `deps` is a vector of RequiredDependency. Check [RequiredDependency] for more context.
///
/// After `deps` are provided helper will check each dependency and check if it is in cache, if not it will try to pull from S3
/// and if it does not work either, it will invoke `install_fn` closure.
/// Closure arguments has dependency name as well as it`s expected path in cache.
/// Closure should return Command that will install dependency to asked place.
pub async fn par_install_language_dependencies<'a>(
deps: Vec<RequiredDependency>,
language_name: &'a str,
installer_executable_name: &'a str,
platform_agnostic: bool,
concurrent_downloads: usize,
stdout_on_err: bool,
install_fn: InstallStrategy,
postinstall_cb: impl AsyncFn(Vec<RequiredDependency>) -> Result<(), error::Error>,
job_id: &'a Uuid,
w_id: &'a str,
worker_name: &'a str,
db: &sqlx::Pool<sqlx::Postgres>,
) -> anyhow::Result<()> {
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
let _ = (platform_agnostic, language_name);
let total_time = std::time::Instant::now();
// Total to install
let mut not_installed = vec![];
let total_to_install;
// let mut not_installed = vec![];
let counter_arc = Arc::new(tokio::sync::Mutex::new(0));
// Append logs with line like this:
// [9/21] + requests==2.32.3 << (S3) | in 57ms
#[allow(unused_assignments)]
async fn print_success(
mut s3_pull: bool,
mut s3_push: bool,
job_id: &Uuid,
w_id: &str,
req: &str,
req_tl: usize,
counter_arc: Arc<tokio::sync::Mutex<usize>>,
total_to_install: usize,
instant: std::time::Instant,
db: Pool<Postgres>,
) {
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
{
(s3_pull, s3_push) = (false, false);
}
#[cfg(all(feature = "enterprise", feature = "parquet"))]
if windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
.read()
.await
.is_none()
{
(s3_pull, s3_push) = (false, false);
}
let mut counter = counter_arc.lock().await;
*counter += 1;
windmill_queue::append_logs(
job_id,
w_id,
format!(
"\n{}+ {}{}{}| in {}ms",
windmill_common::worker::pad_string(
&format!("[{}/{total_to_install}]", counter),
9
),
// Because we want to align to max len [999/999] we take 9
// 123456789
windmill_common::worker::pad_string(&req, req_tl + 1),
// Margin to the right ^
if s3_pull { "<< (S3) " } else { "" },
if s3_push { " > (S3) " } else { "" },
instant.elapsed().as_millis(),
),
db,
)
.await;
// Drop lock, so next print success can fire
}
let mut name_tl = 0;
struct NotInstalledDependency {
path: String,
custom_name: Option<String>,
short_name: Option<String>,
display_name: String,
}
{
let mut to_be_installed_is_used = false;
for RequiredDependency {
path, //
custom_name,
short_name,
} in deps.into_iter()
{
if path.ends_with("/") {
anyhow::bail!("Internal error: path should not end with '/'")
}
let display_name = short_name
.as_ref()
.or(custom_name.as_ref())
.or(path.split("/").last().map(|e| e.to_owned()).as_ref())
.unwrap_or_else(|| {
tracing::warn!(
workspace_id = %w_id,
job_id = %job_id,
"failed to parse top level directory name for {path}, fallback to full path.",
);
&path
})
.to_owned();
{
// Later will help us align text in log console
if display_name.len() > name_tl {
name_tl = display_name.len();
}
}
// Will look like: /tmp/windmill/cache/lang/dependency.valid.windmill
if tokio::fs::metadata(path.clone() + ".valid.windmill")
.await
.is_err()
{
if !to_be_installed_is_used {
windmill_queue::append_logs(
job_id,
w_id,
format!("\n--- INSTALLATION ---\n\nTo be installed:\n\n"),
db.clone(),
)
.await;
to_be_installed_is_used = true;
}
windmill_queue::append_logs(
job_id,
w_id,
format!("- {display_name}\n"),
db.clone(),
)
.await;
not_installed.push(NotInstalledDependency {
path,
custom_name,
short_name,
display_name,
});
}
}
}
total_to_install = not_installed.len();
if total_to_install == 0 {
return Ok(());
}
#[cfg(all(feature = "enterprise", feature = "parquet"))]
let is_not_pro = !matches!(
windmill_common::ee::get_license_plan().await,
windmill_common::ee::LicensePlan::Pro
);
#[cfg(all(feature = "enterprise", feature = "parquet"))]
if is_not_pro && matches!(install_fn, InstallStrategy::AllAtOnce(_)) {
windmill_queue::append_logs(
job_id,
w_id,
format!("\nLooking for packages on S3:\n"),
db.clone(),
)
.await;
}
// Parallelism level (N)
let parallel_limit = // Semaphore will panic if value less then 1
concurrent_downloads.clamp(1, 30);
tracing::info!(
workspace_id = %w_id,
"Install parallel limit: {}, job: {}",
parallel_limit,
job_id
);
let mut handles = vec![];
let semaphore = Arc::new(Semaphore::new(parallel_limit));
let not_pulled = Arc::new(RwLock::new(vec![]));
// let mut handles = Vec::with_capacity(total_to_install);
for NotInstalledDependency {
//
path,
custom_name,
short_name,
display_name,
} in not_installed
{
let permit = semaphore.clone().acquire_owned().await; // Acquire a permit
if let Err(_) = permit {
tracing::error!(
workspace_id = %w_id,
"Cannot acquire permit on semaphore, that can only mean that semaphore has been closed."
);
break;
}
let permit = permit.unwrap();
#[cfg(all(feature = "enterprise", feature = "parquet"))]
let s3_pull_future = if is_not_pro {
if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
.read()
.await
.clone()
{
Some(crate::global_cache::pull_from_tar(
os,
path.clone(),
language_name.to_owned(),
custom_name.clone(),
platform_agnostic,
))
} else {
None
}
} else {
None
};
let child = {
if let InstallStrategy::Single(ref callback, ..) = install_fn {
let cmd = callback(RequiredDependency {
path: path.clone(),
custom_name: custom_name.clone(),
short_name: short_name.clone(),
})?;
tracing::debug!("{:?}", &cmd);
Some(start_child_process(cmd, &installer_executable_name).await?)
} else {
None
}
};
let (
worker_name_2,
path_2,
display_name_2,
custom_name,
job_id_2,
w_id_2,
db_2,
counter_arc,
language_name,
installer_executable_name,
not_pulled,
) = (
worker_name.to_owned(),
path.clone(),
display_name.clone(),
custom_name.clone(),
job_id.clone(),
w_id.to_owned(),
db.clone(),
counter_arc.clone(),
language_name.to_owned(),
installer_executable_name.to_owned(),
not_pulled.clone(),
);
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
let _ = language_name;
let start = std::time::Instant::now();
let handle = tokio::spawn(async move {
let _permit = permit;
#[cfg(all(feature = "enterprise", feature = "parquet"))]
if let Some(s3_pull_future) = s3_pull_future {
if let Err(e) = s3_pull_future.await {
tracing::info!(
workspace_id = %w_id_2,
"No tarball was found for {:?} on S3 or different problem occured {job_id_2}:\n{e}",
&custom_name.clone().unwrap_or(path)
);
} else {
// TODO: Refactor
// Create a file to indicate that installation was successfull
let valid_path = path_2.clone() + ".valid.windmill";
// This is atomic operation, meaning, that it either completes and dependency is valid,
// or it does not and dependency is invalid and will be reinstalled next run
if let Err(e) = File::create(&valid_path).await {
tracing::error!(
workspace_id = %w_id_2,
job_id = %job_id_2,
"Failed to create {}!\n{e}\n
This file needed for jobs to function",
valid_path
);
};
print_success(
true,
false,
&job_id_2,
&w_id_2,
&display_name_2,
name_tl,
counter_arc,
total_to_install,
start,
db_2,
)
.await;
return;
}
}
let Some(child) = child else {
let mut lock = not_pulled.write().await;
lock.push(RequiredDependency {
path: path_2.clone(),
custom_name: custom_name.clone(),
short_name: short_name.clone(),
});
return;
};
if let Err(e) = crate::handle_child::handle_child(
&job_id_2,
&db_2,
// TODO: Return mem_peak
&mut 0,
// TODO: Return canceld_by_ref
&mut None,
child,
!*DISABLE_NSJAIL,
&worker_name_2,
&w_id_2,
&installer_executable_name,
None,
false,
&mut None,
None,
)
.await
{
windmill_queue::append_logs(
&job_id_2,
&w_id_2,
format!("error while installing {}: {e:?}", &display_name_2),
db_2.clone(),
)
.await;
} else {
// if let Some(cb) = postinstall_cb {
// if let Err(e) = cb(vec![RequiredDependency {
// path: path_2.clone(),
// custom_name: custom_name.clone(),
// short_name: short_name.clone(),
// }])
// .await
// {
// tracing::error!(
// workspace_id = %w_id_2,
// job_id = %job_id_2,
// "Postinstall callback failed!\n{e}\n
// This might affect execution",
// );
// }
// }
print_success(
false,
true,
&job_id_2,
&w_id_2,
&display_name_2,
name_tl,
counter_arc,
total_to_install,
start,
db_2,
)
.await;
// TODO: Refactor
// Create a file to indicate that installation was successfull
let valid_path = path_2.clone() + ".valid.windmill";
// This is atomic operation, meaning, that it either completes and dependency is valid,
// or it does not and dependency is invalid and will be reinstalled next run
if let Err(e) = File::create(&valid_path).await {
tracing::error!(
workspace_id = %w_id_2,
job_id = %job_id_2,
"Failed to create {}!\n{e}\n
This file needed for jobs to function",
valid_path
);
};
#[cfg(all(feature = "enterprise", feature = "parquet"))]
{
if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
.read()
.await
.clone()
{
tokio::spawn(async move {
if let Err(e) = crate::global_cache::build_tar_and_push(
os,
path_2,
language_name,
custom_name,
platform_agnostic,
)
.await
{
tracing::warn!("failed to build tar and push: {e:?}");
}
});
}
}
}
});
handles.push(handle);
}
for handle in handles {
if let Err(e) = handle.await {
tracing::error!("Error joining handles: {e:?}");
}
}
if !not_pulled.read().await.is_empty() {
if let InstallStrategy::AllAtOnce(ref callback, ..) = install_fn {
let not_pulled_copy = not_pulled.read().await.clone();
windmill_queue::append_logs(
job_id,
w_id,
format!("\n\nFetching {} packages...\n", not_pulled_copy.len()),
db.clone(),
)
.await;
let cmd = callback(not_pulled_copy.clone())?;
tracing::debug!("{:?}", &cmd);
let child = start_child_process(cmd, &installer_executable_name).await?;
let mut buf = "".to_owned();
let pipe_stdout = if stdout_on_err { Some(&mut buf) } else { None };
if let Err(e) = crate::handle_child::handle_child(
// &job_id,
&Uuid::nil(),
&db,
// TODO: Return mem_peak
&mut 0,
// TODO: Return canceld_by_ref
&mut None,
child,
!*DISABLE_NSJAIL,
&worker_name,
&w_id,
&installer_executable_name,
None,
false,
&mut None,
pipe_stdout,
)
.await
{
bail!(format!(
"error while installing dependencies: {e:?}\n{}",
buf
));
}
{
postinstall_cb(not_pulled_copy.clone()).await?;
}
for RequiredDependency { path, custom_name: _custom_name, .. } in
not_pulled_copy.into_iter()
{
// TODO: Refactor
// Create a file to indicate that installation was successfull
let valid_path = path.clone() + ".valid.windmill";
// This is atomic operation, meaning, that it either completes and dependency is valid,
// or it does not and dependency is invalid and will be reinstalled next run
if let Err(e) = File::create(&valid_path).await {
tracing::error!(
workspace_id = %w_id,
job_id = %job_id,
"Failed to create {}!\n{e}\n
This file needed for jobs to function",
valid_path
);
};
#[cfg(all(feature = "enterprise", feature = "parquet"))]
{
if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
.read()
.await
.clone()
{
let language_name = language_name.to_owned();
tokio::spawn(async move {
if let Err(e) = crate::global_cache::build_tar_and_push(
os,
path,
language_name,
_custom_name,
platform_agnostic,
)
.await
{
tracing::warn!("failed to build tar and push: {e:?}");
}
});
}
}
}
}
}
{
let total_time = total_time.elapsed().as_millis();
windmill_queue::append_logs(
&job_id,
w_id,
format!(
"\nDone. Time spent on installation phase: {}ms\n",
total_time
),
db,
)
.await;
}
Ok(())
}
@@ -126,6 +126,7 @@ pub async fn generate_nuget_lockfile(
None,
false,
&mut Some(occupancy_metrics),
None,
)
.await?;
@@ -381,6 +382,7 @@ async fn build_cs_proj(
None,
false,
&mut Some(occupancy_metrics),
None,
)
.await?;
append_logs(job_id, w_id, "\n\n", db).await;
@@ -400,6 +402,7 @@ async fn build_cs_proj(
&bin_path,
&format!("{CSHARP_OBJECT_STORE_PREFIX}{hash}"),
&target,
false,
)
.await
{
@@ -474,7 +477,8 @@ pub async fn handle_csharp_job(
let bin_path = format!("{}/{hash}", CSHARP_CACHE_DIR);
let remote_path = format!("{CSHARP_OBJECT_STORE_PREFIX}{hash}");
let (cache, cache_logs) = windmill_common::worker::load_cache(&bin_path, &remote_path).await;
let (cache, cache_logs) =
windmill_common::worker::load_cache(&bin_path, &remote_path, false).await;
let cache_logs = if cache {
#[cfg(unix)]
@@ -634,6 +638,7 @@ pub async fn handle_csharp_job(
job.timeout,
false,
&mut Some(occupancy_metrics),
None,
)
.await?;
read_result(job_dir).await
@@ -156,6 +156,7 @@ pub async fn generate_deno_lock(
None,
false,
occupancy_metrics,
None,
)
.await?;
} else {
@@ -414,6 +415,7 @@ try {{
job.timeout,
false,
&mut Some(occupancy_metrics),
None,
)
.await?;
// logs.push_str(format!("execute: {:?}\n", start.elapsed().as_millis()).as_str());
+26 -10
View File
@@ -17,8 +17,9 @@ pub const TARGET: &str = const_format::concatcp!(std::env::consts::OS, "_", std:
pub async fn build_tar_and_push(
s3_client: Arc<dyn ObjectStore>,
folder: String,
// python_311
python_xyz: String,
lang: String,
custom_folder_name: Option<String>,
platform_agnostic: bool,
) -> error::Result<()> {
use object_store::path::Path;
@@ -28,9 +29,13 @@ pub async fn build_tar_and_push(
let start = Instant::now();
// e.g. tiny==1.0.0
let folder_name = folder.split("/").last().unwrap();
let folder_name = if let Some(name) = custom_folder_name {
name
} else {
folder.split("/").last().unwrap().to_owned()
};
let prefix = &format!("{TAR_PYBASE_CACHE_DIR}/{}", python_xyz);
let prefix = &format!("{TAR_PYBASE_CACHE_DIR}/{}", lang);
let tar_path = format!("{prefix}/{folder_name}_tar.tar",);
let tar_file = std::fs::File::create(&tar_path)?;
@@ -51,7 +56,10 @@ pub async fn build_tar_and_push(
// })?;
if let Err(e) = s3_client
.put(
&Path::from(format!("/tar/{TARGET}/{python_xyz}/{folder_name}.tar")),
&Path::from(format!(
"/tar/{}/{lang}/{folder_name}.tar",
if platform_agnostic { "" } else { TARGET }
)),
std::fs::read(&tar_path)?.into(),
)
.await
@@ -79,18 +87,26 @@ pub async fn build_tar_and_push(
pub async fn pull_from_tar(
client: Arc<dyn ObjectStore>,
folder: String,
// python_311
python_xyz: String,
lang: String,
custom_folder_name: Option<String>,
platform_agnostic: bool,
) -> error::Result<()> {
use windmill_common::s3_helpers::attempt_fetch_bytes;
let folder_name = folder.split("/").last().unwrap();
let folder_name = if let Some(name) = custom_folder_name {
name
} else {
folder.split("/").last().unwrap().to_owned()
};
tracing::info!("Attempting to pull piptar {folder_name} from bucket");
tracing::info!("Attempting to pull tar {folder_name} from bucket");
let start = Instant::now();
let tar_path = format!("tar/{TARGET}/{python_xyz}/{folder_name}.tar");
let tar_path = format!(
"tar/{}/{lang}/{folder_name}.tar",
if platform_agnostic { "" } else { TARGET }
);
let bytes = attempt_fetch_bytes(client, &tar_path).await?;
extract_tar(bytes, &folder).map_err(|e| {
+7 -1
View File
@@ -65,7 +65,8 @@ pub async fn handle_go_job(
));
let bin_path = format!("{}/{hash}", GO_BIN_CACHE_DIR);
let remote_path = format!("{GO_OBJECT_STORE_PREFIX}{hash}");
let (cache, cache_logs) = windmill_common::worker::load_cache(&bin_path, &remote_path).await;
let (cache, cache_logs) =
windmill_common::worker::load_cache(&bin_path, &remote_path, false).await;
let (skip_go_mod, skip_tidy) = if cache {
(true, true)
@@ -212,6 +213,7 @@ func Run(req Req) (interface{{}}, error){{
None,
false,
&mut Some(occupation_metrics),
None,
)
.await?;
@@ -219,6 +221,7 @@ func Run(req Req) (interface{{}}, error){{
&bin_path,
&format!("{GO_OBJECT_STORE_PREFIX}{hash}"),
&format!("{job_dir}/main"),
false,
)
.await
{
@@ -314,6 +317,7 @@ func Run(req Req) (interface{{}}, error){{
job.timeout,
false,
&mut Some(occupation_metrics),
None,
)
.await?;
@@ -378,6 +382,7 @@ pub async fn install_go_dependencies(
None,
false,
&mut Some(occupation_metrics),
None,
)
.await?;
@@ -444,6 +449,7 @@ pub async fn install_go_dependencies(
None,
false,
&mut Some(occupation_metrics),
None,
)
.await?;
+10 -3
View File
@@ -103,6 +103,8 @@ pub async fn handle_child(
custom_timeout: Option<i32>,
sigterm: bool,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
// Do not print logs to output, but instead save to string.
pipe_stdout: Option<&mut String>,
) -> error::Result<()> {
let start = Instant::now();
@@ -304,6 +306,8 @@ pub async fn handle_child(
let mut log_total_size: u64 = 0;
let pg_log_total_size = Arc::new(AtomicU32::new(0));
let mut pipe_stdout = pipe_stdout;
while let Some(line) = output.by_ref().next().await {
let do_write_ = do_write.shared();
@@ -385,9 +389,12 @@ pub async fn handle_child(
let worker_name = worker.to_string();
let w_id2 = w_id.to_string();
(do_write, write_result) = tokio::spawn(append_job_logs(job_id, w_id2, joined, db.clone(), compact_logs, pg_log_total_size.clone(), worker_name)).remote_handle();
if let Some(buf) = &mut pipe_stdout {
buf.push_str(&joined);
(do_write, write_result) = tokio::spawn(async { }).remote_handle();
} else {
(do_write, write_result) = tokio::spawn(append_job_logs(job_id, w_id2, joined, db.clone(), compact_logs, pg_log_total_size.clone(), worker_name)).remote_handle();
}
if let Err(err) = result {
tracing::error!(%job_id, %err, "error reading output for job {job_id} '{child_name}': {err}");
@@ -0,0 +1,915 @@
use std::{collections::HashMap, path::PathBuf, process::Stdio, sync::Arc};
use anyhow::{anyhow, bail};
use async_recursion::async_recursion;
use itertools::Itertools;
use serde_json::value::RawValue;
use tokio::{
fs::{create_dir_all, metadata, remove_dir_all, File},
io::AsyncWriteExt,
process::Command,
};
use uuid::Uuid;
use windmill_common::{
error::{self, Error},
utils::calculate_hash,
worker::{copy_dir_recursively, save_cache, write_file},
};
use windmill_parser::Arg;
use windmill_parser_java::parse_java_sig_meta;
use windmill_queue::{append_logs, CanceledBy, MiniPulledJob};
use crate::{
common::{
create_args_and_out_file, get_reserved_variables, par_install_language_dependencies,
read_result, start_child_process, OccupancyMetrics, RequiredDependency,
},
handle_child, AuthedClient, COURSIER_CACHE_DIR, DISABLE_NSJAIL, DISABLE_NUSER, JAVA_CACHE_DIR,
JAVA_REPOSITORY_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, NSJAIL_PATH, PATH_ENV, PROXY_ENVS,
};
lazy_static::lazy_static! {
static ref JAVA_CONCURRENT_DOWNLOADS: usize = std::env::var("JAVA_CONCURRENT_DOWNLOADS").ok().map(|flag| flag.parse().unwrap_or(20)).unwrap_or(20);
static ref JAVA_PATH: String = std::env::var("JAVA_PATH").unwrap_or_else(|_| "/usr/bin/java".to_string());
static ref JAVAC_PATH: String = std::env::var("JAVAC_PATH").unwrap_or_else(|_| "/usr/bin/javac".to_string());
static ref CS_PATH: String = std::env::var("COURSIER_PATH").unwrap_or_else(|_| "/usr/bin/coursier".to_string());
static ref STOREPASS: String = std::env::var("JAVA_STOREPASS").unwrap_or("123456".into());
static ref TRUST_STORE_PATH: String = std::env::var("JAVA_TRUST_STORE_PATH").unwrap_or("/usr/local/share/ca-certificates/truststore.jks".into());
}
const NSJAIL_CONFIG_RUN_JAVA_CONTENT: &str = include_str!("../nsjail/run.java.config.proto");
#[allow(dead_code)]
pub(crate) struct JobHandlerInput<'a> {
pub base_internal_url: &'a str,
pub canceled_by: &'a mut Option<CanceledBy>,
pub client: &'a AuthedClient,
pub parent_runnable_path: Option<String>,
pub db: &'a sqlx::Pool<sqlx::Postgres>,
pub envs: HashMap<String, String>,
pub inner_content: &'a str,
pub job: &'a MiniPulledJob,
pub job_dir: &'a str,
pub mem_peak: &'a mut i32,
pub occupancy_metrics: &'a mut OccupancyMetrics,
pub requirements_o: Option<&'a String>,
pub shared_mount: &'a str,
pub worker_name: &'a str,
}
pub async fn handle_java_job<'a>(mut args: JobHandlerInput<'a>) -> Result<Box<RawValue>, Error> {
// --- Prepare ---
{
prepare(&mut args).await?;
}
// --- Generate Lockfile ---
let deps = resolve(
&args.job.id,
&args.inner_content,
&args.job_dir,
&args.db,
&args.job.workspace_id,
)
.await?;
// --- Install ---
let classpath = install(&mut args, deps).await?;
// --- Build .java files ---
{
compile(&mut args, &classpath).await?;
}
// --- Run ---
{
run(&mut args, &classpath).await?;
}
// --- Retrieve results ---
{
read_result(&args.job_dir).await
}
}
async fn prepare<'a>(
JobHandlerInput { job, db, job_dir, client, inner_content, .. }: &mut JobHandlerInput<'a>,
) -> Result<(), Error> {
// Create needed files
{
create_args_and_out_file(&client, job, job_dir, db).await?;
let app_path = format!("{}/src/main/java/net/script/", job_dir);
create_dir_all(&app_path).await?;
File::create(format!("{app_path}/App.java"))
.await?
.write_all(&wrap(inner_content)?.into_bytes())
.await?;
File::create(format!("{app_path}/Main.java"))
.await?
.write_all(
&format!(
"package net.script;\n{MINI_CLIENT_IMPORTS}\n{}\n{MINI_CLIENT}",
inner_content
)
.into_bytes(),
)
.await?;
}
Ok(())
}
pub async fn resolve<'a>(
job_id: &Uuid,
code: &str,
job_dir: &str,
db: &sqlx::Pool<sqlx::Postgres>,
w_id: &str,
) -> Result<String, Error> {
let deps = {
let find_requirements = code.lines().find_position(|x| {
x.starts_with("//requirements:") || x.starts_with("// requirements:")
});
let specified_deps = if let Some((pos, _)) = find_requirements {
code.lines()
.skip(pos + 1)
.map_while(|x| {
if x.starts_with("//") {
Some(x.replace("//", "").trim().to_owned())
} else {
None
}
})
.collect::<Vec<String>>()
} else {
Default::default()
};
let mut deps = vec![
// Default requirements
"com.fasterxml.jackson.core:jackson-databind:2.9.8".to_owned(),
];
deps.extend(specified_deps);
deps.join("\n")
};
let req_hash = format!("java-{}", calculate_hash(&deps));
if let Some(cached) = sqlx::query_scalar!(
"SELECT lockfile FROM pip_resolution_cache WHERE hash = $1",
req_hash
)
.fetch_optional(db)
.await?
{
return Ok(cached);
}
let lock = {
append_logs(
job_id,
w_id,
format!("\n--- RESOLVING LOCKFILE ---\n"),
db.clone(),
)
.await;
let mut cmd = Command::new(if cfg!(windows) {
"java"
} else {
JAVA_PATH.as_str()
});
cmd.env_clear()
.current_dir(job_dir.to_owned())
.env("PATH", PATH_ENV.as_str())
.envs(PROXY_ENVS.clone());
// Configure proxies
{
let jps = parse_proxy()?;
if let Some(val) = jps.https_host {
cmd.arg(&format!("-Dhttps.proxyHost={}", val));
}
if let Some(val) = jps.https_port {
cmd.arg(&format!("-Dhttps.proxyPort={}", val));
}
if let Some(val) = jps.http_host {
cmd.arg(&format!("-Dhttp.proxyHost={}", val));
}
if let Some(val) = jps.http_port {
cmd.arg(&format!("-Dhttp.proxyPort={}", val));
}
if let Some(val) = jps.no_proxy {
cmd.arg(&format!("-Dhttp.nonProxyHosts=\"{}\"", val));
}
}
if metadata(TRUST_STORE_PATH.clone()).await.is_ok() {
cmd.args(&[
&format!("-Djavax.net.ssl.trustStore={}", *TRUST_STORE_PATH),
&format!("-Djavax.net.ssl.trustStorePassword={}", *STOREPASS),
]);
}
cmd.args(&[
"-jar",
&CS_PATH,
"resolve",
&get_no_default(),
"--parallel",
&format!("{}", *JAVA_CONCURRENT_DOWNLOADS),
"--cache",
COURSIER_CACHE_DIR,
])
.args(&get_repos().await)
.args(&deps.split("\n").collect_vec())
.stderr(Stdio::piped());
#[cfg(windows)]
{
cmd.env("SystemRoot", crate::SYSTEM_ROOT.as_str())
.env("USERPROFILE", crate::USERPROFILE_ENV.as_str())
.env(
"TMP",
std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")),
);
}
let output = cmd.output().await?;
// Check if the command was successful
if output.status.success() {
String::from_utf8(output.stdout).expect("Failed to convert output to String")
} else {
let stderr =
String::from_utf8(output.stderr).expect("Failed to convert error output to String");
return Err(error::Error::internal_err(stderr));
}
};
sqlx::query!(
"INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2",
req_hash,
lock.clone(),
).fetch_optional(db).await?;
append_logs(job_id, w_id, format!("\n{}", &lock), db.clone()).await;
Ok(lock)
}
async fn install<'a>(
JobHandlerInput { worker_name, job, db, job_dir, .. }: &mut JobHandlerInput<'a>,
deps: String,
) -> Result<String, Error> {
let deps = deps
.lines()
.map(|line| {
let unparsed_dep = line.replace(":jar", "").replace(":lib", "");
let mut it = unparsed_dep.split(":");
match (it.next(), it.next(), it.next()) {
(Some(group_id), Some(artifact_id), Some(version)) => {
let path = format!(
"{JAVA_REPOSITORY_DIR}/{}/{artifact_id}/{version}",
group_id.replace(".", "/")
);
Ok(RequiredDependency {
path,
custom_name: Some(format!("{group_id}:{artifact_id}:{version}")),
short_name: Some(format!("{artifact_id}:{version}")),
})
}
_ => anyhow::bail!("{line} is not parsable"),
}
})
.collect::<anyhow::Result<Vec<RequiredDependency>>>()?;
let classpath = deps
.clone()
.into_iter()
.map(|RequiredDependency { path, .. }| path + "/*")
.collect_vec()
.join(":")
+ ":target";
#[cfg(windows)]
let classpath = classpath.replace(":", ";");
tracing::debug!(
workspace_id = %job.workspace_id,
"JAVA classpath: {}", &classpath
);
let (repos, no_default, trust_store_metadata) = (
get_repos().await,
get_no_default(),
metadata(TRUST_STORE_PATH.clone()).await,
);
let job_dir = job_dir.to_owned();
let fetch_dir = format!("{JAVA_CACHE_DIR}/tmp-fetch-{}", Uuid::new_v4());
let fetch_dir2 = fetch_dir.clone();
par_install_language_dependencies(
deps,
"java",
"java",
true,
*JAVA_CONCURRENT_DOWNLOADS,
true,
crate::common::InstallStrategy::AllAtOnce(Arc::new(move |dependencies| {
let mut cmd = Command::new(if cfg!(windows) {
"java"
} else {
JAVA_PATH.as_str()
});
let artifacts = dependencies
.into_iter()
.map(|e| {
e.custom_name.ok_or(anyhow::anyhow!(
"Internal Error: Artifact name should be Some!"
))
})
.collect::<anyhow::Result<Vec<String>>>()?;
cmd.env_clear()
.current_dir(&job_dir)
.env("PATH", PATH_ENV.as_str())
.envs(PROXY_ENVS.clone());
// Configure proxies
{
let jps = parse_proxy()?;
if let Some(val) = jps.https_host {
cmd.arg(&format!("-Dhttps.proxyHost={}", val));
}
if let Some(val) = jps.https_port {
cmd.arg(&format!("-Dhttps.proxyPort={}", val));
}
if let Some(val) = jps.http_host {
cmd.arg(&format!("-Dhttp.proxyHost={}", val));
}
if let Some(val) = jps.http_port {
cmd.arg(&format!("-Dhttp.proxyPort={}", val));
}
if let Some(val) = jps.no_proxy {
cmd.arg(&format!("-Dhttp.nonProxyHosts=\"{}\"", val));
}
}
if trust_store_metadata.is_ok() {
cmd.args(&[
&format!("-Djavax.net.ssl.trustStore={}", *TRUST_STORE_PATH),
&format!("-Djavax.net.ssl.trustStorePassword={}", *STOREPASS),
]);
}
cmd.args(&[
"-jar",
&CS_PATH,
"fetch",
&no_default,
"--quiet",
"--parallel",
&format!("{}", *JAVA_CONCURRENT_DOWNLOADS),
"--cache",
&fetch_dir,
])
.args(&repos)
.arg("--intransitive")
.args(artifacts)
.stdout(Stdio::piped())
.stderr(Stdio::piped());
#[cfg(windows)]
{
cmd.env("SystemRoot", crate::SYSTEM_ROOT.as_str())
.env("USERPROFILE", crate::USERPROFILE_ENV.as_str())
.env(
"TMP",
std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")),
);
}
Ok(cmd)
})),
async move |_| {
move_to_repository(&fetch_dir2, 0).await?;
remove_dir_all(&fetch_dir2).await?;
#[async_recursion]
async fn move_to_repository(path: &str, depth: u8) -> anyhow::Result<()> {
if depth == 3 {
copy_dir_recursively(
&PathBuf::from(path),
&PathBuf::from(JAVA_REPOSITORY_DIR),
)?;
return Ok(());
}
let mut entries = tokio::fs::read_dir(path).await?;
loop {
let Some(entry) = entries.next_entry().await? else {
break Ok(());
};
let path = entry
.path()
.to_str()
.ok_or(anyhow!("Internal Error: Cannot convert Path to Str"))?
.to_owned();
move_to_repository(&path, depth + 1).await?;
}
}
Ok(())
},
&job.id,
&job.workspace_id,
worker_name,
db,
)
.await?;
Ok(classpath)
}
async fn compile<'a>(
JobHandlerInput {
occupancy_metrics,
mem_peak,
canceled_by,
worker_name,
job,
db,
job_dir,
client,
envs,
base_internal_url,
inner_content,
requirements_o,
parent_runnable_path,
..
}: &mut JobHandlerInput<'a>,
classpath: &'a str,
// plugins: Vec<&'a str>,
) -> Result<(), Error> {
fn compute_hash(code: &str, requirements_o: Option<&String>) -> String {
calculate_hash(&format!(
"{}{}",
code,
requirements_o
.as_ref()
.map(|x| x.to_string())
.unwrap_or_default()
))
}
let reserved_variables =
get_reserved_variables(job, &client.token, db, parent_runnable_path.clone()).await?;
let hash = compute_hash(inner_content, *requirements_o);
let bin_path = format!("{}/{hash}", JAVA_CACHE_DIR);
let remote_path = format!("java_jar/{hash}");
let (cache, ..) = windmill_common::worker::load_cache(&bin_path, &remote_path, true).await;
if cache {
let target = format!("{job_dir}/target");
#[cfg(unix)]
let symlink = std::os::unix::fs::symlink(&bin_path, &target);
#[cfg(windows)]
let symlink = copy_dir_recursively(&PathBuf::from(&bin_path), &PathBuf::from(&target));
symlink.map_err(|e| {
Error::ExecutionErr(format!(
"could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}"
))
})?;
} else {
// let plugin_registry = format!("{job_dir}/plugin-registry");
let child = {
append_logs(
&job.id,
&job.workspace_id,
format!("\n--- COMPILING .JAVA FILES\n"),
db.clone(),
)
.await;
let mut cmd = Command::new(if cfg!(windows) {
"javac"
} else {
JAVAC_PATH.as_str()
});
cmd.env_clear()
.current_dir(job_dir.to_owned())
.env("PATH", PATH_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
.envs(envs)
.envs(reserved_variables)
.envs(PROXY_ENVS.clone())
.args(&[
"-classpath",
&classpath,
"src/main/java/net/script/Main.java",
"src/main/java/net/script/App.java",
"-d",
"./target",
])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
#[cfg(windows)]
{
cmd.env("SystemRoot", crate::SYSTEM_ROOT.as_str())
.env("USERPROFILE", crate::USERPROFILE_ENV.as_str())
.env(
"TMP",
std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")),
);
}
start_child_process(cmd, "javac").await?
};
handle_child::handle_child(
&job.id,
db,
mem_peak,
canceled_by,
child,
!*DISABLE_NSJAIL,
worker_name,
&job.workspace_id,
"javac",
job.timeout,
false,
&mut Some(occupancy_metrics),
None,
)
.await?;
match save_cache(
&bin_path,
&format!("java_jar/{hash}"),
&format!("{job_dir}/target"),
true,
)
.await
{
Err(e) => {
let em = format!(
"could not save {bin_path} to {} to java cache: {e:?}",
format!("{job_dir}/main"),
);
tracing::error!(em);
}
Ok(logs) => {
tracing::trace!(logs);
}
}
};
Ok(())
}
async fn run<'a>(
JobHandlerInput {
occupancy_metrics,
mem_peak,
canceled_by,
worker_name,
job,
db,
job_dir,
shared_mount,
client,
envs,
base_internal_url,
parent_runnable_path,
..
}: &mut JobHandlerInput<'a>,
classpath: &'a str,
) -> Result<(), Error> {
let reserved_variables =
get_reserved_variables(job, &client.token, db, parent_runnable_path.clone()).await?;
let child = if !cfg!(windows) && !*DISABLE_NSJAIL {
append_logs(
&job.id,
&job.workspace_id,
format!("\n--- ISOLATED JAVA CODE EXECUTION ---\n"),
db.clone(),
)
.await;
write_file(
job_dir,
"run.config.proto",
&NSJAIL_CONFIG_RUN_JAVA_CONTENT
.replace("{JOB_DIR}", job_dir)
.replace("{CACHE_DIR}", JAVA_CACHE_DIR)
.replace("{SHARED_MOUNT}", &shared_mount)
// .replace("{CACHED_TARGET}", &shared_mount)
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()),
)?;
let mut cmd = Command::new(NSJAIL_PATH.as_str());
cmd.env_clear()
.current_dir(job_dir)
.env("PATH", PATH_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
.envs(envs)
.envs(reserved_variables)
.envs(PROXY_ENVS.clone())
.args(vec![
"--config",
"run.config.proto",
"--",
JAVA_PATH.as_str(),
]);
if metadata(TRUST_STORE_PATH.clone()).await.is_ok() {
cmd.args(&[
&format!("-Djavax.net.ssl.trustStore={}", *TRUST_STORE_PATH),
&format!("-Djavax.net.ssl.trustStorePassword={}", *STOREPASS),
]);
}
// Configure proxies
{
let jps = parse_proxy()?;
if let Some(val) = jps.https_host {
cmd.arg(&format!("-Dhttps.proxyHost={}", val));
}
if let Some(val) = jps.https_port {
cmd.arg(&format!("-Dhttps.proxyPort={}", val));
}
if let Some(val) = jps.http_host {
cmd.arg(&format!("-Dhttp.proxyHost={}", val));
}
if let Some(val) = jps.http_port {
cmd.arg(&format!("-Dhttp.proxyPort={}", val));
}
if let Some(val) = jps.no_proxy {
cmd.arg(&format!("-Dhttp.nonProxyHosts=\"{}\"", val));
}
}
cmd.args(vec!["-classpath", &classpath, "net.script.App"]);
cmd.stdout(Stdio::piped()).stderr(Stdio::piped());
start_child_process(cmd, NSJAIL_PATH.as_str()).await?
} else {
append_logs(
&job.id,
&job.workspace_id,
format!("\n--- JAVA CODE EXECUTION ---\n"),
db.clone(),
)
.await;
let mut cmd = Command::new(if cfg!(windows) {
"java"
} else {
JAVA_PATH.as_str()
});
cmd.env_clear()
.current_dir(job_dir.to_owned())
.env("PATH", PATH_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
.envs(envs)
.envs(reserved_variables);
if metadata(TRUST_STORE_PATH.clone()).await.is_ok() {
cmd.args(&[
&format!("-Djavax.net.ssl.trustStore={}", *TRUST_STORE_PATH),
&format!("-Djavax.net.ssl.trustStorePassword={}", *STOREPASS),
]);
}
// Configure proxies
{
let jps = parse_proxy()?;
if let Some(val) = jps.https_host {
cmd.arg(&format!("-Dhttps.proxyHost={}", val));
}
if let Some(val) = jps.https_port {
cmd.arg(&format!("-Dhttps.proxyPort={}", val));
}
if let Some(val) = jps.http_host {
cmd.arg(&format!("-Dhttp.proxyHost={}", val));
}
if let Some(val) = jps.http_port {
cmd.arg(&format!("-Dhttp.proxyPort={}", val));
}
if let Some(val) = jps.no_proxy {
cmd.arg(&format!("-Dhttp.nonProxyHosts=\"{}\"", val));
}
}
cmd.args(&["-classpath", &classpath, "net.script.App"])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
#[cfg(windows)]
{
cmd.env("SystemRoot", crate::SYSTEM_ROOT.as_str())
.env("USERPROFILE", crate::USERPROFILE_ENV.as_str())
.env(
"TMP",
std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")),
);
}
start_child_process(cmd, "java").await?
};
handle_child::handle_child(
&job.id,
db,
mem_peak,
canceled_by,
child,
!*DISABLE_NSJAIL,
worker_name,
&job.workspace_id,
"java",
job.timeout,
false,
&mut Some(occupancy_metrics),
None,
)
.await
}
#[derive(Default, Debug)]
struct JavaProxySettings {
http_host: Option<String>,
http_port: Option<String>,
https_host: Option<String>,
https_port: Option<String>,
no_proxy: Option<String>,
}
fn parse_proxy() -> anyhow::Result<JavaProxySettings> {
let mut jps = JavaProxySettings::default();
for (ident, mut val) in PROXY_ENVS.clone() {
match ident {
"HTTPS_PROXY" => {
if val.contains("http://") {
bail!("HTTPS_PROXY url cannot contain http scheme.");
}
if !val.contains("https://") {
val = format!("https://{val}");
}
let mut url = url::Url::parse(&val)?;
let port = url.port();
// Make sure port and schema is not included in final url
{
url.set_port(None).unwrap_or_default();
jps.https_host = Some(url.as_str().replace("https://", ""));
if let Some(port) = port {
jps.https_port = Some(format!("{}", port));
}
}
}
"HTTP_PROXY" => {
if val.contains("https://") {
bail!("HTTP_PROXY url cannot contain https scheme.");
}
if !val.contains("http://") {
val = format!("http://{val}");
}
let mut url = url::Url::parse(&val)?;
let port = url.port();
// Make sure port and schema is not included in final url
{
url.set_port(None).unwrap_or_default();
jps.http_host = Some(url.as_str().replace("http://", ""));
if let Some(port) = port {
jps.https_port = Some(format!("{}", port));
}
}
}
// Java uses | instead of ,
"NO_PROXY" => jps.no_proxy = Some(val.replace(",", "|")),
_ => {}
}
}
Ok(jps)
}
async fn get_repos() -> Vec<String> {
MAVEN_REPOS
.read()
.await
.as_ref()
.map(|repos| {
repos
.trim()
.split_whitespace()
.into_iter()
.map(|el| vec!["--repository".to_owned(), el.to_owned()])
.collect_vec()
})
.unwrap_or_default()
.concat()
}
fn get_no_default() -> String {
if NO_DEFAULT_MAVEN.load(std::sync::atomic::Ordering::Relaxed) {
"--no-default"
} else {
// Command does not take empty arguments
"-q"
}
.into()
}
/// Wraps content script
/// that upon execution reads args.json (which are piped and transformed from previous flow step or top level inputs)
/// Also wrapper takes output of program and serializes to result.json (Which windmill will know how to use later)
fn wrap(inner_content: &str) -> Result<String, Error> {
let sig = parse_java_sig_meta(inner_content)?;
let ret_void = sig.returns_void;
let spread = sig
.main_sig
.args
.clone()
.into_iter()
.map(|Arg { name, .. }| {
// Apply additional input transformation
format!(" parsedArgs.{name}")
})
.collect_vec()
.join(",");
let args = sig
.main_sig
.args
.clone()
.into_iter()
.map(|Arg { name, otyp, .. }| {
// Apply additional input transformation
format!("public {} {name};\n", otyp.unwrap())
})
.collect_vec()
.join(" ");
Ok(r#"
package net.script;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.FileOutputStream;
import net.script.Main;
public class App{
public static class Args {ARGS}
public static void main(String[] args) {
try {
InputStream fileInputStream = new FileInputStream("args.json");
ObjectMapper mapper = new ObjectMapper();
Args parsedArgs = mapper.readValue(fileInputStream, Args.class);
fileInputStream.close();
{MAIN_HANDLER}
FileOutputStream fileOutputStream = new FileOutputStream("result.json");
mapper.writeValue(fileOutputStream, res);
fileOutputStream.close();
} catch (Exception e) { // Catching general Exception
e.printStackTrace(); // Handle the exception
}
}
}
"#
.replace(
"{MAIN_HANDLER}",
if ret_void {
"
Main.main(SPREAD);
Object res = null;
"
} else {
"
Object res = Main.main(SPREAD);
"
},
)
.replace("SPREAD", &spread)
.replace("ARGS", &args))
}
const MINI_CLIENT_IMPORTS: &str = r#"
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
"#;
const MINI_CLIENT: &str = r#"
class Wmill {
public static String getVariable(String path) {
var baseUrl = System.getenv("BASE_INTERNAL_URL");
var workspace = System.getenv("WM_WORKSPACE");
var uri = java.text.MessageFormat.format("{0}/api/w/{1}/variables/get_value/{2}", baseUrl, workspace, path);
// Create an HttpRequest
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(uri))
.header("Authorization", "Bearer " + System.getenv("WM_TOKEN")) // Add the Authorization header
.GET() // Set the request method to GET
.build();
// Send the request and get the response
return HttpClient.newHttpClient().sendAsync(request, HttpResponse.BodyHandlers.ofString())
.thenApply(HttpResponse::body)
.join(); // Wait for the completion
}
public static String getResource(String path) {
var baseUrl = System.getenv("BASE_INTERNAL_URL");
var workspace = System.getenv("WM_WORKSPACE");
var uri = java.text.MessageFormat.format("{0}/api/w/{1}/resources/get_value_interpolated/{2}", baseUrl, workspace, path);
// Create an HttpRequest
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(uri))
.header("Authorization", "Bearer " + System.getenv("WM_TOKEN")) // Add the Authorization header
.GET() // Set the request method to GET
.build();
// Send the request and get the response
return HttpClient.newHttpClient().sendAsync(request, HttpResponse.BodyHandlers.ofString())
.thenApply(HttpResponse::body)
.join(); // Wait for the completion
}
}
"#;
+6 -2
View File
@@ -11,6 +11,10 @@ mod bash_executor;
#[cfg(feature = "nu")]
mod nu_executor;
#[cfg(feature = "java")]
mod java_executor;
#[cfg(feature = "benchmark")]
pub mod bench;
mod bun_executor;
@@ -40,11 +44,11 @@ mod python_executor;
mod result_processor;
#[cfg(feature = "rust")]
mod rust_executor;
mod sanitized_sql_params;
mod schema;
mod worker;
mod worker_flow;
mod worker_lockfiles;
mod sanitized_sql_params;
mod schema;
pub use worker::*;
@@ -334,6 +334,7 @@ async fn run<'a>(
job.timeout,
false,
&mut Some(occupancy_metrics),
None,
)
.await
}
@@ -105,6 +105,7 @@ pub async fn composer_install(
None,
false,
&mut Some(occupancy_metrics),
None,
)
.await?;
@@ -339,6 +340,7 @@ try {{
job.timeout,
false,
&mut Some(occupancy_metrics),
None,
)
.await?;
read_result(job_dir).await
+7 -40
View File
@@ -27,7 +27,7 @@ use windmill_common::{
Error::{self},
},
utils::calculate_hash,
worker::{write_file, PythonAnnotations, WORKER_CONFIG},
worker::{copy_dir_recursively, pad_string, write_file, PythonAnnotations, WORKER_CONFIG},
DB,
};
@@ -49,6 +49,7 @@ lazy_static::lazy_static! {
static ref PY_CONCURRENT_DOWNLOADS: usize =
var("PY_CONCURRENT_DOWNLOADS").ok().map(|flag| flag.parse().unwrap_or(20)).unwrap_or(20);
static ref NON_ALPHANUM_CHAR: Regex = regex::Regex::new(r"[^0-9A-Za-z=.-]").unwrap();
static ref TRUSTED_HOST: Option<String> = var("PY_TRUSTED_HOST").ok().or(var("PIP_TRUSTED_HOST").ok());
@@ -350,6 +351,7 @@ impl PyVersion {
None,
false,
occupancy_metrics,
None,
)
.await
}
@@ -644,6 +646,7 @@ pub async fn uv_pip_compile(
None,
false,
occupancy_metrics,
None,
)
.await
.map_err(|e| {
@@ -780,30 +783,6 @@ async fn postinstall(
Ok(())
}
fn copy_dir_recursively(src: &Path, dst: &Path) -> windmill_common::error::Result<()> {
if !dst.exists() {
fs::create_dir_all(dst)?;
}
tracing::debug!("Copying recursively from {:?} to {:?}", src, dst);
for entry in fs::read_dir(src)? {
let entry = entry?;
let src_path = entry.path();
let dst_path = dst.join(entry.file_name());
if src_path.is_dir() && !src_path.is_symlink() {
copy_dir_recursively(&src_path, &dst_path)?;
} else {
fs::copy(&src_path, &dst_path)?;
}
}
tracing::debug!("Finished copying recursively from {:?} to {:?}", src, dst);
Ok(())
}
async fn get_python_path(
py_version: PyVersion,
worker_name: &str,
@@ -1161,6 +1140,7 @@ mount {{
job.timeout,
false,
&mut Some(occupancy_metrics),
None,
)
.await?;
@@ -1709,19 +1689,6 @@ async fn spawn_uv_install(
}
}
/// length = 5
/// value = "foo"
/// output = "foo "
/// 12345
fn pad_string(value: &str, total_length: usize) -> String {
if value.len() >= total_length {
value.to_string() // Return the original string if it's already long enough
} else {
let padding_needed = total_length - value.len();
format!("{value}{}", " ".repeat(padding_needed)) // Pad with spaces
}
}
/// uv pip install, include cached or pull from S3
pub async fn handle_python_reqs(
requirements: Vec<&str>,
@@ -2053,7 +2020,7 @@ pub async fn handle_python_reqs(
tokio::select! {
// Cancel was called on the job
_ = kill_rx.recv() => return Err(anyhow::anyhow!("S3 pull was canceled")),
pull = pull_from_tar(os, venv_p.clone(), py_version.to_cache_dir_top_level()) => {
pull = pull_from_tar(os, venv_p.clone(), py_version.to_cache_dir_top_level(), None, false) => {
if let Err(e) = pull {
tracing::info!(
workspace_id = %w_id,
@@ -2204,7 +2171,7 @@ pub async fn handle_python_reqs(
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
if s3_push {
if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() {
tokio::spawn(build_tar_and_push(os, venv_p.clone(), py_version.to_cache_dir_top_level()));
tokio::spawn(build_tar_and_push(os, venv_p.clone(), py_version.to_cache_dir_top_level(), None, false));
}
}
+6 -1
View File
@@ -164,6 +164,7 @@ pub async fn generate_cargo_lockfile(
None,
false,
&mut Some(occupancy_metrics),
None,
)
.await?;
@@ -226,6 +227,7 @@ pub async fn build_rust_crate(
None,
false,
&mut Some(occupancy_metrics),
None,
)
.await?;
append_logs(job_id, w_id, "\n\n", db).await;
@@ -245,6 +247,7 @@ pub async fn build_rust_crate(
&bin_path,
&format!("{RUST_OBJECT_STORE_PREFIX}{hash}"),
&format!("{job_dir}/main"),
false,
)
.await
{
@@ -294,7 +297,8 @@ pub async fn handle_rust_job(
let bin_path = format!("{}/{hash}", RUST_CACHE_DIR);
let remote_path = format!("{RUST_OBJECT_STORE_PREFIX}{hash}");
let (cache, cache_logs) = windmill_common::worker::load_cache(&bin_path, &remote_path).await;
let (cache, cache_logs) =
windmill_common::worker::load_cache(&bin_path, &remote_path, false).await;
let cache_logs = if cache {
let target = format!("{job_dir}/main");
@@ -407,6 +411,7 @@ pub async fn handle_rust_job(
job.timeout,
false,
&mut Some(occupancy_metrics),
None,
)
.await?;
read_result(job_dir).await
+49 -2
View File
@@ -123,7 +123,10 @@ use crate::{
use crate::rust_executor::handle_rust_job;
#[cfg(feature = "nu")]
use crate::nu_executor::{handle_nu_job, JobHandlerInput};
use crate::nu_executor::{handle_nu_job, JobHandlerInput as JobHandlerInputNu};
#[cfg(feature = "java")]
use crate::java_executor::{handle_java_job, JobHandlerInput as JobHandlerInputJava};
#[cfg(feature = "php")]
use crate::php_executor::handle_php_job;
@@ -264,6 +267,8 @@ pub const TAR_PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_311"
pub const TAR_PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_312");
pub const TAR_PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_313");
pub const TAR_JAVA_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/java");
pub const UV_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "uv");
pub const PY_INSTALL_DIR: &str = concatcp!(ROOT_CACHE_DIR, "py_runtime");
pub const TAR_PYBASE_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar");
@@ -275,6 +280,12 @@ pub const GO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "go");
pub const RUST_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "rust");
pub const NU_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "nu");
pub const CSHARP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "csharp");
// JAVA
pub const JAVA_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "java");
pub const COURSIER_CACHE_DIR: &str = concatcp!(JAVA_CACHE_DIR, "/coursier-cache");
pub const JAVA_REPOSITORY_DIR: &str = concatcp!(JAVA_CACHE_DIR, "/repository");
// KJQXZ
pub const BUN_CACHE_DIR: &str = concatcp!(ROOT_CACHE_NOMOUNT_DIR, "bun");
pub const BUN_BUNDLE_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "bun");
pub const BUN_CODEBASE_BUNDLE_CACHE_DIR: &str = concatcp!(ROOT_CACHE_NOMOUNT_DIR, "script_bundle");
@@ -398,6 +409,11 @@ lazy_static::lazy_static! {
pub static ref NPM_CONFIG_REGISTRY: Arc<RwLock<Option<String>>> = Arc::new(RwLock::new(None));
pub static ref BUNFIG_INSTALL_SCOPES: Arc<RwLock<Option<String>>> = Arc::new(RwLock::new(None));
pub static ref NUGET_CONFIG: Arc<RwLock<Option<String>>> = Arc::new(RwLock::new(None));
pub static ref MAVEN_REPOS: Arc<RwLock<Option<String>>> = Arc::new(RwLock::new(None));
pub static ref NO_DEFAULT_MAVEN: AtomicBool = AtomicBool::new(std::env::var("NO_DEFAULT_MAVEN")
.ok()
.and_then(|x| x.parse::<bool>().ok())
.unwrap_or(false));
pub static ref PIP_EXTRA_INDEX_URL: Arc<RwLock<Option<String>>> = Arc::new(RwLock::new(None));
pub static ref PIP_INDEX_URL: Arc<RwLock<Option<String>>> = Arc::new(RwLock::new(None));
@@ -3078,7 +3094,33 @@ mount {{
);
#[cfg(feature = "nu")]
handle_nu_job(JobHandlerInput {
handle_nu_job(JobHandlerInputNu {
mem_peak,
canceled_by,
job,
db,
client,
parent_runnable_path,
inner_content: &code,
job_dir,
requirements_o: lock.as_ref(),
shared_mount: &shared_mount,
base_internal_url,
worker_name,
envs,
occupancy_metrics,
})
.await
}
Some(ScriptLang::Java) => {
#[cfg(not(feature = "java"))]
return Err(anyhow::anyhow!(
"Java is not available because the feature is not enabled"
)
.into());
#[cfg(feature = "java")]
handle_java_job(JobHandlerInputJava {
mem_peak,
canceled_by,
job,
@@ -3163,6 +3205,11 @@ fn parse_sig_of_lang(
ScriptLang::Nu => Some(windmill_parser_nu::parse_nu_signature(code)?),
#[cfg(not(feature = "nu"))]
ScriptLang::Nu => None,
#[cfg(feature = "java")]
ScriptLang::Java => Some(windmill_parser_java::parse_java_signature(code)?),
#[cfg(not(feature = "java"))]
ScriptLang::Java => None,
// KJQXZ
}
} else {
None
+15 -11
View File
@@ -36,6 +36,9 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob, PushIsolationLevel}
use crate::common::OccupancyMetrics;
use crate::csharp_executor::generate_nuget_lockfile;
#[cfg(feature = "java")]
use crate::java_executor::resolve;
#[cfg(feature = "php")]
use crate::php_executor::{composer_install, parse_php_imports};
#[cfg(feature = "python")]
@@ -2076,16 +2079,17 @@ async fn capture_dependency_job(
)
.await
}
ScriptLang::Postgresql => Ok("".to_owned()),
ScriptLang::Mysql => Ok("".to_owned()),
ScriptLang::Bigquery => Ok("".to_owned()),
ScriptLang::Snowflake => Ok("".to_owned()),
ScriptLang::Mssql => Ok("".to_owned()),
ScriptLang::Graphql => Ok("".to_owned()),
ScriptLang::OracleDB => Ok("".to_owned()),
ScriptLang::Bash => Ok("".to_owned()),
ScriptLang::Nu => Ok("".to_owned()),
ScriptLang::Powershell => Ok("".to_owned()),
ScriptLang::Nativets => Ok("".to_owned()),
#[cfg(feature = "java")]
ScriptLang::Java => {
if raw_deps {
return Err(Error::ExecutionErr(
"Raw dependencies not supported for Java".to_string(),
));
}
resolve(job_id, job_raw_code, job_dir, db, w_id).await
}
// KJQXZ
_ => Ok("".to_owned()),
}
}
+8
View File
@@ -119,4 +119,12 @@ inventory:
debug:
msg: "Hello, world!"
`,
java: `
public class Main {
public static void main() {
System.out.println("Hello World");
}
}
`,
// KJQXZ
};
+1
View File
@@ -69,6 +69,7 @@ await build({
"rust",
"yaml",
"csharp",
"java",
];
for (const l of dirs) {
+1 -1
View File
@@ -8652,4 +8652,4 @@ export const clearIndex = (data: ClearIndexData): CancelablePromise<ClearIndexRe
path: {
idx_name: data.idxName
}
}); };
}); };
+2 -2
View File
@@ -389,7 +389,7 @@ export type MainArgSignature = {
export type type2 = 'Valid' | 'Invalid';
export type ScriptLang = 'python3' | 'deno' | 'go' | 'bash' | 'powershell' | 'postgresql' | 'mysql' | 'bigquery' | 'snowflake' | 'mssql' | 'oracledb' | 'graphql' | 'nativets' | 'bun' | 'php' | 'rust' | 'ansible' | 'csharp' | 'nu';
export type ScriptLang = 'python3' | 'deno' | 'go' | 'bash' | 'powershell' | 'postgresql' | 'mysql' | 'bigquery' | 'snowflake' | 'mssql' | 'oracledb' | 'graphql' | 'nativets' | 'bun' | 'php' | 'rust' | 'ansible' | 'csharp' | 'nu' | 'java';
export type Preview = {
content?: string;
@@ -7521,4 +7521,4 @@ export type ClearIndexData = {
idxName: 'JobIndex' | 'ServiceLogIndex';
};
export type ClearIndexResponse = (string);
export type ClearIndexResponse = (string);
+4 -2
View File
@@ -12,8 +12,6 @@ import {
ScriptMetadata,
defaultScriptMetadata,
} from "./bootstrap/script_bootstrap.ts";
import { Workspace } from "./workspace.ts";
import { SchemaProperty } from "./bootstrap/common.ts";
import { ScriptLanguage } from "./script_common.ts";
@@ -550,6 +548,10 @@ export async function inferSchema(
} else if (language === "ansible") {
const { parse_ansible } = await import("./wasm/yaml/windmill_parser_wasm.js");
inferedSchema = JSON.parse(parse_ansible(content));
} else if (language === "java") {
const { parse_java } = await import("./wasm/java/windmill_parser_wasm.js");
inferedSchema = JSON.parse(parse_java(content));
// KJQXZ
} else {
throw new Error("Invalid language: " + language);
}
+6
View File
@@ -569,6 +569,9 @@ export function filePathExtensionFromContentType(
return ".cs";
} else if (language === "nu") {
return ".nu";
} else if (language === "java") {
return ".java";
// KJQXZ
} else {
throw new Error("Invalid language: " + language);
}
@@ -596,6 +599,9 @@ export const exts = [
".cs",
".nu",
".playbook.yml",
".java"
// KJQXZ
];
export function removeExtensionToPath(path: string): string {
+6 -1
View File
@@ -17,7 +17,9 @@ export type ScriptLanguage =
| "rust"
| "csharp"
| "nu"
| "ansible";
| "ansible"
| "java";
// KJQXZ
export function inferContentTypeFromFilePath(
contentPath: string,
@@ -63,6 +65,9 @@ export function inferContentTypeFromFilePath(
return "ansible";
} else if (contentPath.endsWith(".nu")) {
return "nu";
} else if (contentPath.endsWith(".java")) {
return "java";
// KJQXZ
} else {
throw new Error(
"Invalid language: " + contentPath.substring(contentPath.lastIndexOf("."))
+4
View File
@@ -342,6 +342,8 @@ export function newPathAssigner(defaultTs: "bun" | "deno"): PathAssigner {
else if (language == "csharp") ext = "cs";
else if (language == "nu") ext = "nu";
else if (language == "ansible") ext = "playbook.yml";
else if (language == "java") ext = "java";
// KJQXZ
else ext = "no_ext";
return [`${name}.inline_script.`, ext];
@@ -689,6 +691,8 @@ export async function elementsToMap(
"cs",
"yml",
"nu",
"java",
// KJQXZ
].includes(path.split(".").pop() ?? "") &&
!isFileResource(path)
)
+2
View File
@@ -227,6 +227,8 @@ export function getTypeStrFromPath(
parsed.ext == ".rs" ||
parsed.ext == ".cs" ||
parsed.ext == ".nu" ||
parsed.ext == ".java" ||
// KJQXZ
(parsed.ext == ".yml" && parsed.name.split(".").pop() == "playbook")
) {
return "script";
+3
View File
@@ -0,0 +1,3 @@
/* tslint:disable */
/* eslint-disable */
export function parse_java(code: string): string;
+121
View File
@@ -0,0 +1,121 @@
let WASM_VECTOR_LEN = 0;
let cachedUint8ArrayMemory0 = null;
function getUint8ArrayMemory0() {
if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) {
cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer);
}
return cachedUint8ArrayMemory0;
}
const cachedTextEncoder = (typeof TextEncoder !== 'undefined' ? new TextEncoder('utf-8') : { encode: () => { throw Error('TextEncoder not available') } } );
const encodeString = function (arg, view) {
return cachedTextEncoder.encodeInto(arg, view);
};
function passStringToWasm0(arg, malloc, realloc) {
if (realloc === undefined) {
const buf = cachedTextEncoder.encode(arg);
const ptr = malloc(buf.length, 1) >>> 0;
getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf);
WASM_VECTOR_LEN = buf.length;
return ptr;
}
let len = arg.length;
let ptr = malloc(len, 1) >>> 0;
const mem = getUint8ArrayMemory0();
let offset = 0;
for (; offset < len; offset++) {
const code = arg.charCodeAt(offset);
if (code > 0x7F) break;
mem[ptr + offset] = code;
}
if (offset !== len) {
if (offset !== 0) {
arg = arg.slice(offset);
}
ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0;
const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len);
const ret = encodeString(arg, view);
offset += ret.written;
ptr = realloc(ptr, len, offset, 1) >>> 0;
}
WASM_VECTOR_LEN = offset;
return ptr;
}
const cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } );
if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); };
function getStringFromWasm0(ptr, len) {
ptr = ptr >>> 0;
return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len));
}
/**
* @param {string} code
* @returns {string}
*/
export function parse_java(code) {
let deferred2_0;
let deferred2_1;
try {
const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
const len0 = WASM_VECTOR_LEN;
const ret = wasm.parse_java(ptr0, len0);
deferred2_0 = ret[0];
deferred2_1 = ret[1];
return getStringFromWasm0(ret[0], ret[1]);
} finally {
wasm.__wbindgen_free(deferred2_0, deferred2_1, 1);
}
}
const imports = {
__wbindgen_placeholder__: {
__wbindgen_init_externref_table: function() {
const table = wasm.__wbindgen_export_0;
const offset = table.grow(4);
table.set(0, undefined);
table.set(offset + 0, undefined);
table.set(offset + 1, null);
table.set(offset + 2, true);
table.set(offset + 3, false);
;
},
},
};
const wasm_url = new URL('windmill_parser_wasm_bg.wasm', import.meta.url);
let wasmCode = '';
switch (wasm_url.protocol) {
case 'file:':
wasmCode = await Deno.readFile(wasm_url);
break
case 'https:':
case 'http:':
wasmCode = await (await fetch(wasm_url)).arrayBuffer();
break
default:
throw new Error(`Unsupported protocol: ${wasm_url.protocol}`);
}
const wasmInstance = (await WebAssembly.instantiate(wasmCode, imports)).instance;
const wasm = wasmInstance.exports;
export const __wasm = wasm;
wasm.__wbindgen_start();
Binary file not shown.
+29
View File
@@ -0,0 +1,29 @@
/* tslint:disable */
/* eslint-disable */
export const memory: WebAssembly.Memory;
export const parse_java: (a: number, b: number) => [number, number];
export const abort: () => void;
export const malloc: (a: number) => number;
export const calloc: (a: number, b: number) => number;
export const realloc: (a: number, b: number) => number;
export const free: (a: number) => void;
export const strncmp: (a: number, b: number, c: number) => number;
export const iswspace: (a: number) => number;
export const iswalnum: (a: number) => number;
export const clock: () => bigint;
export const isprint: (a: number) => number;
export const fprintf: (a: number, b: number, c: number) => number;
export const fputs: (a: number, b: number) => number;
export const fputc: (a: number, b: number) => number;
export const fdopen: (a: number, b: number) => number;
export const fclose: (a: number) => number;
export const fwrite: (a: number, b: number, c: number, d: number) => number;
export const vsnprintf: (a: number, b: number, c: number, d: number) => number;
export const clock_gettime: (a: number, b: number) => void;
export const snprintf: () => void;
export const __assert_fail: (a: number, b: number, c: number, d: number) => void;
export const __wbindgen_export_0: WebAssembly.Table;
export const __wbindgen_malloc: (a: number, b: number) => number;
export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
export const __wbindgen_free: (a: number, b: number, c: number) => void;
export const __wbindgen_start: () => void;
+7
View File
@@ -14,3 +14,10 @@ ENV DOTNET_ROOT="/opt/dotnet-sdk/bin"
# Nushell
COPY --from=ghcr.io/nushell/nushell:0.101.0-bookworm /usr/bin/nu /usr/bin/nu
# Java
RUN apt-get -y update && apt-get install -y default-jdk
RUN curl -fLo coursier https://github.com/coursier/coursier/releases/download/v2.1.24/coursier \
&& mv ./coursier /usr/bin/coursier \
&& chmod +x /usr/bin/coursier
RUN /usr/bin/java -jar /usr/bin/coursier about
+7
View File
@@ -37,3 +37,10 @@ RUN echo /opt/oracle/23/lib > /etc/ld.so.conf.d/oracle-instantclient.conf && ldc
# Nushell
COPY --from=ghcr.io/nushell/nushell:0.101.0-bookworm /usr/bin/nu /usr/bin/nu
# Java
RUN apt-get -y update && apt-get install -y default-jdk
RUN curl -fLo coursier https://github.com/coursier/coursier/releases/download/v2.1.24/coursier \
&& mv ./coursier /usr/bin/coursier \
&& chmod +x /usr/bin/coursier
RUN /usr/bin/java -jar /usr/bin/coursier about
+8
View File
@@ -62,4 +62,12 @@ RUN echo /opt/oracle/23/lib > /etc/ld.so.conf.d/oracle-instantclient.conf && ldc
# Nushell
COPY --from=ghcr.io/nushell/nushell:0.101.0-bookworm /usr/bin/nu /usr/bin/nu
# Java
RUN apt-get install -y default-jdk
RUN curl -fLo coursier https://github.com/coursier/coursier/releases/download/v2.1.24/coursier \
&& mv ./coursier /usr/bin/coursier \
&& chmod +x /usr/bin/coursier
RUN /usr/bin/java -jar /usr/bin/coursier about
COPY --from=nsjail /nsjail/nsjail /bin/nsjail
+35 -8
View File
@@ -31,6 +31,13 @@
pkg-config
cmake
];
coursier = pkgs.fetchFromGitHub {
owner = "coursier";
repo = "launchers";
rev = "79d927f7586c09ca6d8cd01862adb0d9f9d88dff";
hash = "sha256-8E0WtDFc7RcqmftDigMyy1xXUkjgL4X4kpf7h1GdE48=";
};
PKG_CONFIG_PATH = pkgs.lib.makeSearchPath "lib/pkgconfig"
(with pkgs; [ openssl.dev libxml2.dev xmlsec.dev libxslt.dev ]);
RUSTY_V8_ARCHIVE = let
@@ -75,6 +82,7 @@
sqlx-cli
flock
sccache
nsjail
deno
python3
python3Packages.pip
@@ -84,7 +92,9 @@
nushell
dotnet-sdk_9
oracle-instantclient
# LSP/Local dev
svelte-language-server
taplo
]);
packages = [
(pkgs.writeScriptBin "wm-caddy" ''
@@ -126,6 +136,25 @@
cd ./frontend
npm run dev $*
'')
(pkgs.writeScriptBin "wm-minio" ''
set -e
cd ./backend
mkdir -p .minio-data/wmill
${pkgs.minio}/bin/minio server ./.minio-data
'')
# Generate keys
# TODO: Do not set new keys if ran multiple times
(pkgs.writeScriptBin "wm-minio-keys" ''
set -e
cd ./backend
${pkgs.minio-client}/bin/mc alias set 'wmill-minio-dev' 'http://localhost:9000' 'minioadmin' 'minioadmin'
${pkgs.minio-client}/bin/mc admin accesskey create myminio | tee .minio-data/secrets.txt
echo ""
echo 'Saving to: ./backend/.minio-data/secrets.txt'
echo "bucket: wmill"
echo "endpoint: http://localhost:9000"
'')
];
inherit PKG_CONFIG_PATH RUSTY_V8_ARCHIVE;
@@ -140,11 +169,16 @@
BUN_PATH = "${pkgs.bun}/bin/bun";
UV_PATH = "${pkgs.uv}/bin/uv";
NU_PATH = "${pkgs.nushell}/bin/nu";
JAVA_PATH = "${pkgs.jdk21}/bin/java";
JAVAC_PATH = "${pkgs.jdk21}/bin/javac";
COURSIER_PATH = "${coursier}/coursier";
# KJQXZ
FLOCK_PATH = "${pkgs.flock}/bin/flock";
CARGO_PATH = "${rust}/bin/cargo";
DOTNET_PATH = "${pkgs.dotnet-sdk_9}/bin/dotnet";
DOTNET_ROOT = "${pkgs.dotnet-sdk_9}/share/dotnet";
ORACLE_LIB_DIR = "${pkgs.oracle-instantclient.lib}/lib";
RUST_LOG = "debug";
};
packages.default = self.packages.${system}.windmill;
packages.windmill-client = pkgs.buildNpmPackage {
@@ -218,7 +252,6 @@
"cloud"
"jemalloc"
"tantivy"
"deno_core"
"license"
"http_trigger"
"zip"
@@ -226,16 +259,10 @@
"kafka"
"otel"
"dind"
"php"
"mysql"
"mssql"
"bigquery"
"websocket"
"python"
"smtp"
"csharp"
"static_frontend"
"rust"
"all_languages"
];
doCheck = false;
+6
View File
@@ -68,6 +68,7 @@
"vscode-ws-jsonrpc": "~3.4.0",
"windmill-parser-wasm-csharp": "^1.437.1",
"windmill-parser-wasm-go": "^1.429.0",
"windmill-parser-wasm-java": "^1.478.1",
"windmill-parser-wasm-nu": "^1.474.1",
"windmill-parser-wasm-php": "^1.429.0",
"windmill-parser-wasm-py": "^1.477.1",
@@ -12528,6 +12529,11 @@
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-go/-/windmill-parser-wasm-go-1.429.0.tgz",
"integrity": "sha512-M3jeGDqeTyPj9HyyX3msdzMrqIIzlfMfxTMsXS8m7MJp4Cm60qifMxD29Ipxb2B4WdzyGwCSlaBjLsXu0b3c5g=="
},
"node_modules/windmill-parser-wasm-java": {
"version": "1.478.1",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-java/-/windmill-parser-wasm-java-1.478.1.tgz",
"integrity": "sha512-2gMr5pXaEExoLjeSMOpHzY4UjDU/JGjvlXSnBiQDHUaEoKSS7FWa5APzBKcmn8cM+mr2p8m4sMCyTXS88At0fQ=="
},
"node_modules/windmill-parser-wasm-nu": {
"version": "1.474.1",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-nu/-/windmill-parser-wasm-nu-1.474.1.tgz",
+1
View File
@@ -144,6 +144,7 @@
"vscode-ws-jsonrpc": "~3.4.0",
"windmill-parser-wasm-csharp": "^1.437.1",
"windmill-parser-wasm-go": "^1.429.0",
"windmill-parser-wasm-java": "^1.478.1",
"windmill-parser-wasm-nu": "^1.474.1",
"windmill-parser-wasm-php": "^1.429.0",
"windmill-parser-wasm-py": "^1.477.1",
@@ -195,6 +195,8 @@
| 'yaml'
| 'csharp'
| 'nu'
| 'java'
// KJQXZ
export let code: string = ''
export let cmdEnterAction: (() => void) | undefined = undefined
export let formatAction: (() => void) | undefined = undefined
+19 -3
View File
@@ -93,7 +93,9 @@
'php',
'rust',
'csharp',
'nu'
'nu',
'java',
// KJQXZ
].includes(lang ?? '')
$: showVarPicker = [
'python3',
@@ -107,7 +109,9 @@
'php',
'rust',
'csharp',
'nu'
'nu',
'java',
// KJQXZ
].includes(lang ?? '')
$: showResourcePicker = [
'python3',
@@ -121,7 +125,9 @@
'php',
'rust',
'csharp',
'nu'
'nu',
'java',
// KJQXZ
].includes(lang ?? '')
$: showResourceTypePicker =
['typescript', 'javascript'].includes(scriptLangToEditorLang(lang)) ||
@@ -373,6 +379,9 @@
editor.insertAtCursor(`Environment.GetEnvironmentVariable("${name}");`)
} else if (lang == 'nu') {
editor.insertAtCursor(`$env.${name}`)
} else if (lang == 'java') {
editor.insertAtCursor(`System.getenv("${name}");`)
// KJQXZ
}
sendUserToast(`${name} inserted at cursor`)
}}
@@ -440,6 +449,9 @@ string ${windmillPathToCamelCaseName(path)} = await client.GetStringAsync(uri);
`)
} else if (lang == 'nu') {
editor.insertAtCursor(`get_variable ${path}`)
} else if (lang == 'java') {
editor.insertAtCursor(`(Wmill.getVariable("${path}"))`)
// KJQXZ
}
sendUserToast(`${name} inserted at cursor`)
}}
@@ -524,7 +536,11 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS
`)
} else if (lang == 'nu') {
editor.insertAtCursor(`get_resource ${path}`)
} else if (lang == 'java') {
editor.insertAtCursor(`(Wmill.getResource("${path}"))`)
// KJQXZ
}
sendUserToast(`${path} inserted at cursor`)
}}
tooltip="Resources represent connections to third party systems. Resources are a good way to define a connection to a frequently used third party system such as a database."
@@ -12,6 +12,7 @@
import rust from 'svelte-highlight/languages/rust'
import csharp from 'svelte-highlight/languages/csharp'
import yaml from 'svelte-highlight/languages/yaml'
import java from 'svelte-highlight/languages/java'
import type { Script } from '$lib/gen'
import { Button } from './common'
import { copyToClipboard } from '$lib/utils'
@@ -65,7 +66,10 @@
case 'nu':
return python
case 'ansible':
return yaml
return yaml;
case 'java':
return java;
// KJQXZ
default:
return typescript
}
@@ -78,7 +78,7 @@
deno: false,
go: false,
ruff: false,
shellcheck: false
shellcheck: false,
}
const dispatch = createEventDispatcher()
@@ -123,7 +123,9 @@
'rust',
'ansible',
'csharp',
'nu'
'nu',
'java'
// KJQXZ
]
const nativeTags = [
'nativets',
@@ -21,6 +21,7 @@
import AnsibleIcon from '$lib/components/icons/AnsibleIcon.svelte'
import CSharpIcon from '$lib/components/icons/CSharpIcon.svelte'
import NuIcon from '$lib/components/icons/NuIcon.svelte'
import JavaIcon from '$lib/components/icons/JavaIcon.svelte'
export let lang:
| SupportedLanguage
@@ -56,7 +57,9 @@
rust: 'Rust',
ansible: 'Ansible Playbook',
csharp: 'C#',
nu: 'Nu'
nu: 'Nu',
java: 'Java'
// KJQXZ
}
const langToComponent: Record<
@@ -87,7 +90,9 @@
rust: RustIcon,
ansible: AnsibleIcon,
csharp: CSharpIcon,
nu: NuIcon
nu: NuIcon,
java: JavaIcon
// KJQXZ
}
let subIconScale = width === 30 ? 0.6 : 0.8
@@ -0,0 +1,6 @@
<script>
export let height = 24
export let width = 24
</script>
<svg xmlns="http://www.w3.org/2000/svg" {width} {height} viewBox="-60.6242 20 378.0764 511.359"><path d="M278.577 426.546h-2.038v-1.14h5.486v1.14h-2.024v5.688h-1.424zm10.942.297h-.032l-2.019 5.392h-.924l-2.006-5.392h-.025v5.392h-1.342v-6.828h1.975l1.86 4.835 1.854-4.835h1.968v6.828h-1.31v-5.392zM96.154 286.925s-14.178 8.245 10.09 11.035c29.4 3.354 44.426 2.873 76.825-3.259 0 0 8.518 5.341 20.414 9.967-72.63 31.128-164.376-1.803-107.329-17.743m-8.875-40.62s-15.902 11.771 8.384 14.283c31.406 3.24 56.208 3.505 99.125-4.759 0 0 5.936 6.018 15.27 9.309-87.814 25.678-185.623 2.025-122.779-18.833" fill="#5382a1"/><path d="M162.098 177.4c17.896 20.604-4.702 39.145-4.702 39.145s45.441-23.458 24.572-52.833c-19.491-27.394-34.438-41.005 46.479-87.934 0 0-127.013 31.722-66.349 101.622" fill="#f8981d"/><path d="M258.157 316.97s10.492 8.645-11.555 15.333c-41.923 12.7-174.488 16.535-211.314.506-13.238-5.759 11.587-13.751 19.396-15.428 8.144-1.766 12.798-1.437 12.798-1.437-14.722-10.371-95.157 20.364-40.857 29.166 148.084 24.015 269.943-10.814 231.532-28.14M102.972 204.218s-67.431 16.016-23.879 21.832c18.389 2.462 55.047 1.905 89.193-.956 27.906-2.354 55.927-7.359 55.927-7.359s-9.84 4.214-16.959 9.075c-68.475 18.009-200.756 9.631-162.674-8.79 32.206-15.568 58.392-13.802 58.392-13.802m120.963 67.614c69.608-36.171 37.424-70.931 14.96-66.248-5.506 1.146-7.961 2.139-7.961 2.139s2.044-3.202 5.948-4.588c44.441-15.624 78.619 46.081-14.346 70.52 0 .001 1.077-.962 1.399-1.823" fill="#5382a1"/><path d="M181.968 0s38.55 38.563-36.563 97.862c-60.233 47.568-13.735 74.69-.025 105.678-35.159-31.722-60.961-59.647-43.651-85.637C127.136 79.752 197.522 61.255 181.968 0" fill="#f8981d"/><path d="M109.812 369.847c66.815 4.277 169.417-2.373 171.847-33.988 0 0-4.671 11.985-55.219 21.503-57.028 10.732-127.364 9.479-169.081 2.601.001-.001 8.54 7.068 52.453 9.884m-10.95 120.802c-6.303 5.467-12.96 8.536-18.934 8.536-8.527 0-13.134-5.113-13.134-13.314 0-8.871 4.936-15.357 24.739-15.357h7.328v20.135m17.393 19.623V449.53c0-15.517-8.85-25.756-30.188-25.756-12.457 0-23.369 3.076-32.238 6.999l2.56 10.752c6.983-2.563 16.022-4.949 24.894-4.949 12.292 0 17.58 4.949 17.58 15.181v7.677h-6.135c-29.865 0-43.337 11.593-43.337 28.994 0 15.017 8.878 23.553 25.594 23.553 10.745 0 18.766-4.436 26.264-10.928l1.361 9.22h13.645zm58.044 0h-21.692l-26.106-84.96h18.943l16.199 52.2 3.601 15.699c8.195-22.698 13.991-45.726 16.89-67.899h18.427c-4.937 27.977-13.821 58.685-26.262 84.96m83.213-19.623c-6.315 5.467-12.984 8.536-18.958 8.536-8.512 0-13.131-5.113-13.131-13.314 0-8.871 4.948-15.357 24.749-15.357h7.34v20.135m17.39 19.623V449.53c0-15.517-8.872-25.756-30.185-25.756-12.466 0-23.382 3.076-32.247 6.999l2.556 10.752c6.986-2.563 16.042-4.949 24.907-4.949 12.283 0 17.579 4.949 17.579 15.181v7.677h-6.145c-29.874 0-43.34 11.593-43.34 28.994 0 15.017 8.871 23.553 25.584 23.553 10.751 0 18.769-4.436 26.28-10.928l1.366 9.22h13.645zM30.32 524.7c-4.958 7.239-12.966 12.966-21.733 16.206L0 530.801c6.673-3.424 12.396-8.954 15.055-14.105 2.3-4.581 3.252-10.485 3.252-24.604v-96.995h18.478v95.666c-.001 18.876-1.51 26.501-6.465 33.937" fill="#5382a1"/></svg>
@@ -302,7 +302,24 @@ export const settings: Record<string, Setting[]> = {
codeAreaLang: 'xml',
storage: 'setting',
ee_only: ''
}
},
{
label: 'Maven/Ivy repositories',
description: 'Add private Maven/Ivy repositories',
key: 'maven_repos',
fieldType: 'text',
placeholder: 'https://user:password@artifacts.foo.com/maven',
storage: 'setting',
ee_only: ''
},
{
label: 'Disable default Maven repository',
description: 'Do not use default Maven repository',
key: 'no_default_maven',
fieldType: 'boolean',
storage: 'setting',
ee_only: ''
},
],
Alerts: [
{
+3
View File
@@ -84,6 +84,9 @@ export function langToExt(lang: string): string {
return 'cs'
case 'nu':
return 'nu'
case 'java':
return 'java'
// KJQXZ
default:
return 'unknown'
}
+9
View File
@@ -24,6 +24,7 @@ import initRustParser, { parse_rust } from 'windmill-parser-wasm-rust'
import initYamlParser, { parse_ansible } from 'windmill-parser-wasm-yaml'
import initCSharpParser, { parse_csharp } from 'windmill-parser-wasm-csharp'
import initNuParser, { parse_nu } from 'windmill-parser-wasm-nu'
import initJavaParser, { parse_java } from 'windmill-parser-wasm-java'
import wasmUrlTs from 'windmill-parser-wasm-ts/windmill_parser_wasm_bg.wasm?url'
import wasmUrlRegex from 'windmill-parser-wasm-regex/windmill_parser_wasm_bg.wasm?url'
@@ -34,6 +35,7 @@ import wasmUrlRust from 'windmill-parser-wasm-rust/windmill_parser_wasm_bg.wasm?
import wasmUrlYaml from 'windmill-parser-wasm-yaml/windmill_parser_wasm_bg.wasm?url'
import wasmUrlCSharp from 'windmill-parser-wasm-csharp/windmill_parser_wasm_bg.wasm?url'
import wasmUrlNu from 'windmill-parser-wasm-nu/windmill_parser_wasm_bg.wasm?url'
import wasmUrlJava from 'windmill-parser-wasm-java/windmill_parser_wasm_bg.wasm?url'
import { workspaceStore } from './stores.js'
import { argSigToJsonSchemaType } from './inferArgSig.js'
@@ -71,6 +73,9 @@ async function initWasmCSharp() {
async function initWasmNu() {
await initNuParser(wasmUrlNu)
}
async function initWasmJava() {
await initJavaParser(wasmUrlJava)
}
export async function inferArgs(
language: SupportedLanguage | 'bunnative' | undefined,
@@ -186,6 +191,10 @@ export async function inferArgs(
} else if (language == 'nu') {
await initWasmNu()
inferedSchema = JSON.parse(parse_nu(code))
} else if (language == 'java') {
await initWasmJava()
inferedSchema = JSON.parse(parse_java(code))
// KJQXZ
} else {
return null
}
+63 -1
View File
@@ -945,6 +945,61 @@ dependencies:
content: "{{ my_result | to_json }}"
dest: result.json
`
const JAVA_INIT_CODE = `//requirements:
//com.google.code.gson:gson:2.8.9
//com.github.ricksbrown:cowsay:1.1.0
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.github.ricksbrown.cowsay.Cowsay;
import com.github.ricksbrown.cowsay.plugin.CowExecutor;
public class Main {
public static class Person {
private String name;
private int age;
// Constructor
public Person(String name, int age) {
this.name = name;
this.age = age;
}
}
public static Object main(
// Primitive
int a,
float b,
// Objects
Integer age,
Float d,
Object e,
String name,
// Lists
String[] f
// No trailing commas!
){
Gson gson = new Gson();
// Get resources
var theme = Wmill.getResource("f/app_themes/theme_0");
System.out.println("Theme: " + theme);
// Create a Person object
Person person = new Person( (name == "") ? "Alice" : name, (age == null) ? 30 : age);
// Serialize the Person object to JSON
String json = gson.toJson(person);
System.out.println("Serialized JSON: " + json);
// Use cowsay
String[] args = new String[]{"-f", "dragon", json };
String result = Cowsay.say(args);
return result;
}
}
`
// KJQXZ
export const INITIAL_CODE = {
bun: {
scriptInitCodeBlock: BUN_INIT_BLOCK,
@@ -1028,7 +1083,11 @@ export const INITIAL_CODE = {
},
bunnative: {
script: BUNNATIVE_INIT_CODE
}
},
java: {
script: JAVA_INIT_CODE
},
// KJQXZ
}
export function isInitialCode(content: string): boolean {
@@ -1132,6 +1191,9 @@ export function initialCode(
return INITIAL_CODE.csharp.script
} else if (language == 'nu') {
return INITIAL_CODE.nu.script
} else if (language == 'java') {
return INITIAL_CODE.java.script
// KJQXZ
} else if (language == 'bun' || language == 'bunnative') {
if (kind == 'trigger') {
return INITIAL_CODE.bun.trigger
+8 -2
View File
@@ -45,6 +45,9 @@ export function scriptLangToEditorLang(
return 'csharp'
} else if (lang == 'nu') {
return 'nu'
} else if (lang == 'java') {
return 'java'
// KJQXZ
} else if (lang == undefined) {
return 'typescript'
} else {
@@ -125,7 +128,9 @@ const scriptLanguagesArray: [SupportedLanguage | 'docker' | 'bunnative', string]
['ansible', 'Ansible Playbook'],
['csharp', 'C#'],
['docker', 'Docker'],
['nu', 'Nu']
['nu', 'Nu'],
['java', 'Java']
// KJQXZ
]
export function processLangs(selected: string | undefined, langs: string[]): string[] {
if (selected === 'nativets') {
@@ -134,7 +139,8 @@ export function processLangs(selected: string | undefined, langs: string[]): str
let ls = langs.filter((lang) => lang !== 'nativets')
//those languages are newer and may not be in the saved list
let nl = ['bunnative', 'rust', 'ansible', 'csharp', 'nu']
let nl = ['bunnative', 'rust', 'ansible', 'csharp', 'nu', 'java']
// KJQXZ
nl.forEach((lang) => {
if (!ls.includes(lang)) {
ls.push(lang)
@@ -755,7 +755,8 @@
<Badge color="blue">priority: {job.priority}</Badge>
</div>
{/if}
{#if job.tag && !['deno', 'python3', 'flow', 'other', 'go', 'postgresql', 'mysql', 'bigquery', 'snowflake', 'mssql', 'graphql', 'oracledb', 'nativets', 'bash', 'powershell', 'php', 'rust', 'other', 'ansible', 'csharp', 'dependency', 'nu'].includes(job.tag)}
{#if job.tag && !['deno', 'python3', 'flow', 'other', 'go', 'postgresql', 'mysql', 'bigquery', 'snowflake', 'mssql', 'graphql', 'oracledb', 'nativets', 'bash', 'powershell', 'php', 'rust', 'other', 'ansible', 'csharp', 'nu', 'java', 'dependency'].includes(job.tag)}
<!-- KJQXZ -->
<div>
<Badge color="indigo">Tag: {job.tag}</Badge>
</div>