mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-08-21 08:02:26 +00:00
feat: add java support (#5458)
* feat: add nu (nushell) support * add worker tests * deactivate tables and non-any types below top-level full support will come in V1 for V0 it's better to keep things minimal and simple * add syntax highlighting used python's grammar, since nushell isn't supported by monaco nor svelte-highlights for V1 nu will get it`s own grammar * add logo * partially implement plugin support * change logo + ability to deploy + nsjail draft * static variables + get_resource + get_variable * lsp/dev.nu + initial nu lsp (not working yet) * make it work with nsjail * nullguard * Much more flexible signature parsing and better error-messages * add init script * rename nulsp to nu * install nu to dockerfile * fix merge * implement Default for MainArgSignature * stage NU_CACHE_DIR * improve dockerfiles * dev.nu for parser-wasm + flake.nix * update code for windows * add nushell to flake * upload Cargo.lock * make build.sh work on nixos * build wasm cli parsers * add docs to README_DEV.md * add helper script docker/dev.nu * improve docker/dev.nu * fix windows * commit frontend/package(lock).json * update cargo.lock * correctly update cargo.lock * remove lsp * update flake.nix to include svelte server and nushell * Revert base.sql to main * remove PLUGIN_USE_RE * make CARGO_PATH private * add nu to cli * Change flags to build wasm-nu-parser * remove flake.nix from parser-wasm * update wasm-build target * remove unused import * add cli support for nu * update github workflows * wasm-build 0.17 -> 0.19 * update build script * update cargo.lock * Fix typographical error * start working on java * do java boilerplate * implement parser for java * update Cargo.lock * update ENV_SETTINGS * use published nu parser * update package.lock * java is S3 + Caching enabled * install nsjail backup * commit v0 * fix nsjail * v0.1 * rewrite parser in tree-sitter * implement parser from scratch * polishing * change init script to match new parser * fix imports * fix cli build * fix cli build * refactor install phase * implement .valid.windmill atomic verification * implement java init functionality * remove quick-xml * fix windows not recognizing 'mvn' * create empty settings.xml if there is no config provided * clean up * change default settings.xml * change classpath format for windows * docs to helper * java copy bin cache instead of symlink * remove comments * merge * fix package.json * fix package.json 2 * minor fixing * migrate to Coursier * update misc * Http(s) Proxy + CA certs * remove unused .wasm * make requirements insensitive to spaces * update handle_child refs * rework save_cache for directories * fix s3 bug * compile .wasm for cli * remove uuid import * fix compilation * use reference * fix zero-dep failure * removing unsafe stuff * remove unneeded imports * revert: we still need winapi * remove nix store from nsjail * do not create cache_nomount * add java to dnt * remove duplicated dependency in init script * fix typos * fix CI * use published parser
This commit is contained in:
@@ -64,7 +64,7 @@ jobs:
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
build-args: |
|
||||
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust,nu
|
||||
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
|
||||
secrets: |
|
||||
rh_username=${{ secrets.RH_USERNAME }}
|
||||
rh_password=${{ secrets.RH_PASSWORD }}
|
||||
@@ -81,7 +81,7 @@ jobs:
|
||||
platforms: linux/arm64
|
||||
push: true
|
||||
build-args: |
|
||||
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust,nu
|
||||
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
|
||||
secrets: |
|
||||
rh_username=${{ secrets.RH_USERNAME }}
|
||||
rh_password=${{ secrets.RH_PASSWORD }}
|
||||
|
||||
@@ -51,8 +51,7 @@ jobs:
|
||||
$env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static"
|
||||
mkdir frontend/build && cd backend
|
||||
New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force
|
||||
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,nats,sqs_trigger,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust,nu
|
||||
|
||||
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,nats,sqs_trigger,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
|
||||
- name: Rename binary with corresponding architecture
|
||||
run: |
|
||||
Rename-Item -Path ".\backend\target\release\windmill.exe" -NewName "windmill-ee.exe"
|
||||
|
||||
@@ -67,7 +67,7 @@ jobs:
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
build-args: |
|
||||
features=embedding,parquet,openidconnect,deno_core,license,http_trigger,zip,oauth2,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust,nu
|
||||
features=embedding,parquet,openidconnect,license,http_trigger,zip,oauth2,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
|
||||
tags: |
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
|
||||
${{ steps.meta-public.outputs.tags }}
|
||||
|
||||
@@ -95,7 +95,7 @@ jobs:
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
build-args: |
|
||||
features=embedding,parquet,openidconnect,jemalloc,deno_core,license,http_trigger,zip,oauth2,dind,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust,nu
|
||||
features=embedding,parquet,openidconnect,jemalloc,license,http_trigger,zip,oauth2,dind,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
|
||||
tags: |
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }}
|
||||
${{ steps.meta-public.outputs.tags }}
|
||||
@@ -157,7 +157,7 @@ jobs:
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
build-args: |
|
||||
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,otel,dind,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust,nu
|
||||
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,otel,dind,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
|
||||
tags: |
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }}
|
||||
${{ steps.meta-ee-public.outputs.tags }}
|
||||
|
||||
@@ -53,8 +53,7 @@ jobs:
|
||||
$env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static"
|
||||
mkdir frontend/build && cd backend
|
||||
New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force
|
||||
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust,nu
|
||||
|
||||
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
|
||||
- name: Rename binary with corresponding architecture
|
||||
run: |
|
||||
Rename-Item -Path ".\backend\target\release\windmill.exe" -NewName "windmill-ee.exe"
|
||||
|
||||
@@ -9,3 +9,4 @@ CaddyfileRemoteMalo
|
||||
.direnv
|
||||
.vscode
|
||||
.dev-docker-wrapper*
|
||||
backend/.minio-data
|
||||
|
||||
Generated
+27
@@ -12792,6 +12792,16 @@ dependencies = [
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-java"
|
||||
version = "0.23.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0aa6cbcdc8c679b214e616fd3300da67da0e492e066df01bcf5a5921a71e90d6"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-language"
|
||||
version = "0.1.5"
|
||||
@@ -13878,6 +13888,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"sha2 0.10.8",
|
||||
"sqlx",
|
||||
"tar",
|
||||
"tempfile",
|
||||
"thiserror 2.0.12",
|
||||
"tikv-jemalloc-ctl",
|
||||
@@ -13998,6 +14009,18 @@ dependencies = [
|
||||
"windmill-parser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-java"
|
||||
version = "1.478.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"serde_json",
|
||||
"tree-sitter",
|
||||
"tree-sitter-java",
|
||||
"wasm-bindgen",
|
||||
"windmill-parser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-nu"
|
||||
version = "1.478.1"
|
||||
@@ -14112,6 +14135,7 @@ dependencies = [
|
||||
"windmill-parser-csharp",
|
||||
"windmill-parser-go",
|
||||
"windmill-parser-graphql",
|
||||
"windmill-parser-java",
|
||||
"windmill-parser-nu",
|
||||
"windmill-parser-php",
|
||||
"windmill-parser-py",
|
||||
@@ -14236,17 +14260,20 @@ dependencies = [
|
||||
"tokio-postgres 0.7.13",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
"url",
|
||||
"urlencoding",
|
||||
"uuid",
|
||||
"winapi",
|
||||
"windmill-audit",
|
||||
"windmill-common",
|
||||
"windmill-git-sync",
|
||||
"windmill-macros",
|
||||
"windmill-parser",
|
||||
"windmill-parser-bash",
|
||||
"windmill-parser-csharp",
|
||||
"windmill-parser-go",
|
||||
"windmill-parser-graphql",
|
||||
"windmill-parser-java",
|
||||
"windmill-parser-nu",
|
||||
"windmill-parser-php",
|
||||
"windmill-parser-py",
|
||||
|
||||
+16
-10
@@ -23,6 +23,7 @@ members = [
|
||||
"./parsers/windmill-parser-rust",
|
||||
"./parsers/windmill-parser-csharp",
|
||||
"./parsers/windmill-parser-nu",
|
||||
"./parsers/windmill-parser-java",
|
||||
"./parsers/windmill-parser-bash",
|
||||
"./parsers/windmill-parser-py",
|
||||
"./parsers/windmill-parser-py-imports",
|
||||
@@ -62,31 +63,34 @@ cloud = ["windmill-queue/cloud", "windmill-worker/cloud", "windmill-common/cloud
|
||||
jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"]
|
||||
tantivy = ["dep:windmill-indexer", "windmill-api/tantivy", "windmill-indexer/enterprise", "windmill-indexer/parquet", "windmill-common/tantivy", "enterprise", "parquet"]
|
||||
sqlx = ["windmill-worker/sqlx"]
|
||||
deno_core = ["windmill-worker/deno_core", "dep:deno_core", "dep:v8"]
|
||||
kafka = ["windmill-api/kafka"]
|
||||
nats = ["windmill-api/nats"]
|
||||
otel = ["windmill-common/otel", "windmill-worker/otel"]
|
||||
dind = ["windmill-worker/dind"]
|
||||
php = ["windmill-worker/php"]
|
||||
rust = ["windmill-worker/rust"]
|
||||
mysql = ["windmill-worker/mysql"]
|
||||
oracledb = ["windmill-worker/oracledb"]
|
||||
mssql = ["windmill-worker/mssql"]
|
||||
bigquery = ["windmill-worker/bigquery"]
|
||||
websocket = ["windmill-api/websocket"]
|
||||
postgres_trigger = ["windmill-api/postgres_trigger"]
|
||||
mqtt_trigger = ["windmill-api/mqtt_trigger"]
|
||||
sqs_trigger = ["windmill-api/sqs_trigger"]
|
||||
python = ["windmill-worker/python"]
|
||||
smtp = ["windmill-api/smtp", "windmill-common/smtp"]
|
||||
csharp = ["windmill-worker/csharp"]
|
||||
nu = ["windmill-worker/nu"]
|
||||
license = ["windmill-api/license"]
|
||||
oauth2 = ["windmill-api/oauth2"]
|
||||
http_trigger = ["windmill-api/http_trigger"]
|
||||
zip = ["windmill-api/zip"]
|
||||
static_frontend = ["windmill-api/static_frontend"]
|
||||
scoped_cache = ["windmill-common/scoped_cache"]
|
||||
# Languages
|
||||
python = ["windmill-worker/python"]
|
||||
deno_core = ["windmill-worker/deno_core", "dep:deno_core", "dep:v8"]
|
||||
rust = ["windmill-worker/rust"]
|
||||
mysql = ["windmill-worker/mysql"]
|
||||
oracledb = ["windmill-worker/oracledb"]
|
||||
mssql = ["windmill-worker/mssql"]
|
||||
bigquery = ["windmill-worker/bigquery"]
|
||||
php = ["windmill-worker/php"]
|
||||
csharp = ["windmill-worker/csharp"]
|
||||
nu = ["windmill-worker/nu"]
|
||||
java = ["windmill-worker/java"]
|
||||
all_languages = [ "python", "deno_core", "rust", "mysql", "oracledb", "mssql", "bigquery", "csharp", "nu", "php", "java"]
|
||||
|
||||
|
||||
[dependencies]
|
||||
@@ -158,6 +162,7 @@ windmill-parser-go = { path = "./parsers/windmill-parser-go" }
|
||||
windmill-parser-rust = { path = "./parsers/windmill-parser-rust" }
|
||||
windmill-parser-yaml = { path = "./parsers/windmill-parser-yaml" }
|
||||
windmill-parser-csharp = { path = "./parsers/windmill-parser-csharp" }
|
||||
windmill-parser-java = { path = "./parsers/windmill-parser-java" }
|
||||
windmill-parser-nu = { path = "./parsers/windmill-parser-nu" }
|
||||
windmill-parser-bash = { path = "./parsers/windmill-parser-bash" }
|
||||
windmill-parser-sql = { path = "./parsers/windmill-parser-sql" }
|
||||
@@ -363,5 +368,6 @@ yaml-rust = "0.4.5"
|
||||
tokio-tungstenite = { version = "0.24.0", features = ["native-tls"] }
|
||||
tree-sitter = { version = "0.23.0", features = [] }
|
||||
tree-sitter-c-sharp = "0.23.0"
|
||||
tree-sitter-java = "0.23.0"
|
||||
oracle = { version = "0.6.3", features = ["chrono"] }
|
||||
rumqttc = { version = "0.24.0", features = ["use-native-tls"]}
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
-- Add down migration script here
|
||||
@@ -0,0 +1,3 @@
|
||||
-- Add up migration script here
|
||||
ALTER TYPE SCRIPT_LANG ADD VALUE IF NOT EXISTS 'java';
|
||||
UPDATE config set config = jsonb_set(config, '{worker_tags}', config->'worker_tags' || '["java"]'::jsonb) where name = 'worker__default' and config @> '{"worker_tags": ["deno", "python3", "go", "bash", "powershell", "dependency", "flow", "hub", "other", "bun", "php", "rust", "ansible", "csharp", "nu"]}'::jsonb AND NOT config->'worker_tags' @> '"java"'::jsonb;
|
||||
@@ -0,0 +1,21 @@
|
||||
[package]
|
||||
name = "windmill-parser-java"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
authors.workspace = true
|
||||
|
||||
[lib]
|
||||
name = "windmill_parser_java"
|
||||
path = "./src/lib.rs"
|
||||
|
||||
[dependencies]
|
||||
windmill-parser.workspace = true
|
||||
tree-sitter.workspace = true
|
||||
tree-sitter-java.workspace = true
|
||||
anyhow.workspace = true
|
||||
wasm-bindgen.workspace = true
|
||||
serde_json.workspace = true
|
||||
# convert_case.workspace = true
|
||||
# lazy_static.workspace = true
|
||||
# regex.workspace = true
|
||||
|
||||
@@ -0,0 +1,446 @@
|
||||
#![cfg_attr(target_arch = "wasm32", feature(c_variadic))]
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
pub mod wasm_libc;
|
||||
|
||||
use anyhow::anyhow;
|
||||
use anyhow::bail;
|
||||
use serde_json::Value;
|
||||
use tree_sitter::Node;
|
||||
use windmill_parser::Arg;
|
||||
use windmill_parser::MainArgSignature;
|
||||
use windmill_parser::Typ;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct JavaMainSigMeta {
|
||||
pub is_public: bool,
|
||||
pub returns_void: bool,
|
||||
pub class_name: Option<String>,
|
||||
pub main_sig: MainArgSignature,
|
||||
}
|
||||
|
||||
pub fn parse_java_sig_meta(code: &str) -> anyhow::Result<JavaMainSigMeta> {
|
||||
let mut parser = tree_sitter::Parser::new();
|
||||
let language = tree_sitter_java::LANGUAGE;
|
||||
parser
|
||||
.set_language(&language.into())
|
||||
.map_err(|e| anyhow!("Error setting Java as language: {e}"))?;
|
||||
|
||||
// Parse code
|
||||
let tree = parser
|
||||
.parse(code, None)
|
||||
.ok_or(anyhow!("Failed to parse code"))?;
|
||||
let root_node = tree.root_node();
|
||||
|
||||
// Traverse the AST to find the Main method signature
|
||||
let main_sig = find_main_signature(root_node, code);
|
||||
let no_main_func = Some(main_sig.is_none());
|
||||
let mut is_public = false;
|
||||
let mut returns_void = false;
|
||||
let mut class_name = None;
|
||||
|
||||
let mut args = vec![];
|
||||
if let Some((sig, name)) = main_sig {
|
||||
class_name = name;
|
||||
for sig_node in sig.children(&mut sig.walk()) {
|
||||
if sig_node.kind() == "modifier" && sig_node.utf8_text(code.as_bytes())? == "public" {
|
||||
is_public = true;
|
||||
}
|
||||
}
|
||||
if let Some(return_type) = sig.child_by_field_name("type") {
|
||||
let return_type = return_type.utf8_text(code.as_bytes())?;
|
||||
|
||||
if return_type == "void" {
|
||||
returns_void = true;
|
||||
}
|
||||
}
|
||||
if let Some(param_list) = sig.child_by_field_name("parameters") {
|
||||
for p_list_node in param_list.children(&mut param_list.walk()) {
|
||||
if p_list_node.kind() == "formal_parameter" {
|
||||
let (otyp, typ, name, default) = parse_java_typ(p_list_node, code)?;
|
||||
args.push(Arg {
|
||||
name,
|
||||
otyp,
|
||||
typ,
|
||||
has_default: default.is_some(),
|
||||
default,
|
||||
oidx: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let main_sig = MainArgSignature {
|
||||
star_args: false,
|
||||
star_kwargs: false,
|
||||
args,
|
||||
has_preprocessor: None,
|
||||
no_main_func,
|
||||
};
|
||||
|
||||
Ok(JavaMainSigMeta { returns_void, class_name, main_sig, is_public })
|
||||
}
|
||||
|
||||
pub fn parse_java_signature(code: &str) -> anyhow::Result<MainArgSignature> {
|
||||
Ok(parse_java_sig_meta(code)?.main_sig)
|
||||
}
|
||||
|
||||
fn find_typ<'a>(typ_node: Node<'a>, code: &str) -> anyhow::Result<(Typ, Option<Value>)> {
|
||||
let null = Some(serde_json::Value::Null);
|
||||
let res = match typ_node.kind() {
|
||||
#[rustfmt::skip]
|
||||
"type_identifier" => {
|
||||
match typ_node.utf8_text(code.as_bytes()) {
|
||||
Ok("String") => (Typ::Str(None), null),
|
||||
Ok("Byte") => (Typ::Bytes, null),
|
||||
Ok("Short") => (Typ::Int, null),
|
||||
Ok("Integer") => (Typ::Int, null),
|
||||
Ok("Long") => (Typ::Int, null),
|
||||
Ok("Float") => (Typ::Float, null),
|
||||
Ok("Double") => (Typ::Float, null),
|
||||
Ok("Boolean") => (Typ::Bool, null),
|
||||
Ok("Character") => (Typ::Str(None), null),
|
||||
Ok("Object") => (Typ::Object(vec![]),null), // TODO: Complete the object type
|
||||
Ok(s) => bail!("Unknown type `{s}`"),
|
||||
Err(e) => bail!("Error getting type name: {}", e),
|
||||
}
|
||||
}
|
||||
#[rustfmt::skip]
|
||||
"integral_type" => {
|
||||
match typ_node.utf8_text(code.as_bytes()) {
|
||||
Ok("byte") => (Typ::Bytes, None),
|
||||
Ok("short") => (Typ::Int, None),
|
||||
Ok("int") => (Typ::Int, None),
|
||||
Ok("long") => (Typ::Int, None),
|
||||
Ok("char") => (Typ::Str(None), None),
|
||||
Ok(s) => bail!("Unknown type `{s}`"),
|
||||
Err(e) => bail!("Error getting type name: {}", e),
|
||||
}
|
||||
}
|
||||
"floating_point_type" => {
|
||||
match typ_node.utf8_text(code.as_bytes()) {
|
||||
Ok("float") => (Typ::Float, None),
|
||||
Ok("double") => (Typ::Float, None),
|
||||
Ok(s) => bail!("Unknown type `{s}`"),
|
||||
Err(e) => bail!("Error getting type name: {}", e),
|
||||
}
|
||||
}
|
||||
"boolean_type" => {
|
||||
match typ_node.utf8_text(code.as_bytes()) {
|
||||
Ok("boolean") => (Typ::Bool, None),
|
||||
Ok(s) => bail!("Unknown type `{s}`"),
|
||||
Err(e) => bail!("Error getting type name: {}", e),
|
||||
}
|
||||
}
|
||||
"array_type" => {
|
||||
let new_typ_node = typ_node
|
||||
.named_child(0)
|
||||
.ok_or(anyhow!("Failed to find inner type of array type"))?;
|
||||
(Typ::List(Box::new(find_typ(new_typ_node, code)?.0)), null)
|
||||
}
|
||||
wc => bail!(
|
||||
"Unexpected Java type node kind: {} for '{}'. This type is not handled by Windmill, please open an issue if this seems to be an error",
|
||||
wc,
|
||||
typ_node.utf8_text(code.as_bytes())?
|
||||
),
|
||||
|
||||
};
|
||||
Ok(res)
|
||||
}
|
||||
|
||||
fn parse_java_typ<'a>(
|
||||
param_node: Node<'a>,
|
||||
code: &str,
|
||||
) -> anyhow::Result<(Option<String>, Typ, String, Option<Value>)> {
|
||||
let name = param_node
|
||||
.child_by_field_name("name")
|
||||
.and_then(|n| n.utf8_text(code.as_bytes()).ok())
|
||||
.unwrap_or("");
|
||||
let otyp_node = param_node.child_by_field_name("type");
|
||||
let otyp = otyp_node
|
||||
.and_then(|n| n.utf8_text(code.as_bytes()).ok())
|
||||
.map(|s| s.to_string());
|
||||
|
||||
let (typ, default) = find_typ(
|
||||
otyp_node.ok_or(anyhow!(
|
||||
"Internal error: Failed to get child by field name 'type'"
|
||||
))?,
|
||||
code,
|
||||
)?;
|
||||
|
||||
Ok((otyp, typ, name.to_string(), default))
|
||||
}
|
||||
|
||||
// Function to find the Main method's signature
|
||||
fn find_main_signature<'a>(root_node: Node<'a>, code: &str) -> Option<(Node<'a>, Option<String>)> {
|
||||
let mut cursor = root_node.walk();
|
||||
for x in root_node.children(&mut cursor) {
|
||||
if x.kind() == "class_declaration" {
|
||||
let class_name = x
|
||||
.child_by_field_name("name")
|
||||
.and_then(|n| n.utf8_text(code.as_bytes()).ok().map(|s| s.to_string()));
|
||||
for c in x.children(&mut x.walk()) {
|
||||
if c.kind() == "class_body" {
|
||||
for w in c.children(&mut c.walk()) {
|
||||
if w.kind() == "method_declaration" {
|
||||
for child in w.children(&mut w.walk()) {
|
||||
if child
|
||||
.utf8_text(code.as_bytes())
|
||||
.map(|name| name == "main")
|
||||
.unwrap_or(false)
|
||||
{
|
||||
return Some((w, class_name));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return None;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
#[test]
|
||||
fn test_parse_java_return_void() {
|
||||
let code = r#"
|
||||
class Main {
|
||||
public static void main() {}
|
||||
}"#;
|
||||
let sig_meta = parse_java_sig_meta(code).unwrap();
|
||||
|
||||
assert_eq!(sig_meta.class_name, Some("Main".to_string()));
|
||||
|
||||
assert!(sig_meta.returns_void);
|
||||
}
|
||||
#[test]
|
||||
fn test_parse_java_return_object() {
|
||||
let code = r#"
|
||||
class Main {
|
||||
public static Object main() {}
|
||||
}"#;
|
||||
let sig_meta = parse_java_sig_meta(code).unwrap();
|
||||
|
||||
assert_eq!(sig_meta.class_name, Some("Main".to_string()));
|
||||
assert!(!sig_meta.returns_void);
|
||||
}
|
||||
#[test]
|
||||
fn test_parse_java_primitive_types() {
|
||||
let code = r#"
|
||||
class Main {
|
||||
public static string main(byte a, short b, int c, long d, float e, double f, boolean g, char h) {}
|
||||
}"#;
|
||||
let sig_meta = parse_java_sig_meta(code).unwrap();
|
||||
|
||||
assert_eq!(sig_meta.class_name, Some("Main".to_string()));
|
||||
|
||||
let ret = sig_meta.main_sig;
|
||||
assert_eq!(
|
||||
ret.args,
|
||||
vec![
|
||||
Arg {
|
||||
name: "a".into(),
|
||||
otyp: Some("byte".into()),
|
||||
typ: Typ::Bytes,
|
||||
default: None,
|
||||
has_default: false,
|
||||
oidx: None
|
||||
},
|
||||
Arg {
|
||||
name: "b".into(),
|
||||
otyp: Some("short".into()),
|
||||
typ: Typ::Int,
|
||||
default: None,
|
||||
has_default: false,
|
||||
oidx: None
|
||||
},
|
||||
Arg {
|
||||
name: "c".into(),
|
||||
otyp: Some("int".into()),
|
||||
typ: Typ::Int,
|
||||
default: None,
|
||||
has_default: false,
|
||||
oidx: None
|
||||
},
|
||||
Arg {
|
||||
name: "d".into(),
|
||||
otyp: Some("long".into()),
|
||||
typ: Typ::Int,
|
||||
default: None,
|
||||
has_default: false,
|
||||
oidx: None
|
||||
},
|
||||
Arg {
|
||||
name: "e".into(),
|
||||
otyp: Some("float".into()),
|
||||
typ: Typ::Float,
|
||||
default: None,
|
||||
has_default: false,
|
||||
oidx: None
|
||||
},
|
||||
Arg {
|
||||
name: "f".into(),
|
||||
otyp: Some("double".into()),
|
||||
typ: Typ::Float,
|
||||
default: None,
|
||||
has_default: false,
|
||||
oidx: None
|
||||
},
|
||||
Arg {
|
||||
name: "g".into(),
|
||||
otyp: Some("boolean".into()),
|
||||
typ: Typ::Bool,
|
||||
default: None,
|
||||
has_default: false,
|
||||
oidx: None
|
||||
},
|
||||
Arg {
|
||||
name: "h".into(),
|
||||
otyp: Some("char".into()),
|
||||
typ: Typ::Str(None),
|
||||
default: None,
|
||||
has_default: false,
|
||||
oidx: None
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_java_objects() {
|
||||
let code = r#"
|
||||
class Main {
|
||||
public static string main(Byte a, Short b, Integer c, Long d, Float e, Double f, Boolean g, Character h, Object i) {}
|
||||
}"#;
|
||||
let sig_meta = parse_java_sig_meta(code).unwrap();
|
||||
|
||||
assert_eq!(sig_meta.class_name, Some("Main".to_string()));
|
||||
|
||||
let ret = sig_meta.main_sig;
|
||||
assert_eq!(
|
||||
ret.args,
|
||||
vec![
|
||||
Arg {
|
||||
name: "a".into(),
|
||||
otyp: Some("Byte".into()),
|
||||
typ: Typ::Bytes,
|
||||
default: Some(json!(null)),
|
||||
has_default: true,
|
||||
oidx: None
|
||||
},
|
||||
Arg {
|
||||
name: "b".into(),
|
||||
otyp: Some("Short".into()),
|
||||
typ: Typ::Int,
|
||||
default: Some(json!(null)),
|
||||
has_default: true,
|
||||
oidx: None
|
||||
},
|
||||
Arg {
|
||||
name: "c".into(),
|
||||
otyp: Some("Integer".into()),
|
||||
typ: Typ::Int,
|
||||
default: Some(json!(null)),
|
||||
has_default: true,
|
||||
oidx: None
|
||||
},
|
||||
Arg {
|
||||
name: "d".into(),
|
||||
otyp: Some("Long".into()),
|
||||
typ: Typ::Int,
|
||||
default: Some(json!(null)),
|
||||
has_default: true,
|
||||
oidx: None
|
||||
},
|
||||
Arg {
|
||||
name: "e".into(),
|
||||
otyp: Some("Float".into()),
|
||||
typ: Typ::Float,
|
||||
default: Some(json!(null)),
|
||||
has_default: true,
|
||||
oidx: None
|
||||
},
|
||||
Arg {
|
||||
name: "f".into(),
|
||||
otyp: Some("Double".into()),
|
||||
typ: Typ::Float,
|
||||
default: Some(json!(null)),
|
||||
has_default: true,
|
||||
oidx: None
|
||||
},
|
||||
Arg {
|
||||
name: "g".into(),
|
||||
otyp: Some("Boolean".into()),
|
||||
typ: Typ::Bool,
|
||||
default: Some(json!(null)),
|
||||
has_default: true,
|
||||
oidx: None
|
||||
},
|
||||
Arg {
|
||||
name: "h".into(),
|
||||
otyp: Some("Character".into()),
|
||||
typ: Typ::Str(None),
|
||||
default: Some(json!(null)),
|
||||
has_default: true,
|
||||
oidx: None
|
||||
},
|
||||
Arg {
|
||||
name: "i".into(),
|
||||
otyp: Some("Object".into()),
|
||||
typ: Typ::Object(vec![]),
|
||||
default: Some(json!(null)),
|
||||
has_default: true,
|
||||
oidx: None
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn test_parse_java_array() {
|
||||
let code = r#"
|
||||
class Main {
|
||||
public static string main(int[] a, Object[] b, String[] c) {}
|
||||
}"#;
|
||||
let sig_meta = parse_java_sig_meta(code).unwrap();
|
||||
|
||||
assert_eq!(sig_meta.class_name, Some("Main".to_string()));
|
||||
|
||||
let ret = sig_meta.main_sig;
|
||||
assert_eq!(
|
||||
ret.args,
|
||||
vec![
|
||||
Arg {
|
||||
name: "a".into(),
|
||||
otyp: Some("int[]".into()),
|
||||
typ: Typ::List(Box::new(Typ::Int)),
|
||||
default: Some(json!(null)),
|
||||
has_default: true,
|
||||
oidx: None
|
||||
},
|
||||
Arg {
|
||||
name: "b".into(),
|
||||
otyp: Some("Object[]".into()),
|
||||
typ: Typ::List(Box::new(Typ::Object(vec![]))),
|
||||
default: Some(json!(null)),
|
||||
has_default: true,
|
||||
oidx: None
|
||||
},
|
||||
Arg {
|
||||
name: "c".into(),
|
||||
otyp: Some("String[]".into()),
|
||||
typ: Typ::List(Box::new(Typ::Str(None))),
|
||||
default: Some(json!(null)),
|
||||
has_default: true,
|
||||
oidx: None
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::{Mutex, OnceLock};
|
||||
use std::{
|
||||
alloc::{self, Layout},
|
||||
ffi::{c_char, c_int, c_void},
|
||||
mem::align_of,
|
||||
ptr,
|
||||
};
|
||||
use wasm_bindgen::prelude::*;
|
||||
|
||||
/* -------------------------------- stdlib.h -------------------------------- */
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn abort() {
|
||||
panic!("Aborted from C");
|
||||
}
|
||||
|
||||
macro_rules! console_log {
|
||||
($($t:tt)*) => (unsafe { log(&format_args!($($t)*).to_string()) })
|
||||
}
|
||||
|
||||
#[wasm_bindgen]
|
||||
extern "C" {
|
||||
#[wasm_bindgen(js_namespace = console)]
|
||||
fn log(a: &str);
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn malloc(size: usize) -> *mut c_void {
|
||||
if size == 0 {
|
||||
return ptr::null_mut();
|
||||
}
|
||||
|
||||
let (layout, offset_to_data) = layout_for_size_prepended(size);
|
||||
let buf = alloc::alloc(layout);
|
||||
store_layout(buf, layout, offset_to_data)
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn calloc(count: usize, size: usize) -> *mut c_void {
|
||||
if count == 0 || size == 0 {
|
||||
return ptr::null_mut();
|
||||
}
|
||||
|
||||
let (layout, offset_to_data) = layout_for_size_prepended(size * count);
|
||||
let buf = alloc::alloc_zeroed(layout);
|
||||
store_layout(buf, layout, offset_to_data)
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn realloc(buf: *mut c_void, new_size: usize) -> *mut c_void {
|
||||
if buf.is_null() {
|
||||
malloc(new_size)
|
||||
} else if new_size == 0 {
|
||||
free(buf);
|
||||
ptr::null_mut()
|
||||
} else {
|
||||
let (old_buf, old_layout) = retrieve_layout(buf);
|
||||
let (new_layout, offset_to_data) = layout_for_size_prepended(new_size);
|
||||
let new_buf = alloc::realloc(old_buf, old_layout, new_layout.size());
|
||||
store_layout(new_buf, new_layout, offset_to_data)
|
||||
}
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn free(buf: *mut c_void) {
|
||||
if buf.is_null() {
|
||||
return;
|
||||
}
|
||||
let (buf, layout) = retrieve_layout(buf);
|
||||
alloc::dealloc(buf, layout);
|
||||
}
|
||||
|
||||
// In all these allocations, we store the layout before the data for later retrieval.
|
||||
// This is because we need to know the layout when deallocating the memory.
|
||||
// Here are some helper methods for that:
|
||||
|
||||
/// Given a pointer to the data, retrieve the layout and the pointer to the layout.
|
||||
unsafe fn retrieve_layout(buf: *mut c_void) -> (*mut u8, Layout) {
|
||||
let (_, layout_offset) = Layout::new::<Layout>()
|
||||
.extend(Layout::from_size_align(0, align_of::<*const u8>() * 2).unwrap())
|
||||
.unwrap();
|
||||
|
||||
let buf = (buf as *mut u8).offset(-(layout_offset as isize));
|
||||
let layout = *(buf as *mut Layout);
|
||||
|
||||
(buf, layout)
|
||||
}
|
||||
|
||||
/// Calculate a layout for a given size with space for storing a layout at the start.
|
||||
/// Returns the layout and the offset to the data.
|
||||
fn layout_for_size_prepended(size: usize) -> (Layout, usize) {
|
||||
Layout::new::<Layout>()
|
||||
.extend(Layout::from_size_align(size, align_of::<*const u8>() * 2).unwrap())
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// Store a layout in the pointer, returning a pointer to where the data should be stored.
|
||||
unsafe fn store_layout(buf: *mut u8, layout: Layout, offset_to_data: usize) -> *mut c_void {
|
||||
*(buf as *mut Layout) = layout;
|
||||
(buf as *mut u8).offset(offset_to_data as isize) as *mut c_void
|
||||
}
|
||||
|
||||
/* -------------------------------- string.h -------------------------------- */
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn strncmp(ptr1: *const c_void, ptr2: *const c_void, n: usize) -> c_int {
|
||||
let s1 = std::slice::from_raw_parts(ptr1 as *const u8, n);
|
||||
let s2 = std::slice::from_raw_parts(ptr2 as *const u8, n);
|
||||
|
||||
for (a, b) in s1.iter().zip(s2.iter()) {
|
||||
if *a != *b || *a == 0 {
|
||||
return (*a as i32) - (*b as i32);
|
||||
}
|
||||
}
|
||||
|
||||
0
|
||||
}
|
||||
|
||||
/* -------------------------------- wctype.h -------------------------------- */
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn iswspace(c: c_int) -> bool {
|
||||
char::from_u32(c as u32).map_or(false, |c| c.is_whitespace())
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn iswalnum(c: c_int) -> bool {
|
||||
char::from_u32(c as u32).map_or(false, |c| c.is_alphanumeric())
|
||||
}
|
||||
|
||||
/* --------------------------------- time.h --------------------------------- */
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn clock() -> u64 {
|
||||
panic!("clock is not supported");
|
||||
}
|
||||
|
||||
/* --------------------------------- ctype.h -------------------------------- */
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn isprint(c: c_int) -> bool {
|
||||
c >= 32 && c <= 126
|
||||
}
|
||||
|
||||
/* --------------------------------- stdio.h -------------------------------- */
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn fprintf(_file: *mut c_void, _format: *const c_void, _args: ...) -> c_int {
|
||||
panic!("fprintf is not supported");
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn fputs(_s: *const c_void, _file: *mut c_void) -> c_int {
|
||||
panic!("fputs is not supported");
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn fputc(_c: c_int, _file: *mut c_void) -> c_int {
|
||||
panic!("fputc is not supported");
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn fdopen(_fd: c_int, _mode: *const c_void) -> *mut c_void {
|
||||
panic!("fdopen is not supported");
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn fclose(_file: *mut c_void) -> c_int {
|
||||
panic!("fclose is not supported");
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn fwrite(
|
||||
_ptr: *const c_void,
|
||||
_size: usize,
|
||||
_nmemb: usize,
|
||||
_stream: *mut c_void,
|
||||
) -> usize {
|
||||
panic!("fwrite is not supported");
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn vsnprintf(
|
||||
_buf: *mut c_char,
|
||||
_size: usize,
|
||||
_format: *const c_char,
|
||||
_args: ...
|
||||
) -> c_int {
|
||||
panic!("vsnprintf is not supported");
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub extern "C" fn clock_gettime(ptr: usize, new_size: usize) {
|
||||
panic!("clock_gettime is not supported");
|
||||
}
|
||||
|
||||
// int snprintf( char* restrict buffer, size_t bufsz, const char* restrict format, ... );
|
||||
#[no_mangle]
|
||||
pub extern "C" fn snprintf() {
|
||||
panic!("snprintf is not supported");
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub extern "C" fn __assert_fail(_: *const i32, _: *const i32, _: *const i32, _: *const i32) {
|
||||
panic!("oh no");
|
||||
}
|
||||
@@ -28,6 +28,7 @@ graphql-parser = [ "dep:windmill-parser-graphql"]
|
||||
ansible-parser = [ "dep:windmill-parser-yaml"]
|
||||
csharp-parser = [ "dep:windmill-parser-csharp"]
|
||||
nu-parser = [ "dep:windmill-parser-nu"]
|
||||
java-parser = [ "dep:windmill-parser-java"]
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
@@ -43,6 +44,7 @@ windmill-parser-rust = { workspace = true, optional = true }
|
||||
windmill-parser-yaml = { workspace = true, optional = true }
|
||||
windmill-parser-csharp = { workspace = true, optional = true }
|
||||
windmill-parser-nu = { workspace = true, optional = true }
|
||||
windmill-parser-java = { workspace = true, optional = true }
|
||||
wasm-bindgen.workspace = true
|
||||
serde_json.workspace = true
|
||||
getrandom = { workspace = true, features = ["js"] }
|
||||
|
||||
@@ -70,6 +70,10 @@ OUT_DIR="../../../cli/wasm/nu"
|
||||
mkdir -p $OUT_DIR
|
||||
wasm-pack build --release --target deno --out-dir $OUT_DIR --features "nu-parser" \
|
||||
-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
|
||||
# sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-nu"/' $OUT_DIR/package.json
|
||||
rm $OUT_DIR/.gitignore
|
||||
|
||||
#-# Java
|
||||
OUT_DIR="../../../cli/wasm/java"
|
||||
mkdir -p $OUT_DIR
|
||||
CFLAGS_wasm32_unknown_unknown="-I$(pwd)/wasm-sysroot -Wbad-function-cast -Wcast-function-type -fno-builtin" RUSTFLAGS="-Zwasm-c-abi=spec" wasm-pack build --release --target deno --out-dir $OUT_DIR --features "java-parser"
|
||||
rm $OUT_DIR/.gitignore
|
||||
|
||||
@@ -61,3 +61,8 @@ OUT_DIR="pkg-nu"
|
||||
wasm-pack build --release --target web --out-dir $OUT_DIR --features "nu-parser" \
|
||||
-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
|
||||
sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-nu"/' $OUT_DIR/package.json
|
||||
|
||||
#-# Java (needs some more stuff to compile C tree sitter into wasm)
|
||||
OUT_DIR="pkg-java"
|
||||
CFLAGS_wasm32_unknown_unknown="-I$(pwd)/wasm-sysroot -Wbad-function-cast -Wcast-function-type -fno-builtin" RUSTFLAGS="-Zwasm-c-abi=spec" wasm-pack build --release --target web --out-dir $OUT_DIR --features "java-parser"
|
||||
sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-java"/' $OUT_DIR/package.json
|
||||
|
||||
@@ -60,3 +60,7 @@ wasm-pack build --release --target web --out-dir $OUT_DIR --features "nu-parser"
|
||||
-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
|
||||
sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-nu"/' $OUT_DIR/package.json
|
||||
|
||||
#-# Java (needs some more stuff to compile C tree sitter into wasm)
|
||||
OUT_DIR="pkg-java"
|
||||
CFLAGS_wasm32_unknown_unknown="-I$(pwd)/wasm-sysroot -Wbad-function-cast -Wcast-function-type -fno-builtin" RUSTFLAGS="-Zwasm-c-abi=spec" wasm-pack build --release --target web --out-dir $OUT_DIR --features "java-parser"
|
||||
sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-java"/' $OUT_DIR/package.json
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eou pipefail
|
||||
|
||||
# Flags are needed to build correctly with C#
|
||||
# NOTE: Currently does not work with nix nor nixos
|
||||
CFLAGS_wasm32_unknown_unknown="-I$(pwd)/wasm-sysroot -Wbad-function-cast -Wcast-function-type -fno-builtin" RUSTFLAGS="-Zwasm-c-abi=spec" deno task wasmbuild --inline --out ../../../cli/wasm/ -p windmill-parser-wasm --all-features
|
||||
|
||||
|
||||
@@ -155,3 +155,11 @@ pub fn parse_csharp(code: &str) -> String {
|
||||
pub fn parse_nu(code: &str) -> String {
|
||||
wrap_sig(windmill_parser_nu::parse_nu_signature(code))
|
||||
}
|
||||
|
||||
#[cfg(feature = "java-parser")]
|
||||
#[wasm_bindgen]
|
||||
pub fn parse_java(code: &str) -> String {
|
||||
wrap_sig(windmill_parser_java::parse_java_signature(code))
|
||||
}
|
||||
|
||||
// KJQXZ
|
||||
|
||||
+19
-9
@@ -9,7 +9,8 @@
|
||||
use anyhow::Context;
|
||||
use monitor::{
|
||||
load_base_url, load_otel, reload_delete_logs_periodically_setting, reload_indexer_config,
|
||||
reload_instance_python_version_setting, reload_nuget_config_setting,
|
||||
reload_instance_python_version_setting, reload_maven_repos_setting,
|
||||
reload_no_default_maven_setting, reload_nuget_config_setting,
|
||||
reload_timeout_wait_result_setting, send_current_log_file_to_object_store,
|
||||
send_logs_to_object_store,
|
||||
};
|
||||
@@ -37,11 +38,12 @@ use windmill_common::{
|
||||
EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
|
||||
HUB_BASE_URL_SETTING, INDEXER_SETTING, INSTANCE_PYTHON_VERSION_SETTING,
|
||||
JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING,
|
||||
LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING,
|
||||
NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING,
|
||||
REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING,
|
||||
RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING,
|
||||
TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
|
||||
LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING,
|
||||
NO_DEFAULT_MAVEN_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING,
|
||||
OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
|
||||
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
|
||||
SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TEAMS_SETTING,
|
||||
TIMEOUT_WAIT_RESULT_SETTING,
|
||||
},
|
||||
scripts::ScriptLang,
|
||||
stats_ee::schedule_stats,
|
||||
@@ -66,9 +68,9 @@ use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING;
|
||||
use windmill_worker::{
|
||||
get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, CSHARP_CACHE_DIR,
|
||||
DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR,
|
||||
NU_CACHE_DIR, POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, PY312_CACHE_DIR,
|
||||
PY313_CACHE_DIR, RUST_CACHE_DIR, TAR_PY310_CACHE_DIR, TAR_PY311_CACHE_DIR, TAR_PY312_CACHE_DIR,
|
||||
TAR_PY313_CACHE_DIR, UV_CACHE_DIR,
|
||||
JAVA_CACHE_DIR, NU_CACHE_DIR, POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR,
|
||||
PY312_CACHE_DIR, PY313_CACHE_DIR, RUST_CACHE_DIR, TAR_JAVA_CACHE_DIR, TAR_PY310_CACHE_DIR,
|
||||
TAR_PY311_CACHE_DIR, TAR_PY312_CACHE_DIR, TAR_PY313_CACHE_DIR, UV_CACHE_DIR,
|
||||
};
|
||||
|
||||
use crate::monitor::{
|
||||
@@ -796,6 +798,12 @@ Windmill Community Edition {GIT_VERSION}
|
||||
NUGET_CONFIG_SETTING => {
|
||||
reload_nuget_config_setting(&db).await
|
||||
},
|
||||
MAVEN_REPOS_SETTING => {
|
||||
reload_maven_repos_setting(&db).await
|
||||
},
|
||||
NO_DEFAULT_MAVEN_SETTING => {
|
||||
reload_no_default_maven_setting(&db).await
|
||||
},
|
||||
KEEP_JOB_DIR_SETTING => {
|
||||
load_keep_job_dir(&db).await;
|
||||
},
|
||||
@@ -1103,6 +1111,8 @@ pub async fn run_workers(
|
||||
NU_CACHE_DIR,
|
||||
HUB_CACHE_DIR,
|
||||
POWERSHELL_CACHE_DIR,
|
||||
JAVA_CACHE_DIR,
|
||||
TAR_JAVA_CACHE_DIR, // KJQXZ
|
||||
] {
|
||||
DirBuilder::new()
|
||||
.recursive(true)
|
||||
|
||||
+17
-3
@@ -52,9 +52,7 @@ use windmill_common::{
|
||||
};
|
||||
use windmill_queue::{cancel_job, MiniPulledJob};
|
||||
use windmill_worker::{
|
||||
create_token_for_owner, handle_job_error, AuthedClient, SameWorkerPayload, SameWorkerSender,
|
||||
SendResult, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR,
|
||||
NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, SCRIPT_TOKEN_EXPIRY,
|
||||
create_token_for_owner, handle_job_error, AuthedClient, SameWorkerPayload, SameWorkerSender, SendResult, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, SCRIPT_TOKEN_EXPIRY
|
||||
};
|
||||
|
||||
#[cfg(feature = "parquet")]
|
||||
@@ -186,6 +184,8 @@ pub async fn initial_load(
|
||||
reload_bunfig_install_scopes_setting(&db).await;
|
||||
reload_instance_python_version_setting(&db).await;
|
||||
reload_nuget_config_setting(&db).await;
|
||||
reload_maven_repos_setting(&db).await;
|
||||
reload_no_default_maven_setting(&db).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -969,6 +969,20 @@ pub async fn reload_nuget_config_setting(db: &DB) {
|
||||
)
|
||||
.await;
|
||||
}
|
||||
pub async fn reload_maven_repos_setting(db: &DB) {
|
||||
reload_option_setting_with_tracing(db, windmill_common::global_settings::MAVEN_REPOS_SETTING, "MAVEN_REPOS", MAVEN_REPOS.clone())
|
||||
.await;
|
||||
}
|
||||
pub async fn reload_no_default_maven_setting(db: &DB) {
|
||||
let value = load_value_from_global_settings(db, windmill_common::global_settings::NO_DEFAULT_MAVEN_SETTING).await;
|
||||
match value {
|
||||
Ok(Some(serde_json::Value::Bool(t))) => NO_DEFAULT_MAVEN.store(t, Ordering::Relaxed),
|
||||
Err(e) => {
|
||||
tracing::error!("Error loading no default maven repository: {e:#}");
|
||||
}
|
||||
_ => (),
|
||||
};
|
||||
}
|
||||
|
||||
pub async fn reload_retention_period_setting(db: &DB) {
|
||||
if let Err(e) = reload_setting(
|
||||
|
||||
@@ -1988,6 +1988,51 @@ def main [
|
||||
assert_eq!(result, serde_json::json!(0));
|
||||
}
|
||||
|
||||
#[cfg(feature = "java")]
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_java_job(db: Pool<Postgres>) {
|
||||
initialize_tracing().await;
|
||||
let server = ApiServer::start(db.clone()).await;
|
||||
let port = server.addr.port();
|
||||
|
||||
let content = r#"
|
||||
public class Main {
|
||||
public static Object main(
|
||||
// Primitive
|
||||
int a,
|
||||
float b,
|
||||
// Objects
|
||||
Integer age,
|
||||
Float d
|
||||
){
|
||||
return "hello world";
|
||||
}
|
||||
}
|
||||
|
||||
"#
|
||||
.to_owned();
|
||||
|
||||
let job = RunJob::from(JobPayload::Code(RawCode {
|
||||
hash: None,
|
||||
content,
|
||||
path: None,
|
||||
lock: None,
|
||||
language: ScriptLang::Java,
|
||||
custom_concurrency_key: None,
|
||||
concurrent_limit: None,
|
||||
concurrency_time_window_s: None,
|
||||
cache_ttl: None,
|
||||
dedicated_worker: None,
|
||||
}))
|
||||
.arg("a", json!(3))
|
||||
.arg("b", json!(3.0))
|
||||
.arg("age", json!(30))
|
||||
.arg("d", json!(3.0))
|
||||
.run_until_complete(&db, port)
|
||||
.await;
|
||||
assert_eq!(job.json_result(), Some(json!("hello world")));
|
||||
}
|
||||
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_python_job(db: Pool<Postgres>) {
|
||||
initialize_tracing().await;
|
||||
|
||||
@@ -13529,7 +13529,9 @@ components:
|
||||
rust,
|
||||
ansible,
|
||||
csharp,
|
||||
nu
|
||||
nu,
|
||||
java
|
||||
# KJQXZ
|
||||
]
|
||||
|
||||
Preview:
|
||||
|
||||
@@ -624,17 +624,20 @@ async fn create_script_internal<'c>(
|
||||
.unwrap_or(json!({}));
|
||||
let lock = if ns.codebase.is_some() {
|
||||
Some(String::new())
|
||||
} else if !(ns.language == ScriptLang::Python3
|
||||
|| ns.language == ScriptLang::Go
|
||||
|| ns.language == ScriptLang::Bun
|
||||
|| ns.language == ScriptLang::Bunnative
|
||||
|| ns.language == ScriptLang::Deno
|
||||
|| ns.language == ScriptLang::Rust
|
||||
|| ns.language == ScriptLang::Ansible
|
||||
|| ns.language == ScriptLang::CSharp
|
||||
|| ns.language == ScriptLang::Nu
|
||||
|| ns.language == ScriptLang::Php)
|
||||
{
|
||||
} else if !(
|
||||
ns.language == ScriptLang::Python3
|
||||
|| ns.language == ScriptLang::Go
|
||||
|| ns.language == ScriptLang::Bun
|
||||
|| ns.language == ScriptLang::Bunnative
|
||||
|| ns.language == ScriptLang::Deno
|
||||
|| ns.language == ScriptLang::Rust
|
||||
|| ns.language == ScriptLang::Ansible
|
||||
|| ns.language == ScriptLang::CSharp
|
||||
|| ns.language == ScriptLang::Nu
|
||||
|| ns.language == ScriptLang::Php
|
||||
|| ns.language == ScriptLang::Java
|
||||
// KJQXZ
|
||||
) {
|
||||
Some(String::new())
|
||||
} else {
|
||||
ns.lock
|
||||
|
||||
@@ -375,6 +375,8 @@ pub(crate) async fn tarball_workspace(
|
||||
ScriptLang::CSharp => "cs",
|
||||
ScriptLang::Nu => "nu",
|
||||
ScriptLang::OracleDB => "odb.sql",
|
||||
ScriptLang::Java => "java",
|
||||
// KJQXZ
|
||||
};
|
||||
archive
|
||||
.write_to_archive(&script.content, &format!("{}.{}", script.path, ext))
|
||||
|
||||
@@ -24,6 +24,7 @@ name = "windmill_common"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[dependencies]
|
||||
tar.workspace = true
|
||||
hmac.workspace = true
|
||||
sha2.workspace = true
|
||||
thiserror.workspace = true
|
||||
|
||||
@@ -11,6 +11,8 @@ pub const LICENSE_KEY_SETTING: &str = "license_key";
|
||||
pub const NPM_CONFIG_REGISTRY_SETTING: &str = "npm_config_registry";
|
||||
pub const BUNFIG_INSTALL_SCOPES_SETTING: &str = "bunfig_install_scopes";
|
||||
pub const NUGET_CONFIG_SETTING: &str = "nuget_config";
|
||||
pub const MAVEN_REPOS_SETTING: &str = "maven_repos";
|
||||
pub const NO_DEFAULT_MAVEN_SETTING: &str = "no_default_maven";
|
||||
|
||||
pub const EXTRA_PIP_INDEX_URL_SETTING: &str = "pip_extra_index_url";
|
||||
pub const PIP_INDEX_URL_SETTING: &str = "pip_index_url";
|
||||
@@ -61,6 +63,8 @@ pub const ENV_SETTINGS: &[&str] = &[
|
||||
"NU_PATH",
|
||||
"DENO_PATH",
|
||||
"GO_PATH",
|
||||
"JAVA_PATH",
|
||||
// KJQXZ
|
||||
"GOPRIVATE",
|
||||
"GOPROXY",
|
||||
"NETRC",
|
||||
|
||||
@@ -345,27 +345,15 @@ fn find_annotation(comm_lit: &str, annotation: &str, code: &str) -> bool {
|
||||
|
||||
pub fn should_validate_schema(code: &str, lang: &ScriptLang) -> bool {
|
||||
let annotation = "schema_validation";
|
||||
match lang {
|
||||
ScriptLang::Nativets | ScriptLang::Bun | ScriptLang::Bunnative | ScriptLang::Deno => {
|
||||
find_annotation("//", annotation, code)
|
||||
}
|
||||
ScriptLang::Python3 => find_annotation("#", annotation, code),
|
||||
ScriptLang::Go => find_annotation("#", annotation, code),
|
||||
ScriptLang::Bash => find_annotation("#", annotation, code),
|
||||
ScriptLang::Powershell => find_annotation("#", annotation, code),
|
||||
ScriptLang::Postgresql => find_annotation("--", annotation, code),
|
||||
ScriptLang::Mysql => find_annotation("--", annotation, code),
|
||||
ScriptLang::Bigquery => find_annotation("--", annotation, code),
|
||||
ScriptLang::Snowflake => find_annotation("--", annotation, code),
|
||||
ScriptLang::Graphql => find_annotation("#", annotation, code),
|
||||
ScriptLang::Mssql => find_annotation("--", annotation, code),
|
||||
ScriptLang::OracleDB => find_annotation("--", annotation, code),
|
||||
ScriptLang::Php => find_annotation("//", annotation, code),
|
||||
ScriptLang::Rust => find_annotation("//!", annotation, code),
|
||||
ScriptLang::Ansible => find_annotation("#", annotation, code),
|
||||
ScriptLang::CSharp => find_annotation("//", annotation, code),
|
||||
ScriptLang::Nu => find_annotation("#", annotation, code),
|
||||
}
|
||||
use ScriptLang::*;
|
||||
let comment = match lang {
|
||||
Nativets | Bun | Bunnative | Deno | Php | CSharp | Java => "//",
|
||||
Python3 | Go | Bash | Powershell | Graphql | Ansible | Nu => "#",
|
||||
Postgresql | Mysql | Bigquery | Snowflake | Mssql | OracleDB => "--",
|
||||
Rust => "//!",
|
||||
// KJQXZ
|
||||
};
|
||||
find_annotation(comment, annotation, code)
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
|
||||
@@ -51,6 +51,8 @@ pub enum ScriptLang {
|
||||
Ansible,
|
||||
CSharp,
|
||||
Nu,
|
||||
Java,
|
||||
// KJQXZ
|
||||
}
|
||||
|
||||
impl ScriptLang {
|
||||
@@ -76,6 +78,8 @@ impl ScriptLang {
|
||||
ScriptLang::Ansible => "ansible",
|
||||
ScriptLang::CSharp => "csharp",
|
||||
ScriptLang::Nu => "nu",
|
||||
ScriptLang::Java => "java",
|
||||
// KJQXZ
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ use serde_json::value::RawValue;
|
||||
use std::{
|
||||
cmp::Reverse,
|
||||
collections::{HashMap, HashSet},
|
||||
fs::File,
|
||||
fs::{self, File},
|
||||
io::Write,
|
||||
path::{Component, Path, PathBuf},
|
||||
str::FromStr,
|
||||
@@ -51,6 +51,8 @@ lazy_static::lazy_static! {
|
||||
"ansible".to_string(),
|
||||
"csharp".to_string(),
|
||||
"nu".to_string(),
|
||||
"java".to_string(),
|
||||
// KJQXZ
|
||||
"dependency".to_string(),
|
||||
"flow".to_string(),
|
||||
"other".to_string()
|
||||
@@ -402,8 +404,43 @@ pub struct SqlAnnotations {
|
||||
pub struct BashAnnotations {
|
||||
pub docker: bool,
|
||||
}
|
||||
/// length = 5
|
||||
/// value = "foo"
|
||||
/// output = "foo "
|
||||
/// 12345
|
||||
pub fn pad_string(value: &str, total_length: usize) -> String {
|
||||
if value.len() >= total_length {
|
||||
value.to_string() // Return the original string if it's already long enough
|
||||
} else {
|
||||
let padding_needed = total_length - value.len();
|
||||
format!("{value}{}", " ".repeat(padding_needed)) // Pad with spaces
|
||||
}
|
||||
}
|
||||
pub fn copy_dir_recursively(src: &Path, dst: &Path) -> error::Result<()> {
|
||||
if !dst.exists() {
|
||||
fs::create_dir_all(dst)?;
|
||||
}
|
||||
|
||||
pub async fn load_cache(bin_path: &str, _remote_path: &str) -> (bool, String) {
|
||||
tracing::debug!("Copying recursively from {:?} to {:?}", src, dst);
|
||||
|
||||
for entry in fs::read_dir(src)? {
|
||||
let entry = entry?;
|
||||
let src_path = entry.path();
|
||||
let dst_path = dst.join(entry.file_name());
|
||||
|
||||
if src_path.is_dir() && !src_path.is_symlink() {
|
||||
copy_dir_recursively(&src_path, &dst_path)?;
|
||||
} else {
|
||||
fs::copy(&src_path, &dst_path)?;
|
||||
}
|
||||
}
|
||||
|
||||
tracing::debug!("Finished copying recursively from {:?} to {:?}", src, dst);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn load_cache(bin_path: &str, _remote_path: &str, is_dir: bool) -> (bool, String) {
|
||||
if tokio::fs::metadata(&bin_path).await.is_ok() {
|
||||
(true, format!("loaded from local cache: {}\n", bin_path))
|
||||
} else {
|
||||
@@ -417,12 +454,22 @@ pub async fn load_cache(bin_path: &str, _remote_path: &str) -> (bool, String) {
|
||||
use crate::s3_helpers::attempt_fetch_bytes;
|
||||
|
||||
if let Ok(mut x) = attempt_fetch_bytes(os, _remote_path).await {
|
||||
if let Err(e) = write_binary_file(bin_path, &mut x) {
|
||||
tracing::error!("could not write bundle/bin file locally: {e:?}");
|
||||
return (
|
||||
false,
|
||||
"error writing bundle/bin file from object store".to_string(),
|
||||
);
|
||||
if is_dir {
|
||||
if let Err(e) = extract_tar(x, bin_path).await {
|
||||
tracing::error!("could not write tar archive locally: {e:?}");
|
||||
return (
|
||||
false,
|
||||
"error writing tar archive from object store".to_string(),
|
||||
);
|
||||
}
|
||||
} else {
|
||||
if let Err(e) = write_binary_file(bin_path, &mut x) {
|
||||
tracing::error!("could not write bundle/bin file locally: {e:?}");
|
||||
return (
|
||||
false,
|
||||
"error writing bundle/bin file from object store".to_string(),
|
||||
);
|
||||
}
|
||||
}
|
||||
tracing::info!("loaded from object store {}", bin_path);
|
||||
return (
|
||||
@@ -435,6 +482,7 @@ pub async fn load_cache(bin_path: &str, _remote_path: &str) -> (bool, String) {
|
||||
);
|
||||
}
|
||||
}
|
||||
let _ = is_dir;
|
||||
(false, "".to_string())
|
||||
}
|
||||
}
|
||||
@@ -462,6 +510,7 @@ pub async fn save_cache(
|
||||
local_cache_path: &str,
|
||||
_remote_cache_path: &str,
|
||||
origin: &str,
|
||||
is_dir: bool,
|
||||
) -> crate::error::Result<String> {
|
||||
let mut _cached_to_s3 = false;
|
||||
#[cfg(all(feature = "enterprise", feature = "parquet"))]
|
||||
@@ -471,11 +520,33 @@ pub async fn save_cache(
|
||||
.clone()
|
||||
{
|
||||
use object_store::path::Path;
|
||||
let file_to_cache = if is_dir {
|
||||
let tar_path = format!(
|
||||
"{ROOT_CACHE_DIR}/tar/{}_tar.tar",
|
||||
local_cache_path
|
||||
.split("/")
|
||||
.last()
|
||||
.unwrap_or(&uuid::Uuid::new_v4().to_string())
|
||||
);
|
||||
let tar_file = std::fs::File::create(&tar_path)?;
|
||||
let mut tar = tar::Builder::new(tar_file);
|
||||
tar.append_dir_all(".", &origin)?;
|
||||
let tar_metadata = tokio::fs::metadata(&tar_path).await;
|
||||
if tar_metadata.is_err() || tar_metadata.as_ref().unwrap().len() == 0 {
|
||||
tracing::info!("Failed to tar cache: {origin}");
|
||||
return Err(error::Error::ExecutionErr(format!(
|
||||
"Failed to tar cache: {origin}"
|
||||
)));
|
||||
}
|
||||
tar_path
|
||||
} else {
|
||||
origin.to_owned()
|
||||
};
|
||||
|
||||
if let Err(e) = os
|
||||
.put(
|
||||
&Path::from(_remote_cache_path),
|
||||
std::fs::read(origin)?.into(),
|
||||
std::fs::read(&file_to_cache)?.into(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -485,12 +556,19 @@ pub async fn save_cache(
|
||||
);
|
||||
} else {
|
||||
_cached_to_s3 = true;
|
||||
if is_dir {
|
||||
tokio::fs::remove_dir_all(&file_to_cache).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// if !*CLOUD_HOSTED {
|
||||
if true {
|
||||
std::fs::copy(origin, local_cache_path)?;
|
||||
if is_dir {
|
||||
copy_dir_recursively(&PathBuf::from(origin), &PathBuf::from(local_cache_path))?;
|
||||
} else {
|
||||
std::fs::copy(origin, local_cache_path)?;
|
||||
}
|
||||
Ok(format!(
|
||||
"\nwrote cached binary: {} (backed by EE distributed object store: {_cached_to_s3})\n",
|
||||
local_cache_path
|
||||
@@ -505,6 +583,31 @@ pub async fn save_cache(
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "enterprise", feature = "parquet"))]
|
||||
pub async fn extract_tar(tar: bytes::Bytes, folder: &str) -> error::Result<()> {
|
||||
use std::time::Instant;
|
||||
|
||||
use bytes::Buf;
|
||||
use tokio::fs::{self};
|
||||
|
||||
let start: Instant = Instant::now();
|
||||
fs::create_dir_all(&folder).await?;
|
||||
|
||||
let mut ar = tar::Archive::new(tar.reader());
|
||||
|
||||
if let Err(e) = ar.unpack(folder) {
|
||||
tracing::info!("Failed to untar to {folder}. Error: {:?}", e);
|
||||
fs::remove_dir_all(&folder).await?;
|
||||
return Err(error::Error::ExecutionErr(format!(
|
||||
"Failed to untar tar {folder}"
|
||||
)));
|
||||
}
|
||||
tracing::info!(
|
||||
"Finished extracting tar to {folder}. Took {}ms",
|
||||
start.elapsed().as_millis(),
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
#[cfg(all(feature = "enterprise", feature = "parquet"))]
|
||||
fn write_binary_file(main_path: &str, byts: &mut bytes::Bytes) -> error::Result<()> {
|
||||
use std::fs::{File, Permissions};
|
||||
|
||||
@@ -30,17 +30,20 @@ python = ["dep:windmill-parser-py", "dep:windmill-parser-py-imports"]
|
||||
csharp = ["dep:windmill-parser-csharp"]
|
||||
rust = ["dep:windmill-parser-rust"]
|
||||
nu = ["dep:windmill-parser-nu"]
|
||||
java = ["dep:windmill-parser-java"]
|
||||
|
||||
[dependencies]
|
||||
windmill-queue.workspace = true
|
||||
windmill-audit.workspace = true # there isn't really a reason for audit-worth actions to happen in the worker.
|
||||
windmill-common = { workspace = true, default-features = false }
|
||||
windmill-macros.workspace = true
|
||||
windmill-parser.workspace = true
|
||||
windmill-parser-ts.workspace = true
|
||||
windmill-parser-go.workspace = true
|
||||
windmill-parser-rust = { workspace = true, optional = true }
|
||||
windmill-parser-csharp = { workspace = true, optional = true }
|
||||
windmill-parser-nu = { workspace = true, optional = true }
|
||||
windmill-parser-java = { workspace = true, optional = true }
|
||||
windmill-parser-py = { workspace = true, optional = true }
|
||||
windmill-parser-yaml.workspace = true
|
||||
windmill-parser-py-imports = { workspace = true, optional = true }
|
||||
@@ -72,6 +75,7 @@ dyn-iter.workspace = true
|
||||
once_cell.workspace = true
|
||||
tokio-postgres.workspace = true
|
||||
bit-vec.workspace = true
|
||||
url.workspace = true
|
||||
deno_telemetry = { workspace = true, optional = true }
|
||||
deno_fetch = { workspace = true, optional = true }
|
||||
deno_webidl = { workspace = true, optional = true }
|
||||
@@ -127,4 +131,4 @@ deno_permissions = { workspace = true, optional = true }
|
||||
deno_io = { workspace = true, optional = true }
|
||||
deno_runtime = { workspace = true, optional = true }
|
||||
deno_telemetry = { workspace = true, optional = true }
|
||||
winapi = { workspace = true, optional = true }
|
||||
winapi = { workspace = true, optional = true }
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
name: "java run script"
|
||||
|
||||
mode: ONCE
|
||||
hostname: "java"
|
||||
log_level: ERROR
|
||||
|
||||
disable_rl: true
|
||||
|
||||
cwd: "/tmp"
|
||||
|
||||
clone_newnet: false
|
||||
clone_newuser: {CLONE_NEWUSER}
|
||||
|
||||
keep_caps: false
|
||||
keep_env: true
|
||||
mount_proc: true
|
||||
|
||||
mount {
|
||||
src: "/bin"
|
||||
dst: "/bin"
|
||||
is_bind: true
|
||||
}
|
||||
|
||||
mount {
|
||||
src: "/lib"
|
||||
dst: "/lib"
|
||||
is_bind: true
|
||||
}
|
||||
|
||||
|
||||
mount {
|
||||
src: "/lib64"
|
||||
dst: "/lib64"
|
||||
is_bind: true
|
||||
mandatory: false
|
||||
}
|
||||
|
||||
|
||||
mount {
|
||||
src: "/usr"
|
||||
dst: "/usr"
|
||||
is_bind: true
|
||||
}
|
||||
|
||||
mount {
|
||||
src: "/dev/null"
|
||||
dst: "/dev/null"
|
||||
is_bind: true
|
||||
rw: true
|
||||
}
|
||||
|
||||
mount {
|
||||
dst: "/tmp"
|
||||
fstype: "tmpfs"
|
||||
rw: true
|
||||
options: "size=500000000"
|
||||
}
|
||||
|
||||
|
||||
mount {
|
||||
src: "{JOB_DIR}/target"
|
||||
dst: "/tmp/target"
|
||||
is_bind: true
|
||||
mandatory: false
|
||||
}
|
||||
|
||||
mount {
|
||||
src: "{JOB_DIR}/args.json"
|
||||
dst: "/tmp/args.json"
|
||||
is_bind: true
|
||||
}
|
||||
|
||||
mount {
|
||||
src: "{JOB_DIR}/result.json"
|
||||
dst: "/tmp/result.json"
|
||||
rw: true
|
||||
is_bind: true
|
||||
}
|
||||
|
||||
mount {
|
||||
src: "{CACHE_DIR}"
|
||||
dst: "{CACHE_DIR}"
|
||||
is_bind: true
|
||||
mandatory: false
|
||||
}
|
||||
|
||||
mount {
|
||||
src: "/etc"
|
||||
dst: "/etc"
|
||||
is_bind: true
|
||||
}
|
||||
|
||||
mount {
|
||||
src: "/dev/random"
|
||||
dst: "/dev/random"
|
||||
is_bind: true
|
||||
}
|
||||
|
||||
mount {
|
||||
src: "/dev/urandom"
|
||||
dst: "/dev/urandom"
|
||||
is_bind: true
|
||||
}
|
||||
|
||||
iface_no_lo: true
|
||||
|
||||
{SHARED_MOUNT}
|
||||
@@ -170,6 +170,7 @@ async fn install_galaxy_collections(
|
||||
None,
|
||||
false,
|
||||
&mut Some(occupancy_metrics),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -432,6 +433,7 @@ fi
|
||||
job.timeout,
|
||||
false,
|
||||
&mut Some(occupancy_metrics),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
read_and_check_result(job_dir).await
|
||||
|
||||
@@ -226,6 +226,7 @@ exit $exit_status
|
||||
job.timeout,
|
||||
true,
|
||||
&mut Some(occupancy_metrics),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -569,6 +570,7 @@ pub async fn handle_powershell_job(
|
||||
job.timeout,
|
||||
false,
|
||||
&mut Some(occupancy_metrics),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -780,6 +782,7 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"",
|
||||
job.timeout,
|
||||
false,
|
||||
&mut Some(occupancy_metrics),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
||||
@@ -166,6 +166,7 @@ pub async fn gen_bun_lockfile(
|
||||
None,
|
||||
false,
|
||||
occupancy_metrics,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
@@ -349,6 +350,7 @@ pub async fn install_bun_lockfile(
|
||||
None,
|
||||
false,
|
||||
occupancy_metrics,
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
@@ -520,6 +522,7 @@ pub async fn generate_wrapper_mjs(
|
||||
timeout,
|
||||
false,
|
||||
occupancy_metrics,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
fs::rename(
|
||||
@@ -570,6 +573,7 @@ pub async fn generate_bun_bundle(
|
||||
timeout,
|
||||
false,
|
||||
occupancy_metrics,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
@@ -594,7 +598,6 @@ pub async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result<()> {
|
||||
|
||||
if std::fs::metadata(&bun_cache_path).is_ok() {
|
||||
tracing::info!("loading {bun_cache_path} from cache");
|
||||
|
||||
extract_saved_codebase(job_dir, &bun_cache_path, is_tar, &dst, false)?;
|
||||
} else {
|
||||
#[cfg(all(feature = "enterprise", feature = "parquet"))]
|
||||
@@ -733,7 +736,7 @@ pub async fn prebundle_bun_script(
|
||||
)
|
||||
.await?;
|
||||
|
||||
save_cache(&local_path, &remote_path, &origin).await?;
|
||||
save_cache(&local_path, &remote_path, &origin, false).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -852,7 +855,8 @@ pub async fn handle_bun_job(
|
||||
)
|
||||
.await;
|
||||
|
||||
let (cache, logs) = windmill_common::worker::load_cache(&local_path, &remote_path).await;
|
||||
let (cache, logs) =
|
||||
windmill_common::worker::load_cache(&local_path, &remote_path, false).await;
|
||||
(cache, logs, local_path, remote_path)
|
||||
} else {
|
||||
(false, "".to_string(), "".to_string(), "".to_string())
|
||||
@@ -1181,7 +1185,14 @@ try {{
|
||||
)
|
||||
.await?;
|
||||
if !local_path.is_empty() {
|
||||
match save_cache(&local_path, &remote_path, &format!("{job_dir}/main.js")).await {
|
||||
match save_cache(
|
||||
&local_path,
|
||||
&remote_path,
|
||||
&format!("{job_dir}/main.js"),
|
||||
false,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Err(e) => {
|
||||
let em = format!("could not save {local_path} to bundle cache: {e:?}");
|
||||
tracing::error!(em)
|
||||
@@ -1421,6 +1432,7 @@ try {{
|
||||
job.timeout,
|
||||
false,
|
||||
&mut Some(occupancy_metrics),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ use sha2::Digest;
|
||||
use sqlx::types::Json;
|
||||
use sqlx::{Pool, Postgres};
|
||||
use tokio::process::Command;
|
||||
use tokio::sync::{RwLock, Semaphore};
|
||||
use tokio::{fs::File, io::AsyncReadExt};
|
||||
|
||||
#[cfg(feature = "parquet")]
|
||||
@@ -28,7 +29,7 @@ use windmill_common::{
|
||||
variables::ContextualVariable,
|
||||
};
|
||||
|
||||
use anyhow::{anyhow, Result};
|
||||
use anyhow::{anyhow, bail, Result};
|
||||
use windmill_queue::MiniPulledJob;
|
||||
|
||||
use std::path::Path;
|
||||
@@ -39,7 +40,10 @@ use windmill_common::{variables, DB};
|
||||
|
||||
use tokio::{io::AsyncWriteExt, process::Child, time::Instant};
|
||||
|
||||
use crate::{AuthedClient, JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, MAX_TIMEOUT_DURATION, PATH_ENV};
|
||||
use crate::{
|
||||
AuthedClient, DISABLE_NSJAIL, JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, MAX_TIMEOUT_DURATION,
|
||||
PATH_ENV,
|
||||
};
|
||||
|
||||
pub async fn build_args_map<'a>(
|
||||
job: &'a MiniPulledJob,
|
||||
@@ -946,3 +950,575 @@ pub fn build_http_client(timeout_duration: std::time::Duration) -> error::Result
|
||||
.build()
|
||||
.map_err(|e| Error::internal_err(format!("Error building http client: {e:#}")))
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct RequiredDependency {
|
||||
/// Expected directory of dependency in cache
|
||||
/// For example:
|
||||
/// /tmp/windmill/cache/python_311/rich==0.0.0
|
||||
/// IMPORTANT!: path should not end with '/'
|
||||
pub path: String,
|
||||
/// Name to use for S3 tars
|
||||
/// If not specified will use top level directory of path.
|
||||
pub custom_name: Option<String>,
|
||||
/// Display name
|
||||
/// Name that will be used for console output and logging
|
||||
/// If not specified will either use custom_name or top level directory of path.
|
||||
pub short_name: Option<String>,
|
||||
}
|
||||
|
||||
pub enum InstallStrategy {
|
||||
/// Will invoke callback to install single dependency
|
||||
Single(Arc<dyn Fn(RequiredDependency) -> Result<Command, error::Error> + Send + Sync>),
|
||||
/// Will try to pull S3 first and will invoke closure to install the rest
|
||||
AllAtOnce(Arc<dyn Fn(Vec<RequiredDependency>) -> Result<Command, error::Error> + Send + Sync>),
|
||||
}
|
||||
/// # General
|
||||
///
|
||||
/// Languages that compile usually include dependencies in final executable.
|
||||
/// When dynamic languages do not and runtime dependencies provided separately.
|
||||
///
|
||||
/// This helper implies that the language is dynamic.
|
||||
/// Python, Ruby, Java are dynamic and they can use this helper.
|
||||
///
|
||||
/// # Features
|
||||
///
|
||||
/// This helper will install all specified dependencies in parallel and if it is EE, cache to S3
|
||||
/// It has atomic success file, allowing to distinguish failed installations from succesfull.
|
||||
///
|
||||
/// Besides that it provides console output and does logging.
|
||||
///
|
||||
/// # Usage
|
||||
///
|
||||
/// Most important arguments in this helper are `deps` and `install_fn`
|
||||
///
|
||||
/// In `deps` you specify all dependencies that are needed to be on worker in order to execute script.
|
||||
/// You don't know which are actually installed and which are not.
|
||||
///
|
||||
/// `deps` is a vector of RequiredDependency. Check [RequiredDependency] for more context.
|
||||
///
|
||||
/// After `deps` are provided helper will check each dependency and check if it is in cache, if not it will try to pull from S3
|
||||
/// and if it does not work either, it will invoke `install_fn` closure.
|
||||
/// Closure arguments has dependency name as well as it`s expected path in cache.
|
||||
/// Closure should return Command that will install dependency to asked place.
|
||||
pub async fn par_install_language_dependencies<'a>(
|
||||
deps: Vec<RequiredDependency>,
|
||||
language_name: &'a str,
|
||||
installer_executable_name: &'a str,
|
||||
platform_agnostic: bool,
|
||||
concurrent_downloads: usize,
|
||||
stdout_on_err: bool,
|
||||
install_fn: InstallStrategy,
|
||||
postinstall_cb: impl AsyncFn(Vec<RequiredDependency>) -> Result<(), error::Error>,
|
||||
job_id: &'a Uuid,
|
||||
w_id: &'a str,
|
||||
worker_name: &'a str,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
) -> anyhow::Result<()> {
|
||||
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
|
||||
let _ = (platform_agnostic, language_name);
|
||||
|
||||
let total_time = std::time::Instant::now();
|
||||
|
||||
// Total to install
|
||||
let mut not_installed = vec![];
|
||||
let total_to_install;
|
||||
// let mut not_installed = vec![];
|
||||
let counter_arc = Arc::new(tokio::sync::Mutex::new(0));
|
||||
// Append logs with line like this:
|
||||
// [9/21] + requests==2.32.3 << (S3) | in 57ms
|
||||
#[allow(unused_assignments)]
|
||||
async fn print_success(
|
||||
mut s3_pull: bool,
|
||||
mut s3_push: bool,
|
||||
job_id: &Uuid,
|
||||
w_id: &str,
|
||||
req: &str,
|
||||
req_tl: usize,
|
||||
counter_arc: Arc<tokio::sync::Mutex<usize>>,
|
||||
total_to_install: usize,
|
||||
instant: std::time::Instant,
|
||||
db: Pool<Postgres>,
|
||||
) {
|
||||
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
|
||||
{
|
||||
(s3_pull, s3_push) = (false, false);
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "enterprise", feature = "parquet"))]
|
||||
if windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
|
||||
.read()
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
(s3_pull, s3_push) = (false, false);
|
||||
}
|
||||
|
||||
let mut counter = counter_arc.lock().await;
|
||||
*counter += 1;
|
||||
|
||||
windmill_queue::append_logs(
|
||||
job_id,
|
||||
w_id,
|
||||
format!(
|
||||
"\n{}+ {}{}{}| in {}ms",
|
||||
windmill_common::worker::pad_string(
|
||||
&format!("[{}/{total_to_install}]", counter),
|
||||
9
|
||||
),
|
||||
// Because we want to align to max len [999/999] we take 9
|
||||
// 123456789
|
||||
windmill_common::worker::pad_string(&req, req_tl + 1),
|
||||
// Margin to the right ^
|
||||
if s3_pull { "<< (S3) " } else { "" },
|
||||
if s3_push { " > (S3) " } else { "" },
|
||||
instant.elapsed().as_millis(),
|
||||
),
|
||||
db,
|
||||
)
|
||||
.await;
|
||||
// Drop lock, so next print success can fire
|
||||
}
|
||||
|
||||
let mut name_tl = 0;
|
||||
struct NotInstalledDependency {
|
||||
path: String,
|
||||
custom_name: Option<String>,
|
||||
short_name: Option<String>,
|
||||
display_name: String,
|
||||
}
|
||||
{
|
||||
let mut to_be_installed_is_used = false;
|
||||
for RequiredDependency {
|
||||
path, //
|
||||
custom_name,
|
||||
short_name,
|
||||
} in deps.into_iter()
|
||||
{
|
||||
if path.ends_with("/") {
|
||||
anyhow::bail!("Internal error: path should not end with '/'")
|
||||
}
|
||||
let display_name = short_name
|
||||
.as_ref()
|
||||
.or(custom_name.as_ref())
|
||||
.or(path.split("/").last().map(|e| e.to_owned()).as_ref())
|
||||
.unwrap_or_else(|| {
|
||||
tracing::warn!(
|
||||
workspace_id = %w_id,
|
||||
job_id = %job_id,
|
||||
"failed to parse top level directory name for {path}, fallback to full path.",
|
||||
);
|
||||
|
||||
&path
|
||||
})
|
||||
.to_owned();
|
||||
{
|
||||
// Later will help us align text in log console
|
||||
if display_name.len() > name_tl {
|
||||
name_tl = display_name.len();
|
||||
}
|
||||
}
|
||||
// Will look like: /tmp/windmill/cache/lang/dependency.valid.windmill
|
||||
if tokio::fs::metadata(path.clone() + ".valid.windmill")
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
if !to_be_installed_is_used {
|
||||
windmill_queue::append_logs(
|
||||
job_id,
|
||||
w_id,
|
||||
format!("\n--- INSTALLATION ---\n\nTo be installed:\n\n"),
|
||||
db.clone(),
|
||||
)
|
||||
.await;
|
||||
to_be_installed_is_used = true;
|
||||
}
|
||||
windmill_queue::append_logs(
|
||||
job_id,
|
||||
w_id,
|
||||
format!("- {display_name}\n"),
|
||||
db.clone(),
|
||||
)
|
||||
.await;
|
||||
not_installed.push(NotInstalledDependency {
|
||||
path,
|
||||
custom_name,
|
||||
short_name,
|
||||
display_name,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
total_to_install = not_installed.len();
|
||||
if total_to_install == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
#[cfg(all(feature = "enterprise", feature = "parquet"))]
|
||||
let is_not_pro = !matches!(
|
||||
windmill_common::ee::get_license_plan().await,
|
||||
windmill_common::ee::LicensePlan::Pro
|
||||
);
|
||||
#[cfg(all(feature = "enterprise", feature = "parquet"))]
|
||||
if is_not_pro && matches!(install_fn, InstallStrategy::AllAtOnce(_)) {
|
||||
windmill_queue::append_logs(
|
||||
job_id,
|
||||
w_id,
|
||||
format!("\nLooking for packages on S3:\n"),
|
||||
db.clone(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
// Parallelism level (N)
|
||||
let parallel_limit = // Semaphore will panic if value less then 1
|
||||
concurrent_downloads.clamp(1, 30);
|
||||
|
||||
tracing::info!(
|
||||
workspace_id = %w_id,
|
||||
"Install parallel limit: {}, job: {}",
|
||||
parallel_limit,
|
||||
job_id
|
||||
);
|
||||
|
||||
let mut handles = vec![];
|
||||
let semaphore = Arc::new(Semaphore::new(parallel_limit));
|
||||
let not_pulled = Arc::new(RwLock::new(vec![]));
|
||||
// let mut handles = Vec::with_capacity(total_to_install);
|
||||
for NotInstalledDependency {
|
||||
//
|
||||
path,
|
||||
custom_name,
|
||||
short_name,
|
||||
display_name,
|
||||
} in not_installed
|
||||
{
|
||||
let permit = semaphore.clone().acquire_owned().await; // Acquire a permit
|
||||
|
||||
if let Err(_) = permit {
|
||||
tracing::error!(
|
||||
workspace_id = %w_id,
|
||||
"Cannot acquire permit on semaphore, that can only mean that semaphore has been closed."
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
let permit = permit.unwrap();
|
||||
|
||||
#[cfg(all(feature = "enterprise", feature = "parquet"))]
|
||||
let s3_pull_future = if is_not_pro {
|
||||
if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
|
||||
.read()
|
||||
.await
|
||||
.clone()
|
||||
{
|
||||
Some(crate::global_cache::pull_from_tar(
|
||||
os,
|
||||
path.clone(),
|
||||
language_name.to_owned(),
|
||||
custom_name.clone(),
|
||||
platform_agnostic,
|
||||
))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let child = {
|
||||
if let InstallStrategy::Single(ref callback, ..) = install_fn {
|
||||
let cmd = callback(RequiredDependency {
|
||||
path: path.clone(),
|
||||
custom_name: custom_name.clone(),
|
||||
short_name: short_name.clone(),
|
||||
})?;
|
||||
tracing::debug!("{:?}", &cmd);
|
||||
Some(start_child_process(cmd, &installer_executable_name).await?)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
|
||||
let (
|
||||
worker_name_2,
|
||||
path_2,
|
||||
display_name_2,
|
||||
custom_name,
|
||||
job_id_2,
|
||||
w_id_2,
|
||||
db_2,
|
||||
counter_arc,
|
||||
language_name,
|
||||
installer_executable_name,
|
||||
not_pulled,
|
||||
) = (
|
||||
worker_name.to_owned(),
|
||||
path.clone(),
|
||||
display_name.clone(),
|
||||
custom_name.clone(),
|
||||
job_id.clone(),
|
||||
w_id.to_owned(),
|
||||
db.clone(),
|
||||
counter_arc.clone(),
|
||||
language_name.to_owned(),
|
||||
installer_executable_name.to_owned(),
|
||||
not_pulled.clone(),
|
||||
);
|
||||
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
|
||||
let _ = language_name;
|
||||
|
||||
let start = std::time::Instant::now();
|
||||
let handle = tokio::spawn(async move {
|
||||
let _permit = permit;
|
||||
|
||||
#[cfg(all(feature = "enterprise", feature = "parquet"))]
|
||||
if let Some(s3_pull_future) = s3_pull_future {
|
||||
if let Err(e) = s3_pull_future.await {
|
||||
tracing::info!(
|
||||
workspace_id = %w_id_2,
|
||||
"No tarball was found for {:?} on S3 or different problem occured {job_id_2}:\n{e}",
|
||||
&custom_name.clone().unwrap_or(path)
|
||||
);
|
||||
} else {
|
||||
// TODO: Refactor
|
||||
// Create a file to indicate that installation was successfull
|
||||
let valid_path = path_2.clone() + ".valid.windmill";
|
||||
// This is atomic operation, meaning, that it either completes and dependency is valid,
|
||||
// or it does not and dependency is invalid and will be reinstalled next run
|
||||
if let Err(e) = File::create(&valid_path).await {
|
||||
tracing::error!(
|
||||
workspace_id = %w_id_2,
|
||||
job_id = %job_id_2,
|
||||
"Failed to create {}!\n{e}\n
|
||||
This file needed for jobs to function",
|
||||
valid_path
|
||||
);
|
||||
};
|
||||
print_success(
|
||||
true,
|
||||
false,
|
||||
&job_id_2,
|
||||
&w_id_2,
|
||||
&display_name_2,
|
||||
name_tl,
|
||||
counter_arc,
|
||||
total_to_install,
|
||||
start,
|
||||
db_2,
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let Some(child) = child else {
|
||||
let mut lock = not_pulled.write().await;
|
||||
lock.push(RequiredDependency {
|
||||
path: path_2.clone(),
|
||||
custom_name: custom_name.clone(),
|
||||
short_name: short_name.clone(),
|
||||
});
|
||||
return;
|
||||
};
|
||||
if let Err(e) = crate::handle_child::handle_child(
|
||||
&job_id_2,
|
||||
&db_2,
|
||||
// TODO: Return mem_peak
|
||||
&mut 0,
|
||||
// TODO: Return canceld_by_ref
|
||||
&mut None,
|
||||
child,
|
||||
!*DISABLE_NSJAIL,
|
||||
&worker_name_2,
|
||||
&w_id_2,
|
||||
&installer_executable_name,
|
||||
None,
|
||||
false,
|
||||
&mut None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
windmill_queue::append_logs(
|
||||
&job_id_2,
|
||||
&w_id_2,
|
||||
format!("error while installing {}: {e:?}", &display_name_2),
|
||||
db_2.clone(),
|
||||
)
|
||||
.await;
|
||||
} else {
|
||||
// if let Some(cb) = postinstall_cb {
|
||||
// if let Err(e) = cb(vec![RequiredDependency {
|
||||
// path: path_2.clone(),
|
||||
// custom_name: custom_name.clone(),
|
||||
// short_name: short_name.clone(),
|
||||
// }])
|
||||
// .await
|
||||
// {
|
||||
// tracing::error!(
|
||||
// workspace_id = %w_id_2,
|
||||
// job_id = %job_id_2,
|
||||
// "Postinstall callback failed!\n{e}\n
|
||||
// This might affect execution",
|
||||
// );
|
||||
// }
|
||||
// }
|
||||
print_success(
|
||||
false,
|
||||
true,
|
||||
&job_id_2,
|
||||
&w_id_2,
|
||||
&display_name_2,
|
||||
name_tl,
|
||||
counter_arc,
|
||||
total_to_install,
|
||||
start,
|
||||
db_2,
|
||||
)
|
||||
.await;
|
||||
// TODO: Refactor
|
||||
// Create a file to indicate that installation was successfull
|
||||
let valid_path = path_2.clone() + ".valid.windmill";
|
||||
// This is atomic operation, meaning, that it either completes and dependency is valid,
|
||||
// or it does not and dependency is invalid and will be reinstalled next run
|
||||
if let Err(e) = File::create(&valid_path).await {
|
||||
tracing::error!(
|
||||
workspace_id = %w_id_2,
|
||||
job_id = %job_id_2,
|
||||
"Failed to create {}!\n{e}\n
|
||||
This file needed for jobs to function",
|
||||
valid_path
|
||||
);
|
||||
};
|
||||
#[cfg(all(feature = "enterprise", feature = "parquet"))]
|
||||
{
|
||||
if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
|
||||
.read()
|
||||
.await
|
||||
.clone()
|
||||
{
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = crate::global_cache::build_tar_and_push(
|
||||
os,
|
||||
path_2,
|
||||
language_name,
|
||||
custom_name,
|
||||
platform_agnostic,
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::warn!("failed to build tar and push: {e:?}");
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
handles.push(handle);
|
||||
}
|
||||
|
||||
for handle in handles {
|
||||
if let Err(e) = handle.await {
|
||||
tracing::error!("Error joining handles: {e:?}");
|
||||
}
|
||||
}
|
||||
if !not_pulled.read().await.is_empty() {
|
||||
if let InstallStrategy::AllAtOnce(ref callback, ..) = install_fn {
|
||||
let not_pulled_copy = not_pulled.read().await.clone();
|
||||
windmill_queue::append_logs(
|
||||
job_id,
|
||||
w_id,
|
||||
format!("\n\nFetching {} packages...\n", not_pulled_copy.len()),
|
||||
db.clone(),
|
||||
)
|
||||
.await;
|
||||
let cmd = callback(not_pulled_copy.clone())?;
|
||||
tracing::debug!("{:?}", &cmd);
|
||||
let child = start_child_process(cmd, &installer_executable_name).await?;
|
||||
let mut buf = "".to_owned();
|
||||
let pipe_stdout = if stdout_on_err { Some(&mut buf) } else { None };
|
||||
if let Err(e) = crate::handle_child::handle_child(
|
||||
// &job_id,
|
||||
&Uuid::nil(),
|
||||
&db,
|
||||
// TODO: Return mem_peak
|
||||
&mut 0,
|
||||
// TODO: Return canceld_by_ref
|
||||
&mut None,
|
||||
child,
|
||||
!*DISABLE_NSJAIL,
|
||||
&worker_name,
|
||||
&w_id,
|
||||
&installer_executable_name,
|
||||
None,
|
||||
false,
|
||||
&mut None,
|
||||
pipe_stdout,
|
||||
)
|
||||
.await
|
||||
{
|
||||
bail!(format!(
|
||||
"error while installing dependencies: {e:?}\n{}",
|
||||
buf
|
||||
));
|
||||
}
|
||||
{
|
||||
postinstall_cb(not_pulled_copy.clone()).await?;
|
||||
}
|
||||
for RequiredDependency { path, custom_name: _custom_name, .. } in
|
||||
not_pulled_copy.into_iter()
|
||||
{
|
||||
// TODO: Refactor
|
||||
// Create a file to indicate that installation was successfull
|
||||
let valid_path = path.clone() + ".valid.windmill";
|
||||
// This is atomic operation, meaning, that it either completes and dependency is valid,
|
||||
// or it does not and dependency is invalid and will be reinstalled next run
|
||||
if let Err(e) = File::create(&valid_path).await {
|
||||
tracing::error!(
|
||||
workspace_id = %w_id,
|
||||
job_id = %job_id,
|
||||
"Failed to create {}!\n{e}\n
|
||||
This file needed for jobs to function",
|
||||
valid_path
|
||||
);
|
||||
};
|
||||
#[cfg(all(feature = "enterprise", feature = "parquet"))]
|
||||
{
|
||||
if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
|
||||
.read()
|
||||
.await
|
||||
.clone()
|
||||
{
|
||||
let language_name = language_name.to_owned();
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = crate::global_cache::build_tar_and_push(
|
||||
os,
|
||||
path,
|
||||
language_name,
|
||||
_custom_name,
|
||||
platform_agnostic,
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::warn!("failed to build tar and push: {e:?}");
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
let total_time = total_time.elapsed().as_millis();
|
||||
windmill_queue::append_logs(
|
||||
&job_id,
|
||||
w_id,
|
||||
format!(
|
||||
"\nDone. Time spent on installation phase: {}ms\n",
|
||||
total_time
|
||||
),
|
||||
db,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -126,6 +126,7 @@ pub async fn generate_nuget_lockfile(
|
||||
None,
|
||||
false,
|
||||
&mut Some(occupancy_metrics),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -381,6 +382,7 @@ async fn build_cs_proj(
|
||||
None,
|
||||
false,
|
||||
&mut Some(occupancy_metrics),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
append_logs(job_id, w_id, "\n\n", db).await;
|
||||
@@ -400,6 +402,7 @@ async fn build_cs_proj(
|
||||
&bin_path,
|
||||
&format!("{CSHARP_OBJECT_STORE_PREFIX}{hash}"),
|
||||
&target,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -474,7 +477,8 @@ pub async fn handle_csharp_job(
|
||||
let bin_path = format!("{}/{hash}", CSHARP_CACHE_DIR);
|
||||
let remote_path = format!("{CSHARP_OBJECT_STORE_PREFIX}{hash}");
|
||||
|
||||
let (cache, cache_logs) = windmill_common::worker::load_cache(&bin_path, &remote_path).await;
|
||||
let (cache, cache_logs) =
|
||||
windmill_common::worker::load_cache(&bin_path, &remote_path, false).await;
|
||||
|
||||
let cache_logs = if cache {
|
||||
#[cfg(unix)]
|
||||
@@ -634,6 +638,7 @@ pub async fn handle_csharp_job(
|
||||
job.timeout,
|
||||
false,
|
||||
&mut Some(occupancy_metrics),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
read_result(job_dir).await
|
||||
|
||||
@@ -156,6 +156,7 @@ pub async fn generate_deno_lock(
|
||||
None,
|
||||
false,
|
||||
occupancy_metrics,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
@@ -414,6 +415,7 @@ try {{
|
||||
job.timeout,
|
||||
false,
|
||||
&mut Some(occupancy_metrics),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
// logs.push_str(format!("execute: {:?}\n", start.elapsed().as_millis()).as_str());
|
||||
|
||||
@@ -17,8 +17,9 @@ pub const TARGET: &str = const_format::concatcp!(std::env::consts::OS, "_", std:
|
||||
pub async fn build_tar_and_push(
|
||||
s3_client: Arc<dyn ObjectStore>,
|
||||
folder: String,
|
||||
// python_311
|
||||
python_xyz: String,
|
||||
lang: String,
|
||||
custom_folder_name: Option<String>,
|
||||
platform_agnostic: bool,
|
||||
) -> error::Result<()> {
|
||||
use object_store::path::Path;
|
||||
|
||||
@@ -28,9 +29,13 @@ pub async fn build_tar_and_push(
|
||||
let start = Instant::now();
|
||||
|
||||
// e.g. tiny==1.0.0
|
||||
let folder_name = folder.split("/").last().unwrap();
|
||||
let folder_name = if let Some(name) = custom_folder_name {
|
||||
name
|
||||
} else {
|
||||
folder.split("/").last().unwrap().to_owned()
|
||||
};
|
||||
|
||||
let prefix = &format!("{TAR_PYBASE_CACHE_DIR}/{}", python_xyz);
|
||||
let prefix = &format!("{TAR_PYBASE_CACHE_DIR}/{}", lang);
|
||||
let tar_path = format!("{prefix}/{folder_name}_tar.tar",);
|
||||
|
||||
let tar_file = std::fs::File::create(&tar_path)?;
|
||||
@@ -51,7 +56,10 @@ pub async fn build_tar_and_push(
|
||||
// })?;
|
||||
if let Err(e) = s3_client
|
||||
.put(
|
||||
&Path::from(format!("/tar/{TARGET}/{python_xyz}/{folder_name}.tar")),
|
||||
&Path::from(format!(
|
||||
"/tar/{}/{lang}/{folder_name}.tar",
|
||||
if platform_agnostic { "" } else { TARGET }
|
||||
)),
|
||||
std::fs::read(&tar_path)?.into(),
|
||||
)
|
||||
.await
|
||||
@@ -79,18 +87,26 @@ pub async fn build_tar_and_push(
|
||||
pub async fn pull_from_tar(
|
||||
client: Arc<dyn ObjectStore>,
|
||||
folder: String,
|
||||
// python_311
|
||||
python_xyz: String,
|
||||
lang: String,
|
||||
custom_folder_name: Option<String>,
|
||||
platform_agnostic: bool,
|
||||
) -> error::Result<()> {
|
||||
use windmill_common::s3_helpers::attempt_fetch_bytes;
|
||||
|
||||
let folder_name = folder.split("/").last().unwrap();
|
||||
let folder_name = if let Some(name) = custom_folder_name {
|
||||
name
|
||||
} else {
|
||||
folder.split("/").last().unwrap().to_owned()
|
||||
};
|
||||
|
||||
tracing::info!("Attempting to pull piptar {folder_name} from bucket");
|
||||
tracing::info!("Attempting to pull tar {folder_name} from bucket");
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
let tar_path = format!("tar/{TARGET}/{python_xyz}/{folder_name}.tar");
|
||||
let tar_path = format!(
|
||||
"tar/{}/{lang}/{folder_name}.tar",
|
||||
if platform_agnostic { "" } else { TARGET }
|
||||
);
|
||||
let bytes = attempt_fetch_bytes(client, &tar_path).await?;
|
||||
|
||||
extract_tar(bytes, &folder).map_err(|e| {
|
||||
|
||||
@@ -65,7 +65,8 @@ pub async fn handle_go_job(
|
||||
));
|
||||
let bin_path = format!("{}/{hash}", GO_BIN_CACHE_DIR);
|
||||
let remote_path = format!("{GO_OBJECT_STORE_PREFIX}{hash}");
|
||||
let (cache, cache_logs) = windmill_common::worker::load_cache(&bin_path, &remote_path).await;
|
||||
let (cache, cache_logs) =
|
||||
windmill_common::worker::load_cache(&bin_path, &remote_path, false).await;
|
||||
|
||||
let (skip_go_mod, skip_tidy) = if cache {
|
||||
(true, true)
|
||||
@@ -212,6 +213,7 @@ func Run(req Req) (interface{{}}, error){{
|
||||
None,
|
||||
false,
|
||||
&mut Some(occupation_metrics),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -219,6 +221,7 @@ func Run(req Req) (interface{{}}, error){{
|
||||
&bin_path,
|
||||
&format!("{GO_OBJECT_STORE_PREFIX}{hash}"),
|
||||
&format!("{job_dir}/main"),
|
||||
false,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -314,6 +317,7 @@ func Run(req Req) (interface{{}}, error){{
|
||||
job.timeout,
|
||||
false,
|
||||
&mut Some(occupation_metrics),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -378,6 +382,7 @@ pub async fn install_go_dependencies(
|
||||
None,
|
||||
false,
|
||||
&mut Some(occupation_metrics),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -444,6 +449,7 @@ pub async fn install_go_dependencies(
|
||||
None,
|
||||
false,
|
||||
&mut Some(occupation_metrics),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
||||
@@ -103,6 +103,8 @@ pub async fn handle_child(
|
||||
custom_timeout: Option<i32>,
|
||||
sigterm: bool,
|
||||
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
|
||||
// Do not print logs to output, but instead save to string.
|
||||
pipe_stdout: Option<&mut String>,
|
||||
) -> error::Result<()> {
|
||||
let start = Instant::now();
|
||||
|
||||
@@ -304,6 +306,8 @@ pub async fn handle_child(
|
||||
let mut log_total_size: u64 = 0;
|
||||
let pg_log_total_size = Arc::new(AtomicU32::new(0));
|
||||
|
||||
let mut pipe_stdout = pipe_stdout;
|
||||
|
||||
while let Some(line) = output.by_ref().next().await {
|
||||
|
||||
let do_write_ = do_write.shared();
|
||||
@@ -385,9 +389,12 @@ pub async fn handle_child(
|
||||
|
||||
let worker_name = worker.to_string();
|
||||
let w_id2 = w_id.to_string();
|
||||
(do_write, write_result) = tokio::spawn(append_job_logs(job_id, w_id2, joined, db.clone(), compact_logs, pg_log_total_size.clone(), worker_name)).remote_handle();
|
||||
|
||||
|
||||
if let Some(buf) = &mut pipe_stdout {
|
||||
buf.push_str(&joined);
|
||||
(do_write, write_result) = tokio::spawn(async { }).remote_handle();
|
||||
} else {
|
||||
(do_write, write_result) = tokio::spawn(append_job_logs(job_id, w_id2, joined, db.clone(), compact_logs, pg_log_total_size.clone(), worker_name)).remote_handle();
|
||||
}
|
||||
|
||||
if let Err(err) = result {
|
||||
tracing::error!(%job_id, %err, "error reading output for job {job_id} '{child_name}': {err}");
|
||||
|
||||
@@ -0,0 +1,915 @@
|
||||
use std::{collections::HashMap, path::PathBuf, process::Stdio, sync::Arc};
|
||||
|
||||
use anyhow::{anyhow, bail};
|
||||
use async_recursion::async_recursion;
|
||||
use itertools::Itertools;
|
||||
use serde_json::value::RawValue;
|
||||
use tokio::{
|
||||
fs::{create_dir_all, metadata, remove_dir_all, File},
|
||||
io::AsyncWriteExt,
|
||||
process::Command,
|
||||
};
|
||||
use uuid::Uuid;
|
||||
use windmill_common::{
|
||||
error::{self, Error},
|
||||
utils::calculate_hash,
|
||||
worker::{copy_dir_recursively, save_cache, write_file},
|
||||
};
|
||||
use windmill_parser::Arg;
|
||||
use windmill_parser_java::parse_java_sig_meta;
|
||||
use windmill_queue::{append_logs, CanceledBy, MiniPulledJob};
|
||||
|
||||
use crate::{
|
||||
common::{
|
||||
create_args_and_out_file, get_reserved_variables, par_install_language_dependencies,
|
||||
read_result, start_child_process, OccupancyMetrics, RequiredDependency,
|
||||
},
|
||||
handle_child, AuthedClient, COURSIER_CACHE_DIR, DISABLE_NSJAIL, DISABLE_NUSER, JAVA_CACHE_DIR,
|
||||
JAVA_REPOSITORY_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, NSJAIL_PATH, PATH_ENV, PROXY_ENVS,
|
||||
};
|
||||
lazy_static::lazy_static! {
|
||||
static ref JAVA_CONCURRENT_DOWNLOADS: usize = std::env::var("JAVA_CONCURRENT_DOWNLOADS").ok().map(|flag| flag.parse().unwrap_or(20)).unwrap_or(20);
|
||||
static ref JAVA_PATH: String = std::env::var("JAVA_PATH").unwrap_or_else(|_| "/usr/bin/java".to_string());
|
||||
static ref JAVAC_PATH: String = std::env::var("JAVAC_PATH").unwrap_or_else(|_| "/usr/bin/javac".to_string());
|
||||
static ref CS_PATH: String = std::env::var("COURSIER_PATH").unwrap_or_else(|_| "/usr/bin/coursier".to_string());
|
||||
static ref STOREPASS: String = std::env::var("JAVA_STOREPASS").unwrap_or("123456".into());
|
||||
static ref TRUST_STORE_PATH: String = std::env::var("JAVA_TRUST_STORE_PATH").unwrap_or("/usr/local/share/ca-certificates/truststore.jks".into());
|
||||
}
|
||||
|
||||
const NSJAIL_CONFIG_RUN_JAVA_CONTENT: &str = include_str!("../nsjail/run.java.config.proto");
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub(crate) struct JobHandlerInput<'a> {
|
||||
pub base_internal_url: &'a str,
|
||||
pub canceled_by: &'a mut Option<CanceledBy>,
|
||||
pub client: &'a AuthedClient,
|
||||
pub parent_runnable_path: Option<String>,
|
||||
pub db: &'a sqlx::Pool<sqlx::Postgres>,
|
||||
pub envs: HashMap<String, String>,
|
||||
pub inner_content: &'a str,
|
||||
pub job: &'a MiniPulledJob,
|
||||
pub job_dir: &'a str,
|
||||
pub mem_peak: &'a mut i32,
|
||||
pub occupancy_metrics: &'a mut OccupancyMetrics,
|
||||
pub requirements_o: Option<&'a String>,
|
||||
pub shared_mount: &'a str,
|
||||
pub worker_name: &'a str,
|
||||
}
|
||||
|
||||
pub async fn handle_java_job<'a>(mut args: JobHandlerInput<'a>) -> Result<Box<RawValue>, Error> {
|
||||
// --- Prepare ---
|
||||
{
|
||||
prepare(&mut args).await?;
|
||||
}
|
||||
// --- Generate Lockfile ---
|
||||
|
||||
let deps = resolve(
|
||||
&args.job.id,
|
||||
&args.inner_content,
|
||||
&args.job_dir,
|
||||
&args.db,
|
||||
&args.job.workspace_id,
|
||||
)
|
||||
.await?;
|
||||
|
||||
// --- Install ---
|
||||
|
||||
let classpath = install(&mut args, deps).await?;
|
||||
|
||||
// --- Build .java files ---
|
||||
{
|
||||
compile(&mut args, &classpath).await?;
|
||||
}
|
||||
// --- Run ---
|
||||
{
|
||||
run(&mut args, &classpath).await?;
|
||||
}
|
||||
// --- Retrieve results ---
|
||||
{
|
||||
read_result(&args.job_dir).await
|
||||
}
|
||||
}
|
||||
|
||||
async fn prepare<'a>(
|
||||
JobHandlerInput { job, db, job_dir, client, inner_content, .. }: &mut JobHandlerInput<'a>,
|
||||
) -> Result<(), Error> {
|
||||
// Create needed files
|
||||
{
|
||||
create_args_and_out_file(&client, job, job_dir, db).await?;
|
||||
let app_path = format!("{}/src/main/java/net/script/", job_dir);
|
||||
create_dir_all(&app_path).await?;
|
||||
File::create(format!("{app_path}/App.java"))
|
||||
.await?
|
||||
.write_all(&wrap(inner_content)?.into_bytes())
|
||||
.await?;
|
||||
File::create(format!("{app_path}/Main.java"))
|
||||
.await?
|
||||
.write_all(
|
||||
&format!(
|
||||
"package net.script;\n{MINI_CLIENT_IMPORTS}\n{}\n{MINI_CLIENT}",
|
||||
inner_content
|
||||
)
|
||||
.into_bytes(),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn resolve<'a>(
|
||||
job_id: &Uuid,
|
||||
code: &str,
|
||||
job_dir: &str,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
w_id: &str,
|
||||
) -> Result<String, Error> {
|
||||
let deps = {
|
||||
let find_requirements = code.lines().find_position(|x| {
|
||||
x.starts_with("//requirements:") || x.starts_with("// requirements:")
|
||||
});
|
||||
|
||||
let specified_deps = if let Some((pos, _)) = find_requirements {
|
||||
code.lines()
|
||||
.skip(pos + 1)
|
||||
.map_while(|x| {
|
||||
if x.starts_with("//") {
|
||||
Some(x.replace("//", "").trim().to_owned())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect::<Vec<String>>()
|
||||
} else {
|
||||
Default::default()
|
||||
};
|
||||
|
||||
let mut deps = vec![
|
||||
// Default requirements
|
||||
"com.fasterxml.jackson.core:jackson-databind:2.9.8".to_owned(),
|
||||
];
|
||||
deps.extend(specified_deps);
|
||||
deps.join("\n")
|
||||
};
|
||||
|
||||
let req_hash = format!("java-{}", calculate_hash(&deps));
|
||||
if let Some(cached) = sqlx::query_scalar!(
|
||||
"SELECT lockfile FROM pip_resolution_cache WHERE hash = $1",
|
||||
req_hash
|
||||
)
|
||||
.fetch_optional(db)
|
||||
.await?
|
||||
{
|
||||
return Ok(cached);
|
||||
}
|
||||
let lock = {
|
||||
append_logs(
|
||||
job_id,
|
||||
w_id,
|
||||
format!("\n--- RESOLVING LOCKFILE ---\n"),
|
||||
db.clone(),
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut cmd = Command::new(if cfg!(windows) {
|
||||
"java"
|
||||
} else {
|
||||
JAVA_PATH.as_str()
|
||||
});
|
||||
cmd.env_clear()
|
||||
.current_dir(job_dir.to_owned())
|
||||
.env("PATH", PATH_ENV.as_str())
|
||||
.envs(PROXY_ENVS.clone());
|
||||
|
||||
// Configure proxies
|
||||
{
|
||||
let jps = parse_proxy()?;
|
||||
if let Some(val) = jps.https_host {
|
||||
cmd.arg(&format!("-Dhttps.proxyHost={}", val));
|
||||
}
|
||||
if let Some(val) = jps.https_port {
|
||||
cmd.arg(&format!("-Dhttps.proxyPort={}", val));
|
||||
}
|
||||
if let Some(val) = jps.http_host {
|
||||
cmd.arg(&format!("-Dhttp.proxyHost={}", val));
|
||||
}
|
||||
if let Some(val) = jps.http_port {
|
||||
cmd.arg(&format!("-Dhttp.proxyPort={}", val));
|
||||
}
|
||||
if let Some(val) = jps.no_proxy {
|
||||
cmd.arg(&format!("-Dhttp.nonProxyHosts=\"{}\"", val));
|
||||
}
|
||||
}
|
||||
if metadata(TRUST_STORE_PATH.clone()).await.is_ok() {
|
||||
cmd.args(&[
|
||||
&format!("-Djavax.net.ssl.trustStore={}", *TRUST_STORE_PATH),
|
||||
&format!("-Djavax.net.ssl.trustStorePassword={}", *STOREPASS),
|
||||
]);
|
||||
}
|
||||
cmd.args(&[
|
||||
"-jar",
|
||||
&CS_PATH,
|
||||
"resolve",
|
||||
&get_no_default(),
|
||||
"--parallel",
|
||||
&format!("{}", *JAVA_CONCURRENT_DOWNLOADS),
|
||||
"--cache",
|
||||
COURSIER_CACHE_DIR,
|
||||
])
|
||||
.args(&get_repos().await)
|
||||
.args(&deps.split("\n").collect_vec())
|
||||
.stderr(Stdio::piped());
|
||||
|
||||
#[cfg(windows)]
|
||||
{
|
||||
cmd.env("SystemRoot", crate::SYSTEM_ROOT.as_str())
|
||||
.env("USERPROFILE", crate::USERPROFILE_ENV.as_str())
|
||||
.env(
|
||||
"TMP",
|
||||
std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")),
|
||||
);
|
||||
}
|
||||
let output = cmd.output().await?;
|
||||
// Check if the command was successful
|
||||
if output.status.success() {
|
||||
String::from_utf8(output.stdout).expect("Failed to convert output to String")
|
||||
} else {
|
||||
let stderr =
|
||||
String::from_utf8(output.stderr).expect("Failed to convert error output to String");
|
||||
return Err(error::Error::internal_err(stderr));
|
||||
}
|
||||
};
|
||||
|
||||
sqlx::query!(
|
||||
"INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2",
|
||||
req_hash,
|
||||
lock.clone(),
|
||||
).fetch_optional(db).await?;
|
||||
|
||||
append_logs(job_id, w_id, format!("\n{}", &lock), db.clone()).await;
|
||||
Ok(lock)
|
||||
}
|
||||
|
||||
async fn install<'a>(
|
||||
JobHandlerInput { worker_name, job, db, job_dir, .. }: &mut JobHandlerInput<'a>,
|
||||
deps: String,
|
||||
) -> Result<String, Error> {
|
||||
let deps = deps
|
||||
.lines()
|
||||
.map(|line| {
|
||||
let unparsed_dep = line.replace(":jar", "").replace(":lib", "");
|
||||
let mut it = unparsed_dep.split(":");
|
||||
|
||||
match (it.next(), it.next(), it.next()) {
|
||||
(Some(group_id), Some(artifact_id), Some(version)) => {
|
||||
let path = format!(
|
||||
"{JAVA_REPOSITORY_DIR}/{}/{artifact_id}/{version}",
|
||||
group_id.replace(".", "/")
|
||||
);
|
||||
Ok(RequiredDependency {
|
||||
path,
|
||||
custom_name: Some(format!("{group_id}:{artifact_id}:{version}")),
|
||||
short_name: Some(format!("{artifact_id}:{version}")),
|
||||
})
|
||||
}
|
||||
_ => anyhow::bail!("{line} is not parsable"),
|
||||
}
|
||||
})
|
||||
.collect::<anyhow::Result<Vec<RequiredDependency>>>()?;
|
||||
|
||||
let classpath = deps
|
||||
.clone()
|
||||
.into_iter()
|
||||
.map(|RequiredDependency { path, .. }| path + "/*")
|
||||
.collect_vec()
|
||||
.join(":")
|
||||
+ ":target";
|
||||
|
||||
#[cfg(windows)]
|
||||
let classpath = classpath.replace(":", ";");
|
||||
|
||||
tracing::debug!(
|
||||
workspace_id = %job.workspace_id,
|
||||
"JAVA classpath: {}", &classpath
|
||||
);
|
||||
let (repos, no_default, trust_store_metadata) = (
|
||||
get_repos().await,
|
||||
get_no_default(),
|
||||
metadata(TRUST_STORE_PATH.clone()).await,
|
||||
);
|
||||
let job_dir = job_dir.to_owned();
|
||||
let fetch_dir = format!("{JAVA_CACHE_DIR}/tmp-fetch-{}", Uuid::new_v4());
|
||||
let fetch_dir2 = fetch_dir.clone();
|
||||
par_install_language_dependencies(
|
||||
deps,
|
||||
"java",
|
||||
"java",
|
||||
true,
|
||||
*JAVA_CONCURRENT_DOWNLOADS,
|
||||
true,
|
||||
crate::common::InstallStrategy::AllAtOnce(Arc::new(move |dependencies| {
|
||||
let mut cmd = Command::new(if cfg!(windows) {
|
||||
"java"
|
||||
} else {
|
||||
JAVA_PATH.as_str()
|
||||
});
|
||||
let artifacts = dependencies
|
||||
.into_iter()
|
||||
.map(|e| {
|
||||
e.custom_name.ok_or(anyhow::anyhow!(
|
||||
"Internal Error: Artifact name should be Some!"
|
||||
))
|
||||
})
|
||||
.collect::<anyhow::Result<Vec<String>>>()?;
|
||||
cmd.env_clear()
|
||||
.current_dir(&job_dir)
|
||||
.env("PATH", PATH_ENV.as_str())
|
||||
.envs(PROXY_ENVS.clone());
|
||||
// Configure proxies
|
||||
{
|
||||
let jps = parse_proxy()?;
|
||||
if let Some(val) = jps.https_host {
|
||||
cmd.arg(&format!("-Dhttps.proxyHost={}", val));
|
||||
}
|
||||
if let Some(val) = jps.https_port {
|
||||
cmd.arg(&format!("-Dhttps.proxyPort={}", val));
|
||||
}
|
||||
if let Some(val) = jps.http_host {
|
||||
cmd.arg(&format!("-Dhttp.proxyHost={}", val));
|
||||
}
|
||||
if let Some(val) = jps.http_port {
|
||||
cmd.arg(&format!("-Dhttp.proxyPort={}", val));
|
||||
}
|
||||
if let Some(val) = jps.no_proxy {
|
||||
cmd.arg(&format!("-Dhttp.nonProxyHosts=\"{}\"", val));
|
||||
}
|
||||
}
|
||||
|
||||
if trust_store_metadata.is_ok() {
|
||||
cmd.args(&[
|
||||
&format!("-Djavax.net.ssl.trustStore={}", *TRUST_STORE_PATH),
|
||||
&format!("-Djavax.net.ssl.trustStorePassword={}", *STOREPASS),
|
||||
]);
|
||||
}
|
||||
cmd.args(&[
|
||||
"-jar",
|
||||
&CS_PATH,
|
||||
"fetch",
|
||||
&no_default,
|
||||
"--quiet",
|
||||
"--parallel",
|
||||
&format!("{}", *JAVA_CONCURRENT_DOWNLOADS),
|
||||
"--cache",
|
||||
&fetch_dir,
|
||||
])
|
||||
.args(&repos)
|
||||
.arg("--intransitive")
|
||||
.args(artifacts)
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped());
|
||||
#[cfg(windows)]
|
||||
{
|
||||
cmd.env("SystemRoot", crate::SYSTEM_ROOT.as_str())
|
||||
.env("USERPROFILE", crate::USERPROFILE_ENV.as_str())
|
||||
.env(
|
||||
"TMP",
|
||||
std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")),
|
||||
);
|
||||
}
|
||||
|
||||
Ok(cmd)
|
||||
})),
|
||||
async move |_| {
|
||||
move_to_repository(&fetch_dir2, 0).await?;
|
||||
remove_dir_all(&fetch_dir2).await?;
|
||||
#[async_recursion]
|
||||
async fn move_to_repository(path: &str, depth: u8) -> anyhow::Result<()> {
|
||||
if depth == 3 {
|
||||
copy_dir_recursively(
|
||||
&PathBuf::from(path),
|
||||
&PathBuf::from(JAVA_REPOSITORY_DIR),
|
||||
)?;
|
||||
|
||||
return Ok(());
|
||||
}
|
||||
let mut entries = tokio::fs::read_dir(path).await?;
|
||||
loop {
|
||||
let Some(entry) = entries.next_entry().await? else {
|
||||
break Ok(());
|
||||
};
|
||||
|
||||
let path = entry
|
||||
.path()
|
||||
.to_str()
|
||||
.ok_or(anyhow!("Internal Error: Cannot convert Path to Str"))?
|
||||
.to_owned();
|
||||
|
||||
move_to_repository(&path, depth + 1).await?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
&job.id,
|
||||
&job.workspace_id,
|
||||
worker_name,
|
||||
db,
|
||||
)
|
||||
.await?;
|
||||
Ok(classpath)
|
||||
}
|
||||
|
||||
async fn compile<'a>(
|
||||
JobHandlerInput {
|
||||
occupancy_metrics,
|
||||
mem_peak,
|
||||
canceled_by,
|
||||
worker_name,
|
||||
job,
|
||||
db,
|
||||
job_dir,
|
||||
client,
|
||||
envs,
|
||||
base_internal_url,
|
||||
inner_content,
|
||||
requirements_o,
|
||||
parent_runnable_path,
|
||||
..
|
||||
}: &mut JobHandlerInput<'a>,
|
||||
classpath: &'a str,
|
||||
// plugins: Vec<&'a str>,
|
||||
) -> Result<(), Error> {
|
||||
fn compute_hash(code: &str, requirements_o: Option<&String>) -> String {
|
||||
calculate_hash(&format!(
|
||||
"{}{}",
|
||||
code,
|
||||
requirements_o
|
||||
.as_ref()
|
||||
.map(|x| x.to_string())
|
||||
.unwrap_or_default()
|
||||
))
|
||||
}
|
||||
let reserved_variables =
|
||||
get_reserved_variables(job, &client.token, db, parent_runnable_path.clone()).await?;
|
||||
let hash = compute_hash(inner_content, *requirements_o);
|
||||
let bin_path = format!("{}/{hash}", JAVA_CACHE_DIR);
|
||||
let remote_path = format!("java_jar/{hash}");
|
||||
let (cache, ..) = windmill_common::worker::load_cache(&bin_path, &remote_path, true).await;
|
||||
|
||||
if cache {
|
||||
let target = format!("{job_dir}/target");
|
||||
|
||||
#[cfg(unix)]
|
||||
let symlink = std::os::unix::fs::symlink(&bin_path, &target);
|
||||
#[cfg(windows)]
|
||||
let symlink = copy_dir_recursively(&PathBuf::from(&bin_path), &PathBuf::from(&target));
|
||||
|
||||
symlink.map_err(|e| {
|
||||
Error::ExecutionErr(format!(
|
||||
"could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}"
|
||||
))
|
||||
})?;
|
||||
} else {
|
||||
// let plugin_registry = format!("{job_dir}/plugin-registry");
|
||||
let child = {
|
||||
append_logs(
|
||||
&job.id,
|
||||
&job.workspace_id,
|
||||
format!("\n--- COMPILING .JAVA FILES\n"),
|
||||
db.clone(),
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut cmd = Command::new(if cfg!(windows) {
|
||||
"javac"
|
||||
} else {
|
||||
JAVAC_PATH.as_str()
|
||||
});
|
||||
cmd.env_clear()
|
||||
.current_dir(job_dir.to_owned())
|
||||
.env("PATH", PATH_ENV.as_str())
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
.envs(envs)
|
||||
.envs(reserved_variables)
|
||||
.envs(PROXY_ENVS.clone())
|
||||
.args(&[
|
||||
"-classpath",
|
||||
&classpath,
|
||||
"src/main/java/net/script/Main.java",
|
||||
"src/main/java/net/script/App.java",
|
||||
"-d",
|
||||
"./target",
|
||||
])
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped());
|
||||
|
||||
#[cfg(windows)]
|
||||
{
|
||||
cmd.env("SystemRoot", crate::SYSTEM_ROOT.as_str())
|
||||
.env("USERPROFILE", crate::USERPROFILE_ENV.as_str())
|
||||
.env(
|
||||
"TMP",
|
||||
std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")),
|
||||
);
|
||||
}
|
||||
start_child_process(cmd, "javac").await?
|
||||
};
|
||||
handle_child::handle_child(
|
||||
&job.id,
|
||||
db,
|
||||
mem_peak,
|
||||
canceled_by,
|
||||
child,
|
||||
!*DISABLE_NSJAIL,
|
||||
worker_name,
|
||||
&job.workspace_id,
|
||||
"javac",
|
||||
job.timeout,
|
||||
false,
|
||||
&mut Some(occupancy_metrics),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
match save_cache(
|
||||
&bin_path,
|
||||
&format!("java_jar/{hash}"),
|
||||
&format!("{job_dir}/target"),
|
||||
true,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Err(e) => {
|
||||
let em = format!(
|
||||
"could not save {bin_path} to {} to java cache: {e:?}",
|
||||
format!("{job_dir}/main"),
|
||||
);
|
||||
tracing::error!(em);
|
||||
}
|
||||
Ok(logs) => {
|
||||
tracing::trace!(logs);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
Ok(())
|
||||
}
|
||||
async fn run<'a>(
|
||||
JobHandlerInput {
|
||||
occupancy_metrics,
|
||||
mem_peak,
|
||||
canceled_by,
|
||||
worker_name,
|
||||
job,
|
||||
db,
|
||||
job_dir,
|
||||
shared_mount,
|
||||
client,
|
||||
envs,
|
||||
base_internal_url,
|
||||
parent_runnable_path,
|
||||
..
|
||||
}: &mut JobHandlerInput<'a>,
|
||||
classpath: &'a str,
|
||||
) -> Result<(), Error> {
|
||||
let reserved_variables =
|
||||
get_reserved_variables(job, &client.token, db, parent_runnable_path.clone()).await?;
|
||||
|
||||
let child = if !cfg!(windows) && !*DISABLE_NSJAIL {
|
||||
append_logs(
|
||||
&job.id,
|
||||
&job.workspace_id,
|
||||
format!("\n--- ISOLATED JAVA CODE EXECUTION ---\n"),
|
||||
db.clone(),
|
||||
)
|
||||
.await;
|
||||
|
||||
write_file(
|
||||
job_dir,
|
||||
"run.config.proto",
|
||||
&NSJAIL_CONFIG_RUN_JAVA_CONTENT
|
||||
.replace("{JOB_DIR}", job_dir)
|
||||
.replace("{CACHE_DIR}", JAVA_CACHE_DIR)
|
||||
.replace("{SHARED_MOUNT}", &shared_mount)
|
||||
// .replace("{CACHED_TARGET}", &shared_mount)
|
||||
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()),
|
||||
)?;
|
||||
let mut cmd = Command::new(NSJAIL_PATH.as_str());
|
||||
cmd.env_clear()
|
||||
.current_dir(job_dir)
|
||||
.env("PATH", PATH_ENV.as_str())
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
.envs(envs)
|
||||
.envs(reserved_variables)
|
||||
.envs(PROXY_ENVS.clone())
|
||||
.args(vec![
|
||||
"--config",
|
||||
"run.config.proto",
|
||||
"--",
|
||||
JAVA_PATH.as_str(),
|
||||
]);
|
||||
if metadata(TRUST_STORE_PATH.clone()).await.is_ok() {
|
||||
cmd.args(&[
|
||||
&format!("-Djavax.net.ssl.trustStore={}", *TRUST_STORE_PATH),
|
||||
&format!("-Djavax.net.ssl.trustStorePassword={}", *STOREPASS),
|
||||
]);
|
||||
}
|
||||
// Configure proxies
|
||||
{
|
||||
let jps = parse_proxy()?;
|
||||
if let Some(val) = jps.https_host {
|
||||
cmd.arg(&format!("-Dhttps.proxyHost={}", val));
|
||||
}
|
||||
if let Some(val) = jps.https_port {
|
||||
cmd.arg(&format!("-Dhttps.proxyPort={}", val));
|
||||
}
|
||||
if let Some(val) = jps.http_host {
|
||||
cmd.arg(&format!("-Dhttp.proxyHost={}", val));
|
||||
}
|
||||
if let Some(val) = jps.http_port {
|
||||
cmd.arg(&format!("-Dhttp.proxyPort={}", val));
|
||||
}
|
||||
if let Some(val) = jps.no_proxy {
|
||||
cmd.arg(&format!("-Dhttp.nonProxyHosts=\"{}\"", val));
|
||||
}
|
||||
}
|
||||
cmd.args(vec!["-classpath", &classpath, "net.script.App"]);
|
||||
cmd.stdout(Stdio::piped()).stderr(Stdio::piped());
|
||||
|
||||
start_child_process(cmd, NSJAIL_PATH.as_str()).await?
|
||||
} else {
|
||||
append_logs(
|
||||
&job.id,
|
||||
&job.workspace_id,
|
||||
format!("\n--- JAVA CODE EXECUTION ---\n"),
|
||||
db.clone(),
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut cmd = Command::new(if cfg!(windows) {
|
||||
"java"
|
||||
} else {
|
||||
JAVA_PATH.as_str()
|
||||
});
|
||||
cmd.env_clear()
|
||||
.current_dir(job_dir.to_owned())
|
||||
.env("PATH", PATH_ENV.as_str())
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
.envs(envs)
|
||||
.envs(reserved_variables);
|
||||
if metadata(TRUST_STORE_PATH.clone()).await.is_ok() {
|
||||
cmd.args(&[
|
||||
&format!("-Djavax.net.ssl.trustStore={}", *TRUST_STORE_PATH),
|
||||
&format!("-Djavax.net.ssl.trustStorePassword={}", *STOREPASS),
|
||||
]);
|
||||
}
|
||||
// Configure proxies
|
||||
{
|
||||
let jps = parse_proxy()?;
|
||||
if let Some(val) = jps.https_host {
|
||||
cmd.arg(&format!("-Dhttps.proxyHost={}", val));
|
||||
}
|
||||
if let Some(val) = jps.https_port {
|
||||
cmd.arg(&format!("-Dhttps.proxyPort={}", val));
|
||||
}
|
||||
if let Some(val) = jps.http_host {
|
||||
cmd.arg(&format!("-Dhttp.proxyHost={}", val));
|
||||
}
|
||||
if let Some(val) = jps.http_port {
|
||||
cmd.arg(&format!("-Dhttp.proxyPort={}", val));
|
||||
}
|
||||
if let Some(val) = jps.no_proxy {
|
||||
cmd.arg(&format!("-Dhttp.nonProxyHosts=\"{}\"", val));
|
||||
}
|
||||
}
|
||||
cmd.args(&["-classpath", &classpath, "net.script.App"])
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped());
|
||||
|
||||
#[cfg(windows)]
|
||||
{
|
||||
cmd.env("SystemRoot", crate::SYSTEM_ROOT.as_str())
|
||||
.env("USERPROFILE", crate::USERPROFILE_ENV.as_str())
|
||||
.env(
|
||||
"TMP",
|
||||
std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")),
|
||||
);
|
||||
}
|
||||
start_child_process(cmd, "java").await?
|
||||
};
|
||||
handle_child::handle_child(
|
||||
&job.id,
|
||||
db,
|
||||
mem_peak,
|
||||
canceled_by,
|
||||
child,
|
||||
!*DISABLE_NSJAIL,
|
||||
worker_name,
|
||||
&job.workspace_id,
|
||||
"java",
|
||||
job.timeout,
|
||||
false,
|
||||
&mut Some(occupancy_metrics),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[derive(Default, Debug)]
|
||||
struct JavaProxySettings {
|
||||
http_host: Option<String>,
|
||||
http_port: Option<String>,
|
||||
https_host: Option<String>,
|
||||
https_port: Option<String>,
|
||||
no_proxy: Option<String>,
|
||||
}
|
||||
fn parse_proxy() -> anyhow::Result<JavaProxySettings> {
|
||||
let mut jps = JavaProxySettings::default();
|
||||
for (ident, mut val) in PROXY_ENVS.clone() {
|
||||
match ident {
|
||||
"HTTPS_PROXY" => {
|
||||
if val.contains("http://") {
|
||||
bail!("HTTPS_PROXY url cannot contain http scheme.");
|
||||
}
|
||||
if !val.contains("https://") {
|
||||
val = format!("https://{val}");
|
||||
}
|
||||
let mut url = url::Url::parse(&val)?;
|
||||
let port = url.port();
|
||||
// Make sure port and schema is not included in final url
|
||||
{
|
||||
url.set_port(None).unwrap_or_default();
|
||||
jps.https_host = Some(url.as_str().replace("https://", ""));
|
||||
if let Some(port) = port {
|
||||
jps.https_port = Some(format!("{}", port));
|
||||
}
|
||||
}
|
||||
}
|
||||
"HTTP_PROXY" => {
|
||||
if val.contains("https://") {
|
||||
bail!("HTTP_PROXY url cannot contain https scheme.");
|
||||
}
|
||||
if !val.contains("http://") {
|
||||
val = format!("http://{val}");
|
||||
}
|
||||
let mut url = url::Url::parse(&val)?;
|
||||
let port = url.port();
|
||||
// Make sure port and schema is not included in final url
|
||||
{
|
||||
url.set_port(None).unwrap_or_default();
|
||||
jps.http_host = Some(url.as_str().replace("http://", ""));
|
||||
if let Some(port) = port {
|
||||
jps.https_port = Some(format!("{}", port));
|
||||
}
|
||||
}
|
||||
}
|
||||
// Java uses | instead of ,
|
||||
"NO_PROXY" => jps.no_proxy = Some(val.replace(",", "|")),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(jps)
|
||||
}
|
||||
async fn get_repos() -> Vec<String> {
|
||||
MAVEN_REPOS
|
||||
.read()
|
||||
.await
|
||||
.as_ref()
|
||||
.map(|repos| {
|
||||
repos
|
||||
.trim()
|
||||
.split_whitespace()
|
||||
.into_iter()
|
||||
.map(|el| vec!["--repository".to_owned(), el.to_owned()])
|
||||
.collect_vec()
|
||||
})
|
||||
.unwrap_or_default()
|
||||
.concat()
|
||||
}
|
||||
|
||||
fn get_no_default() -> String {
|
||||
if NO_DEFAULT_MAVEN.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
"--no-default"
|
||||
} else {
|
||||
// Command does not take empty arguments
|
||||
"-q"
|
||||
}
|
||||
.into()
|
||||
}
|
||||
|
||||
/// Wraps content script
|
||||
/// that upon execution reads args.json (which are piped and transformed from previous flow step or top level inputs)
|
||||
/// Also wrapper takes output of program and serializes to result.json (Which windmill will know how to use later)
|
||||
fn wrap(inner_content: &str) -> Result<String, Error> {
|
||||
let sig = parse_java_sig_meta(inner_content)?;
|
||||
let ret_void = sig.returns_void;
|
||||
let spread = sig
|
||||
.main_sig
|
||||
.args
|
||||
.clone()
|
||||
.into_iter()
|
||||
.map(|Arg { name, .. }| {
|
||||
// Apply additional input transformation
|
||||
format!(" parsedArgs.{name}")
|
||||
})
|
||||
.collect_vec()
|
||||
.join(",");
|
||||
let args = sig
|
||||
.main_sig
|
||||
.args
|
||||
.clone()
|
||||
.into_iter()
|
||||
.map(|Arg { name, otyp, .. }| {
|
||||
// Apply additional input transformation
|
||||
format!("public {} {name};\n", otyp.unwrap())
|
||||
})
|
||||
.collect_vec()
|
||||
.join(" ");
|
||||
Ok(r#"
|
||||
package net.script;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import net.script.Main;
|
||||
|
||||
public class App{
|
||||
|
||||
public static class Args {ARGS}
|
||||
|
||||
public static void main(String[] args) {
|
||||
try {
|
||||
InputStream fileInputStream = new FileInputStream("args.json");
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
Args parsedArgs = mapper.readValue(fileInputStream, Args.class);
|
||||
fileInputStream.close();
|
||||
{MAIN_HANDLER}
|
||||
FileOutputStream fileOutputStream = new FileOutputStream("result.json");
|
||||
mapper.writeValue(fileOutputStream, res);
|
||||
fileOutputStream.close();
|
||||
|
||||
} catch (Exception e) { // Catching general Exception
|
||||
e.printStackTrace(); // Handle the exception
|
||||
}
|
||||
}
|
||||
}
|
||||
"#
|
||||
.replace(
|
||||
"{MAIN_HANDLER}",
|
||||
if ret_void {
|
||||
"
|
||||
Main.main(SPREAD);
|
||||
Object res = null;
|
||||
"
|
||||
} else {
|
||||
"
|
||||
Object res = Main.main(SPREAD);
|
||||
"
|
||||
},
|
||||
)
|
||||
.replace("SPREAD", &spread)
|
||||
.replace("ARGS", &args))
|
||||
}
|
||||
const MINI_CLIENT_IMPORTS: &str = r#"
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
"#;
|
||||
const MINI_CLIENT: &str = r#"
|
||||
class Wmill {
|
||||
public static String getVariable(String path) {
|
||||
var baseUrl = System.getenv("BASE_INTERNAL_URL");
|
||||
var workspace = System.getenv("WM_WORKSPACE");
|
||||
var uri = java.text.MessageFormat.format("{0}/api/w/{1}/variables/get_value/{2}", baseUrl, workspace, path);
|
||||
|
||||
// Create an HttpRequest
|
||||
HttpRequest request = HttpRequest.newBuilder()
|
||||
.uri(URI.create(uri))
|
||||
.header("Authorization", "Bearer " + System.getenv("WM_TOKEN")) // Add the Authorization header
|
||||
.GET() // Set the request method to GET
|
||||
.build();
|
||||
|
||||
// Send the request and get the response
|
||||
return HttpClient.newHttpClient().sendAsync(request, HttpResponse.BodyHandlers.ofString())
|
||||
.thenApply(HttpResponse::body)
|
||||
.join(); // Wait for the completion
|
||||
}
|
||||
public static String getResource(String path) {
|
||||
var baseUrl = System.getenv("BASE_INTERNAL_URL");
|
||||
var workspace = System.getenv("WM_WORKSPACE");
|
||||
var uri = java.text.MessageFormat.format("{0}/api/w/{1}/resources/get_value_interpolated/{2}", baseUrl, workspace, path);
|
||||
|
||||
// Create an HttpRequest
|
||||
HttpRequest request = HttpRequest.newBuilder()
|
||||
.uri(URI.create(uri))
|
||||
.header("Authorization", "Bearer " + System.getenv("WM_TOKEN")) // Add the Authorization header
|
||||
.GET() // Set the request method to GET
|
||||
.build();
|
||||
|
||||
// Send the request and get the response
|
||||
return HttpClient.newHttpClient().sendAsync(request, HttpResponse.BodyHandlers.ofString())
|
||||
.thenApply(HttpResponse::body)
|
||||
.join(); // Wait for the completion
|
||||
}
|
||||
}
|
||||
"#;
|
||||
@@ -11,6 +11,10 @@ mod bash_executor;
|
||||
|
||||
#[cfg(feature = "nu")]
|
||||
mod nu_executor;
|
||||
|
||||
#[cfg(feature = "java")]
|
||||
mod java_executor;
|
||||
|
||||
#[cfg(feature = "benchmark")]
|
||||
pub mod bench;
|
||||
mod bun_executor;
|
||||
@@ -40,11 +44,11 @@ mod python_executor;
|
||||
mod result_processor;
|
||||
#[cfg(feature = "rust")]
|
||||
mod rust_executor;
|
||||
mod sanitized_sql_params;
|
||||
mod schema;
|
||||
mod worker;
|
||||
mod worker_flow;
|
||||
mod worker_lockfiles;
|
||||
mod sanitized_sql_params;
|
||||
mod schema;
|
||||
|
||||
pub use worker::*;
|
||||
|
||||
|
||||
@@ -334,6 +334,7 @@ async fn run<'a>(
|
||||
job.timeout,
|
||||
false,
|
||||
&mut Some(occupancy_metrics),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -105,6 +105,7 @@ pub async fn composer_install(
|
||||
None,
|
||||
false,
|
||||
&mut Some(occupancy_metrics),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -339,6 +340,7 @@ try {{
|
||||
job.timeout,
|
||||
false,
|
||||
&mut Some(occupancy_metrics),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
read_result(job_dir).await
|
||||
|
||||
@@ -27,7 +27,7 @@ use windmill_common::{
|
||||
Error::{self},
|
||||
},
|
||||
utils::calculate_hash,
|
||||
worker::{write_file, PythonAnnotations, WORKER_CONFIG},
|
||||
worker::{copy_dir_recursively, pad_string, write_file, PythonAnnotations, WORKER_CONFIG},
|
||||
DB,
|
||||
};
|
||||
|
||||
@@ -49,6 +49,7 @@ lazy_static::lazy_static! {
|
||||
static ref PY_CONCURRENT_DOWNLOADS: usize =
|
||||
var("PY_CONCURRENT_DOWNLOADS").ok().map(|flag| flag.parse().unwrap_or(20)).unwrap_or(20);
|
||||
|
||||
|
||||
static ref NON_ALPHANUM_CHAR: Regex = regex::Regex::new(r"[^0-9A-Za-z=.-]").unwrap();
|
||||
|
||||
static ref TRUSTED_HOST: Option<String> = var("PY_TRUSTED_HOST").ok().or(var("PIP_TRUSTED_HOST").ok());
|
||||
@@ -350,6 +351,7 @@ impl PyVersion {
|
||||
None,
|
||||
false,
|
||||
occupancy_metrics,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -644,6 +646,7 @@ pub async fn uv_pip_compile(
|
||||
None,
|
||||
false,
|
||||
occupancy_metrics,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
@@ -780,30 +783,6 @@ async fn postinstall(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn copy_dir_recursively(src: &Path, dst: &Path) -> windmill_common::error::Result<()> {
|
||||
if !dst.exists() {
|
||||
fs::create_dir_all(dst)?;
|
||||
}
|
||||
|
||||
tracing::debug!("Copying recursively from {:?} to {:?}", src, dst);
|
||||
|
||||
for entry in fs::read_dir(src)? {
|
||||
let entry = entry?;
|
||||
let src_path = entry.path();
|
||||
let dst_path = dst.join(entry.file_name());
|
||||
|
||||
if src_path.is_dir() && !src_path.is_symlink() {
|
||||
copy_dir_recursively(&src_path, &dst_path)?;
|
||||
} else {
|
||||
fs::copy(&src_path, &dst_path)?;
|
||||
}
|
||||
}
|
||||
|
||||
tracing::debug!("Finished copying recursively from {:?} to {:?}", src, dst);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn get_python_path(
|
||||
py_version: PyVersion,
|
||||
worker_name: &str,
|
||||
@@ -1161,6 +1140,7 @@ mount {{
|
||||
job.timeout,
|
||||
false,
|
||||
&mut Some(occupancy_metrics),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -1709,19 +1689,6 @@ async fn spawn_uv_install(
|
||||
}
|
||||
}
|
||||
|
||||
/// length = 5
|
||||
/// value = "foo"
|
||||
/// output = "foo "
|
||||
/// 12345
|
||||
fn pad_string(value: &str, total_length: usize) -> String {
|
||||
if value.len() >= total_length {
|
||||
value.to_string() // Return the original string if it's already long enough
|
||||
} else {
|
||||
let padding_needed = total_length - value.len();
|
||||
format!("{value}{}", " ".repeat(padding_needed)) // Pad with spaces
|
||||
}
|
||||
}
|
||||
|
||||
/// uv pip install, include cached or pull from S3
|
||||
pub async fn handle_python_reqs(
|
||||
requirements: Vec<&str>,
|
||||
@@ -2053,7 +2020,7 @@ pub async fn handle_python_reqs(
|
||||
tokio::select! {
|
||||
// Cancel was called on the job
|
||||
_ = kill_rx.recv() => return Err(anyhow::anyhow!("S3 pull was canceled")),
|
||||
pull = pull_from_tar(os, venv_p.clone(), py_version.to_cache_dir_top_level()) => {
|
||||
pull = pull_from_tar(os, venv_p.clone(), py_version.to_cache_dir_top_level(), None, false) => {
|
||||
if let Err(e) = pull {
|
||||
tracing::info!(
|
||||
workspace_id = %w_id,
|
||||
@@ -2204,7 +2171,7 @@ pub async fn handle_python_reqs(
|
||||
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
|
||||
if s3_push {
|
||||
if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() {
|
||||
tokio::spawn(build_tar_and_push(os, venv_p.clone(), py_version.to_cache_dir_top_level()));
|
||||
tokio::spawn(build_tar_and_push(os, venv_p.clone(), py_version.to_cache_dir_top_level(), None, false));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -164,6 +164,7 @@ pub async fn generate_cargo_lockfile(
|
||||
None,
|
||||
false,
|
||||
&mut Some(occupancy_metrics),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -226,6 +227,7 @@ pub async fn build_rust_crate(
|
||||
None,
|
||||
false,
|
||||
&mut Some(occupancy_metrics),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
append_logs(job_id, w_id, "\n\n", db).await;
|
||||
@@ -245,6 +247,7 @@ pub async fn build_rust_crate(
|
||||
&bin_path,
|
||||
&format!("{RUST_OBJECT_STORE_PREFIX}{hash}"),
|
||||
&format!("{job_dir}/main"),
|
||||
false,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -294,7 +297,8 @@ pub async fn handle_rust_job(
|
||||
let bin_path = format!("{}/{hash}", RUST_CACHE_DIR);
|
||||
let remote_path = format!("{RUST_OBJECT_STORE_PREFIX}{hash}");
|
||||
|
||||
let (cache, cache_logs) = windmill_common::worker::load_cache(&bin_path, &remote_path).await;
|
||||
let (cache, cache_logs) =
|
||||
windmill_common::worker::load_cache(&bin_path, &remote_path, false).await;
|
||||
|
||||
let cache_logs = if cache {
|
||||
let target = format!("{job_dir}/main");
|
||||
@@ -407,6 +411,7 @@ pub async fn handle_rust_job(
|
||||
job.timeout,
|
||||
false,
|
||||
&mut Some(occupancy_metrics),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
read_result(job_dir).await
|
||||
|
||||
@@ -123,7 +123,10 @@ use crate::{
|
||||
use crate::rust_executor::handle_rust_job;
|
||||
|
||||
#[cfg(feature = "nu")]
|
||||
use crate::nu_executor::{handle_nu_job, JobHandlerInput};
|
||||
use crate::nu_executor::{handle_nu_job, JobHandlerInput as JobHandlerInputNu};
|
||||
|
||||
#[cfg(feature = "java")]
|
||||
use crate::java_executor::{handle_java_job, JobHandlerInput as JobHandlerInputJava};
|
||||
|
||||
#[cfg(feature = "php")]
|
||||
use crate::php_executor::handle_php_job;
|
||||
@@ -264,6 +267,8 @@ pub const TAR_PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_311"
|
||||
pub const TAR_PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_312");
|
||||
pub const TAR_PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_313");
|
||||
|
||||
pub const TAR_JAVA_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/java");
|
||||
|
||||
pub const UV_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "uv");
|
||||
pub const PY_INSTALL_DIR: &str = concatcp!(ROOT_CACHE_DIR, "py_runtime");
|
||||
pub const TAR_PYBASE_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar");
|
||||
@@ -275,6 +280,12 @@ pub const GO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "go");
|
||||
pub const RUST_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "rust");
|
||||
pub const NU_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "nu");
|
||||
pub const CSHARP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "csharp");
|
||||
|
||||
// JAVA
|
||||
pub const JAVA_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "java");
|
||||
pub const COURSIER_CACHE_DIR: &str = concatcp!(JAVA_CACHE_DIR, "/coursier-cache");
|
||||
pub const JAVA_REPOSITORY_DIR: &str = concatcp!(JAVA_CACHE_DIR, "/repository");
|
||||
// KJQXZ
|
||||
pub const BUN_CACHE_DIR: &str = concatcp!(ROOT_CACHE_NOMOUNT_DIR, "bun");
|
||||
pub const BUN_BUNDLE_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "bun");
|
||||
pub const BUN_CODEBASE_BUNDLE_CACHE_DIR: &str = concatcp!(ROOT_CACHE_NOMOUNT_DIR, "script_bundle");
|
||||
@@ -398,6 +409,11 @@ lazy_static::lazy_static! {
|
||||
pub static ref NPM_CONFIG_REGISTRY: Arc<RwLock<Option<String>>> = Arc::new(RwLock::new(None));
|
||||
pub static ref BUNFIG_INSTALL_SCOPES: Arc<RwLock<Option<String>>> = Arc::new(RwLock::new(None));
|
||||
pub static ref NUGET_CONFIG: Arc<RwLock<Option<String>>> = Arc::new(RwLock::new(None));
|
||||
pub static ref MAVEN_REPOS: Arc<RwLock<Option<String>>> = Arc::new(RwLock::new(None));
|
||||
pub static ref NO_DEFAULT_MAVEN: AtomicBool = AtomicBool::new(std::env::var("NO_DEFAULT_MAVEN")
|
||||
.ok()
|
||||
.and_then(|x| x.parse::<bool>().ok())
|
||||
.unwrap_or(false));
|
||||
|
||||
pub static ref PIP_EXTRA_INDEX_URL: Arc<RwLock<Option<String>>> = Arc::new(RwLock::new(None));
|
||||
pub static ref PIP_INDEX_URL: Arc<RwLock<Option<String>>> = Arc::new(RwLock::new(None));
|
||||
@@ -3078,7 +3094,33 @@ mount {{
|
||||
);
|
||||
|
||||
#[cfg(feature = "nu")]
|
||||
handle_nu_job(JobHandlerInput {
|
||||
handle_nu_job(JobHandlerInputNu {
|
||||
mem_peak,
|
||||
canceled_by,
|
||||
job,
|
||||
db,
|
||||
client,
|
||||
parent_runnable_path,
|
||||
inner_content: &code,
|
||||
job_dir,
|
||||
requirements_o: lock.as_ref(),
|
||||
shared_mount: &shared_mount,
|
||||
base_internal_url,
|
||||
worker_name,
|
||||
envs,
|
||||
occupancy_metrics,
|
||||
})
|
||||
.await
|
||||
}
|
||||
Some(ScriptLang::Java) => {
|
||||
#[cfg(not(feature = "java"))]
|
||||
return Err(anyhow::anyhow!(
|
||||
"Java is not available because the feature is not enabled"
|
||||
)
|
||||
.into());
|
||||
|
||||
#[cfg(feature = "java")]
|
||||
handle_java_job(JobHandlerInputJava {
|
||||
mem_peak,
|
||||
canceled_by,
|
||||
job,
|
||||
@@ -3163,6 +3205,11 @@ fn parse_sig_of_lang(
|
||||
ScriptLang::Nu => Some(windmill_parser_nu::parse_nu_signature(code)?),
|
||||
#[cfg(not(feature = "nu"))]
|
||||
ScriptLang::Nu => None,
|
||||
#[cfg(feature = "java")]
|
||||
ScriptLang::Java => Some(windmill_parser_java::parse_java_signature(code)?),
|
||||
#[cfg(not(feature = "java"))]
|
||||
ScriptLang::Java => None,
|
||||
// KJQXZ
|
||||
}
|
||||
} else {
|
||||
None
|
||||
|
||||
@@ -36,6 +36,9 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob, PushIsolationLevel}
|
||||
use crate::common::OccupancyMetrics;
|
||||
use crate::csharp_executor::generate_nuget_lockfile;
|
||||
|
||||
#[cfg(feature = "java")]
|
||||
use crate::java_executor::resolve;
|
||||
|
||||
#[cfg(feature = "php")]
|
||||
use crate::php_executor::{composer_install, parse_php_imports};
|
||||
#[cfg(feature = "python")]
|
||||
@@ -2076,16 +2079,17 @@ async fn capture_dependency_job(
|
||||
)
|
||||
.await
|
||||
}
|
||||
ScriptLang::Postgresql => Ok("".to_owned()),
|
||||
ScriptLang::Mysql => Ok("".to_owned()),
|
||||
ScriptLang::Bigquery => Ok("".to_owned()),
|
||||
ScriptLang::Snowflake => Ok("".to_owned()),
|
||||
ScriptLang::Mssql => Ok("".to_owned()),
|
||||
ScriptLang::Graphql => Ok("".to_owned()),
|
||||
ScriptLang::OracleDB => Ok("".to_owned()),
|
||||
ScriptLang::Bash => Ok("".to_owned()),
|
||||
ScriptLang::Nu => Ok("".to_owned()),
|
||||
ScriptLang::Powershell => Ok("".to_owned()),
|
||||
ScriptLang::Nativets => Ok("".to_owned()),
|
||||
#[cfg(feature = "java")]
|
||||
ScriptLang::Java => {
|
||||
if raw_deps {
|
||||
return Err(Error::ExecutionErr(
|
||||
"Raw dependencies not supported for Java".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
resolve(job_id, job_raw_code, job_dir, db, w_id).await
|
||||
}
|
||||
// KJQXZ
|
||||
_ => Ok("".to_owned()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,4 +119,12 @@ inventory:
|
||||
debug:
|
||||
msg: "Hello, world!"
|
||||
`,
|
||||
java: `
|
||||
public class Main {
|
||||
public static void main() {
|
||||
System.out.println("Hello World");
|
||||
}
|
||||
}
|
||||
`,
|
||||
// KJQXZ
|
||||
};
|
||||
|
||||
@@ -69,6 +69,7 @@ await build({
|
||||
"rust",
|
||||
"yaml",
|
||||
"csharp",
|
||||
"java",
|
||||
];
|
||||
|
||||
for (const l of dirs) {
|
||||
|
||||
@@ -8652,4 +8652,4 @@ export const clearIndex = (data: ClearIndexData): CancelablePromise<ClearIndexRe
|
||||
path: {
|
||||
idx_name: data.idxName
|
||||
}
|
||||
}); };
|
||||
}); };
|
||||
|
||||
@@ -389,7 +389,7 @@ export type MainArgSignature = {
|
||||
|
||||
export type type2 = 'Valid' | 'Invalid';
|
||||
|
||||
export type ScriptLang = 'python3' | 'deno' | 'go' | 'bash' | 'powershell' | 'postgresql' | 'mysql' | 'bigquery' | 'snowflake' | 'mssql' | 'oracledb' | 'graphql' | 'nativets' | 'bun' | 'php' | 'rust' | 'ansible' | 'csharp' | 'nu';
|
||||
export type ScriptLang = 'python3' | 'deno' | 'go' | 'bash' | 'powershell' | 'postgresql' | 'mysql' | 'bigquery' | 'snowflake' | 'mssql' | 'oracledb' | 'graphql' | 'nativets' | 'bun' | 'php' | 'rust' | 'ansible' | 'csharp' | 'nu' | 'java';
|
||||
|
||||
export type Preview = {
|
||||
content?: string;
|
||||
@@ -7521,4 +7521,4 @@ export type ClearIndexData = {
|
||||
idxName: 'JobIndex' | 'ServiceLogIndex';
|
||||
};
|
||||
|
||||
export type ClearIndexResponse = (string);
|
||||
export type ClearIndexResponse = (string);
|
||||
|
||||
+4
-2
@@ -12,8 +12,6 @@ import {
|
||||
ScriptMetadata,
|
||||
defaultScriptMetadata,
|
||||
} from "./bootstrap/script_bootstrap.ts";
|
||||
|
||||
|
||||
import { Workspace } from "./workspace.ts";
|
||||
import { SchemaProperty } from "./bootstrap/common.ts";
|
||||
import { ScriptLanguage } from "./script_common.ts";
|
||||
@@ -550,6 +548,10 @@ export async function inferSchema(
|
||||
} else if (language === "ansible") {
|
||||
const { parse_ansible } = await import("./wasm/yaml/windmill_parser_wasm.js");
|
||||
inferedSchema = JSON.parse(parse_ansible(content));
|
||||
} else if (language === "java") {
|
||||
const { parse_java } = await import("./wasm/java/windmill_parser_wasm.js");
|
||||
inferedSchema = JSON.parse(parse_java(content));
|
||||
// KJQXZ
|
||||
} else {
|
||||
throw new Error("Invalid language: " + language);
|
||||
}
|
||||
|
||||
@@ -569,6 +569,9 @@ export function filePathExtensionFromContentType(
|
||||
return ".cs";
|
||||
} else if (language === "nu") {
|
||||
return ".nu";
|
||||
} else if (language === "java") {
|
||||
return ".java";
|
||||
// KJQXZ
|
||||
} else {
|
||||
throw new Error("Invalid language: " + language);
|
||||
}
|
||||
@@ -596,6 +599,9 @@ export const exts = [
|
||||
".cs",
|
||||
".nu",
|
||||
".playbook.yml",
|
||||
".java"
|
||||
// KJQXZ
|
||||
|
||||
];
|
||||
|
||||
export function removeExtensionToPath(path: string): string {
|
||||
|
||||
@@ -17,7 +17,9 @@ export type ScriptLanguage =
|
||||
| "rust"
|
||||
| "csharp"
|
||||
| "nu"
|
||||
| "ansible";
|
||||
| "ansible"
|
||||
| "java";
|
||||
// KJQXZ
|
||||
|
||||
export function inferContentTypeFromFilePath(
|
||||
contentPath: string,
|
||||
@@ -63,6 +65,9 @@ export function inferContentTypeFromFilePath(
|
||||
return "ansible";
|
||||
} else if (contentPath.endsWith(".nu")) {
|
||||
return "nu";
|
||||
} else if (contentPath.endsWith(".java")) {
|
||||
return "java";
|
||||
// KJQXZ
|
||||
} else {
|
||||
throw new Error(
|
||||
"Invalid language: " + contentPath.substring(contentPath.lastIndexOf("."))
|
||||
|
||||
@@ -342,6 +342,8 @@ export function newPathAssigner(defaultTs: "bun" | "deno"): PathAssigner {
|
||||
else if (language == "csharp") ext = "cs";
|
||||
else if (language == "nu") ext = "nu";
|
||||
else if (language == "ansible") ext = "playbook.yml";
|
||||
else if (language == "java") ext = "java";
|
||||
// KJQXZ
|
||||
else ext = "no_ext";
|
||||
|
||||
return [`${name}.inline_script.`, ext];
|
||||
@@ -689,6 +691,8 @@ export async function elementsToMap(
|
||||
"cs",
|
||||
"yml",
|
||||
"nu",
|
||||
"java",
|
||||
// KJQXZ
|
||||
].includes(path.split(".").pop() ?? "") &&
|
||||
!isFileResource(path)
|
||||
)
|
||||
|
||||
@@ -227,6 +227,8 @@ export function getTypeStrFromPath(
|
||||
parsed.ext == ".rs" ||
|
||||
parsed.ext == ".cs" ||
|
||||
parsed.ext == ".nu" ||
|
||||
parsed.ext == ".java" ||
|
||||
// KJQXZ
|
||||
(parsed.ext == ".yml" && parsed.name.split(".").pop() == "playbook")
|
||||
) {
|
||||
return "script";
|
||||
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
/* tslint:disable */
|
||||
/* eslint-disable */
|
||||
export function parse_java(code: string): string;
|
||||
@@ -0,0 +1,121 @@
|
||||
|
||||
|
||||
let WASM_VECTOR_LEN = 0;
|
||||
|
||||
let cachedUint8ArrayMemory0 = null;
|
||||
|
||||
function getUint8ArrayMemory0() {
|
||||
if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) {
|
||||
cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer);
|
||||
}
|
||||
return cachedUint8ArrayMemory0;
|
||||
}
|
||||
|
||||
const cachedTextEncoder = (typeof TextEncoder !== 'undefined' ? new TextEncoder('utf-8') : { encode: () => { throw Error('TextEncoder not available') } } );
|
||||
|
||||
const encodeString = function (arg, view) {
|
||||
return cachedTextEncoder.encodeInto(arg, view);
|
||||
};
|
||||
|
||||
function passStringToWasm0(arg, malloc, realloc) {
|
||||
|
||||
if (realloc === undefined) {
|
||||
const buf = cachedTextEncoder.encode(arg);
|
||||
const ptr = malloc(buf.length, 1) >>> 0;
|
||||
getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf);
|
||||
WASM_VECTOR_LEN = buf.length;
|
||||
return ptr;
|
||||
}
|
||||
|
||||
let len = arg.length;
|
||||
let ptr = malloc(len, 1) >>> 0;
|
||||
|
||||
const mem = getUint8ArrayMemory0();
|
||||
|
||||
let offset = 0;
|
||||
|
||||
for (; offset < len; offset++) {
|
||||
const code = arg.charCodeAt(offset);
|
||||
if (code > 0x7F) break;
|
||||
mem[ptr + offset] = code;
|
||||
}
|
||||
|
||||
if (offset !== len) {
|
||||
if (offset !== 0) {
|
||||
arg = arg.slice(offset);
|
||||
}
|
||||
ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0;
|
||||
const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len);
|
||||
const ret = encodeString(arg, view);
|
||||
|
||||
offset += ret.written;
|
||||
ptr = realloc(ptr, len, offset, 1) >>> 0;
|
||||
}
|
||||
|
||||
WASM_VECTOR_LEN = offset;
|
||||
return ptr;
|
||||
}
|
||||
|
||||
const cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } );
|
||||
|
||||
if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); };
|
||||
|
||||
function getStringFromWasm0(ptr, len) {
|
||||
ptr = ptr >>> 0;
|
||||
return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len));
|
||||
}
|
||||
/**
|
||||
* @param {string} code
|
||||
* @returns {string}
|
||||
*/
|
||||
export function parse_java(code) {
|
||||
let deferred2_0;
|
||||
let deferred2_1;
|
||||
try {
|
||||
const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
||||
const len0 = WASM_VECTOR_LEN;
|
||||
const ret = wasm.parse_java(ptr0, len0);
|
||||
deferred2_0 = ret[0];
|
||||
deferred2_1 = ret[1];
|
||||
return getStringFromWasm0(ret[0], ret[1]);
|
||||
} finally {
|
||||
wasm.__wbindgen_free(deferred2_0, deferred2_1, 1);
|
||||
}
|
||||
}
|
||||
|
||||
const imports = {
|
||||
__wbindgen_placeholder__: {
|
||||
__wbindgen_init_externref_table: function() {
|
||||
const table = wasm.__wbindgen_export_0;
|
||||
const offset = table.grow(4);
|
||||
table.set(0, undefined);
|
||||
table.set(offset + 0, undefined);
|
||||
table.set(offset + 1, null);
|
||||
table.set(offset + 2, true);
|
||||
table.set(offset + 3, false);
|
||||
;
|
||||
},
|
||||
},
|
||||
|
||||
};
|
||||
|
||||
const wasm_url = new URL('windmill_parser_wasm_bg.wasm', import.meta.url);
|
||||
let wasmCode = '';
|
||||
switch (wasm_url.protocol) {
|
||||
case 'file:':
|
||||
wasmCode = await Deno.readFile(wasm_url);
|
||||
break
|
||||
case 'https:':
|
||||
case 'http:':
|
||||
wasmCode = await (await fetch(wasm_url)).arrayBuffer();
|
||||
break
|
||||
default:
|
||||
throw new Error(`Unsupported protocol: ${wasm_url.protocol}`);
|
||||
}
|
||||
|
||||
const wasmInstance = (await WebAssembly.instantiate(wasmCode, imports)).instance;
|
||||
const wasm = wasmInstance.exports;
|
||||
export const __wasm = wasm;
|
||||
|
||||
wasm.__wbindgen_start();
|
||||
|
||||
Binary file not shown.
+29
@@ -0,0 +1,29 @@
|
||||
/* tslint:disable */
|
||||
/* eslint-disable */
|
||||
export const memory: WebAssembly.Memory;
|
||||
export const parse_java: (a: number, b: number) => [number, number];
|
||||
export const abort: () => void;
|
||||
export const malloc: (a: number) => number;
|
||||
export const calloc: (a: number, b: number) => number;
|
||||
export const realloc: (a: number, b: number) => number;
|
||||
export const free: (a: number) => void;
|
||||
export const strncmp: (a: number, b: number, c: number) => number;
|
||||
export const iswspace: (a: number) => number;
|
||||
export const iswalnum: (a: number) => number;
|
||||
export const clock: () => bigint;
|
||||
export const isprint: (a: number) => number;
|
||||
export const fprintf: (a: number, b: number, c: number) => number;
|
||||
export const fputs: (a: number, b: number) => number;
|
||||
export const fputc: (a: number, b: number) => number;
|
||||
export const fdopen: (a: number, b: number) => number;
|
||||
export const fclose: (a: number) => number;
|
||||
export const fwrite: (a: number, b: number, c: number, d: number) => number;
|
||||
export const vsnprintf: (a: number, b: number, c: number, d: number) => number;
|
||||
export const clock_gettime: (a: number, b: number) => void;
|
||||
export const snprintf: () => void;
|
||||
export const __assert_fail: (a: number, b: number, c: number, d: number) => void;
|
||||
export const __wbindgen_export_0: WebAssembly.Table;
|
||||
export const __wbindgen_malloc: (a: number, b: number) => number;
|
||||
export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
|
||||
export const __wbindgen_free: (a: number, b: number, c: number) => void;
|
||||
export const __wbindgen_start: () => void;
|
||||
@@ -14,3 +14,10 @@ ENV DOTNET_ROOT="/opt/dotnet-sdk/bin"
|
||||
|
||||
# Nushell
|
||||
COPY --from=ghcr.io/nushell/nushell:0.101.0-bookworm /usr/bin/nu /usr/bin/nu
|
||||
|
||||
# Java
|
||||
RUN apt-get -y update && apt-get install -y default-jdk
|
||||
RUN curl -fLo coursier https://github.com/coursier/coursier/releases/download/v2.1.24/coursier \
|
||||
&& mv ./coursier /usr/bin/coursier \
|
||||
&& chmod +x /usr/bin/coursier
|
||||
RUN /usr/bin/java -jar /usr/bin/coursier about
|
||||
|
||||
@@ -37,3 +37,10 @@ RUN echo /opt/oracle/23/lib > /etc/ld.so.conf.d/oracle-instantclient.conf && ldc
|
||||
|
||||
# Nushell
|
||||
COPY --from=ghcr.io/nushell/nushell:0.101.0-bookworm /usr/bin/nu /usr/bin/nu
|
||||
|
||||
# Java
|
||||
RUN apt-get -y update && apt-get install -y default-jdk
|
||||
RUN curl -fLo coursier https://github.com/coursier/coursier/releases/download/v2.1.24/coursier \
|
||||
&& mv ./coursier /usr/bin/coursier \
|
||||
&& chmod +x /usr/bin/coursier
|
||||
RUN /usr/bin/java -jar /usr/bin/coursier about
|
||||
|
||||
@@ -62,4 +62,12 @@ RUN echo /opt/oracle/23/lib > /etc/ld.so.conf.d/oracle-instantclient.conf && ldc
|
||||
# Nushell
|
||||
COPY --from=ghcr.io/nushell/nushell:0.101.0-bookworm /usr/bin/nu /usr/bin/nu
|
||||
|
||||
# Java
|
||||
RUN apt-get install -y default-jdk
|
||||
RUN curl -fLo coursier https://github.com/coursier/coursier/releases/download/v2.1.24/coursier \
|
||||
&& mv ./coursier /usr/bin/coursier \
|
||||
&& chmod +x /usr/bin/coursier
|
||||
RUN /usr/bin/java -jar /usr/bin/coursier about
|
||||
|
||||
COPY --from=nsjail /nsjail/nsjail /bin/nsjail
|
||||
|
||||
|
||||
@@ -31,6 +31,13 @@
|
||||
pkg-config
|
||||
cmake
|
||||
];
|
||||
coursier = pkgs.fetchFromGitHub {
|
||||
owner = "coursier";
|
||||
repo = "launchers";
|
||||
rev = "79d927f7586c09ca6d8cd01862adb0d9f9d88dff";
|
||||
hash = "sha256-8E0WtDFc7RcqmftDigMyy1xXUkjgL4X4kpf7h1GdE48=";
|
||||
};
|
||||
|
||||
PKG_CONFIG_PATH = pkgs.lib.makeSearchPath "lib/pkgconfig"
|
||||
(with pkgs; [ openssl.dev libxml2.dev xmlsec.dev libxslt.dev ]);
|
||||
RUSTY_V8_ARCHIVE = let
|
||||
@@ -75,6 +82,7 @@
|
||||
sqlx-cli
|
||||
flock
|
||||
sccache
|
||||
nsjail
|
||||
deno
|
||||
python3
|
||||
python3Packages.pip
|
||||
@@ -84,7 +92,9 @@
|
||||
nushell
|
||||
dotnet-sdk_9
|
||||
oracle-instantclient
|
||||
# LSP/Local dev
|
||||
svelte-language-server
|
||||
taplo
|
||||
]);
|
||||
packages = [
|
||||
(pkgs.writeScriptBin "wm-caddy" ''
|
||||
@@ -126,6 +136,25 @@
|
||||
cd ./frontend
|
||||
npm run dev $*
|
||||
'')
|
||||
(pkgs.writeScriptBin "wm-minio" ''
|
||||
set -e
|
||||
cd ./backend
|
||||
mkdir -p .minio-data/wmill
|
||||
${pkgs.minio}/bin/minio server ./.minio-data
|
||||
'')
|
||||
# Generate keys
|
||||
# TODO: Do not set new keys if ran multiple times
|
||||
(pkgs.writeScriptBin "wm-minio-keys" ''
|
||||
set -e
|
||||
cd ./backend
|
||||
${pkgs.minio-client}/bin/mc alias set 'wmill-minio-dev' 'http://localhost:9000' 'minioadmin' 'minioadmin'
|
||||
${pkgs.minio-client}/bin/mc admin accesskey create myminio | tee .minio-data/secrets.txt
|
||||
echo ""
|
||||
echo 'Saving to: ./backend/.minio-data/secrets.txt'
|
||||
echo "bucket: wmill"
|
||||
echo "endpoint: http://localhost:9000"
|
||||
'')
|
||||
|
||||
];
|
||||
|
||||
inherit PKG_CONFIG_PATH RUSTY_V8_ARCHIVE;
|
||||
@@ -140,11 +169,16 @@
|
||||
BUN_PATH = "${pkgs.bun}/bin/bun";
|
||||
UV_PATH = "${pkgs.uv}/bin/uv";
|
||||
NU_PATH = "${pkgs.nushell}/bin/nu";
|
||||
JAVA_PATH = "${pkgs.jdk21}/bin/java";
|
||||
JAVAC_PATH = "${pkgs.jdk21}/bin/javac";
|
||||
COURSIER_PATH = "${coursier}/coursier";
|
||||
# KJQXZ
|
||||
FLOCK_PATH = "${pkgs.flock}/bin/flock";
|
||||
CARGO_PATH = "${rust}/bin/cargo";
|
||||
DOTNET_PATH = "${pkgs.dotnet-sdk_9}/bin/dotnet";
|
||||
DOTNET_ROOT = "${pkgs.dotnet-sdk_9}/share/dotnet";
|
||||
ORACLE_LIB_DIR = "${pkgs.oracle-instantclient.lib}/lib";
|
||||
RUST_LOG = "debug";
|
||||
};
|
||||
packages.default = self.packages.${system}.windmill;
|
||||
packages.windmill-client = pkgs.buildNpmPackage {
|
||||
@@ -218,7 +252,6 @@
|
||||
"cloud"
|
||||
"jemalloc"
|
||||
"tantivy"
|
||||
"deno_core"
|
||||
"license"
|
||||
"http_trigger"
|
||||
"zip"
|
||||
@@ -226,16 +259,10 @@
|
||||
"kafka"
|
||||
"otel"
|
||||
"dind"
|
||||
"php"
|
||||
"mysql"
|
||||
"mssql"
|
||||
"bigquery"
|
||||
"websocket"
|
||||
"python"
|
||||
"smtp"
|
||||
"csharp"
|
||||
"static_frontend"
|
||||
"rust"
|
||||
"all_languages"
|
||||
];
|
||||
doCheck = false;
|
||||
|
||||
|
||||
Generated
+6
@@ -68,6 +68,7 @@
|
||||
"vscode-ws-jsonrpc": "~3.4.0",
|
||||
"windmill-parser-wasm-csharp": "^1.437.1",
|
||||
"windmill-parser-wasm-go": "^1.429.0",
|
||||
"windmill-parser-wasm-java": "^1.478.1",
|
||||
"windmill-parser-wasm-nu": "^1.474.1",
|
||||
"windmill-parser-wasm-php": "^1.429.0",
|
||||
"windmill-parser-wasm-py": "^1.477.1",
|
||||
@@ -12528,6 +12529,11 @@
|
||||
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-go/-/windmill-parser-wasm-go-1.429.0.tgz",
|
||||
"integrity": "sha512-M3jeGDqeTyPj9HyyX3msdzMrqIIzlfMfxTMsXS8m7MJp4Cm60qifMxD29Ipxb2B4WdzyGwCSlaBjLsXu0b3c5g=="
|
||||
},
|
||||
"node_modules/windmill-parser-wasm-java": {
|
||||
"version": "1.478.1",
|
||||
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-java/-/windmill-parser-wasm-java-1.478.1.tgz",
|
||||
"integrity": "sha512-2gMr5pXaEExoLjeSMOpHzY4UjDU/JGjvlXSnBiQDHUaEoKSS7FWa5APzBKcmn8cM+mr2p8m4sMCyTXS88At0fQ=="
|
||||
},
|
||||
"node_modules/windmill-parser-wasm-nu": {
|
||||
"version": "1.474.1",
|
||||
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-nu/-/windmill-parser-wasm-nu-1.474.1.tgz",
|
||||
|
||||
@@ -144,6 +144,7 @@
|
||||
"vscode-ws-jsonrpc": "~3.4.0",
|
||||
"windmill-parser-wasm-csharp": "^1.437.1",
|
||||
"windmill-parser-wasm-go": "^1.429.0",
|
||||
"windmill-parser-wasm-java": "^1.478.1",
|
||||
"windmill-parser-wasm-nu": "^1.474.1",
|
||||
"windmill-parser-wasm-php": "^1.429.0",
|
||||
"windmill-parser-wasm-py": "^1.477.1",
|
||||
|
||||
@@ -195,6 +195,8 @@
|
||||
| 'yaml'
|
||||
| 'csharp'
|
||||
| 'nu'
|
||||
| 'java'
|
||||
// KJQXZ
|
||||
export let code: string = ''
|
||||
export let cmdEnterAction: (() => void) | undefined = undefined
|
||||
export let formatAction: (() => void) | undefined = undefined
|
||||
|
||||
@@ -93,7 +93,9 @@
|
||||
'php',
|
||||
'rust',
|
||||
'csharp',
|
||||
'nu'
|
||||
'nu',
|
||||
'java',
|
||||
// KJQXZ
|
||||
].includes(lang ?? '')
|
||||
$: showVarPicker = [
|
||||
'python3',
|
||||
@@ -107,7 +109,9 @@
|
||||
'php',
|
||||
'rust',
|
||||
'csharp',
|
||||
'nu'
|
||||
'nu',
|
||||
'java',
|
||||
// KJQXZ
|
||||
].includes(lang ?? '')
|
||||
$: showResourcePicker = [
|
||||
'python3',
|
||||
@@ -121,7 +125,9 @@
|
||||
'php',
|
||||
'rust',
|
||||
'csharp',
|
||||
'nu'
|
||||
'nu',
|
||||
'java',
|
||||
// KJQXZ
|
||||
].includes(lang ?? '')
|
||||
$: showResourceTypePicker =
|
||||
['typescript', 'javascript'].includes(scriptLangToEditorLang(lang)) ||
|
||||
@@ -373,6 +379,9 @@
|
||||
editor.insertAtCursor(`Environment.GetEnvironmentVariable("${name}");`)
|
||||
} else if (lang == 'nu') {
|
||||
editor.insertAtCursor(`$env.${name}`)
|
||||
} else if (lang == 'java') {
|
||||
editor.insertAtCursor(`System.getenv("${name}");`)
|
||||
// KJQXZ
|
||||
}
|
||||
sendUserToast(`${name} inserted at cursor`)
|
||||
}}
|
||||
@@ -440,6 +449,9 @@ string ${windmillPathToCamelCaseName(path)} = await client.GetStringAsync(uri);
|
||||
`)
|
||||
} else if (lang == 'nu') {
|
||||
editor.insertAtCursor(`get_variable ${path}`)
|
||||
} else if (lang == 'java') {
|
||||
editor.insertAtCursor(`(Wmill.getVariable("${path}"))`)
|
||||
// KJQXZ
|
||||
}
|
||||
sendUserToast(`${name} inserted at cursor`)
|
||||
}}
|
||||
@@ -524,7 +536,11 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS
|
||||
`)
|
||||
} else if (lang == 'nu') {
|
||||
editor.insertAtCursor(`get_resource ${path}`)
|
||||
} else if (lang == 'java') {
|
||||
editor.insertAtCursor(`(Wmill.getResource("${path}"))`)
|
||||
// KJQXZ
|
||||
}
|
||||
|
||||
sendUserToast(`${path} inserted at cursor`)
|
||||
}}
|
||||
tooltip="Resources represent connections to third party systems. Resources are a good way to define a connection to a frequently used third party system such as a database."
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
import rust from 'svelte-highlight/languages/rust'
|
||||
import csharp from 'svelte-highlight/languages/csharp'
|
||||
import yaml from 'svelte-highlight/languages/yaml'
|
||||
import java from 'svelte-highlight/languages/java'
|
||||
import type { Script } from '$lib/gen'
|
||||
import { Button } from './common'
|
||||
import { copyToClipboard } from '$lib/utils'
|
||||
@@ -65,7 +66,10 @@
|
||||
case 'nu':
|
||||
return python
|
||||
case 'ansible':
|
||||
return yaml
|
||||
return yaml;
|
||||
case 'java':
|
||||
return java;
|
||||
// KJQXZ
|
||||
default:
|
||||
return typescript
|
||||
}
|
||||
|
||||
@@ -78,7 +78,7 @@
|
||||
deno: false,
|
||||
go: false,
|
||||
ruff: false,
|
||||
shellcheck: false
|
||||
shellcheck: false,
|
||||
}
|
||||
|
||||
const dispatch = createEventDispatcher()
|
||||
|
||||
@@ -123,7 +123,9 @@
|
||||
'rust',
|
||||
'ansible',
|
||||
'csharp',
|
||||
'nu'
|
||||
'nu',
|
||||
'java'
|
||||
// KJQXZ
|
||||
]
|
||||
const nativeTags = [
|
||||
'nativets',
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
import AnsibleIcon from '$lib/components/icons/AnsibleIcon.svelte'
|
||||
import CSharpIcon from '$lib/components/icons/CSharpIcon.svelte'
|
||||
import NuIcon from '$lib/components/icons/NuIcon.svelte'
|
||||
import JavaIcon from '$lib/components/icons/JavaIcon.svelte'
|
||||
|
||||
export let lang:
|
||||
| SupportedLanguage
|
||||
@@ -56,7 +57,9 @@
|
||||
rust: 'Rust',
|
||||
ansible: 'Ansible Playbook',
|
||||
csharp: 'C#',
|
||||
nu: 'Nu'
|
||||
nu: 'Nu',
|
||||
java: 'Java'
|
||||
// KJQXZ
|
||||
}
|
||||
|
||||
const langToComponent: Record<
|
||||
@@ -87,7 +90,9 @@
|
||||
rust: RustIcon,
|
||||
ansible: AnsibleIcon,
|
||||
csharp: CSharpIcon,
|
||||
nu: NuIcon
|
||||
nu: NuIcon,
|
||||
java: JavaIcon
|
||||
// KJQXZ
|
||||
}
|
||||
|
||||
let subIconScale = width === 30 ? 0.6 : 0.8
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
<script>
|
||||
export let height = 24
|
||||
export let width = 24
|
||||
</script>
|
||||
|
||||
<svg xmlns="http://www.w3.org/2000/svg" {width} {height} viewBox="-60.6242 20 378.0764 511.359"><path d="M278.577 426.546h-2.038v-1.14h5.486v1.14h-2.024v5.688h-1.424zm10.942.297h-.032l-2.019 5.392h-.924l-2.006-5.392h-.025v5.392h-1.342v-6.828h1.975l1.86 4.835 1.854-4.835h1.968v6.828h-1.31v-5.392zM96.154 286.925s-14.178 8.245 10.09 11.035c29.4 3.354 44.426 2.873 76.825-3.259 0 0 8.518 5.341 20.414 9.967-72.63 31.128-164.376-1.803-107.329-17.743m-8.875-40.62s-15.902 11.771 8.384 14.283c31.406 3.24 56.208 3.505 99.125-4.759 0 0 5.936 6.018 15.27 9.309-87.814 25.678-185.623 2.025-122.779-18.833" fill="#5382a1"/><path d="M162.098 177.4c17.896 20.604-4.702 39.145-4.702 39.145s45.441-23.458 24.572-52.833c-19.491-27.394-34.438-41.005 46.479-87.934 0 0-127.013 31.722-66.349 101.622" fill="#f8981d"/><path d="M258.157 316.97s10.492 8.645-11.555 15.333c-41.923 12.7-174.488 16.535-211.314.506-13.238-5.759 11.587-13.751 19.396-15.428 8.144-1.766 12.798-1.437 12.798-1.437-14.722-10.371-95.157 20.364-40.857 29.166 148.084 24.015 269.943-10.814 231.532-28.14M102.972 204.218s-67.431 16.016-23.879 21.832c18.389 2.462 55.047 1.905 89.193-.956 27.906-2.354 55.927-7.359 55.927-7.359s-9.84 4.214-16.959 9.075c-68.475 18.009-200.756 9.631-162.674-8.79 32.206-15.568 58.392-13.802 58.392-13.802m120.963 67.614c69.608-36.171 37.424-70.931 14.96-66.248-5.506 1.146-7.961 2.139-7.961 2.139s2.044-3.202 5.948-4.588c44.441-15.624 78.619 46.081-14.346 70.52 0 .001 1.077-.962 1.399-1.823" fill="#5382a1"/><path d="M181.968 0s38.55 38.563-36.563 97.862c-60.233 47.568-13.735 74.69-.025 105.678-35.159-31.722-60.961-59.647-43.651-85.637C127.136 79.752 197.522 61.255 181.968 0" fill="#f8981d"/><path d="M109.812 369.847c66.815 4.277 169.417-2.373 171.847-33.988 0 0-4.671 11.985-55.219 21.503-57.028 10.732-127.364 9.479-169.081 2.601.001-.001 8.54 7.068 52.453 9.884m-10.95 120.802c-6.303 5.467-12.96 8.536-18.934 8.536-8.527 0-13.134-5.113-13.134-13.314 0-8.871 4.936-15.357 24.739-15.357h7.328v20.135m17.393 19.623V449.53c0-15.517-8.85-25.756-30.188-25.756-12.457 0-23.369 3.076-32.238 6.999l2.56 10.752c6.983-2.563 16.022-4.949 24.894-4.949 12.292 0 17.58 4.949 17.58 15.181v7.677h-6.135c-29.865 0-43.337 11.593-43.337 28.994 0 15.017 8.878 23.553 25.594 23.553 10.745 0 18.766-4.436 26.264-10.928l1.361 9.22h13.645zm58.044 0h-21.692l-26.106-84.96h18.943l16.199 52.2 3.601 15.699c8.195-22.698 13.991-45.726 16.89-67.899h18.427c-4.937 27.977-13.821 58.685-26.262 84.96m83.213-19.623c-6.315 5.467-12.984 8.536-18.958 8.536-8.512 0-13.131-5.113-13.131-13.314 0-8.871 4.948-15.357 24.749-15.357h7.34v20.135m17.39 19.623V449.53c0-15.517-8.872-25.756-30.185-25.756-12.466 0-23.382 3.076-32.247 6.999l2.556 10.752c6.986-2.563 16.042-4.949 24.907-4.949 12.283 0 17.579 4.949 17.579 15.181v7.677h-6.145c-29.874 0-43.34 11.593-43.34 28.994 0 15.017 8.871 23.553 25.584 23.553 10.751 0 18.769-4.436 26.28-10.928l1.366 9.22h13.645zM30.32 524.7c-4.958 7.239-12.966 12.966-21.733 16.206L0 530.801c6.673-3.424 12.396-8.954 15.055-14.105 2.3-4.581 3.252-10.485 3.252-24.604v-96.995h18.478v95.666c-.001 18.876-1.51 26.501-6.465 33.937" fill="#5382a1"/></svg>
|
||||
@@ -302,7 +302,24 @@ export const settings: Record<string, Setting[]> = {
|
||||
codeAreaLang: 'xml',
|
||||
storage: 'setting',
|
||||
ee_only: ''
|
||||
}
|
||||
},
|
||||
{
|
||||
label: 'Maven/Ivy repositories',
|
||||
description: 'Add private Maven/Ivy repositories',
|
||||
key: 'maven_repos',
|
||||
fieldType: 'text',
|
||||
placeholder: 'https://user:password@artifacts.foo.com/maven',
|
||||
storage: 'setting',
|
||||
ee_only: ''
|
||||
},
|
||||
{
|
||||
label: 'Disable default Maven repository',
|
||||
description: 'Do not use default Maven repository',
|
||||
key: 'no_default_maven',
|
||||
fieldType: 'boolean',
|
||||
storage: 'setting',
|
||||
ee_only: ''
|
||||
},
|
||||
],
|
||||
Alerts: [
|
||||
{
|
||||
|
||||
@@ -84,6 +84,9 @@ export function langToExt(lang: string): string {
|
||||
return 'cs'
|
||||
case 'nu':
|
||||
return 'nu'
|
||||
case 'java':
|
||||
return 'java'
|
||||
// KJQXZ
|
||||
default:
|
||||
return 'unknown'
|
||||
}
|
||||
|
||||
@@ -24,6 +24,7 @@ import initRustParser, { parse_rust } from 'windmill-parser-wasm-rust'
|
||||
import initYamlParser, { parse_ansible } from 'windmill-parser-wasm-yaml'
|
||||
import initCSharpParser, { parse_csharp } from 'windmill-parser-wasm-csharp'
|
||||
import initNuParser, { parse_nu } from 'windmill-parser-wasm-nu'
|
||||
import initJavaParser, { parse_java } from 'windmill-parser-wasm-java'
|
||||
|
||||
import wasmUrlTs from 'windmill-parser-wasm-ts/windmill_parser_wasm_bg.wasm?url'
|
||||
import wasmUrlRegex from 'windmill-parser-wasm-regex/windmill_parser_wasm_bg.wasm?url'
|
||||
@@ -34,6 +35,7 @@ import wasmUrlRust from 'windmill-parser-wasm-rust/windmill_parser_wasm_bg.wasm?
|
||||
import wasmUrlYaml from 'windmill-parser-wasm-yaml/windmill_parser_wasm_bg.wasm?url'
|
||||
import wasmUrlCSharp from 'windmill-parser-wasm-csharp/windmill_parser_wasm_bg.wasm?url'
|
||||
import wasmUrlNu from 'windmill-parser-wasm-nu/windmill_parser_wasm_bg.wasm?url'
|
||||
import wasmUrlJava from 'windmill-parser-wasm-java/windmill_parser_wasm_bg.wasm?url'
|
||||
import { workspaceStore } from './stores.js'
|
||||
import { argSigToJsonSchemaType } from './inferArgSig.js'
|
||||
|
||||
@@ -71,6 +73,9 @@ async function initWasmCSharp() {
|
||||
async function initWasmNu() {
|
||||
await initNuParser(wasmUrlNu)
|
||||
}
|
||||
async function initWasmJava() {
|
||||
await initJavaParser(wasmUrlJava)
|
||||
}
|
||||
|
||||
export async function inferArgs(
|
||||
language: SupportedLanguage | 'bunnative' | undefined,
|
||||
@@ -186,6 +191,10 @@ export async function inferArgs(
|
||||
} else if (language == 'nu') {
|
||||
await initWasmNu()
|
||||
inferedSchema = JSON.parse(parse_nu(code))
|
||||
} else if (language == 'java') {
|
||||
await initWasmJava()
|
||||
inferedSchema = JSON.parse(parse_java(code))
|
||||
// KJQXZ
|
||||
} else {
|
||||
return null
|
||||
}
|
||||
|
||||
@@ -945,6 +945,61 @@ dependencies:
|
||||
content: "{{ my_result | to_json }}"
|
||||
dest: result.json
|
||||
`
|
||||
const JAVA_INIT_CODE = `//requirements:
|
||||
//com.google.code.gson:gson:2.8.9
|
||||
//com.github.ricksbrown:cowsay:1.1.0
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.GsonBuilder;
|
||||
import com.github.ricksbrown.cowsay.Cowsay;
|
||||
import com.github.ricksbrown.cowsay.plugin.CowExecutor;
|
||||
|
||||
public class Main {
|
||||
public static class Person {
|
||||
private String name;
|
||||
private int age;
|
||||
|
||||
// Constructor
|
||||
public Person(String name, int age) {
|
||||
this.name = name;
|
||||
this.age = age;
|
||||
}
|
||||
}
|
||||
|
||||
public static Object main(
|
||||
// Primitive
|
||||
int a,
|
||||
float b,
|
||||
// Objects
|
||||
Integer age,
|
||||
Float d,
|
||||
Object e,
|
||||
String name,
|
||||
// Lists
|
||||
String[] f
|
||||
// No trailing commas!
|
||||
){
|
||||
Gson gson = new Gson();
|
||||
|
||||
// Get resources
|
||||
var theme = Wmill.getResource("f/app_themes/theme_0");
|
||||
System.out.println("Theme: " + theme);
|
||||
|
||||
// Create a Person object
|
||||
Person person = new Person( (name == "") ? "Alice" : name, (age == null) ? 30 : age);
|
||||
|
||||
// Serialize the Person object to JSON
|
||||
String json = gson.toJson(person);
|
||||
System.out.println("Serialized JSON: " + json);
|
||||
|
||||
// Use cowsay
|
||||
String[] args = new String[]{"-f", "dragon", json };
|
||||
String result = Cowsay.say(args);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
`
|
||||
// KJQXZ
|
||||
export const INITIAL_CODE = {
|
||||
bun: {
|
||||
scriptInitCodeBlock: BUN_INIT_BLOCK,
|
||||
@@ -1028,7 +1083,11 @@ export const INITIAL_CODE = {
|
||||
},
|
||||
bunnative: {
|
||||
script: BUNNATIVE_INIT_CODE
|
||||
}
|
||||
},
|
||||
java: {
|
||||
script: JAVA_INIT_CODE
|
||||
},
|
||||
// KJQXZ
|
||||
}
|
||||
|
||||
export function isInitialCode(content: string): boolean {
|
||||
@@ -1132,6 +1191,9 @@ export function initialCode(
|
||||
return INITIAL_CODE.csharp.script
|
||||
} else if (language == 'nu') {
|
||||
return INITIAL_CODE.nu.script
|
||||
} else if (language == 'java') {
|
||||
return INITIAL_CODE.java.script
|
||||
// KJQXZ
|
||||
} else if (language == 'bun' || language == 'bunnative') {
|
||||
if (kind == 'trigger') {
|
||||
return INITIAL_CODE.bun.trigger
|
||||
|
||||
@@ -45,6 +45,9 @@ export function scriptLangToEditorLang(
|
||||
return 'csharp'
|
||||
} else if (lang == 'nu') {
|
||||
return 'nu'
|
||||
} else if (lang == 'java') {
|
||||
return 'java'
|
||||
// KJQXZ
|
||||
} else if (lang == undefined) {
|
||||
return 'typescript'
|
||||
} else {
|
||||
@@ -125,7 +128,9 @@ const scriptLanguagesArray: [SupportedLanguage | 'docker' | 'bunnative', string]
|
||||
['ansible', 'Ansible Playbook'],
|
||||
['csharp', 'C#'],
|
||||
['docker', 'Docker'],
|
||||
['nu', 'Nu']
|
||||
['nu', 'Nu'],
|
||||
['java', 'Java']
|
||||
// KJQXZ
|
||||
]
|
||||
export function processLangs(selected: string | undefined, langs: string[]): string[] {
|
||||
if (selected === 'nativets') {
|
||||
@@ -134,7 +139,8 @@ export function processLangs(selected: string | undefined, langs: string[]): str
|
||||
let ls = langs.filter((lang) => lang !== 'nativets')
|
||||
|
||||
//those languages are newer and may not be in the saved list
|
||||
let nl = ['bunnative', 'rust', 'ansible', 'csharp', 'nu']
|
||||
let nl = ['bunnative', 'rust', 'ansible', 'csharp', 'nu', 'java']
|
||||
// KJQXZ
|
||||
nl.forEach((lang) => {
|
||||
if (!ls.includes(lang)) {
|
||||
ls.push(lang)
|
||||
|
||||
@@ -755,7 +755,8 @@
|
||||
<Badge color="blue">priority: {job.priority}</Badge>
|
||||
</div>
|
||||
{/if}
|
||||
{#if job.tag && !['deno', 'python3', 'flow', 'other', 'go', 'postgresql', 'mysql', 'bigquery', 'snowflake', 'mssql', 'graphql', 'oracledb', 'nativets', 'bash', 'powershell', 'php', 'rust', 'other', 'ansible', 'csharp', 'dependency', 'nu'].includes(job.tag)}
|
||||
{#if job.tag && !['deno', 'python3', 'flow', 'other', 'go', 'postgresql', 'mysql', 'bigquery', 'snowflake', 'mssql', 'graphql', 'oracledb', 'nativets', 'bash', 'powershell', 'php', 'rust', 'other', 'ansible', 'csharp', 'nu', 'java', 'dependency'].includes(job.tag)}
|
||||
<!-- KJQXZ -->
|
||||
<div>
|
||||
<Badge color="indigo">Tag: {job.tag}</Badge>
|
||||
</div>
|
||||
|
||||
Reference in New Issue
Block a user