Merge remote-tracking branch 'origin/main' into fork-datatable-schema-export

This commit is contained in:
Diego Imbert
2026-04-01 13:49:37 +02:00
77 changed files with 3348 additions and 338 deletions
+23
View File
@@ -0,0 +1,23 @@
You are reviewing a GitHub pull request for this repository.
Review policy:
- Read `CLAUDE.md` before reviewing code.
- Only report issues you are confident are real and introduced by this pull request.
- Focus on bugs, security problems, and clear `CLAUDE.md` violations.
- Do not report style nits, speculative concerns, pre-existing issues, or problems that a normal linter/typechecker would obviously catch.
- Keep the review high signal. If there is no clear issue, return no findings.
Repository context:
- Read `./.github/codex/pr-review-context.md` for the PR metadata and the exact diff commands to use.
- Review only the changes introduced by this PR.
- Read additional files only when the diff is not enough to validate a finding.
- Do not modify any files.
Output requirements:
- Return a GitHub PR comment in markdown, not JSON.
- Start with `## Codex Review`.
- Give a short overall summary first.
- If you found high-signal issues, list them in a short numbered list with file paths and line numbers when you know them confidently.
- If you found no high-signal issues, say that explicitly.
- End with a `### Reproduction instructions` section containing a short descriptive paragraph for a tester explaining how to navigate the app to observe the change. Do not make it a numbered list. If the diff is not enough to infer this safely, say that plainly.
- Prefer at most 10 findings.
+145
View File
@@ -0,0 +1,145 @@
name: Codex Auto Review
on:
pull_request:
types: [ready_for_review, opened]
concurrency:
group: codex-review-${{ github.event.pull_request.number }}
cancel-in-progress: true
jobs:
codex-review:
runs-on: ubicloud-standard-2
timeout-minutes: 30
if: github.event.pull_request.draft == false && github.event.pull_request.head.repo.fork == false
permissions:
contents: read
issues: write
steps:
- name: Check Codex configuration
id: codex_config
env:
CODEX_AUTH_JSON: ${{ secrets.CODEX_AUTH_JSON }}
run: |
if [ -n "$CODEX_AUTH_JSON" ]; then
echo "enabled=true" >> "$GITHUB_OUTPUT"
else
echo "enabled=false" >> "$GITHUB_OUTPUT"
echo "CODEX_AUTH_JSON is not configured; skipping Codex review."
fi
- name: Checkout repository
if: steps.codex_config.outputs.enabled == 'true'
uses: actions/checkout@v5
with:
ref: refs/pull/${{ github.event.pull_request.number }}/merge
fetch-depth: 1
- name: Set up Node.js
if: steps.codex_config.outputs.enabled == 'true'
uses: actions/setup-node@v4
with:
node-version: 22
- name: Install Codex CLI
if: steps.codex_config.outputs.enabled == 'true'
run: npm install --global @openai/codex@0.117.0
- name: Configure file-backed Codex auth
if: steps.codex_config.outputs.enabled == 'true'
env:
CODEX_AUTH_JSON: ${{ secrets.CODEX_AUTH_JSON }}
run: |
CODEX_HOME="$HOME/.codex"
echo "CODEX_HOME=$CODEX_HOME" >> "$GITHUB_ENV"
mkdir -p "$CODEX_HOME"
chmod 700 "$CODEX_HOME"
cat > "$CODEX_HOME/config.toml" <<'EOF'
cli_auth_credentials_store = "file"
EOF
printf '%s' "$CODEX_AUTH_JSON" > "$CODEX_HOME/auth.json"
chmod 600 "$CODEX_HOME/auth.json"
node -e 'JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"))' "$CODEX_HOME/auth.json"
- name: Pre-fetch base and head refs for the PR
if: steps.codex_config.outputs.enabled == 'true'
env:
PR_BASE_REF: ${{ github.event.pull_request.base.ref }}
PR_NUMBER: ${{ github.event.pull_request.number }}
run: |
git fetch --no-tags origin \
"$PR_BASE_REF" \
"+refs/pull/$PR_NUMBER/head"
- name: Write Codex review context
if: steps.codex_config.outputs.enabled == 'true'
env:
PR_REPOSITORY: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
PR_TITLE: ${{ github.event.pull_request.title }}
PR_BODY: ${{ github.event.pull_request.body || '' }}
run: |
mkdir -p .github/codex
node <<'NODE'
const fs = require('fs');
const lines = [
`Repository: ${process.env.PR_REPOSITORY}`,
`PR number: ${process.env.PR_NUMBER}`,
`Base SHA: ${process.env.PR_BASE_SHA}`,
`Head SHA: ${process.env.PR_HEAD_SHA}`,
'',
'PR title:',
process.env.PR_TITLE || '(empty)',
'',
'PR body:',
process.env.PR_BODY || '(empty)',
'',
'Changed commits command:',
`git log --oneline ${process.env.PR_BASE_SHA}...${process.env.PR_HEAD_SHA}`,
'',
'Changed files command:',
`git diff --stat ${process.env.PR_BASE_SHA}...${process.env.PR_HEAD_SHA}`,
'',
'Full review diff command:',
`git diff --unified=0 ${process.env.PR_BASE_SHA}...${process.env.PR_HEAD_SHA}`
];
fs.writeFileSync('.github/codex/pr-review-context.md', `${lines.join('\n')}\n`);
NODE
- name: Run Codex review
if: steps.codex_config.outputs.enabled == 'true'
run: |
codex exec \
-C "$GITHUB_WORKSPACE" \
-m gpt-5.4 \
-c 'model_reasoning_effort="xhigh"' \
-s read-only \
-o codex-final-message.md \
- < .github/codex/pr-review.prompt.md
- name: Post Codex review comment
if: steps.codex_config.outputs.enabled == 'true'
uses: actions/github-script@v7
with:
github-token: ${{ github.token }}
script: |
const fs = require('fs');
const path = `${process.env.GITHUB_WORKSPACE}/codex-final-message.md`;
if (!fs.existsSync(path)) {
core.info('Codex did not produce a final message; skipping PR comment.');
return;
}
const body = fs.readFileSync(path, 'utf8').trim();
if (!body) {
core.info('Codex final message was empty; skipping PR comment.');
return;
}
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.payload.pull_request.number,
body,
});
+2 -2
View File
@@ -43,7 +43,7 @@ profiles:
- Pane 0: this pane (claude agent)
- Pane 1: backend (cargo watch -x run)
- Pane 2: frontend (npm run dev)
To check logs, use: \`tmux capture-pane -t .1 -p -S -50\` (backend) or \`tmux capture-pane -t .2 -p -S -50\` (frontend).
To check logs, use: \`tmux capture-pane -t $(tmux display-message -t "$TMUX_PANE" -p '#{session_name}:#{window_name}').1 -p -S -50\` (backend) or \`tmux capture-pane -t $(tmux display-message -t "$TMUX_PANE" -p '#{session_name}:#{window_name}').2 -p -S -50\` (frontend).
For this window specifically, backend is running on: ${BACKEND_PORT} and frontend is running on: ${FRONTEND_PORT}.
To connect to the database, use this connection string: ${DATABASE_URL}
Because we are running backend with cargo watch, to verify your changes, just check the logs in the backend pane. No need for cargo check.
@@ -72,7 +72,7 @@ profiles:
Pane layout (current window):
- Pane 0: this pane (claude agent)
- Pane 1: frontend (npm run dev)
To check logs, use: \`tmux capture-pane -t .1 -p -S -50\` (frontend).
To check logs, use: \`tmux capture-pane -t $(tmux display-message -t "$TMUX_PANE" -p '#{session_name}:#{window_name}').1 -p -S -50\` (frontend).
On this window specifically, frontend is running on: ${FRONTEND_PORT}.
To connect to the database, use this connection string: ${DATABASE_URL}
Because we are running frontend with npm run dev, to verify your changes, just check the logs in the frontend pane. No need for npm run build.
+14
View File
@@ -1,5 +1,19 @@
# Changelog
## [1.671.0](https://github.com/windmill-labs/windmill/compare/v1.670.0...v1.671.0) (2026-03-31)
### Features
* add configurable preview job tag override in default tags settings ([#8649](https://github.com/windmill-labs/windmill/issues/8649)) ([da8886b](https://github.com/windmill-labs/windmill/commit/da8886be8575dd925b6d24c55ab379bc6984c5f8))
* improve CLI flow log streaming and job inspection ([#8644](https://github.com/windmill-labs/windmill/issues/8644)) ([6c3c971](https://github.com/windmill-labs/windmill/commit/6c3c971af5aa1362632ee0deeddf91b8bc47c853))
* support hub flows in raw app runnables ([#8627](https://github.com/windmill-labs/windmill/issues/8627)) ([040a199](https://github.com/windmill-labs/windmill/commit/040a199685cea5c99c944bacb5584a381d6ec829))
### Bug Fixes
* return default_args/enums in approval info and fix subflow resume buttons ([#8648](https://github.com/windmill-labs/windmill/issues/8648)) ([852c59e](https://github.com/windmill-labs/windmill/commit/852c59efbb04510e5e6f99919707effcf6769a2f))
## [1.670.0](https://github.com/windmill-labs/windmill/compare/v1.669.1...v1.670.0) (2026-03-31)
+147 -125
View File
File diff suppressed because it is too large Load Diff
+9 -3
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.670.0"
version = "1.671.0"
authors.workspace = true
edition.workspace = true
@@ -66,10 +66,13 @@ members = [
"./parsers/windmill-parser-nu",
"./parsers/windmill-parser-java",
"./parsers/windmill-parser-ruby",
"./parsers/windmill-parser-r",
"./parsers/windmill-parser-bash",
"./parsers/windmill-parser-py",
"./parsers/windmill-parser-py-asset",
"./parsers/windmill-parser-py-imports",
# Uncomment to build wasm parsers:
# "./parsers/windmill-parser-wasm",
"./parsers/windmill-parser-wac",
"./parsers/windmill-parser-sql",
"./parsers/windmill-parser-sql-asset",
@@ -82,7 +85,7 @@ members = [
exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"]
[workspace.package]
version = "1.670.0"
version = "1.671.0"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -163,7 +166,8 @@ csharp = ["windmill-worker/csharp"]
nu = ["windmill-worker/nu"]
java = ["windmill-worker/java"]
ruby = ["windmill-worker/ruby"]
all_languages = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql-kerberos", "bigquery", "csharp", "nu", "php", "java", "ruby"]
rlang = ["windmill-worker/rlang"]
all_languages = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql-kerberos", "bigquery", "csharp", "nu", "php", "java", "ruby", "rlang"]
# For windows we have another set of languages enabled
all_languages_windows = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql-winauth", "bigquery", "csharp", "nu", "php", "java"]
# Edition meta-features: shared groups
@@ -347,6 +351,7 @@ windmill-parser-yaml = { path = "./parsers/windmill-parser-yaml" }
windmill-parser-csharp = { path = "./parsers/windmill-parser-csharp" }
windmill-parser-java = { path = "./parsers/windmill-parser-java" }
windmill-parser-ruby = { path = "./parsers/windmill-parser-ruby" }
windmill-parser-r = { path = "./parsers/windmill-parser-r" }
windmill-parser-nu = { path = "./parsers/windmill-parser-nu" }
windmill-parser-bash = { path = "./parsers/windmill-parser-bash" }
windmill-parser-sql = { path = "./parsers/windmill-parser-sql" }
@@ -613,6 +618,7 @@ tree-sitter = { version = "0.23.0", features = [] }
tree-sitter-c-sharp = "0.23.0"
tree-sitter-java = "0.23.0"
tree-sitter-ruby = "0.23.0"
tree-sitter-r = "1.2.0"
oracle = { version = "0.6.3", features = ["chrono"] }
rumqttc = { version = "0.24.0", features = ["use-native-tls"]}
strum = { version = "0.27", features = ["derive"] }
@@ -0,0 +1,2 @@
ALTER TYPE SCRIPT_LANG ADD VALUE IF NOT EXISTS 'rlang';
UPDATE config SET config = jsonb_set(config, '{worker_tags}', config->'worker_tags' || '["rlang"]'::jsonb) WHERE name = 'worker__default' AND config @> '{"worker_tags": ["deno", "python3", "go", "bash", "powershell", "dependency", "flow", "hub", "other", "bun", "php", "rust", "ansible", "csharp", "nu", "java", "duckdb", "ruby"]}'::jsonb AND NOT config->'worker_tags' @> '"rlang"'::jsonb;
@@ -0,0 +1,17 @@
[package]
name = "windmill-parser-r"
version.workspace = true
edition.workspace = true
authors.workspace = true
[lib]
name = "windmill_parser_r"
path = "./src/lib.rs"
[dependencies]
windmill-parser.workspace = true
tree-sitter.workspace = true
tree-sitter-r.workspace = true
anyhow.workspace = true
wasm-bindgen.workspace = true
serde_json.workspace = true
@@ -0,0 +1,363 @@
#![cfg_attr(target_arch = "wasm32", feature(c_variadic))]
#[cfg(target_arch = "wasm32")]
pub mod wasm_libc;
use anyhow::anyhow;
use serde_json::Value;
use tree_sitter::Node;
use tree_sitter::Range;
use windmill_parser::json_to_typ;
use windmill_parser::Arg;
use windmill_parser::MainArgSignature;
pub fn parse_r_sig_meta(code: &str) -> anyhow::Result<MainArgSignature> {
let mut parser = tree_sitter::Parser::new();
let language = tree_sitter_r::LANGUAGE;
parser
.set_language(&language.into())
.map_err(|e| anyhow!("Error setting R as language: {e}"))?;
let tree = parser
.parse(code, None)
.ok_or(anyhow!("Failed to parse code"))?;
let root_node = tree.root_node();
let args = find_main_signature(root_node, code)?;
let main_sig = MainArgSignature {
star_args: false,
star_kwargs: false,
args: args.unwrap_or_default(),
has_preprocessor: None,
auto_kind: None,
};
Ok(main_sig)
}
pub fn parse_r_signature(code: &str) -> anyhow::Result<MainArgSignature> {
Ok(parse_r_sig_meta(code)?)
}
/// Extract package names from `library(...)` and `require(...)` calls in R code.
/// Returns a newline-separated list of package names.
pub fn parse_r_requirements(code: &str) -> anyhow::Result<String> {
let mut parser = tree_sitter::Parser::new();
let language = tree_sitter_r::LANGUAGE;
parser
.set_language(&language.into())
.map_err(|e| anyhow!("Error setting R as language: {e}"))?;
let tree = parser
.parse(code, None)
.ok_or(anyhow!("Failed to parse code"))?;
let root_node = tree.root_node();
let mut packages = vec![];
find_library_calls(root_node, code, &mut packages);
// Deduplicate and exclude base packages
packages.sort();
packages.dedup();
packages.retain(|p| !is_base_package(p));
Ok(packages.join("\n"))
}
fn find_library_calls(node: Node, code: &str, packages: &mut Vec<String>) {
let mut cursor = node.walk();
for child in node.children(&mut cursor) {
if child.kind() == "call" {
// call node: child 0 is the function name, child 1 is arguments
if let (Some(func_node), Some(args_node)) = (child.child(0), child.child(1)) {
let func_name = func_node.utf8_text(code.as_bytes()).unwrap_or("");
if func_name == "library" || func_name == "require" {
// AST: arguments → ( + argument → identifier/string + )
if args_node.kind() == "arguments" {
let mut args_cursor = args_node.walk();
for arg in args_node.children(&mut args_cursor) {
if arg.kind() == "argument" {
// The argument node wraps the actual value
if let Some(value_node) = arg.child(0) {
let pkg = value_node
.utf8_text(code.as_bytes())
.unwrap_or("")
.trim_matches('"')
.trim_matches('\'');
if !pkg.is_empty() {
packages.push(pkg.to_string());
}
}
break; // only first arg
}
}
}
}
}
}
// Recurse into children to find nested library() calls
find_library_calls(child, code, packages);
}
}
fn is_base_package(pkg: &str) -> bool {
matches!(
pkg,
"base"
| "compiler"
| "datasets"
| "grDevices"
| "graphics"
| "grid"
| "methods"
| "parallel"
| "splines"
| "stats"
| "stats4"
| "tcltk"
| "tools"
| "utils"
)
}
/// Find the main function signature in R code.
/// R function definitions look like: `main <- function(x, y = 10) { ... }`
/// In the tree-sitter-r AST, this is a `binary_operator` node with:
/// - child 0: identifier "main"
/// - child 1: "<-" or "="
/// - child 2: function_definition node
fn find_main_signature<'a>(root_node: Node<'a>, code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
let mut cursor = root_node.walk();
for x in root_node.children(&mut cursor) {
if x.kind() == "binary_operator" {
let child_count = x.child_count();
if child_count < 3 {
continue;
}
// First child should be identifier "main"
let ident_node = x.child(0).unwrap();
if ident_node.kind() != "identifier" {
continue;
}
let ident = ident_node.utf8_text(code.as_bytes()).unwrap_or("");
if ident != "main" {
continue;
}
// Second child should be "<-" or "="
let op_node = x.child(1).unwrap();
let op = op_node.utf8_text(code.as_bytes()).unwrap_or("");
if op != "<-" && op != "=" {
continue;
}
// Third child should be the function_definition
let func_node = x.child(2).unwrap();
if func_node.kind() != "function_definition" {
continue;
}
return Ok(Some(parse_function_params(func_node, code)?));
}
}
Ok(None)
}
/// Parse parameters from a function_definition node.
/// function_definition has children: "function", parameters, body
/// Each parameter node has:
/// - 1 child (identifier) for positional args
/// - 3 children (identifier, "=", value) for default args
fn parse_function_params(func_node: Node, code: &str) -> anyhow::Result<Vec<Arg>> {
let mut args = vec![];
let mut func_cursor = func_node.walk();
for child in func_node.children(&mut func_cursor) {
if child.kind() == "parameters" {
let mut param_cursor = child.walk();
for param in child.children(&mut param_cursor) {
if param.kind() != "parameter" {
continue;
}
let param_child_count = param.child_count();
if param_child_count == 1 {
// Simple parameter: just identifier
let ident_node = param.child(0).unwrap();
let name = ident_node.utf8_text(code.as_bytes())?;
args.push(Arg { name: name.to_owned(), ..Default::default() });
} else if param_child_count >= 3 {
// Default parameter: identifier = value
let ident_node = param.child(0).unwrap();
let value_node = param.child(2).unwrap();
let name = ident_node.utf8_text(code.as_bytes())?;
let Range { start_byte, end_byte, .. } = value_node.range();
let raw = &code[start_byte..end_byte];
// Convert R literals to JSON
let unparsed = raw
.replace("NULL", "null")
.replace("TRUE", "true")
.replace("FALSE", "false");
match serde_json::from_str::<Value>(&unparsed) {
Ok(default) => {
args.push(Arg {
name: name.to_owned(),
typ: json_to_typ(&default, true),
default: Some(default),
has_default: true,
..Default::default()
});
}
Err(_) => {
args.push(Arg {
name: name.to_owned(),
has_default: true,
..Default::default()
});
}
}
}
}
}
}
Ok(args)
}
#[cfg(test)]
mod test {
use serde_json::json;
use windmill_parser::Typ;
use super::parse_r_sig_meta as parse;
#[test]
fn test_parse_r_no_main() {
let code = r#"
not_main <- function() {}
helper <- function(x) { x + 1 }
"#;
let sig = parse(code).unwrap();
assert_eq!(
sig,
windmill_parser::MainArgSignature { auto_kind: None, ..Default::default() }
);
}
#[test]
fn test_parse_r_no_args() {
let code = r#"
main <- function() {
return(42)
}
"#;
let sig = parse(code).unwrap();
assert_eq!(
sig,
windmill_parser::MainArgSignature { auto_kind: None, ..Default::default() }
);
}
#[test]
fn test_parse_r_positional_args() {
let code = r#"main <- function(a, b, c) { a + b + c }"#;
let sig = parse(code).unwrap();
assert_eq!(
sig,
windmill_parser::MainArgSignature {
args: vec![
windmill_parser::Arg { name: "a".into(), ..Default::default() },
windmill_parser::Arg { name: "b".into(), ..Default::default() },
windmill_parser::Arg { name: "c".into(), ..Default::default() },
],
auto_kind: None,
..Default::default()
}
);
}
#[test]
fn test_parse_r_default_args() {
let code = r#"main <- function(a = 10, b = "hey", c = FALSE) { }"#;
let sig = parse(code).unwrap();
assert_eq!(sig.args.len(), 3);
assert_eq!(sig.args[0].name, "a");
assert_eq!(sig.args[0].default, Some(json!(10)));
assert_eq!(sig.args[0].typ, Typ::Int);
assert_eq!(sig.args[1].name, "b");
assert_eq!(sig.args[1].default, Some(json!("hey")));
assert_eq!(sig.args[1].typ, Typ::Str(None));
assert_eq!(sig.args[2].name, "c");
assert_eq!(sig.args[2].default, Some(json!(false)));
assert_eq!(sig.args[2].typ, Typ::Bool);
}
#[test]
fn test_parse_r_equals_assignment() {
let code = r#"main = function(x, y = 5) { x + y }"#;
let sig = parse(code).unwrap();
assert_eq!(sig.args.len(), 2);
assert_eq!(sig.args[0].name, "x");
assert_eq!(sig.args[1].name, "y");
assert_eq!(sig.args[1].default, Some(json!(5)));
}
#[test]
fn test_parse_r_null_default() {
let code = r#"main <- function(x = NULL) { x }"#;
let sig = parse(code).unwrap();
assert_eq!(sig.args.len(), 1);
assert_eq!(sig.args[0].name, "x");
assert_eq!(sig.args[0].default, Some(json!(null)));
}
#[test]
fn test_parse_r_requirements() {
use super::parse_r_requirements;
let code = r#"
library(dplyr)
library(ggplot2)
require(tidyr)
library(stats)
main <- function(x) {
library(stringr)
x
}
"#;
let reqs = parse_r_requirements(code).unwrap();
let pkgs: Vec<&str> = reqs.lines().collect();
assert!(pkgs.contains(&"dplyr"));
assert!(pkgs.contains(&"ggplot2"));
assert!(pkgs.contains(&"tidyr"));
assert!(pkgs.contains(&"stringr"));
assert!(!pkgs.contains(&"stats")); // base package excluded
}
#[test]
fn test_parse_r_requirements_string_args() {
use super::parse_r_requirements;
let code = r#"
library("data.table")
require("jsonlite")
main <- function() { }
"#;
let reqs = parse_r_requirements(code).unwrap();
let pkgs: Vec<&str> = reqs.lines().collect();
assert!(pkgs.contains(&"data.table"));
assert!(pkgs.contains(&"jsonlite"));
}
#[test]
fn test_parse_r_requirements_no_deps() {
use super::parse_r_requirements;
let code = r#"main <- function(x) { x + 1 }"#;
let reqs = parse_r_requirements(code).unwrap();
assert!(reqs.is_empty());
}
}
@@ -0,0 +1,293 @@
use std::collections::BTreeMap;
use std::sync::{Mutex, OnceLock};
use std::{
alloc::{self, Layout},
ffi::{c_char, c_int, c_void},
mem::align_of,
ptr,
};
use wasm_bindgen::prelude::*;
/* -------------------------------- stdlib.h -------------------------------- */
#[no_mangle]
pub unsafe extern "C" fn abort() {
panic!("Aborted from C");
}
macro_rules! console_log {
($($t:tt)*) => (unsafe { log(&format_args!($($t)*).to_string()) })
}
#[wasm_bindgen]
extern "C" {
#[wasm_bindgen(js_namespace = console)]
fn log(a: &str);
}
#[no_mangle]
pub unsafe extern "C" fn malloc(size: usize) -> *mut c_void {
if size == 0 {
return ptr::null_mut();
}
let (layout, offset_to_data) = layout_for_size_prepended(size);
let buf = alloc::alloc(layout);
store_layout(buf, layout, offset_to_data)
}
#[no_mangle]
pub unsafe extern "C" fn calloc(count: usize, size: usize) -> *mut c_void {
if count == 0 || size == 0 {
return ptr::null_mut();
}
let (layout, offset_to_data) = layout_for_size_prepended(size * count);
let buf = alloc::alloc_zeroed(layout);
store_layout(buf, layout, offset_to_data)
}
#[no_mangle]
pub unsafe extern "C" fn realloc(buf: *mut c_void, new_size: usize) -> *mut c_void {
if buf.is_null() {
malloc(new_size)
} else if new_size == 0 {
free(buf);
ptr::null_mut()
} else {
let (old_buf, old_layout) = retrieve_layout(buf);
let (new_layout, offset_to_data) = layout_for_size_prepended(new_size);
let new_buf = alloc::realloc(old_buf, old_layout, new_layout.size());
store_layout(new_buf, new_layout, offset_to_data)
}
}
#[no_mangle]
pub unsafe extern "C" fn free(buf: *mut c_void) {
if buf.is_null() {
return;
}
let (buf, layout) = retrieve_layout(buf);
alloc::dealloc(buf, layout);
}
// In all these allocations, we store the layout before the data for later retrieval.
// This is because we need to know the layout when deallocating the memory.
// Here are some helper methods for that:
/// Given a pointer to the data, retrieve the layout and the pointer to the layout.
unsafe fn retrieve_layout(buf: *mut c_void) -> (*mut u8, Layout) {
let (_, layout_offset) = Layout::new::<Layout>()
.extend(Layout::from_size_align(0, align_of::<*const u8>() * 2).unwrap())
.unwrap();
let buf = (buf as *mut u8).offset(-(layout_offset as isize));
let layout = *(buf as *mut Layout);
(buf, layout)
}
/// Calculate a layout for a given size with space for storing a layout at the start.
/// Returns the layout and the offset to the data.
fn layout_for_size_prepended(size: usize) -> (Layout, usize) {
Layout::new::<Layout>()
.extend(Layout::from_size_align(size, align_of::<*const u8>() * 2).unwrap())
.unwrap()
}
/// Store a layout in the pointer, returning a pointer to where the data should be stored.
unsafe fn store_layout(buf: *mut u8, layout: Layout, offset_to_data: usize) -> *mut c_void {
*(buf as *mut Layout) = layout;
(buf as *mut u8).offset(offset_to_data as isize) as *mut c_void
}
/* -------------------------------- string.h -------------------------------- */
#[no_mangle]
pub unsafe extern "C" fn strncmp(ptr1: *const c_void, ptr2: *const c_void, n: usize) -> c_int {
let s1 = std::slice::from_raw_parts(ptr1 as *const u8, n);
let s2 = std::slice::from_raw_parts(ptr2 as *const u8, n);
for (a, b) in s1.iter().zip(s2.iter()) {
if *a != *b || *a == 0 {
return (*a as i32) - (*b as i32);
}
}
0
}
// Implementation by AI:
pub type size_t = usize;
use std::slice;
#[no_mangle]
pub unsafe extern "C" fn memchr(haystack: *const c_void, needle: c_int, len: usize) -> *mut c_void {
if haystack.is_null() || len == 0 {
return ptr::null_mut(); // Return null if the input pointer is null or length is zero
}
let needle_byte = needle as u8; // Convert needle to a byte
// Create a pointer to the start of the haystack
let mut current = haystack as *const u8;
// Iterate through the memory block
for _ in 0..len {
if *current == needle_byte {
return current as *mut c_void; // Return the pointer to the found byte
}
current = current.add(1); // Move to the next byte
}
ptr::null_mut() // Return null if the byte was not found
}
#[no_mangle]
pub unsafe extern "C" fn strchr(mut s: *const c_char, c: c_int) -> *mut c_char {
if s.is_null() {
return std::ptr::null_mut(); // Return null if the input string is null
}
let target = c as u8 as char; // Convert c to a char
let mut current = s;
// Iterate through the string until we find the character or reach the end
while *current != 0 {
if *current as u8 as char == target {
return current as *mut c_char; // Return the pointer to the found character
}
current = current.add(1); // Move to the next character
}
std::ptr::null_mut() // Return null if the character was not found
}
// End of AI implemetation
/* -------------------------------- wctype.h -------------------------------- */
#[no_mangle]
pub unsafe extern "C" fn iswspace(c: c_int) -> bool {
char::from_u32(c as u32).map_or(false, |c| c.is_whitespace())
}
#[no_mangle]
pub unsafe extern "C" fn iswalnum(c: c_int) -> bool {
char::from_u32(c as u32).map_or(false, |c| c.is_alphanumeric())
}
// Implementation by AI:
pub type wint_t = u32;
#[no_mangle]
pub extern "C" fn iswdigit(wc: wint_t) -> c_int {
// Check if the character is a digit ('0' to '9')
if wc >= '0' as wint_t && wc <= '9' as wint_t {
return 1; // Return true (1)
}
0 // Return false (0)
}
#[no_mangle]
pub extern "C" fn iswupper(wc: wint_t) -> c_int {
// Check if the character is an uppercase letter ('A' to 'Z')
if wc >= 'A' as wint_t && wc <= 'Z' as wint_t {
return 1; // Return true (1)
}
0 // Return false (0)
}
#[no_mangle]
pub extern "C" fn iswalpha(wc: wint_t) -> c_int {
// Check if the character is an alphabetic character ('A' to 'Z' or 'a' to 'z')
if (wc >= 'A' as wint_t && wc <= 'Z' as wint_t) || (wc >= 'a' as wint_t && wc <= 'z' as wint_t)
{
return 1; // Return true (1)
}
0 // Return false (0)
}
#[no_mangle]
pub extern "C" fn iswlower(wc: wint_t) -> c_int {
// Check if the character is a lowercase letter ('a' to 'z')
if wc >= 'a' as wint_t && wc <= 'z' as wint_t {
return 1; // Return true (1)
}
0 // Return false (0)
}
// End of AI implemetation
/* --------------------------------- time.h --------------------------------- */
#[no_mangle]
pub unsafe extern "C" fn clock() -> u64 {
panic!("clock is not supported");
}
/* --------------------------------- ctype.h -------------------------------- */
#[no_mangle]
pub unsafe extern "C" fn isprint(c: c_int) -> bool {
c >= 32 && c <= 126
}
/* --------------------------------- stdio.h -------------------------------- */
#[no_mangle]
pub unsafe extern "C" fn fprintf(_file: *mut c_void, _format: *const c_void, _args: ...) -> c_int {
panic!("fprintf is not supported");
}
#[no_mangle]
pub unsafe extern "C" fn fputs(_s: *const c_void, _file: *mut c_void) -> c_int {
panic!("fputs is not supported");
}
#[no_mangle]
pub unsafe extern "C" fn fputc(_c: c_int, _file: *mut c_void) -> c_int {
panic!("fputc is not supported");
}
#[no_mangle]
pub unsafe extern "C" fn fdopen(_fd: c_int, _mode: *const c_void) -> *mut c_void {
panic!("fdopen is not supported");
}
#[no_mangle]
pub unsafe extern "C" fn fclose(_file: *mut c_void) -> c_int {
panic!("fclose is not supported");
}
#[no_mangle]
pub unsafe extern "C" fn fwrite(
_ptr: *const c_void,
_size: usize,
_nmemb: usize,
_stream: *mut c_void,
) -> usize {
panic!("fwrite is not supported");
}
#[no_mangle]
pub unsafe extern "C" fn vsnprintf(
_buf: *mut c_char,
_size: usize,
_format: *const c_char,
_args: ...
) -> c_int {
panic!("vsnprintf is not supported");
}
#[no_mangle]
pub extern "C" fn clock_gettime(ptr: usize, new_size: usize) {
panic!("clock_gettime is not supported");
}
// int snprintf( char* restrict buffer, size_t bufsz, const char* restrict format, ... );
#[no_mangle]
pub extern "C" fn snprintf() {
panic!("snprintf is not supported");
}
#[no_mangle]
pub extern "C" fn __assert_fail(_: *const i32, _: *const i32, _: *const i32, _: *const i32) {
panic!("oh no");
}
@@ -38,6 +38,7 @@ csharp-parser = [ "dep:windmill-parser-csharp"]
nu-parser = [ "dep:windmill-parser-nu"]
java-parser = [ "dep:windmill-parser-java"]
ruby-parser = [ "dep:windmill-parser-ruby"]
r-parser = [ "dep:windmill-parser-r"]
wac-parser = [ "dep:windmill-parser-wac"]
asset-parser = [ "dep:windmill-parser-ts-asset", "dep:windmill-parser-py-asset", "dep:windmill-parser-sql-asset"]
py-imports-parser = [ "dep:windmill-parser-py-imports"]
@@ -58,6 +59,7 @@ windmill-parser-csharp = { workspace = true, optional = true }
windmill-parser-nu = { workspace = true, optional = true }
windmill-parser-java = { workspace = true, optional = true }
windmill-parser-ruby = { workspace = true, optional = true }
windmill-parser-r = { workspace = true, optional = true }
windmill-parser-wac = { workspace = true, optional = true }
windmill-parser-ts-asset = { workspace = true, optional = true }
windmill-parser-py-asset = { workspace = true, optional = true }
@@ -55,6 +55,11 @@ const targets = [
desc: "Ruby",
features: "ruby-parser",
env: "tree-sitter",
}, {
ident: "r",
desc: "R",
features: "r-parser",
env: "tree-sitter",
},
{
ident: "wac",
@@ -198,6 +198,12 @@ pub fn parse_ruby(code: &str) -> String {
wrap_sig(windmill_parser_ruby::parse_ruby_signature(code))
}
#[cfg(feature = "r-parser")]
#[wasm_bindgen]
pub fn parse_r(code: &str) -> String {
wrap_sig(windmill_parser_r::parse_r_signature(code))
}
#[cfg(feature = "asset-parser")]
#[wasm_bindgen]
pub fn parse_assets_sql(code: &str) -> String {
@@ -1,5 +1,7 @@
#pragma once
#include <stdint.h>
void *memcpy(void *dest, const void *src, unsigned long n);
void *memmove(void *dest, const void *src, unsigned long n);
void *memset(void *s, int c, unsigned long n);
+2 -1
View File
@@ -95,7 +95,7 @@ use windmill_worker::{
BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, CSHARP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS,
DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, JAVA_CACHE_DIR, NU_CACHE_DIR,
POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, PY312_CACHE_DIR, PY313_CACHE_DIR,
RUBY_CACHE_DIR, RUST_CACHE_DIR, TAR_JAVA_CACHE_DIR, UV_CACHE_DIR,
RUBY_CACHE_DIR, RUST_CACHE_DIR, R_CACHE_DIR, TAR_JAVA_CACHE_DIR, UV_CACHE_DIR,
};
use crate::monitor::{
@@ -2011,6 +2011,7 @@ pub async fn run_workers(
&*POWERSHELL_CACHE_DIR,
&*JAVA_CACHE_DIR,
&*RUBY_CACHE_DIR,
&*R_CACHE_DIR,
&*TAR_JAVA_CACHE_DIR, // for related places search: ADD_NEW_LANG
] {
DirBuilder::new()
+1 -1
View File
@@ -26,7 +26,7 @@ native_trigger_service: nextcloud
request_type: sync, async, sync_sse
runnable_type: ScriptHash, ScriptPath, FlowPath
script_kind: script, trigger, failure, command, approval, preprocessor
script_lang: python3, deno, go, bash, postgresql, nativets, bun, mysql, bigquery, snowflake, graphql, powershell, mssql, php, bunnative, rust, ansible, csharp, oracledb, nu, java, duckdb, ruby
script_lang: python3, deno, go, bash, postgresql, nativets, bun, mysql, bigquery, snowflake, graphql, powershell, mssql, php, bunnative, rust, ansible, csharp, oracledb, nu, java, duckdb, ruby, rlang
trigger_kind: webhook, http, websocket, kafka, email, nats, postgres, sqs, mqtt, gcp, default_email, nextcloud
trigger_mode: enabled, disabled, suspended
workspace_key_kind: cloud
+109
View File
@@ -1081,6 +1081,115 @@ echo "$result"
Ok(())
}
#[cfg(feature = "rlang")]
#[sqlx::test(fixtures("base"))]
async fn test_r_job(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let content = r#"
main <- function(msg) {
return(paste("hello", msg))
}
"#
.to_owned();
let result = RunJob::from(JobPayload::Code(RawCode {
hash: None,
content,
path: None,
lock: None,
language: ScriptLang::Rlang,
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
}))
.arg("msg", json!("world"))
.run_until_complete(&db, false, port)
.await
.json_result()
.unwrap();
assert_eq!(result, json!("hello world"));
Ok(())
}
#[cfg(feature = "rlang")]
#[sqlx::test(fixtures("base", "wmill_cli_test"))]
async fn test_r_get_variable(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let content = r#"
main <- function() {
return(get_variable("u/test-user/test_var"))
}
"#
.to_owned();
let result = RunJob::from(JobPayload::Code(RawCode {
hash: None,
content,
path: None,
lock: None,
language: ScriptLang::Rlang,
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
}))
.run_until_complete(&db, false, port)
.await
.json_result()
.unwrap();
assert_eq!(result, json!("hello from variable"));
Ok(())
}
#[cfg(feature = "rlang")]
#[sqlx::test(fixtures("base", "wmill_cli_test"))]
async fn test_r_get_resource(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let content = r#"
main <- function() {
return(get_resource("u/test-user/test_res"))
}
"#
.to_owned();
let result = RunJob::from(JobPayload::Code(RawCode {
hash: None,
content,
path: None,
lock: None,
language: ScriptLang::Rlang,
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
}))
.run_until_complete(&db, false, port)
.await
.json_result()
.unwrap();
assert_eq!(result, json!({"host": "localhost", "port": 5432}));
Ok(())
}
#[cfg(feature = "nu")]
#[sqlx::test(fixtures("base"))]
async fn test_nu_job(db: Pool<Postgres>) -> anyhow::Result<()> {
@@ -820,6 +820,7 @@ async fn create_script_internal<'c>(
|| ns.language == ScriptLang::Php
|| ns.language == ScriptLang::Java
|| ns.language == ScriptLang::Ruby
|| ns.language == ScriptLang::Rlang
// for related places search: ADD_NEW_LANG
) {
Some(String::new())
+2 -1
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.670.0
version: 1.671.0
title: Windmill API
contact:
@@ -20940,6 +20940,7 @@ components:
nu,
java,
ruby,
rlang,
duckdb,
bunnative,
# for related places search: ADD_NEW_LANG
@@ -484,6 +484,7 @@ pub(crate) async fn tarball_workspace(
ScriptLang::OracleDB => "odb.sql",
ScriptLang::Java => "java",
ScriptLang::Ruby => "rb",
ScriptLang::Rlang => "r",
// for related places search: ADD_NEW_LANG
};
archive
@@ -100,6 +100,7 @@ pub const ENV_SETTINGS: &[&str] = &[
"BUNDLE_PATH",
"GEM_PATH",
"RUBY_CONCURRENT_DOWNLOADS",
"RSCRIPT_PATH",
// for related places search: ADD_NEW_LANG
"GOPRIVATE",
"GOPROXY",
@@ -597,6 +597,7 @@ pub enum ScriptLang {
Nu,
Java,
Ruby,
Rlang,
}
// ---------------------------------------------------------------------------
+8
View File
@@ -184,6 +184,7 @@ lazy_static::lazy_static! {
"nu".to_string(),
"java".to_string(),
"ruby".to_string(),
"rlang".to_string(),
"duckdb".to_string(),
// for related places search: ADD_NEW_LANG
"dependency".to_string(),
@@ -728,6 +729,13 @@ pub struct RubyAnnotations {
pub verbose: bool,
}
#[annotations("#")]
pub struct RlangAnnotations {
pub renv_verbose: bool,
pub renv_install_verbose: bool,
pub sandbox: bool,
}
#[annotations("#")]
pub struct PythonAnnotations {
pub no_cache: bool,
@@ -3000,6 +3000,7 @@ var $RawScript = {
"nativets",
"duckdb",
"ruby",
"rlang",
// for related places search: ADD_NEW_LANG
],
},
+4 -1
View File
@@ -65,6 +65,7 @@ pub enum ScriptLang {
Nu,
Java,
Ruby,
Rlang,
// for related places search: ADD_NEW_LANG
}
@@ -94,6 +95,7 @@ impl ScriptLang {
ScriptLang::Nu => "nu",
ScriptLang::Java => "java",
ScriptLang::Ruby => "ruby",
ScriptLang::Rlang => "rlang",
// for related places search: ADD_NEW_LANG
}
}
@@ -132,7 +134,7 @@ impl ScriptLang {
use ScriptLang::*;
match self {
Nativets | Bun | Bunnative | Deno | Go | Php | CSharp | Java => "//",
Python3 | Bash | Powershell | Graphql | Ansible | Nu | Ruby => "#",
Python3 | Bash | Powershell | Graphql | Ansible | Nu | Ruby | Rlang => "#",
Postgresql | Mysql | Bigquery | Snowflake | Mssql | OracleDB | DuckDb => "--",
Rust => "//!",
// for related places search: ADD_NEW_LANG
@@ -167,6 +169,7 @@ impl FromStr for ScriptLang {
"nu" => ScriptLang::Nu,
"java" => ScriptLang::Java,
"ruby" => ScriptLang::Ruby,
"rlang" => ScriptLang::Rlang,
// for related places search: ADD_NEW_LANG
language => return Err(anyhow::anyhow!("{} is currently not supported", language)),
};
+2
View File
@@ -36,6 +36,7 @@ rust = ["dep:windmill-parser-rust"]
nu = ["dep:windmill-parser-nu"]
java = ["dep:windmill-parser-java"]
ruby = ["dep:windmill-parser-ruby"]
rlang = ["dep:windmill-parser-r"]
duckdb = ["dep:libloading"]
quickjs = ["windmill-jseval/quickjs"]
bedrock = ["dep:aws-sdk-bedrockruntime", "windmill-common/bedrock", "dep:aws-config", "dep:aws-credential-types", "dep:aws-smithy-types"]
@@ -60,6 +61,7 @@ windmill-parser-csharp = { workspace = true, optional = true }
windmill-parser-nu = { workspace = true, optional = true }
windmill-parser-java = { workspace = true, optional = true }
windmill-parser-ruby = { workspace = true, optional = true }
windmill-parser-r = { workspace = true, optional = true }
windmill-parser-py = { workspace = true, optional = true }
windmill-parser-yaml.workspace = true
windmill-parser-py-imports = { workspace = true, optional = true }
@@ -0,0 +1,100 @@
name: "r install"
mode: ONCE
hostname: "r"
log_level: ERROR
time_limit: 900
disable_rl: true
envar: "HOME=/tmp"
envar: "R_INSTALL_TAR=/usr/bin/tar --no-same-owner"
cwd: "/tmp"
clone_newnet: false
clone_newuser: {CLONE_NEWUSER}
skip_setsid: true
keep_caps: true
keep_env: true
mount_proc: true
mount {
src: "/bin"
dst: "/bin"
is_bind: true
mandatory: false
}
mount {
src: "/lib"
dst: "/lib"
is_bind: true
mandatory: false
}
mount {
src: "/lib64"
dst: "/lib64"
is_bind: true
mandatory: false
}
mount {
src: "/usr"
dst: "/usr"
is_bind: true
mandatory: false
}
mount {
src: "/etc"
dst: "/etc"
is_bind: true
}
mount {
src: "/dev/null"
dst: "/dev/null"
is_bind: true
rw: true
}
mount {
src: "{JOB_DIR}"
dst: "/tmp"
is_bind: true
mandatory: false
rw: true
}
mount {
src: "{PKG_DIR}"
dst: "/install"
is_bind: true
rw: true
}
mount {
src: "/sys/devices/system/cpu"
dst: "/sys/devices/system/cpu"
is_bind: true
mandatory: false
}
mount {
src: "/dev/urandom"
dst: "/dev/urandom"
is_bind: true
}
mount {
src: "{TRACING_PROXY_CA_CERT_PATH}"
dst: "{TRACING_PROXY_CA_CERT_PATH}"
is_bind: true
mandatory: false
}
#{DEV}
@@ -0,0 +1,125 @@
name: "r run script"
mode: ONCE
hostname: "r"
log_level: ERROR
disable_rl: true
cwd: "/tmp"
clone_newnet: false
clone_newuser: {CLONE_NEWUSER}
skip_setsid: true
keep_caps: false
keep_env: true
# mount_proc: true
mount {
src: "/bin"
dst: "/bin"
is_bind: true
mandatory: false
}
mount {
src: "/lib"
dst: "/lib"
is_bind: true
mandatory: false
}
mount {
src: "/lib64"
dst: "/lib64"
is_bind: true
mandatory: false
}
mount {
src: "/usr"
dst: "/usr"
is_bind: true
mandatory: false
}
mount {
src: "/dev/null"
dst: "/dev/null"
is_bind: true
rw: true
}
mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size=500000000"
}
mount {
src: "{JOB_DIR}/main.r"
dst: "/tmp/main.r"
is_bind: true
mandatory: false
}
mount {
src: "{JOB_DIR}/args.json"
dst: "/tmp/args.json"
is_bind: true
}
mount {
src: "{JOB_DIR}/result.json"
dst: "/tmp/result.json"
rw: true
is_bind: true
}
mount {
src: "{R_CACHE_DIR}"
dst: "{R_CACHE_DIR}"
is_bind: true
mandatory: false
}
mount {
src: "/etc"
dst: "/etc"
is_bind: true
}
mount {
src: "/sys/devices/system/cpu"
dst: "/sys/devices/system/cpu"
is_bind: true
mandatory: false
}
mount {
src: "/dev/random"
dst: "/dev/random"
is_bind: true
}
mount {
src: "/dev/urandom"
dst: "/dev/urandom"
is_bind: true
}
iface_no_lo: true
{SHARED_MOUNT}
mount {
src: "{TRACING_PROXY_CA_CERT_PATH}"
dst: "{TRACING_PROXY_CA_CERT_PATH}"
is_bind: true
mandatory: false
}
#{DEV}
+3
View File
@@ -17,6 +17,9 @@ mod java_executor;
#[cfg(feature = "ruby")]
mod ruby_executor;
#[cfg(feature = "rlang")]
mod r_executor;
mod ai;
mod ai_executor;
mod bun_executor;
+715
View File
@@ -0,0 +1,715 @@
use std::{collections::HashMap, process::Stdio};
use itertools::Itertools;
use tokio::{
fs::{self, File},
io::{AsyncReadExt, AsyncWriteExt},
process::Command,
};
use uuid::Uuid;
use windmill_common::{
client::AuthedClient,
error::Error,
utils::calculate_hash,
worker::{write_file, Connection, RlangAnnotations},
};
use windmill_parser::Arg;
use windmill_parser_r::{parse_r_requirements, parse_r_signature};
use windmill_queue::{append_logs, CanceledBy, MiniPulledJob};
use crate::{
common::{
build_command_with_isolation, create_args_and_out_file, get_reserved_variables,
read_result, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL,
},
get_proxy_envs_for_lang,
handle_child::{self},
is_sandboxing_enabled,
universal_pkg_installer::{
par_install_language_dependencies_seq, DependencyGraph, InstallDeps, RequiredDependency,
},
DISABLE_NUSER, NSJAIL_AVAILABLE, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, R_CACHE_DIR,
TRACING_PROXY_CA_CERT_PATH,
};
use windmill_common::scripts::ScriptLang;
lazy_static::lazy_static! {
static ref RSCRIPT_PATH: String = std::env::var("RSCRIPT_PATH").unwrap_or_else(|_| "/usr/bin/Rscript".to_string());
static ref R_CONCURRENT_DOWNLOADS: usize = std::env::var("R_CONCURRENT_DOWNLOADS").ok().map(|flag| flag.parse().unwrap_or(5)).unwrap_or(5);
static ref R_PROXY_ENVS: Vec<(String, String)> = {
PROXY_ENVS
.clone()
.into_iter()
.map(|(k, v)| (k.to_lowercase(), v))
.collect()
};
}
const NSJAIL_CONFIG_RUN_R_CONTENT: &str = include_str!("../nsjail/run.r.config.proto");
const NSJAIL_CONFIG_INSTALL_R_CONTENT: &str = include_str!("../nsjail/install.r.config.proto");
#[allow(dead_code)]
pub(crate) struct JobHandlerInput<'a> {
pub base_internal_url: &'a str,
pub canceled_by: &'a mut Option<CanceledBy>,
pub client: &'a AuthedClient,
pub parent_runnable_path: Option<String>,
pub conn: &'a Connection,
pub envs: HashMap<String, String>,
pub inner_content: &'a str,
pub job: &'a MiniPulledJob,
pub job_dir: &'a str,
pub mem_peak: &'a mut i32,
pub occupancy_metrics: &'a mut OccupancyMetrics,
pub requirements_o: Option<&'a String>,
pub shared_mount: &'a str,
pub worker_name: &'a str,
}
pub async fn handle_r_job<'a>(
mut args: JobHandlerInput<'a>,
) -> Result<Box<sqlx::types::JsonRawValue>, Error> {
let annotation = RlangAnnotations::parse(args.inner_content);
if !std::path::Path::new(RSCRIPT_PATH.as_str()).exists() {
return Err(Error::ExecutionErr(format!(
"Rscript binary not found at '{}'. R is only available in the windmill-full (CE) or windmill-ee-full (EE) Docker images.",
*RSCRIPT_PATH
)));
}
if annotation.sandbox && NSJAIL_AVAILABLE.is_none() {
return Err(Error::ExecutionErr(
"Script has #sandbox annotation but nsjail is not available on this worker. \
Please ensure nsjail is installed or remove the #sandbox annotation."
.to_string(),
));
}
// --- Prepare ---
{
prepare(&args).await?;
}
// --- Resolve lockfile ---
let lockfile = resolve(
&args.job.id,
args.inner_content,
args.mem_peak,
args.canceled_by,
args.job_dir,
args.conn,
args.worker_name,
&args.job.workspace_id,
annotation.renv_verbose,
)
.await?;
// --- Install ---
let lib_path = if !lockfile.is_empty() {
Some(
install(
&mut args,
&lockfile,
annotation.renv_verbose,
annotation.renv_install_verbose,
)
.await?,
)
} else {
None
};
// --- Execute ---
{
run(&mut args, lib_path.as_deref(), annotation.sandbox).await?;
}
// --- Retrieve results ---
{
read_result(&args.job_dir, None).await
}
}
pub async fn prepare<'a>(
JobHandlerInput { job, conn, job_dir, inner_content, client, .. }: &JobHandlerInput<'a>,
) -> Result<(), Error> {
create_args_and_out_file(&client, job, job_dir, conn).await?;
File::create(format!("{}/main.r", job_dir))
.await?
.write_all(&wrap(inner_content)?.into_bytes())
.await?;
// Create windmill client library for R
let wm_lib_path = format!("{}/r_libs", *R_CACHE_DIR);
fs::create_dir_all(&wm_lib_path).await?;
{
File::create(format!("{}/windmill.r", &wm_lib_path))
.await?
.write_all(
r##"
# Windmill mini client methods for R
# Uses base R url() + readLines() to avoid requiring any extra R packages
.wm_fetch_raw <- function(url) {
token <- Sys.getenv("WM_TOKEN")
con <- url(url, headers = c(Authorization = paste("Bearer", token)))
on.exit(close(con))
paste(readLines(con, warn = FALSE), collapse = "\n")
}
get_variable <- function(path) {
base_url <- Sys.getenv("BASE_INTERNAL_URL")
workspace <- Sys.getenv("WM_WORKSPACE")
url <- paste0(base_url, "/api/w/", workspace, "/variables/get_value/", path)
jsonlite::fromJSON(.wm_fetch_raw(url))
}
get_resource <- function(path) {
base_url <- Sys.getenv("BASE_INTERNAL_URL")
workspace <- Sys.getenv("WM_WORKSPACE")
url <- paste0(base_url, "/api/w/", workspace, "/resources/get_value_interpolated/", path)
jsonlite::fromJSON(.wm_fetch_raw(url))
}
"##
.as_bytes(),
)
.await?;
}
Ok(())
}
pub async fn resolve<'a>(
job_id: &Uuid,
inner_content: &str,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
job_dir: &str,
conn: &Connection,
worker_name: &str,
w_id: &str,
verbose: bool,
) -> Result<String, Error> {
let mut packages = parse_r_requirements(inner_content)?;
// jsonlite is always needed by the wrapper for JSON arg parsing and result serialization
let has_jsonlite = packages.lines().any(|l| l.trim() == "jsonlite");
if !has_jsonlite {
if packages.is_empty() {
packages = "jsonlite".to_string();
} else {
packages.push_str("\njsonlite");
}
}
// Check cache
let req_hash = format!("r-{}", calculate_hash(&packages));
if let Some(db) = conn.as_sql() {
if let Some(cached) = sqlx::query_scalar!(
"SELECT lockfile FROM pip_resolution_cache WHERE hash = $1",
req_hash
)
.fetch_optional(db)
.await?
{
return Ok(cached);
}
}
append_logs(
job_id,
w_id,
format!("\n--- RESOLVING R PACKAGES ---\n"),
conn,
)
.await;
// main.r is already written by prepare() and contains the library() calls.
// renv will scan it to detect dependencies.
// Disable renv's own package cache — Windmill manages its own install cache.
let resolve_script = format!(
r#"options(
repos = c(CRAN = "https://cloud.r-project.org"),
renv.verbose = {verbose_r},
renv.config.cache.enabled = FALSE,
renv.config.restart.enabled = FALSE,
renv.config.synchronized.check = FALSE
)
renv::consent(provided = TRUE)
suppressMessages(renv::init(bare = TRUE, restart = FALSE))
suppressMessages(renv::install(prompt = FALSE))
suppressMessages(renv::snapshot(type = "implicit", prompt = FALSE))
"#,
verbose_r = if verbose { "TRUE" } else { "FALSE" },
);
let mut file = File::create(format!("{}/resolve.r", job_dir)).await?;
file.write_all(resolve_script.as_bytes()).await?;
let child = {
let renv_root = format!("{}/renv", *R_CACHE_DIR);
let rscript_executable = if cfg!(windows) {
"Rscript.exe"
} else {
RSCRIPT_PATH.as_str()
};
let mut cmd = Command::new(rscript_executable);
cmd.current_dir(job_dir)
.env("PATH", PATH_ENV.as_str())
.env("RENV_PATHS_ROOT", &renv_root)
.arg("resolve.r")
.stdout(Stdio::piped())
.stderr(Stdio::piped());
start_child_process(cmd, rscript_executable, false).await?
};
handle_child::handle_child(
job_id,
conn,
mem_peak,
canceled_by,
child,
false,
worker_name,
w_id,
"r resolve",
None,
false,
&mut None,
None,
None,
)
.await?;
let lock_path = format!("{}/renv.lock", job_dir);
let mut lock_file = File::open(&lock_path).await?;
let mut lock = String::new();
lock_file.read_to_string(&mut lock).await?;
// Cache the lockfile
if let Some(db) = conn.as_sql() {
sqlx::query!(
"INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = EXCLUDED.lockfile",
req_hash,
lock.clone(),
).fetch_optional(db).await?;
}
// Log a compact summary instead of the entire renv.lock JSON
let pkg_count = serde_json::from_str::<serde_json::Value>(&lock)
.ok()
.and_then(|v| v.get("Packages")?.as_object().map(|o| o.len()))
.unwrap_or(0);
append_logs(
job_id,
w_id,
format!("resolved {} packages\n", pkg_count),
conn,
)
.await;
Ok(lock)
}
struct RenvPackage {
name: String,
version: String,
repo_url: String,
/// Package names from Imports + Depends fields
dependencies: Vec<String>,
}
/// Parse renv.lock JSON and extract package info including dependency edges.
fn parse_renv_lock(lockfile: &str) -> Result<Vec<RenvPackage>, Error> {
let lock: serde_json::Value = serde_json::from_str(lockfile)
.map_err(|e| Error::ExecutionErr(format!("Failed to parse renv.lock: {}", e)))?;
// Build repo name -> URL map from R.Repositories
let mut repo_urls: HashMap<String, String> = HashMap::new();
if let Some(repos) = lock
.get("R")
.and_then(|r| r.get("Repositories"))
.and_then(|r| r.as_array())
{
for repo in repos {
if let (Some(name), Some(url)) = (
repo.get("Name").and_then(|v| v.as_str()),
repo.get("URL").and_then(|v| v.as_str()),
) {
repo_urls.insert(name.to_string(), url.to_string());
}
}
}
let packages = lock
.get("Packages")
.and_then(|p| p.as_object())
.ok_or_else(|| Error::ExecutionErr("renv.lock missing Packages field".to_string()))?;
let mut result = vec![];
for (_name, pkg) in packages {
let pkg_name = pkg
.get("Package")
.and_then(|v| v.as_str())
.unwrap_or_default()
.to_string();
let version = pkg
.get("Version")
.and_then(|v| v.as_str())
.unwrap_or_default()
.to_string();
let repo_name = pkg
.get("Repository")
.and_then(|v| v.as_str())
.unwrap_or("CRAN");
let repo_url = repo_urls
.get(repo_name)
.cloned()
.unwrap_or_else(|| "https://cloud.r-project.org".to_string());
let mut dependencies = vec![];
if let Some(imports) = pkg.get("Imports").and_then(|v| v.as_array()) {
for entry in imports {
if let Some(s) = entry.as_str() {
// Entries look like "cli (>= 3.6.2)" — take just the name
let name = s.split_whitespace().next().unwrap_or("");
if !name.is_empty() && name != "R" {
dependencies.push(name.to_string());
}
}
}
}
// Skip renv itself — it's already loaded and reinstalling it while
// loaded triggers a noisy "Restart your R session" message.
if !pkg_name.is_empty() && !version.is_empty() && pkg_name != "renv" {
result.push(RenvPackage { name: pkg_name, version, repo_url, dependencies });
}
}
Ok(result)
}
async fn install<'a>(
args: &mut JobHandlerInput<'a>,
lockfile: &str,
verbose: bool,
install_verbose: bool,
) -> Result<String, Error> {
let lib_path = format!("{}/r_site_library", *R_CACHE_DIR);
fs::create_dir_all(&lib_path).await?;
let packages = parse_renv_lock(lockfile)?;
if packages.is_empty() {
return Ok(lib_path);
}
#[derive(Clone, Debug)]
struct RPackagePayload {
pkg: String,
version: String,
#[allow(dead_code)]
repo_url: String,
}
// Build dependency graph for topological layering
let mut graph = DependencyGraph::new();
for renv_pkg in &packages {
let handle = format!("{}-{}", renv_pkg.name, renv_pkg.version);
// renv uses staged installation: it builds to a temp dir then rename()s onto
// the target. If the target is a bind mount point, rename fails with
// "target file already exists". We work around this by mounting the parent
// (wrapper) dir at /install so renv can freely create /install/{pkg}/ via rename.
let pkg_outer = format!("{}/{}_outer", lib_path, renv_pkg.name);
let path = format!("{}/{}", pkg_outer, renv_pkg.name);
graph.insert(
renv_pkg.name.clone(),
RequiredDependency {
path,
_s3_handle: handle,
display_name: format!("{} ({})", renv_pkg.name, renv_pkg.version),
custom_payload: RPackagePayload {
pkg: renv_pkg.name.clone(),
version: renv_pkg.version.clone(),
repo_url: renv_pkg.repo_url.clone(),
},
},
renv_pkg.dependencies.clone(),
);
}
let jailed = !cfg!(windows) && is_sandboxing_enabled();
let job_dir = args.job_dir.to_owned();
par_install_language_dependencies_seq(
InstallDeps::Layered(graph),
"r",
"Rscript",
false,
*R_CONCURRENT_DOWNLOADS,
move |dependency| {
let lib_path_c = lib_path.clone();
let job_dir = job_dir.clone();
let pkg_name = &dependency.custom_payload.pkg;
// pkg_outer is the wrapper dir mounted rw at /install inside nsjail.
// renv creates /install/{pkg}/ inside it via staged rename.
let pkg_outer = format!("{}/{}_outer", lib_path_c, pkg_name);
std::fs::create_dir_all(&pkg_outer)?;
let mut cmd = if jailed {
let nsjail_proto = format!("{}.install.config.proto", Uuid::new_v4());
let config_content = NSJAIL_CONFIG_INSTALL_R_CONTENT
.replace("{JOB_DIR}", &job_dir)
.replace("{PKG_DIR}", &pkg_outer)
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
.replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
.replace("#{DEV}", DEV_CONF_NSJAIL);
let _ = write_file(
&job_dir,
&nsjail_proto,
&config_content,
)?;
let mut cmd = Command::new(NSJAIL_PATH.as_str());
cmd.args(vec![
"--config",
&nsjail_proto,
"--",
RSCRIPT_PATH.as_str(),
]);
cmd
} else {
Command::new(if cfg!(windows) {
"Rscript.exe"
} else {
RSCRIPT_PATH.as_str()
})
};
let verbose_r = if verbose { "TRUE" } else { "FALSE" };
let install_verbose_r = if install_verbose { "TRUE" } else { "FALSE" };
let install_lib = if jailed { "/install".to_string() } else { pkg_outer.clone() };
cmd.env_clear()
.current_dir(&job_dir)
.env("PATH", PATH_ENV.as_str())
.envs(R_PROXY_ENVS.clone());
cmd
.args(&[
"-e",
&format!(
r#"options(renv.verbose = {verbose_r}, renv.config.install.verbose = {install_verbose_r}, renv.config.restart.enabled = FALSE); renv::install("{pkg}@{version}", library = "{lib}", dependencies = FALSE)"#,
verbose_r = verbose_r,
install_verbose_r = install_verbose_r,
pkg = dependency.custom_payload.pkg,
version = dependency.custom_payload.version,
lib = install_lib,
),
// install.packages fallback (no version pinning):
// &format!(
// r#"install.packages("{pkg}", lib = "{lib}", repos = "{repo}", dependencies = FALSE, quiet = {quiet}, INSTALL_opts = "--no-test-load --no-lock")"#,
// pkg = dependency.custom_payload.pkg,
// lib = install_lib,
// repo = dependency.custom_payload.repo_url,
// quiet = quiet_flag,
// ),
])
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped());
Ok(cmd)
},
None,
&args.job.id,
&args.job.workspace_id,
args.worker_name,
jailed,
args.conn,
)
.await?;
Ok(format!("{}/r_site_library", *R_CACHE_DIR))
}
/// Build R_LIBS_USER from lib_path by listing *_outer subdirs.
/// Each package wrapper dir ({pkg}_outer) is added so R finds {pkg}_outer/{pkg}/DESCRIPTION.
fn r_libs_user(lib_path: &str) -> String {
std::fs::read_dir(lib_path)
.into_iter()
.flatten()
.filter_map(|e| e.ok())
.filter(|e| {
e.file_type().map(|t| t.is_dir()).unwrap_or(false)
&& e.file_name().to_string_lossy().ends_with("_outer")
})
.map(|e| e.path().to_string_lossy().to_string())
.collect::<Vec<_>>()
.join(":")
}
async fn run<'a>(
JobHandlerInput {
occupancy_metrics,
mem_peak,
canceled_by,
worker_name,
job,
conn,
job_dir,
shared_mount,
client,
envs,
base_internal_url,
parent_runnable_path,
..
}: &mut JobHandlerInput<'a>,
lib_path: Option<&str>,
sandbox: bool,
) -> Result<(), Error> {
let reserved_variables =
get_reserved_variables(job, &client.token, conn, parent_runnable_path.clone()).await?;
let nsjail = !cfg!(windows) && (is_sandboxing_enabled() || sandbox);
let child = if nsjail {
append_logs(
&job.id,
&job.workspace_id,
"\n--- R CODE EXECUTION (nsjail) ---\n".to_string(),
conn,
)
.await;
write_file(
job_dir,
"run.config.proto",
&NSJAIL_CONFIG_RUN_R_CONTENT
.replace("{JOB_DIR}", job_dir)
.replace("{SHARED_MOUNT}", &shared_mount)
.replace("{R_CACHE_DIR}", &*R_CACHE_DIR)
.replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
.replace("#{DEV}", DEV_CONF_NSJAIL)
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()),
)?;
let mut cmd = Command::new(NSJAIL_PATH.as_str());
cmd.env_clear()
.current_dir(job_dir)
.env("PATH", PATH_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
.envs(envs)
.envs(reserved_variables)
.envs(R_PROXY_ENVS.clone())
.envs(
get_proxy_envs_for_lang(&ScriptLang::Rlang, &job.id, &job.workspace_id, conn)
.await?,
);
if let Some(lp) = lib_path {
cmd.env("R_LIBS_USER", r_libs_user(lp));
}
cmd.args(vec![
"--config",
"run.config.proto",
"--",
RSCRIPT_PATH.as_str(),
"main.r",
]);
cmd.stdout(Stdio::piped()).stderr(Stdio::piped());
start_child_process(cmd, NSJAIL_PATH.as_str(), false).await?
} else {
append_logs(
&job.id,
&job.workspace_id,
format!("\n--- R CODE EXECUTION ---\n"),
conn,
)
.await;
let rscript_executable = if cfg!(windows) {
"Rscript.exe"
} else {
RSCRIPT_PATH.as_str()
};
let args = vec!["main.r"];
let mut cmd = build_command_with_isolation(rscript_executable, &args);
cmd.env_clear()
.current_dir(job_dir.to_owned())
.env("PATH", PATH_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
.envs(reserved_variables)
.envs(R_PROXY_ENVS.clone())
.envs(
get_proxy_envs_for_lang(&ScriptLang::Rlang, &job.id, &job.workspace_id, conn)
.await?,
)
.envs(envs);
if let Some(lp) = lib_path {
cmd.env("R_LIBS_USER", r_libs_user(lp));
}
cmd.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped());
#[cfg(windows)]
{
cmd.env("SystemRoot", crate::SYSTEM_ROOT.as_str())
.env("USERPROFILE", crate::USERPROFILE_ENV.as_str())
.env(
"TMP",
std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")),
);
}
start_child_process(cmd, rscript_executable, false).await?
};
handle_child::handle_child(
&job.id,
conn,
mem_peak,
canceled_by,
child,
nsjail,
worker_name,
&job.workspace_id,
"r",
job.timeout,
false,
&mut Some(occupancy_metrics),
None,
None,
)
.await?;
Ok(())
}
fn wrap(inner_content: &str) -> Result<String, Error> {
let sig = parse_r_signature(inner_content)?;
let spread = sig
.args
.clone()
.into_iter()
.map(|Arg { name, .. }| format!("{name} = args${name}", name = name))
.collect_vec()
.join(", ");
let wm_lib_path = format!("{}/r_libs/windmill.r", *R_CACHE_DIR);
Ok(format!(
r#"source("{wm_lib_path}")
suppressPackageStartupMessages({{
{inner_content}
}})
library(jsonlite)
args <- fromJSON("args.json")
tryCatch({{
res <- main({spread})
write(toJSON(res, auto_unbox = TRUE, null = "null"), "result.json")
}}, error = function(e) {{
error_obj <- list(
name = class(e)[1],
message = conditionMessage(e),
stack = paste(capture.output(traceback()), collapse = "\n")
)
write(toJSON(error_obj, auto_unbox = TRUE), "result.json")
stop(e)
}})
"#,
wm_lib_path = wm_lib_path,
inner_content = inner_content,
spread = spread,
))
}
+3 -3
View File
@@ -29,7 +29,7 @@ use crate::{
get_proxy_envs_for_lang,
handle_child::{self},
is_sandboxing_enabled, read_ee_registry_url_list_with_workspace_override,
universal_pkg_installer::{par_install_language_dependencies_seq, RequiredDependency},
universal_pkg_installer::{par_install_language_dependencies_seq, InstallDeps, RequiredDependency},
DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, RUBY_CACHE_DIR, RUBY_REPOS,
TRACING_PROXY_CA_CERT_PATH,
};
@@ -618,7 +618,7 @@ async fn install<'a>(
get_reserved_variables(job, &client.token, conn, parent_runnable_path.clone()).await?,
);
par_install_language_dependencies_seq(
deps.clone(),
InstallDeps::Flat(deps.clone()),
"ruby",
"gem",
false,
@@ -721,7 +721,7 @@ async fn install<'a>(
Ok(cmd)
},
// async move |_| Ok(()),
None,
&job.id,
&job.workspace_id,
worker_name,
@@ -1,3 +1,4 @@
use std::collections::{HashMap, HashSet, VecDeque};
use std::sync::Arc;
use anyhow::bail;
@@ -31,6 +32,153 @@ pub struct RequiredDependency<T: Clone + Send + Sync> {
pub custom_payload: T,
}
/// Generic dependency graph that produces topologically sorted layers via Kahn's algorithm.
/// Each layer's packages only depend on packages from earlier layers, enabling parallel install
/// per layer.
#[allow(dead_code)]
pub struct DependencyGraph<T: Clone + Send + Sync> {
nodes: HashMap<String, RequiredDependency<T>>,
deps: HashMap<String, HashSet<String>>,
}
#[allow(dead_code)]
impl<T: Clone + Send + Sync> DependencyGraph<T> {
pub fn new() -> Self {
Self { nodes: HashMap::new(), deps: HashMap::new() }
}
/// Insert a dependency and the names of packages it depends on.
/// References to packages not in the graph are silently ignored during layering.
pub fn insert(
&mut self,
key: impl Into<String>,
dep: RequiredDependency<T>,
depends_on: Vec<String>,
) {
let key = key.into();
self.nodes.insert(key.clone(), dep);
self.deps.insert(key, depends_on.into_iter().collect());
}
/// Render a dependency tree string. Each package appears once, nested under the first parent
/// that pulls it in. Only includes packages present in `filter` (if provided).
pub fn print_tree(&self, filter: Option<&HashSet<String>>) -> String {
// Find roots: packages nothing else in the graph depends on
let mut depended_on: HashSet<&str> = HashSet::new();
for dep_set in self.deps.values() {
for to in dep_set {
if self.nodes.contains_key(to) {
depended_on.insert(to.as_str());
}
}
}
let roots: Vec<&String> = self
.nodes
.keys()
.filter(|k| !depended_on.contains(k.as_str()))
.filter(|k| filter.map_or(true, |f| f.contains(*k)))
.sorted()
.collect();
let mut out = String::new();
let mut seen = HashSet::new();
for root in roots {
self.print_tree_node(root, 0, &mut seen, filter, &mut out);
}
out
}
fn print_tree_node(
&self,
key: &str,
depth: usize,
seen: &mut HashSet<String>,
filter: Option<&HashSet<String>>,
out: &mut String,
) {
if !seen.insert(key.to_string()) {
return;
}
if let Some(dep) = self.nodes.get(key) {
let indent = " ".repeat(depth);
out.push_str(&format!("{}- {}\n", indent, dep.display_name));
if let Some(children) = self.deps.get(key) {
for child in children.iter().sorted() {
if self.nodes.contains_key(child)
&& filter.map_or(true, |f| f.contains(child))
&& !seen.contains(child)
{
self.print_tree_node(child, depth + 1, seen, filter, out);
}
}
}
}
}
/// Produce topologically sorted layers.
pub fn layers(self) -> Vec<Vec<RequiredDependency<T>>> {
let mut in_degree: HashMap<String, usize> =
self.nodes.keys().map(|k| (k.clone(), 0)).collect();
let mut reverse: HashMap<String, Vec<String>> = HashMap::new();
for (from, dep_set) in &self.deps {
for to in dep_set {
if self.nodes.contains_key(to) {
*in_degree.entry(from.clone()).or_default() += 1;
reverse.entry(to.clone()).or_default().push(from.clone());
}
}
}
let mut queue: VecDeque<String> = in_degree
.iter()
.filter(|(_, &d)| d == 0)
.map(|(k, _)| k.clone())
.sorted()
.collect();
let mut result = vec![];
let mut nodes = self.nodes;
while !queue.is_empty() {
let mut layer = vec![];
let mut next = VecDeque::new();
for key in queue {
if let Some(dep) = nodes.remove(&key) {
layer.push(dep);
}
if let Some(dependents) = reverse.get(&key) {
for d in dependents {
if let Some(deg) = in_degree.get_mut(d) {
*deg -= 1;
if *deg == 0 {
next.push_back(d.clone());
}
}
}
}
}
if !layer.is_empty() {
result.push(layer);
}
queue = next.into_iter().sorted().collect();
}
result
}
}
#[allow(dead_code)]
pub enum InstallDeps<T: Clone + Send + Sync> {
/// Flat list of dependencies — installed in one parallel batch (existing behavior).
Flat(Vec<RequiredDependency<T>>),
/// Dependency graph — split into topological layers, each installed in parallel.
/// A `--- Layer N ---` separator is printed between layers.
Layered(DependencyGraph<T>),
}
#[allow(dead_code)]
pub enum InstallStrategy<T: Clone + Send + Sync> {
/// Will invoke callback to install single dependency
@@ -105,8 +253,10 @@ pub async fn par_install_language_dependencies_all_at_once<
.await;
}
let total_time = std::time::Instant::now();
let (missing, name_max_length) = filter_to_missing(deps, job_id, w_id, jailed, conn).await?;
if missing.is_empty() {
let (layers, name_max_length, total_missing) =
filter_to_missing(InstallDeps::Flat(deps), job_id, w_id, jailed, conn).await?;
let missing: Vec<RequiredDependency<T>> = layers.into_iter().flatten().collect();
if total_missing == 0 {
return Ok(());
}
let to_batch_install = Arc::new(RwLock::new(vec![]));
@@ -122,6 +272,9 @@ pub async fn par_install_language_dependencies_all_at_once<
conn,
_language_name,
_platform_agnostic,
None,
None,
None,
)
.await?;
let installation_res = process_handles(handles, w_id).await;
@@ -231,18 +384,26 @@ pub async fn par_install_language_dependencies_seq<
'a,
T: Clone + std::marker::Send + Sync + 'a + 'static,
>(
deps: Vec<RequiredDependency<T>>,
install_deps: InstallDeps<T>,
_language_name: &'a str,
installer_executable_name: &'a str,
_platform_agnostic: bool,
concurrent_downloads: usize,
callback: impl Fn(RequiredDependency<T>) -> Result<Command, error::Error> + Send + Sync + 'static,
post_install: Option<Arc<dyn Fn(&RequiredDependency<T>) -> anyhow::Result<()> + Send + Sync + 'static>>,
job_id: &'a Uuid,
w_id: &'a str,
worker_name: &'a str,
jailed: bool,
conn: &'a Connection,
) -> anyhow::Result<()> {
let total_time = std::time::Instant::now();
let (layers, name_max_length, total_missing) =
filter_to_missing(install_deps, job_id, w_id, jailed, conn).await?;
if total_missing == 0 {
return Ok(());
}
#[cfg(all(feature = "enterprise", feature = "parquet"))]
let is_not_pro = !matches!(
windmill_common::ee_oss::get_license_plan().await,
@@ -258,65 +419,133 @@ pub async fn par_install_language_dependencies_seq<
)
.await;
}
let total_time = std::time::Instant::now();
let (missing, name_max_length) = filter_to_missing(deps, job_id, w_id, jailed, conn).await?;
if missing.is_empty() {
return Ok(());
}
let handles = spawn_wrapped_installation_threads(
missing,
name_max_length,
InstallStrategy::Single(Arc::new(callback)),
installer_executable_name,
concurrent_downloads,
let is_layered = layers.len() > 1;
let callback = Arc::new(callback);
let mut offset = 0usize;
windmill_queue::append_logs(
job_id,
w_id,
worker_name,
if jailed {
format!(
"\nStarting isolated installation... ({} tasks in parallel)\n",
concurrent_downloads
)
} else {
format!(
"\nStarting installation... ({} tasks in parallel)\n",
concurrent_downloads
)
},
conn,
_language_name,
_platform_agnostic,
)
.await?;
.await;
for (i, layer_deps) in layers.into_iter().enumerate() {
if layer_deps.is_empty() {
continue;
}
if is_layered && offset > 0 {
windmill_queue::append_logs(
job_id,
w_id,
format!("\n\n--- Layer {} ---", i + 1),
conn,
)
.await;
}
let layer_size = layer_deps.len();
tracing::info!("Layer {}: spawning {} installs", i + 1, layer_size);
let handles = spawn_wrapped_installation_threads(
layer_deps,
name_max_length,
InstallStrategy::Single(callback.clone()),
installer_executable_name,
concurrent_downloads,
job_id,
w_id,
worker_name,
conn,
_language_name,
_platform_agnostic,
Some(offset),
Some(total_missing),
post_install.clone(),
)
.await?;
tracing::info!("Layer {}: all spawned, waiting for handles", i + 1);
process_handles(handles, w_id).await?;
tracing::info!("Layer {}: done", i + 1);
offset += layer_size;
}
let installation_res = process_handles(handles, w_id).await;
finish_installation(total_time, job_id, w_id, conn).await;
installation_res
Ok(())
}
type NameMaxLength = usize;
/// Returns (layers of missing deps, name_max_length, total_missing).
/// Prints the "To be installed" header once with all missing packages.
/// For `Layered`, prints a dependency tree; for `Flat`, prints a flat list.
async fn filter_to_missing<'a, T: Clone + std::marker::Send + Sync + 'a + 'static>(
mut deps: Vec<RequiredDependency<T>>,
install_deps: InstallDeps<T>,
job_id: &Uuid,
w_id: &str,
jailed: bool,
conn: &Connection,
) -> anyhow::Result<(Vec<RequiredDependency<T>>, NameMaxLength)> {
// Unique to flatten all same values
deps = deps.into_iter().unique_by(|rd| rd.path.clone()).collect();
// Total to install
let mut missing = vec![];
// Name max length
let mut name_ml = 0;
for rd in deps.into_iter() {
let display_name = rd.display_name.clone();
if rd.path.ends_with("/") {
anyhow::bail!("Internal error: path should not end with '/'")
) -> anyhow::Result<(Vec<Vec<RequiredDependency<T>>>, NameMaxLength, usize)> {
let (mut layers, tree_data) = match install_deps {
InstallDeps::Flat(deps) => (vec![deps], None),
InstallDeps::Layered(graph) => {
let deps_map = graph.deps.clone();
let nodes_display: HashMap<String, String> = graph
.nodes
.iter()
.map(|(k, v)| (k.clone(), v.display_name.clone()))
.collect();
let layers = graph.layers();
(layers, Some((deps_map, nodes_display)))
}
{
// Later will help us align text in log console
if display_name.len() > name_ml {
};
let mut name_ml = 0;
let mut missing_keys: HashSet<String> = HashSet::new();
let mut total_missing = 0;
for layer in layers.iter_mut() {
*layer = std::mem::take(layer)
.into_iter()
.unique_by(|rd| rd.path.clone())
.collect();
let mut missing = vec![];
for rd in std::mem::take(layer) {
if rd.path.ends_with("/") {
anyhow::bail!("Internal error: path should not end with '/'")
}
if rd.display_name.len() > name_ml {
name_ml = rd.display_name.len();
}
if tokio::fs::metadata(rd.path.clone() + ".valid.windmill")
.await
.is_err()
{
if let Some(key) = rd.path.rsplit('/').next() {
missing_keys.insert(key.to_string());
}
missing.push(rd);
}
}
// Will look like: /tmp/windmill/cache/lang/dependency.valid.windmill
if tokio::fs::metadata(rd.path.clone() + ".valid.windmill")
.await
.is_err()
{
missing.push(rd);
}
total_missing += missing.len();
*layer = missing;
}
if !missing.is_empty() {
if total_missing > 0 {
windmill_queue::append_logs(
job_id,
w_id,
@@ -328,15 +557,40 @@ async fn filter_to_missing<'a, T: Clone + std::marker::Send + Sync + 'a + 'stati
conn,
)
.await;
let to_log = missing
.iter()
.map(|rd| format!("- {}", &rd.display_name))
.join("\n")
+ "\n";
let to_log = if let Some((deps_map, nodes_display)) = tree_data {
let mut print_graph: DependencyGraph<()> = DependencyGraph::new();
for (key, display) in &nodes_display {
if missing_keys.contains(key) {
print_graph.insert(
key.clone(),
RequiredDependency {
path: String::new(),
_s3_handle: String::new(),
display_name: display.clone(),
custom_payload: (),
},
deps_map
.get(key)
.map(|s| s.iter().cloned().collect())
.unwrap_or_default(),
);
}
}
print_graph.print_tree(Some(&missing_keys))
} else {
layers
.iter()
.flat_map(|l| l.iter())
.map(|rd| format!("- {}", &rd.display_name))
.join("\n")
+ "\n"
};
windmill_queue::append_logs(job_id, w_id, to_log, conn).await;
}
Ok((missing, name_ml))
Ok((layers, name_ml, total_missing))
}
enum Action<T: Clone + Send + Sync> {
@@ -369,6 +623,9 @@ async fn spawn_wrapped_installation_threads<
conn: &Connection,
_language_name: &str,
_platform_agnostic: bool,
counter_offset: Option<usize>,
total_override: Option<usize>,
post_install: Option<Arc<dyn Fn(&RequiredDependency<T>) -> anyhow::Result<()> + Send + Sync + 'static>>,
) -> anyhow::Result<(
Vec<JoinHandle<anyhow::Result<TaskKiller>>>,
tokio::sync::broadcast::Sender<()>,
@@ -382,11 +639,11 @@ async fn spawn_wrapped_installation_threads<
job_id
);
let (mut handles, semaphore, total_to_install, counter_arc) = (
let total_to_install = total_override.unwrap_or(missing.len());
let (mut handles, semaphore, counter_arc) = (
vec![],
Arc::new(Semaphore::new(parallel_limit)),
missing.len(),
Arc::new(tokio::sync::Mutex::new(0)),
Arc::new(tokio::sync::Mutex::new(counter_offset.unwrap_or(0))),
);
// Pretty sensitive. Single drop will fail installation
@@ -426,6 +683,7 @@ async fn spawn_wrapped_installation_threads<
),
InstallStrategy::AllAtOnce(ref rw_lock) => Action::AddToBulk(Arc::clone(rw_lock)),
};
let post_install_c = post_install.clone();
let task_fut = try_install_one_detached(
dep,
installer_executable_name.to_owned(),
@@ -441,6 +699,7 @@ async fn spawn_wrapped_installation_threads<
_platform_agnostic,
permit,
TaskKiller(kill_tx),
post_install_c,
);
handles.push(tokio::spawn(async move {
tokio::select! {
@@ -513,6 +772,7 @@ async fn try_install_one_detached<'a, T: Clone + std::marker::Send + Sync + 'a +
// If dropped the entire installation fails and all installation threads are being stopped
// That's why we just pass it to return so it is not being dropped
kill_all_tasks: TaskKiller,
post_install: Option<Arc<dyn Fn(&RequiredDependency<T>) -> anyhow::Result<()> + Send + Sync + 'static>>,
) -> anyhow::Result<TaskKiller> {
let start = std::time::Instant::now();
@@ -607,6 +867,9 @@ async fn try_install_one_detached<'a, T: Clone + std::marker::Send + Sync + 'a +
&dep.display_name
));
} else {
if let Some(ref cb) = post_install {
cb(&dep)?;
}
mark_success(dep.path.clone(), &job_id, &w_id).await;
print_success(
false,
+44 -1
View File
@@ -164,6 +164,9 @@ use crate::java_executor::{handle_java_job, JobHandlerInput as JobHandlerInputJa
#[cfg(feature = "ruby")]
use crate::ruby_executor::{handle_ruby_job, JobHandlerInput as JobHandlerInputRuby};
#[cfg(feature = "rlang")]
use crate::r_executor::{handle_r_job, JobHandlerInput as JobHandlerInputRlang};
#[cfg(feature = "php")]
use crate::php_executor::handle_php_job;
@@ -230,6 +233,9 @@ lazy_static::lazy_static! {
// Ruby
pub static ref RUBY_CACHE_DIR: String = format!("{}ruby", *ROOT_CACHE_DIR);
// R
pub static ref R_CACHE_DIR: String = format!("{}rlang", *ROOT_CACHE_DIR);
// for related places search: ADD_NEW_LANG
pub static ref BUN_CACHE_DIR: String = format!("{}bun", *ROOT_CACHE_NOMOUNT_DIR);
pub static ref BUN_BUNDLE_CACHE_DIR: String = format!("{}bun", *ROOT_CACHE_DIR);
@@ -4602,7 +4608,8 @@ mount {{
| ScriptLang::Bash
| ScriptLang::Powershell
| ScriptLang::Ansible
| ScriptLang::Ruby => "#",
| ScriptLang::Ruby
| ScriptLang::Rlang => "#",
ScriptLang::Deno
| ScriptLang::Bun
| ScriptLang::Bunnative
@@ -5114,6 +5121,38 @@ mount {{
.await
}
}
ScriptLang::Rlang => {
#[cfg(not(feature = "rlang"))]
return Err(
anyhow::anyhow!("R is not available because the feature is not enabled").into(),
);
#[cfg(feature = "rlang")]
{
if run_inline {
return Err(Error::internal_err(
"Inline execution is not yet supported for this language".to_string(),
));
}
Box::pin(handle_r_job(JobHandlerInputRlang {
mem_peak,
canceled_by,
job,
conn,
client,
parent_runnable_path,
inner_content: &code,
job_dir,
requirements_o: lock.as_ref(),
shared_mount: &shared_mount,
base_internal_url,
worker_name,
envs,
occupancy_metrics,
}))
.await
}
}
// for related places search: ADD_NEW_LANG
_ => panic!("unreachable, language is not supported: {language:#?}"),
};
@@ -5247,6 +5286,10 @@ pub fn parse_sig_of_lang(
ScriptLang::Ruby => Some(windmill_parser_ruby::parse_ruby_signature(code)?),
#[cfg(not(feature = "ruby"))]
ScriptLang::Ruby => None,
#[cfg(feature = "rlang")]
ScriptLang::Rlang => Some(windmill_parser_r::parse_r_signature(code)?),
#[cfg(not(feature = "rlang"))]
ScriptLang::Rlang => None,
// for related places search: ADD_NEW_LANG
}
} else {
@@ -61,6 +61,8 @@ use crate::csharp_executor::generate_nuget_lockfile;
#[cfg(feature = "java")]
use crate::java_executor;
#[cfg(feature = "rlang")]
use crate::r_executor;
#[cfg(feature = "ruby")]
use crate::ruby_executor;
@@ -2763,6 +2765,21 @@ async fn capture_dependency_job(
)
.await?
}
#[cfg(feature = "rlang")]
ScriptLang::Rlang => {
r_executor::resolve(
job_id,
job_raw_code,
mem_peak,
canceled_by,
job_dir,
&Connection::Sql(db.clone()),
worker_name,
w_id,
false,
)
.await?
}
// for related places search: ADD_NEW_LANG
_ => "".to_owned(),
};
+1 -1
View File
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
export const VERSION = "v1.670.0";
export const VERSION = "v1.671.0";
export async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({
+5
View File
@@ -134,6 +134,11 @@ public class Main {
def main a, b, c
puts a, b, c
end
`,
rlang: `
main <- function(x, name = "default") {
return(list(result = x, name = name))
}
`,
// for related places search: ADD_NEW_LANG
};
+3
View File
@@ -832,6 +832,8 @@ export function filePathExtensionFromContentType(
return ".java";
} else if (language === "ruby") {
return ".rb";
} else if (language === "rlang") {
return ".r";
// for related places search: ADD_NEW_LANG
} else {
throw new Error("Invalid language: " + language);
@@ -863,6 +865,7 @@ export const exts = [
".playbook.yml",
".java",
".rb",
".r",
// for related places search: ADD_NEW_LANG
];
+1
View File
@@ -1306,6 +1306,7 @@ export async function elementsToMap(
"nu",
"java",
"rb",
"r",
// for related places search: ADD_NEW_LANG
].includes(path.split(".").pop() ?? "")
) {
File diff suppressed because one or more lines are too long
+1 -1
View File
@@ -78,7 +78,7 @@ export {
token,
};
export const VERSION = "1.670.0";
export const VERSION = "1.671.0";
// Re-exported from constants.ts to maintain backwards compatibility
export { WM_FORK_PREFIX } from "./core/constants.ts";
+1
View File
@@ -300,6 +300,7 @@ export function getTypeStrFromPath(
parsed.ext == ".nu" ||
parsed.ext == ".java" ||
parsed.ext == ".rb" ||
parsed.ext == ".r" ||
// for related places search: ADD_NEW_LANG
(parsed.ext == ".yml" && parsed.name.split(".").pop() == "playbook")
) {
+3
View File
@@ -868,6 +868,9 @@ export async function inferSchema(
} else if (language === "ruby") {
const { parse_ruby } = await loadParser("windmill-parser-wasm-ruby");
inferedSchema = JSON.parse(parse_ruby(content));
} else if (language === "rlang") {
const { parse_r } = await loadParser("windmill-parser-wasm-r");
inferedSchema = JSON.parse(parse_r(content));
// for related places search: ADD_NEW_LANG
} else {
throw new Error("Invalid language: " + language);
+3
View File
@@ -20,6 +20,7 @@ export type ScriptLanguage =
| "nu"
| "ansible"
| "ruby"
| "rlang"
| "java";
// for related places search: ADD_NEW_LANG
@@ -105,6 +106,8 @@ export function inferContentTypeFromFilePath(
return "java";
} else if (contentPath.endsWith(".rb")) {
return "ruby";
} else if (contentPath.endsWith(".r")) {
return "rlang";
// for related places search: ADD_NEW_LANG
} else {
throw new Error(
@@ -35,6 +35,7 @@ export const LANGUAGE_EXTENSIONS: Record<SupportedLanguage, string> = {
duckdb: "duckdb.sql",
bunnative: "ts",
ruby: "rb",
rlang: "r",
// for related places search: ADD_NEW_LANG
};
+4
View File
@@ -27,6 +27,10 @@ RUN /usr/bin/java -jar /usr/bin/coursier about
# Ruby
RUN apt-get install -y ruby ruby-bundler
# R
RUN apt-get install -y r-base-dev \
&& Rscript -e 'install.packages("renv", lib="/usr/lib/R/library", repos="https://cloud.r-project.org")'
# Fix UV cache permissions for non-root user support (uid 1000, etc.)
# The uv tool install ansible command populates the UV cache with root-owned files
RUN chmod -R a+rw /tmp/windmill/cache/uv && \
+4
View File
@@ -51,6 +51,10 @@ RUN /usr/bin/java -jar /usr/bin/coursier about
# Ruby
RUN apt-get install -y ruby ruby-bundler
# R
RUN apt-get install -y r-base-dev \
&& Rscript -e 'install.packages("renv", lib="/usr/lib/R/library", repos="https://cloud.r-project.org")'
# iptables
RUN apt-get install -y iptables
+122 -55
View File
@@ -27,6 +27,16 @@
extensions = [ "rust-src" "rust-analyzer" "rustfmt" ];
};
patchedClang = pkgs.llvmPackages_18.clang.overrideAttrs (oldAttrs: {
postFixup = ''
# Copy the original postFixup logic but skip add-hardening.sh
${oldAttrs.postFixup or ""}
# Remove the line that substitutes add-hardening.sh
sed -i 's/.*source.*add-hardening\.sh.*//' $out/bin/clang
'';
});
# ---------------------------------------------------------------
# Native C/C++ dependencies (required to compile the backend)
# ---------------------------------------------------------------
@@ -72,14 +82,16 @@
version = "130.0.7";
target = stdenv.hostPlatform.rust.rustcTarget;
sha256 = {
x86_64-linux = "sha256-pkdsuU6bAkcIHEZUJOt5PXdzK424CEgTLXjLtQ80t10=";
x86_64-linux =
"sha256-pkdsuU6bAkcIHEZUJOt5PXdzK424CEgTLXjLtQ80t10=";
aarch64-linux = lib.fakeHash;
x86_64-darwin = lib.fakeHash;
aarch64-darwin = lib.fakeHash;
}.${system};
in pkgs.fetchurl {
name = "librusty_v8-${version}";
url = "https://github.com/denoland/rusty_v8/releases/download/v${version}/librusty_v8_release_${target}.a.gz";
url =
"https://github.com/denoland/rusty_v8/releases/download/v${version}/librusty_v8_release_${target}.a.gz";
inherit sha256;
};
@@ -87,15 +99,28 @@
# pkg-config search path for native libraries
# ---------------------------------------------------------------
pkgConfigPath = lib.makeSearchPath "lib/pkgconfig"
(with pkgs; [ openssl.dev libxml2.dev xmlsec.dev libxslt.dev cyrus_sasl.dev krb5.dev ]);
pkgConfigPath = lib.makeSearchPath "lib/pkgconfig" (with pkgs; [
openssl.dev
libxml2.dev
xmlsec.dev
libxslt.dev
cyrus_sasl.dev
krb5.dev
]);
# ---------------------------------------------------------------
# RPATH — embed Nix store library paths into compiled binaries
# ---------------------------------------------------------------
rpathLibs = lib.makeLibraryPath (with pkgs; [
openssl libffi cyrus_sasl krb5 libxml2 xmlsec libxslt stdenv.cc.cc.lib
openssl
libffi
cyrus_sasl
krb5
libxml2
xmlsec
libxslt
stdenv.cc.cc.lib
]);
# ---------------------------------------------------------------
@@ -113,11 +138,17 @@
(builtins.readFile "${stdenv.cc}/nix-support/libcxx-cxxflags")
"-idirafter ${pkgs.libiconv}/include"
] ++ lib.optionals stdenv.cc.isClang [
"-idirafter ${stdenv.cc.cc}/lib/clang/${lib.getVersion stdenv.cc.cc}/include"
"-idirafter ${stdenv.cc.cc}/lib/clang/${
lib.getVersion stdenv.cc.cc
}/include"
] ++ lib.optionals stdenv.cc.isGNU [
"-isystem ${stdenv.cc.cc}/include/c++/${lib.getVersion stdenv.cc.cc}"
"-isystem ${stdenv.cc.cc}/include/c++/${lib.getVersion stdenv.cc.cc}/${stdenv.hostPlatform.config}"
"-idirafter ${stdenv.cc.cc}/lib/gcc/${stdenv.hostPlatform.config}/${lib.getVersion stdenv.cc.cc}/include"
"-isystem ${stdenv.cc.cc}/include/c++/${
lib.getVersion stdenv.cc.cc
}/${stdenv.hostPlatform.config}"
"-idirafter ${stdenv.cc.cc}/lib/gcc/${stdenv.hostPlatform.config}/${
lib.getVersion stdenv.cc.cc
}/include"
]);
# ---------------------------------------------------------------
@@ -131,12 +162,16 @@
BINDGEN_EXTRA_CLANG_ARGS = bindgenClangArgs;
# Force clang 18 as cargo linker (stdenv may bring a newer clang that causes SIGSEGV with mold)
CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER = "${pkgs.llvmPackages_18.clang}/bin/clang";
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER = "${pkgs.llvmPackages_18.clang}/bin/clang";
CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER =
"${pkgs.llvmPackages_18.clang}/bin/clang";
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER =
"${pkgs.llvmPackages_18.clang}/bin/clang";
# Embed rpath so binaries find Nix store .so files at runtime
CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS = "-C link-arg=-fuse-ld=mold -C link-arg=-Wl,-rpath,${rpathLibs}";
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS = "-C link-arg=-fuse-ld=mold -C link-arg=-Wl,-rpath,${rpathLibs}";
CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS =
"-C link-arg=-fuse-ld=mold -C link-arg=-Wl,-rpath,${rpathLibs}";
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS =
"-C link-arg=-fuse-ld=mold -C link-arg=-Wl,-rpath,${rpathLibs}";
CARGO_HOST_RUSTFLAGS = "-C link-arg=-Wl,-rpath,${rpathLibs}";
# https://github.com/NixOS/nixpkgs/issues/370494 — jemalloc build fix
@@ -197,6 +232,10 @@
hash = "sha256-8E0WtDFc7RcqmftDigMyy1xXUkjgL4X4kpf7h1GdE48=";
};
rWithPackages = pkgs.rWrapper.override {
packages = with pkgs.rPackages; [ renv ];
};
extraRuntimes = with pkgs; [
dotnet-sdk_9
php
@@ -222,6 +261,7 @@
ANSIBLE_PLAYBOOK_PATH = "${pkgs.ansible}/bin/ansible-playbook";
ANSIBLE_GALAXY_PATH = "${pkgs.ansible}/bin/ansible-galaxy";
CARGO_SWEEP_PATH = "${pkgs.cargo-sweep}/bin/cargo-sweep";
RSCRIPT_PATH = "${rWithPackages}/bin/Rscript";
};
# ---------------------------------------------------------------
@@ -251,13 +291,23 @@
(pkgs.writeScriptBin "wm" ''
cd ./frontend
npm install
npm run ${if stdenv.isDarwin then "generate-backend-client-mac" else "generate-backend-client"}
npm run ${
if stdenv.isDarwin then
"generate-backend-client-mac"
else
"generate-backend-client"
}
npm run dev "$@"
'')
(pkgs.writeScriptBin "wm-build" ''
cd ./frontend
npm install
npm run ${if stdenv.isDarwin then "generate-backend-client-mac" else "generate-backend-client"}
npm run ${
if stdenv.isDarwin then
"generate-backend-client-mac"
else
"generate-backend-client"
}
npm run build "$@"
'')
(pkgs.writeScriptBin "wm-migrate" ''
@@ -322,22 +372,20 @@
# Shared inputs and settings for default + full shells
# ---------------------------------------------------------------
coreBuildInputs = nativeBuildDeps ++ commonRuntimes ++ [
rustStable
openapi-generator-cli
] ++ (with pkgs; [
nodejs
git
sqlx-cli
cargo-watch
jq
gnused
coreBuildInputs = nativeBuildDeps ++ commonRuntimes
++ [ rustStable openapi-generator-cli ] ++ (with pkgs; [
nodejs
git
sqlx-cli
cargo-watch
jq
gnused
# CLI tools (for AI agents and dev workflow)
gh
asciinema
mermaid-cli
]);
# CLI tools (for AI agents and dev workflow)
gh
asciinema
mermaid-cli
]);
# Playwright: use Nix-provided browsers (version-matched to playwright-driver)
# Mermaid/Puppeteer: point at Nix chromium (Puppeteer respects this env var)
@@ -380,16 +428,26 @@
sandboxEnv = pkgs.buildEnv {
name = "windmill-sandbox";
paths = coreBuildInputs ++ helperScriptsBase
++ [ playwrightWrapper sandboxEnvScript pkgConfigWrapper pkgs.chromium ];
paths = coreBuildInputs ++ helperScriptsBase ++ [
playwrightWrapper
sandboxEnvScript
pkgConfigWrapper
pkgs.chromium
];
};
sandboxFullEnv = pkgs.buildEnv {
name = "windmill-sandbox-full";
paths = coreBuildInputs ++ extraRuntimes
++ helperScriptsBase ++ helperScriptsFull
++ [ playwrightWrapper sandboxEnvScript pkgConfigWrapper pkgs.chromium
pkgs.cargo-sweep pkgs.xcaddy pkgs.nsjail ];
paths = coreBuildInputs ++ extraRuntimes ++ helperScriptsBase
++ helperScriptsFull ++ [
playwrightWrapper
sandboxEnvScript
pkgConfigWrapper
pkgs.chromium
pkgs.cargo-sweep
pkgs.xcaddy
pkgs.nsjail
];
};
in {
@@ -412,8 +470,8 @@
shellHook = devShellHook;
buildInputs = coreBuildInputs;
packages = helperScriptsBase ++ [ playwrightWrapper ];
});
packages = helperScriptsBase ++ [ playwrightWrapper ];
});
# =============================================================
# full — all language runtimes, k8s tooling, specialized scripts
@@ -428,27 +486,28 @@
pyright
openapi-python-client
# LSP / editor
svelte-language-server
taplo
# LSP / editor
svelte-language-server
taplo
# Extra dev tools
cargo-sweep
# Extra dev tools
cargo-sweep
# Kubernetes
minikube
kubectl
kubernetes-helm
conntrack-tools
cri-tools
# Kubernetes
minikube
kubectl
kubernetes-helm
conntrack-tools
cri-tools
# Extra
xcaddy
nsjail
]);
# Extra
xcaddy
nsjail
]);
packages = helperScriptsBase ++ helperScriptsFull ++ [ playwrightWrapper ];
});
packages = helperScriptsBase ++ helperScriptsFull
++ [ playwrightWrapper ];
});
# =============================================================
# wasm — WASM target compilation (nightly Rust)
@@ -458,15 +517,23 @@
devShells.wasm = pkgs.mkShell (buildEnvVars // {
hardeningDisable = [ "all" ];
# Explicitly set paths for headers and linker
# DO NOT REMOVE - if absent, breaks wasm builds on NixOS.
shellHook = ''
export CC=${patchedClang}/bin/clang
'';
buildInputs = nativeBuildDeps ++ (with pkgs; [
(rust-bin.nightly.latest.default.override {
extensions = [ "rust-src" "rust-analyzer" ];
targets = [ "wasm32-unknown-unknown" "wasm32-unknown-emscripten" ];
targets =
[ "wasm32-unknown-unknown" "wasm32-unknown-emscripten" ];
})
wasm-pack
deno
emscripten
nushell
nodejs
glibc_multi
]);
});
+10 -50
View File
@@ -1,12 +1,12 @@
{
"name": "windmill-components",
"version": "1.670.0",
"version": "1.671.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "windmill-components",
"version": "1.670.0",
"version": "1.671.0",
"hasInstallScript": true,
"license": "AGPL-3.0",
"dependencies": {
@@ -83,6 +83,7 @@
"windmill-parser-wasm-nu": "1.510.1",
"windmill-parser-wasm-php": "1.647.1",
"windmill-parser-wasm-py": "1.657.2",
"windmill-parser-wasm-r": "^1.668.1",
"windmill-parser-wasm-regex": "1.653.0",
"windmill-parser-wasm-ruby": "1.526.1",
"windmill-parser-wasm-rust": "1.647.1",
@@ -843,7 +844,6 @@
"version": "1.9.0",
"resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.9.0.tgz",
"integrity": "sha512-0DQ98G9ZQZOxfUcQn1waV2yS8aWdZ6kJMbYCJB3oUBecjWYO1fqJ+a1DRfPF3O5JEkwqwP1A9QEN/9mYm2Yd0w==",
"dev": true,
"license": "MIT",
"optional": true,
"dependencies": {
@@ -855,7 +855,6 @@
"version": "1.9.0",
"resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.9.0.tgz",
"integrity": "sha512-QN75eB0IH2ywSpRpNddCRfQIhmJYBCJ1x5Lb3IscKAL8bMnVAKnRg8dCoXbHzVLLH7P38N2Z3mtulB7W0J0FKw==",
"dev": true,
"license": "MIT",
"optional": true,
"dependencies": {
@@ -866,7 +865,6 @@
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.0.tgz",
"integrity": "sha512-N10dEJNSsUx41Z6pZsXU8FjPjpBEplgH24sfkmITrBED1/U2Esum9F3lfLrMjKHHjmi557zQn7kR9R+XWXu5Rg==",
"dev": true,
"license": "MIT",
"optional": true,
"dependencies": {
@@ -1356,7 +1354,6 @@
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.1.tgz",
"integrity": "sha512-p64ah1M1ld8xjWv3qbvFwHiFVWrq1yFvV4f7w+mzaqiR4IlSgkqhcRdHwsGgomwzBH51sRY4NEowLxnaBjcW/A==",
"dev": true,
"license": "MIT",
"optional": true,
"dependencies": {
@@ -1513,7 +1510,6 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -1530,7 +1526,6 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -1547,7 +1542,6 @@
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -1564,7 +1558,6 @@
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -1581,7 +1574,6 @@
"cpu": [
"arm"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -1598,7 +1590,6 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -1615,7 +1606,6 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -1632,7 +1622,6 @@
"cpu": [
"ppc64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -1649,7 +1638,6 @@
"cpu": [
"s390x"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -1666,7 +1654,6 @@
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -1683,7 +1670,6 @@
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -1700,7 +1686,6 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -1717,7 +1702,6 @@
"cpu": [
"wasm32"
],
"dev": true,
"license": "MIT",
"optional": true,
"dependencies": {
@@ -1734,7 +1718,6 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -1751,7 +1734,6 @@
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
@@ -2057,7 +2039,6 @@
"version": "0.10.1",
"resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz",
"integrity": "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==",
"dev": true,
"license": "MIT",
"optional": true,
"dependencies": {
@@ -6866,7 +6847,7 @@
"version": "1.21.7",
"resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.7.tgz",
"integrity": "sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==",
"dev": true,
"devOptional": true,
"license": "MIT",
"bin": {
"jiti": "bin/jiti.js"
@@ -7365,7 +7346,6 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -7386,7 +7366,6 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -7407,7 +7386,6 @@
"cpu": [
"x64"
],
"dev": true,
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -7428,7 +7406,6 @@
"cpu": [
"x64"
],
"dev": true,
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -7449,7 +7426,6 @@
"cpu": [
"arm"
],
"dev": true,
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -7470,7 +7446,6 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -7491,7 +7466,6 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -7512,7 +7486,6 @@
"cpu": [
"x64"
],
"dev": true,
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -7533,7 +7506,6 @@
"cpu": [
"x64"
],
"dev": true,
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -7554,7 +7526,6 @@
"cpu": [
"arm64"
],
"dev": true,
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -7575,7 +7546,6 @@
"cpu": [
"x64"
],
"dev": true,
"license": "MPL-2.0",
"optional": true,
"os": [
@@ -12151,21 +12121,6 @@
}
}
},
"node_modules/svelte-check/node_modules/picomatch": {
"version": "4.0.4",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz",
"integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==",
"dev": true,
"license": "MIT",
"optional": true,
"peer": true,
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/sponsors/jonschlinkert"
}
},
"node_modules/svelte-eslint-parser": {
"version": "0.43.0",
"resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-0.43.0.tgz",
@@ -12896,7 +12851,7 @@
"version": "5.9.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
"integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
"dev": true,
"devOptional": true,
"license": "Apache-2.0",
"bin": {
"tsc": "bin/tsc",
@@ -13685,6 +13640,11 @@
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.657.2.tgz",
"integrity": "sha512-3CN2rziafgCWcZri812+CkzuaE3P3/7dXmV9lSDpK9ma6Esd4zkHRXUFSyRzQE/R7Fxj5mSmSNX6xTff8eX5mw=="
},
"node_modules/windmill-parser-wasm-r": {
"version": "1.668.1",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-r/-/windmill-parser-wasm-r-1.668.1.tgz",
"integrity": "sha512-5YNeUibxpNBvYrxCgQcz1PxGhTFx2CyEpg2udtIhq7bx0d4gF/KDZVupMeQmAObmrEtTSFGUWNRJ4zXSWNrSpQ=="
},
"node_modules/windmill-parser-wasm-regex": {
"version": "1.653.0",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.653.0.tgz",
+2 -1
View File
@@ -1,6 +1,6 @@
{
"name": "windmill-components",
"version": "1.670.0",
"version": "1.671.0",
"scripts": {
"dev": "vite dev",
"build": "vite build",
@@ -156,6 +156,7 @@
"windmill-parser-wasm-nu": "1.510.1",
"windmill-parser-wasm-php": "1.647.1",
"windmill-parser-wasm-py": "1.657.2",
"windmill-parser-wasm-r": "1.668.1",
"windmill-parser-wasm-regex": "1.653.0",
"windmill-parser-wasm-ruby": "1.526.1",
"windmill-parser-wasm-rust": "1.647.1",
+11 -2
View File
@@ -156,6 +156,7 @@
'nu',
'java',
'ruby',
'rlang',
'postgresql',
'mysql',
'bigquery',
@@ -182,7 +183,8 @@
'csharp',
'nu',
'java',
'ruby'
'ruby',
'rlang'
// for related places search: ADD_NEW_LANG
].includes(lang ?? '')
)
@@ -202,7 +204,8 @@
'csharp',
'nu',
'java',
'ruby'
'ruby',
'rlang'
// for related places search: ADD_NEW_LANG
].includes(lang ?? '')
)
@@ -515,6 +518,8 @@
// for related places search: ADD_NEW_LANG
} else if (lang == 'ruby') {
editor.insertAtCursor(`ENV['${name}']`)
} else if (lang == 'rlang') {
editor.insertAtCursor(`Sys.getenv("${name}")`)
} else if (
['postgresql', 'mysql', 'bigquery', 'mssql', 'oracledb', 'snowflake', 'duckdb'].includes(
lang ?? ''
@@ -583,6 +588,8 @@ string ${windmillPathToCamelCaseName(path)} = await client.GetStringAsync(uri);
editor.insertAtBeginning("require 'windmill/mini'\n")
}
editor.insertAtCursor(`get_variable("${path}")`)
} else if (lang == 'rlang') {
editor.insertAtCursor(`get_variable("${path}")`)
}
sendUserToast(`${name} inserted at cursor`)
}}
@@ -662,6 +669,8 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS
editor.insertAtBeginning("require 'windmill/mini'\n")
}
editor.insertAtCursor(`get_resource("${path}")`)
} else if (lang == 'rlang') {
editor.insertAtCursor(`get_resource("${path}")`)
} else if (lang == 'duckdb') {
let t = { postgresql: 'postgres', mysql: 'mysql', bigquery: 'bigquery' }[resType]
if (!t) {
@@ -3,7 +3,7 @@
import { type Job, JobService } from '$lib/gen'
import { workspaceStore } from '$lib/stores'
import { sendUserToast } from '$lib/toast'
import { X } from 'lucide-svelte'
import { ExternalLink, X } from 'lucide-svelte'
import DisplayResult from './DisplayResult.svelte'
import Tooltip from './Tooltip.svelte'
import { Button } from './common'
@@ -23,6 +23,7 @@
let default_payload: object = $state({})
let description: any = $state(undefined)
let hide_cancel = $state(false)
let approvalPageUrl: string | undefined = $state(undefined)
let defaultValues = $state({})
@@ -47,6 +48,8 @@
defaultValues = JSON.parse(JSON.stringify(args))
default_payload = args
approvalPageUrl = job_result?.['approvalPage']
actionTaken = false
hide_cancel = job?.raw_flow?.modules?.[approvalStep]?.suspend?.hide_cancel ?? false
schema = mergeSchema(
job?.raw_flow?.modules?.[approvalStep]?.suspend?.resume_form?.schema ?? {},
@@ -55,6 +58,7 @@
}
let loading = $state(false)
let actionTaken = $state(false)
async function continu(approve: boolean) {
loading = true
try {
@@ -66,6 +70,7 @@
approved: approve
}
})
actionTaken = true
} catch (e: any) {
sendUserToast(e?.body ?? e?.message ?? 'Failed', true)
} finally {
@@ -84,7 +89,7 @@
<div class="mt-2"></div>
{/if}
<div>
<div class={twMerge('flex gap-2', light ? 'flex-col' : 'flex-row ')}>
<div class={twMerge('flex gap-2 items-center', light ? 'flex-col' : 'flex-row ')}>
{#if !hide_cancel}
<div>
<Button
@@ -92,7 +97,7 @@
iconOnly
startIcon={{ icon: X }}
variant="default"
disabled={loading}
disabled={loading || actionTaken}
destructive
unifiedSize="md"
on:click={() => continu(false)}
@@ -100,12 +105,28 @@
</div>
{/if}
<div>
<Button variant="accent" onClick={() => continu(true)} disabled={loading} unifiedSize="md">
<Button
variant="accent"
onClick={() => continu(true)}
disabled={loading || actionTaken}
unifiedSize="md"
>
Resume
<Tooltip class="text-white">Resume or approve this suspended step</Tooltip>
</Button>
</div>
{#if approvalPageUrl}
<a
href={approvalPageUrl}
target="_blank"
rel="noreferrer"
class="text-accent flex items-center gap-1 whitespace-nowrap"
>
Approval page <ExternalLink size={12} />
</a>
{/if}
{#if job?.raw_flow?.modules?.[approvalStep]?.suspend?.resume_form?.schema}
<div
class={twMerge(
@@ -14,6 +14,7 @@
import yaml from 'svelte-highlight/languages/yaml'
import java from 'svelte-highlight/languages/java'
import ruby from 'svelte-highlight/languages/ruby'
import r from 'svelte-highlight/languages/r'
import type { Script } from '$lib/gen'
import { Button } from './common'
import { copyToClipboard } from '$lib/utils'
@@ -91,6 +92,8 @@
return java
case 'ruby':
return ruby
case 'rlang':
return r
case 'json':
return json
// for related places search: ADD_NEW_LANG
+9 -3
View File
@@ -19,7 +19,7 @@
min = 0,
max = 100,
initialValue = 0,
value = $bindable(typeof initialValue === 'string' ? parseInt(initialValue) : initialValue),
value = $bindable(),
disabled = false,
defaultValue = undefined,
format = (v) => `${v}`,
@@ -36,8 +36,14 @@
}
run(() => {
if (value === null) {
value = 0
if (value === null || value === undefined || Number.isNaN(value)) {
const fallback =
initialValue !== undefined
? typeof initialValue === 'string'
? parseInt(initialValue)
: initialValue
: (min ?? 0)
value = Number.isNaN(fallback) ? (min ?? 0) : fallback
}
})
@@ -1268,7 +1268,7 @@
} as ButtonType.Icon}
>
<span class="truncate">{label}</span>
{#if lang === 'ruby'}
{#if lang === 'rlang'}
<span class="text-primary !text-xs"> BETA </span>
{/if}
</Button>
@@ -25,6 +25,7 @@
import JavaIcon from '$lib/components/icons/JavaIcon.svelte'
import DuckDbIcon from '$lib/components/icons/DuckDbIcon.svelte'
import RubyIcon from '$lib/components/icons/RubyIcon.svelte'
import RIcon from '$lib/components/icons/RIcon.svelte'
import ClaudeIcon from '$lib/components/icons/ClaudeIcon.svelte'
interface Props {
@@ -72,6 +73,7 @@
nu: 'Nu',
java: 'Java',
ruby: 'Ruby',
rlang: 'R',
claudesandbox: 'Claude Sandbox'
// for related places search: ADD_NEW_LANG
}
@@ -107,6 +109,7 @@
nu: NuIcon,
java: JavaIcon,
ruby: RubyIcon,
rlang: RIcon,
duckdb: DuckDbIcon,
claudesandbox: TypeScriptIcon
// for related places search: ADD_NEW_LANG
@@ -0,0 +1,37 @@
<script lang="ts">
interface Props {
height?: number
width?: number
}
let { height = 24, width = 24 }: Props = $props()
</script>
<svg
xmlns="http://www.w3.org/2000/svg"
{width}
{height}
viewBox="0 0 724 561"
preserveAspectRatio="xMidYMid"
>
<defs>
<linearGradient id="r-grad-1" x1="0" x2="1" y1="0" y2="1">
<stop offset="0" stop-color="rgb(203,206,208)" />
<stop offset="1" stop-color="rgb(132,131,139)" />
</linearGradient>
<linearGradient id="r-grad-2" x1="0" x2="1" y1="0" y2="1">
<stop offset="0" stop-color="rgb(39,109,195)" />
<stop offset="1" stop-color="rgb(22,92,170)" />
</linearGradient>
</defs>
<path
d="M361.453,485.937 C162.329,485.937 0.906,377.828 0.906,244.469 C0.906,111.109 162.329,3.000 361.453,3.000 C560.578,3.000 722.000,111.109 722.000,244.469 C722.000,377.828 560.578,485.937 361.453,485.937 ZM416.641,97.406 C265.289,97.406 142.594,171.314 142.594,262.484 C142.594,353.654 265.289,427.562 416.641,427.562 C567.992,427.562 679.687,377.033 679.687,262.484 C679.687,147.971 567.992,97.406 416.641,97.406 Z"
fill="url(#r-grad-1)"
fill-rule="evenodd"
/>
<path
d="M550.000,377.000 C550.000,377.000 571.822,383.585 584.500,390.000 C588.899,392.226 596.510,396.668 602.000,402.500 C607.378,408.212 610.000,414.000 610.000,414.000 L696.000,559.000 L557.000,559.062 L492.000,437.000 C492.000,437.000 478.690,414.131 470.500,407.500 C463.668,401.969 460.755,400.000 454.000,400.000 C449.298,400.000 420.974,400.000 420.974,400.000 L421.000,558.974 L298.000,559.026 L298.000,152.938 L545.000,152.938 C545.000,152.938 657.500,154.967 657.500,262.000 C657.500,369.033 550.000,377.000 550.000,377.000 ZM496.500,241.024 L422.037,240.976 L422.000,310.026 L496.500,310.002 C496.500,310.002 531.000,309.895 531.000,274.877 C531.000,239.155 496.500,241.024 496.500,241.024 Z"
fill="url(#r-grad-2)"
fill-rule="evenodd"
/>
</svg>
@@ -58,6 +58,7 @@ export const defaultTags = [
'nu',
'java',
'ruby',
'rlang',
'duckdb'
// for related places search: ADD_NEW_LANG
]
+2
View File
@@ -99,6 +99,8 @@ export function extToLang(ext: string) {
return 'java'
case 'rb':
return 'ruby'
case 'r':
return 'r'
// for related places search: ADD_NEW_LANG
default:
return 'unknown'
+86
View File
@@ -42,6 +42,7 @@ import initCSharpParser, { parse_csharp } from 'windmill-parser-wasm-csharp'
import initNuParser, { parse_nu } from 'windmill-parser-wasm-nu'
import initJavaParser, { parse_java } from 'windmill-parser-wasm-java'
import initRubyParser, { parse_ruby } from 'windmill-parser-wasm-ruby'
import initRParser, { parse_r } from 'windmill-parser-wasm-r'
import wasmUrlTs from 'windmill-parser-wasm-ts/windmill_parser_wasm_bg.wasm?url'
import wasmUrlRegex from 'windmill-parser-wasm-regex/windmill_parser_wasm_bg.wasm?url'
@@ -54,6 +55,7 @@ import wasmUrlCSharp from 'windmill-parser-wasm-csharp/windmill_parser_wasm_bg.w
import wasmUrlNu from 'windmill-parser-wasm-nu/windmill_parser_wasm_bg.wasm?url'
import wasmUrlJava from 'windmill-parser-wasm-java/windmill_parser_wasm_bg.wasm?url'
import wasmUrlRuby from 'windmill-parser-wasm-ruby/windmill_parser_wasm_bg.wasm?url'
import wasmUrlR from 'windmill-parser-wasm-r/windmill_parser_wasm_bg.wasm?url'
import wasmUrlAsset from 'windmill-parser-wasm-asset/windmill_parser_wasm_bg.wasm?url'
import initWacParser, { parse_workflow_as_code } from 'windmill-parser-wasm-wac'
import wasmUrlWac from 'windmill-parser-wasm-wac/windmill_parser_wasm_bg.wasm?url'
@@ -101,6 +103,9 @@ async function initWasmJava() {
async function initWasmRuby() {
await initRubyParser(wasmUrlRuby)
}
async function initWasmR() {
await initRParser(wasmUrlR)
}
async function initWasmAsset() {
await initAssetParser(wasmUrlAsset)
}
@@ -211,6 +216,7 @@ function getCommentPrefix(language: SupportedLanguage | undefined): string | und
case 'powershell':
case 'ansible':
case 'ruby':
case 'rlang':
return '#'
case 'deno':
case 'bun':
@@ -418,6 +424,13 @@ export async function inferArgs(
} else if (language == 'ruby') {
await initWasmRuby()
inferedSchema = JSON.parse(parse_ruby(code))
} else if (language == 'rlang') {
try {
await initWasmR()
inferedSchema = JSON.parse(parse_r(code))
} catch {
inferedSchema = parseRSignatureFallback(code)
}
// for related places search: ADD_NEW_LANG
} else {
return null
@@ -550,3 +563,76 @@ export async function parseOutputs(
}
return outputs.error ? [] : outputs.outputs
}
/** JS fallback parser for R main() signatures when WASM parser is unavailable. */
function parseRSignatureFallback(code: string): MainArgSignature {
const result: MainArgSignature = {
type: 'Valid',
error: '',
star_args: false,
star_kwargs: false,
args: [],
has_preprocessor: null,
auto_kind: null
}
const mainMatch = code.match(/\bmain\s*(?:<-|=)\s*function\s*\(([^)]*)\)/)
if (!mainMatch) {
return result
}
const paramsStr = mainMatch[1].trim()
if (!paramsStr) return result
// Split params respecting nested parens
const params: string[] = []
let depth = 0
let current = ''
for (const ch of paramsStr) {
if ('([{'.includes(ch)) {
depth++
current += ch
} else if (')]}'.includes(ch)) {
depth--
current += ch
} else if (ch === ',' && depth === 0) {
params.push(current)
current = ''
} else {
current += ch
}
}
if (current.trim()) params.push(current)
for (const param of params) {
const trimmed = param.trim()
if (!trimmed) continue
const eqIndex = trimmed.indexOf('=')
if (eqIndex === -1) {
result.args.push({ name: trimmed, typ: 'unknown', has_default: false, default: undefined })
} else {
const name = trimmed.slice(0, eqIndex).trim()
const raw = trimmed.slice(eqIndex + 1).trim()
const parsed = parseRDefault(raw)
result.args.push({ name, typ: parsed.typ, has_default: true, default: parsed.value })
}
}
return result
}
function parseRDefault(raw: string): { value: unknown; typ: MainArgSignature['args'][0]['typ'] } {
if (raw === 'TRUE' || raw === 'true') return { value: true, typ: 'bool' }
if (raw === 'FALSE' || raw === 'false') return { value: false, typ: 'bool' }
if (raw === 'NULL') return { value: null, typ: 'unknown' }
if (/^-?\d+(\.\d+)?$/.test(raw)) {
const num = Number(raw)
if (Number.isInteger(num) && !raw.includes('.')) return { value: num, typ: 'int' }
return { value: num, typ: 'float' }
}
const strMatch = raw.match(/^"((?:[^"\\]|\\.)*)"$/) || raw.match(/^'((?:[^'\\]|\\.)*)'$/)
if (strMatch) return { value: strMatch[1], typ: { str: null } }
if (raw.startsWith('list(') || raw.startsWith('c(')) return { value: null, typ: { list: null } }
return { value: null, typ: 'unknown' }
}
+25
View File
@@ -1283,6 +1283,26 @@ def main(
return result
end
`
const R_INIT_CODE = `library(dplyr)
library(jsonlite)
main <- function(
x,
name = "default",
age = 25,
data = list(1, 2, 3),
flag = TRUE
) {
# Use Windmill helpers:
# var <- get_variable("f/my_var")
# res <- get_resource("f/my_resource")
df <- tibble(name = name, age = age, x = x)
result <- df %>% mutate(greeting = paste("Hello", name))
return(toJSON(result, auto_unbox = TRUE))
}
`
// for related places search: ADD_NEW_LANG
export const INITIAL_CODE = {
bun: {
@@ -1378,6 +1398,9 @@ export const INITIAL_CODE = {
ruby: {
script: RUBY_INIT_CODE
},
rlang: {
script: R_INIT_CODE
},
claudesandbox: {
script: CLAUDE_SANDBOX_INIT_CODE
},
@@ -1507,6 +1530,8 @@ export function initialCode(
return INITIAL_CODE.java.script
} else if (language == 'ruby') {
return INITIAL_CODE.ruby.script
} else if (language == 'rlang') {
return INITIAL_CODE.rlang.script
// for related places search: ADD_NEW_LANG
} else if (language == 'bun' || language == 'bunnative') {
if (subkind === 'claudesandbox') {
+5 -2
View File
@@ -61,6 +61,8 @@ export function scriptLangToEditorLang(
return 'nu'
} else if (lang == 'java') {
return 'java'
} else if (lang == 'rlang') {
return 'r'
// for related places search: ADD_NEW_LANG
} else if (lang == undefined) {
return 'typescript'
@@ -163,7 +165,8 @@ const scriptLanguagesArray: [SupportedLanguage | 'docker' | 'bunnative', string]
['nu', 'Nu'],
['java', 'Java'],
['duckdb', 'DuckDB'],
['ruby', 'Ruby']
['ruby', 'Ruby'],
['rlang', 'R']
// for related places search: ADD_NEW_LANG
]
export function processLangs(selected: string | undefined, langs: string[]): string[] {
@@ -173,7 +176,7 @@ export function processLangs(selected: string | undefined, langs: string[]): str
let ls = langs.filter((lang) => lang !== 'nativets')
//those languages are newer and may not be in the saved list
let nl = ['bunnative', 'rust', 'ansible', 'csharp', 'nu', 'java', 'duckdb', 'ruby']
let nl = ['bunnative', 'rust', 'ansible', 'csharp', 'nu', 'java', 'duckdb', 'ruby', 'rlang']
// for related places search: ADD_NEW_LANG
nl.forEach((lang) => {
if (!ls.includes(lang)) {
@@ -33,6 +33,7 @@
let default_payload: any = $state({})
let loading = $state(false)
let valid = $state(true)
let actionTaken: 'approved' | 'denied' | undefined = $state(undefined)
let pollInterval: number | undefined = undefined
let scheduleEditor: ScheduleEditor | undefined = $state(undefined)
@@ -81,6 +82,9 @@
id: page.params.job ?? ''
})) as Job
completed = job?.type === 'CompletedJob'
if (completed) {
pollInterval && clearInterval(pollInterval)
}
} catch {
// Job details are optional — page works with just approvalInfo
}
@@ -103,7 +107,7 @@
}
})
sendUserToast('Flow approved')
pollInterval && clearInterval(pollInterval)
actionTaken = 'approved'
loadData()
} catch (e: any) {
sendUserToast(e?.body ?? e?.message ?? 'Failed to approve', true)
@@ -125,7 +129,7 @@
}
})
sendUserToast('Flow denied!')
pollInterval && clearInterval(pollInterval)
actionTaken = 'denied'
loadData()
} catch (e: any) {
sendUserToast(e?.body ?? e?.message ?? 'Failed to cancel', true)
@@ -259,6 +263,12 @@
<Alert type="info" title="Flow completed">
The flow is not running anymore. You cannot cancel or resume it.
</Alert>
{:else if actionTaken}
<Alert type="info" title={actionTaken === 'approved' ? 'Flow approved' : 'Flow denied'}>
{actionTaken === 'approved'
? 'You have approved this flow. Waiting for it to complete...'
: 'You have denied this flow. Waiting for it to complete...'}
</Alert>
{/if}
{#if approvalInfo.description != undefined}
@@ -279,31 +289,20 @@
{/if}
{/if}
{#if !completed && approvalInfo.can_approve}
{#if !completed && !actionTaken && approvalInfo.can_approve}
<div class="w-max-md flex flex-row gap-x-4 gap-y-4 justify-between w-full flex-wrap">
{#if approvalInfo.hide_cancel !== true}
<Button
variant="accent"
destructive
onclick={cancel}
size="lg"
disabled={completed || loading}
>
<Button variant="accent" destructive onclick={cancel} size="lg" disabled={loading}>
Deny
</Button>
{:else}
<div></div>
{/if}
<Button
variant="accent"
onclick={resume}
size="lg"
disabled={completed || !valid || loading}
>
<Button variant="accent" onclick={resume} size="lg" disabled={!valid || loading}>
Approve
</Button>
</div>
{:else if !completed && !approvalInfo.can_approve}
{:else if !completed && !actionTaken && !approvalInfo.can_approve}
{#if approvalInfo.user_auth_required && !$userStore}
<Login {rd} />
{:else}
+1 -1
View File
@@ -4,7 +4,7 @@ verify_ssl = true
name = "pypi"
[packages]
wmill = ">=1.670.0"
wmill = ">=1.671.0"
sendgrid = "*"
mysql-connector-python = "*"
pymongo = "*"
+2 -1
View File
@@ -1,7 +1,7 @@
openapi: '3.0.3'
info:
version: 1.670.0
version: 1.671.0
title: OpenFlow Spec
contact:
name: Ruben Fiszel
@@ -661,6 +661,7 @@ components:
- nu
- java
- ruby
- rlang
- duckdb
# for related places search: ADD_NEW_LANG
path:
@@ -12,7 +12,7 @@
RootModule = 'WindmillClient.psm1'
# Version number of this module.
ModuleVersion = '1.670.0'
ModuleVersion = '1.671.0'
# Supported PSEditions
# CompatiblePSEditions = @()
+1 -1
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "wmill"
version = "1.670.0"
version = "1.671.0"
description = "A client library for accessing Windmill server wrapping the Windmill client API"
license = "Apache-2.0"
homepage = "https://windmill.dev"
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+87
View File
@@ -1032,6 +1032,93 @@ result: S3Object = wmill.write_s3_file(
```
# R
## Structure
Define a `main` function using `<-` or `=` assignment. Parameters become the script inputs:
```r
library(dplyr)
library(jsonlite)
main <- function(x, name = "default", flag = TRUE) {
df <- tibble(x = x, name = name)
result <- df %>% mutate(greeting = paste("Hello", name))
return(toJSON(result, auto_unbox = TRUE))
}
```
**Important:**
- The `main` function is required
- Use `library()` to load packages — they are resolved and installed automatically
- `jsonlite` is always available (used internally for argument parsing)
- Return values must be JSON-serializable
## Parameters
R types map to Windmill types:
- `numeric` → float/int
- `character` → string
- `logical` → bool (use `TRUE`/`FALSE`)
- `list` → object/dict
- `NULL` → null
Default values are inferred from the function signature:
```r
main <- function(
name, # required string
count = 10, # optional int, default 10
verbose = FALSE # optional bool, default FALSE
) {
# ...
}
```
## Resources and Variables
Use the built-in Windmill helpers (no import needed):
```r
main <- function() {
# Get a variable
api_key <- get_variable("f/my_folder/api_key")
# Get a resource (returns a list)
db <- get_resource("f/my_folder/postgres_config")
host <- db$host
port <- db$port
return(list(host = host, port = port))
}
```
## Output
Return any JSON-serializable value from `main`. The return value becomes the step result:
```r
main <- function(x) {
# Return a scalar
return(x + 1)
# Or a list (becomes JSON object)
return(list(result = x + 1, status = "ok"))
}
```
## Annotations
Control execution behavior with comment annotations:
```r
#renv_verbose = true # Show verbose renv output during resolution
#renv_install_verbose = true # Show verbose output during package installation
#sandbox = true # Run in nsjail sandbox (requires nsjail)
```
# Rust
## Structure
File diff suppressed because one or more lines are too long
@@ -0,0 +1,100 @@
---
name: write-script-rlang
description: MUST use when writing R scripts.
---
## CLI Commands
Place scripts in a folder. After writing, tell the user they can run:
- `wmill script generate-metadata` - Generate .script.yaml and .lock files
- `wmill sync push` - Deploy to Windmill
Do NOT run these commands yourself. Instead, inform the user that they should run them.
Use `wmill resource-type list --schema` to discover available resource types.
# R
## Structure
Define a `main` function using `<-` or `=` assignment. Parameters become the script inputs:
```r
library(dplyr)
library(jsonlite)
main <- function(x, name = "default", flag = TRUE) {
df <- tibble(x = x, name = name)
result <- df %>% mutate(greeting = paste("Hello", name))
return(toJSON(result, auto_unbox = TRUE))
}
```
**Important:**
- The `main` function is required
- Use `library()` to load packages — they are resolved and installed automatically
- `jsonlite` is always available (used internally for argument parsing)
- Return values must be JSON-serializable
## Parameters
R types map to Windmill types:
- `numeric` → float/int
- `character` → string
- `logical` → bool (use `TRUE`/`FALSE`)
- `list` → object/dict
- `NULL` → null
Default values are inferred from the function signature:
```r
main <- function(
name, # required string
count = 10, # optional int, default 10
verbose = FALSE # optional bool, default FALSE
) {
# ...
}
```
## Resources and Variables
Use the built-in Windmill helpers (no import needed):
```r
main <- function() {
# Get a variable
api_key <- get_variable("f/my_folder/api_key")
# Get a resource (returns a list)
db <- get_resource("f/my_folder/postgres_config")
host <- db$host
port <- db$port
return(list(host = host, port = port))
}
```
## Output
Return any JSON-serializable value from `main`. The return value becomes the step result:
```r
main <- function(x) {
# Return a scalar
return(x + 1)
# Or a list (becomes JSON object)
return(list(result = x + 1, status = "ok"))
}
```
## Annotations
Control execution behavior with comment annotations:
```r
#renv_verbose = true # Show verbose renv output during resolution
#renv_install_verbose = true # Show verbose output during package installation
#sandbox = true # Run in nsjail sandbox (requires nsjail)
```
+85
View File
@@ -0,0 +1,85 @@
# R
## Structure
Define a `main` function using `<-` or `=` assignment. Parameters become the script inputs:
```r
library(dplyr)
library(jsonlite)
main <- function(x, name = "default", flag = TRUE) {
df <- tibble(x = x, name = name)
result <- df %>% mutate(greeting = paste("Hello", name))
return(toJSON(result, auto_unbox = TRUE))
}
```
**Important:**
- The `main` function is required
- Use `library()` to load packages — they are resolved and installed automatically
- `jsonlite` is always available (used internally for argument parsing)
- Return values must be JSON-serializable
## Parameters
R types map to Windmill types:
- `numeric` → float/int
- `character` → string
- `logical` → bool (use `TRUE`/`FALSE`)
- `list` → object/dict
- `NULL` → null
Default values are inferred from the function signature:
```r
main <- function(
name, # required string
count = 10, # optional int, default 10
verbose = FALSE # optional bool, default FALSE
) {
# ...
}
```
## Resources and Variables
Use the built-in Windmill helpers (no import needed):
```r
main <- function() {
# Get a variable
api_key <- get_variable("f/my_folder/api_key")
# Get a resource (returns a list)
db <- get_resource("f/my_folder/postgres_config")
host <- db$host
port <- db$port
return(list(host = host, port = port))
}
```
## Output
Return any JSON-serializable value from `main`. The return value becomes the step result:
```r
main <- function(x) {
# Return a scalar
return(x + 1)
# Or a list (becomes JSON object)
return(list(result = x + 1, status = "ok"))
}
```
## Annotations
Control execution behavior with comment annotations:
```r
#renv_verbose = true # Show verbose renv output during resolution
#renv_install_verbose = true # Show verbose output during package installation
#sandbox = true # Run in nsjail sandbox (requires nsjail)
```
+5
View File
@@ -177,6 +177,11 @@ LANGUAGE_METADATA = {
'description': 'MUST use when writing Java scripts.',
'use_cases': 'Java automation, enterprise integrations'
},
'rlang': {
'name': 'R',
'description': 'MUST use when writing R scripts.',
'use_cases': 'R statistical computing, data analysis, visualization'
},
}
# Languages that use TypeScript SDK
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@windmill/windmill",
"version": "1.670.0",
"version": "1.671.0",
"exports": "./src/index.ts",
"publish": {
"exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"]
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "windmill-client",
"description": "Windmill SDK client for browsers and Node.js",
"version": "1.670.0",
"version": "1.671.0",
"author": "Ruben Fiszel",
"license": "Apache 2.0",
"sideEffects": false,
+1 -1
View File
@@ -1 +1 @@
1.670.0
1.671.0