chore: merge main into asset graph view

This commit is contained in:
Ruben Fiszel
2026-05-20 15:16:31 +00:00
208 changed files with 10602 additions and 2456 deletions
+18 -1
View File
@@ -16,6 +16,23 @@ command="$(echo "$input" | jq -r '.tool_input.command // empty')"
if [[ "$command" =~ ^git\ (push|reset|revert|checkout|merge|rebase|commit|add) ]]; then
branch="$(git rev-parse --abbrev-ref HEAD 2>/dev/null || true)"
if [[ "$branch" == "main" ]]; then
echo "BLOCK: You are on the main branch. Create or switch to a feature branch first."
echo "BLOCK: You are on the main branch. Create or switch to a feature branch first." >&2
exit 2
fi
fi
# Block force-push targeting main from any branch.
if [[ "$command" =~ ^git[[:space:]]+push([[:space:]]|$) ]]; then
has_force=false
if [[ "$command" =~ (--force([[:space:]]|=|$)|--force-with-lease|[[:space:]]-f([[:space:]]|$)) ]]; then
has_force=true
fi
# `+ref` refspec syntax is also a force push.
if [[ "$command" =~ [[:space:]]\+[A-Za-z] ]]; then
has_force=true
fi
if $has_force && [[ "$command" =~ (^|[[:space:]:])\+?main([[:space:]]|$) ]]; then
echo "BLOCK: Force-push to main is not allowed via Claude. Run it yourself if you really mean to." >&2
exit 2
fi
fi
+26 -2
View File
@@ -44,7 +44,25 @@
"Bash(git merge:*)",
"Bash(git rebase:*)",
"Bash(git add:*)",
"Bash(git commit:*)"
"Bash(git commit:*)",
"Read(/tmp/**)",
"Write(/tmp/**)",
"Edit(/tmp/**)",
"Bash(rm:/tmp/*)",
"Bash(rm:/tmp/**)",
"Bash(rmdir:/tmp/*)",
"Bash(mkdir:/tmp/*)",
"Bash(mkdir:/tmp/**)",
"Bash(cp:/tmp/*)",
"Bash(cp:/tmp/**)",
"Bash(mv:/tmp/*)",
"Bash(mv:/tmp/**)",
"Bash(touch:/tmp/*)",
"Bash(touch:/tmp/**)",
"Bash(chmod:/tmp/*)",
"Bash(chmod:/tmp/**)",
"Bash(tar * /tmp/*)",
"Bash(unzip * /tmp/*)"
],
"deny": [
"Read(.env)",
@@ -72,7 +90,13 @@
"Bash(chown:*)",
"Bash(truncate:*)",
"Bash(shred:*)",
"Bash(unlink:*)"
"Bash(unlink:*)",
"mcp__claude_ai_Stripe",
"mcp__claude_ai_Gmail",
"mcp__claude_ai_Google_Calendar",
"mcp__claude_ai_Google_Drive",
"mcp__claude_ai_Slack",
"mcp__claude_ai_Linear"
]
},
"enableAllProjectMcpServers": true,
+43
View File
@@ -100,10 +100,53 @@ profiles:
integrations:
github:
autoRemoveOnMerge: true
linkedRepos:
- repo: windmill-labs/windmill-ee-private
alias: ee-private
dir: ../windmill-ee-private__worktrees
linear:
enabled: true
autoCreateWorktrees: true
watchTeams: [WIN,GIT]
oneshot:
systemPrompt: |
You are running in webmux ONESHOT mode.
# No interactive user
There is NO interactive user — nobody is watching the chat or will respond
to questions, approvals, or status checks. Any message asking the user to
review, approve, confirm, take a look, or "let you know" is wasted output:
it will not be answered.
# Your job
Take the task to its real conclusion without pausing:
1. Make the change.
2. Validate it (run the relevant tests, typecheck, build, or quick
manual check).
3. Commit.
4. Push.
5. Open a pull request.
Only then are you done.
# Decisions
When something is ambiguous, pick the most reasonable default and proceed.
When you would normally ask "should I X or Y?", just pick one and continue
— note the choice in the PR description if it matters.
# PR readiness
Default to opening the PR as a draft. If you are highly confident in the
change — the scope is small and well-understood, validation passed
cleanly, and you would not change anything if a reviewer pushed back —
open the PR as ready-for-review directly (omit `--draft` when invoking
`gh pr create`, or call `gh pr ready <number>` after creation). Err on
the side of draft when validation was partial, the change touches
public APIs or shared infrastructure, or you made a non-obvious judgment
call.
# Ending your turn
Never end your turn with a question, a suggestion to "take a look", or a
request for approval. Stop only when the PR is open, or when you hit a
technical error you cannot recover from yourself (in which case clearly
state the blocker).
@@ -46,11 +46,11 @@
]
},
"nullable": [
false,
false,
false,
false,
false,
true,
true,
true,
true,
true,
true,
true
]
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO variable\n (workspace_id, path, value, is_secret, description, account, is_oauth, expires_at, labels, edited_by)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar",
"Bool",
"Varchar",
"Int4",
"Bool",
"Timestamptz",
"TextArray",
"Varchar"
]
},
"nullable": []
},
"hash": "295a88070e1762255cdd7680ba2e7bb2a2ffd66a7c432dd318e73e0f81ea9622"
}
@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO draft\n (workspace_id, path, value, typ)\n VALUES ($1, $2, $3::text::json, $4)\n ON CONFLICT (workspace_id, path, typ)\n DO UPDATE SET value = EXCLUDED.value, created_at = now()",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Text",
{
"Custom": {
"name": "draft_type",
"kind": {
"Enum": [
"script",
"flow",
"app"
]
}
}
}
]
},
"nullable": []
},
"hash": "5104cf045dc9b7b82d0028af11cfb5c2f6fd58e518085caf8e8d189951f7c4d8"
}
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE variable SET labels = $1, edited_at = now(), edited_by = $4 WHERE path = $2 AND workspace_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"TextArray",
"Text",
"Text",
"Varchar"
]
},
"nullable": []
},
"hash": "5494652553c59b72ca5db4350a8ba3d8bbf1608519b08f2544c64ecfe130c537"
}
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT a.path FROM app_script s JOIN app a ON a.id = s.app\n WHERE s.id = $1 AND a.workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Int8",
"Text"
]
},
"nullable": [
false
]
},
"hash": "8816bbe1ea9ea4f359e7a95857d349fa8bd7d23e4589b6936e0d000745cb3f34"
}
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_status s\n SET flow_status = JSONB_SET(s.flow_status, ARRAY['modules', s.flow_status->>'step', 'progress'], $1)\n FROM v2_job j\n WHERE s.id = $2 AND j.id = s.id AND j.workspace_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb",
"Uuid",
"Text"
]
},
"nullable": []
},
"hash": "d4af472614e1b6af8defa40a79d5f584c8b114ef24fdb67a5eefb40ce17acdef"
}
+1
View File
@@ -15667,6 +15667,7 @@ dependencies = [
"regex",
"reqwest 0.13.1",
"reqwest-middleware",
"rsa",
"rust_decimal",
"serde",
"serde_json",
+1 -1
View File
@@ -1 +1 @@
ec3cd353245e1cdf6a290528dbd7f2ac2498386c
daffe7bb81cfcaca666c61de1ee838a44d60ebc2
@@ -0,0 +1,3 @@
ALTER TABLE variable
DROP COLUMN IF EXISTS edited_by,
DROP COLUMN IF EXISTS edited_at;
@@ -0,0 +1,15 @@
-- Add `edited_at` and `edited_by` so the UI can detect when a variable has
-- been modified remotely while a local autosave was in flight (see the
-- UserDraft staleness check). Mirrors what `resource` already has.
--
-- Backfill: existing rows get `edited_at = now()` via the column's DEFAULT.
-- All pre-migration variables therefore appear to share a single edit
-- timestamp (the migration time). The staleness check only consumes
-- `edited_at` as an opaque rev string — it doesn't display or sort on it —
-- and only after the user edits a variable forward at least once. So the
-- collision is harmless: no UI flow looks at the pre-migration timestamp
-- before it gets overwritten by a real edit.
ALTER TABLE variable
ADD COLUMN edited_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
ADD COLUMN edited_by VARCHAR(50);
@@ -0,0 +1,5 @@
-- Symmetric to the up migration: Postgres's default `TIMESTAMPTZ -> TIMESTAMP`
-- cast strips the timezone by representing the instant in the session's
-- current timezone, mirroring how the original `now()` values were
-- truncated on insert.
ALTER TABLE draft ALTER COLUMN created_at TYPE TIMESTAMP;
@@ -0,0 +1,12 @@
-- `draft.created_at` was originally created as `TIMESTAMP` (no timezone). The
-- new `*WithDraft` API responses surface it as `chrono::DateTime<Utc>` for the
-- frontend's staleness check, which requires `TIMESTAMPTZ`.
--
-- We rely on Postgres's default `TIMESTAMP -> TIMESTAMPTZ` cast (no explicit
-- USING), which interprets each existing wall-clock value in the session's
-- current timezone. That's the exact semantics under which the original
-- `INSERT ... DEFAULT now()` values were truncated to TIMESTAMP — so the
-- conversion is a no-op on UTC servers (the common case) and correctly
-- recovers the original instant on non-UTC servers, instead of shifting all
-- pre-migration timestamps by the server's tz offset.
ALTER TABLE draft ALTER COLUMN created_at TYPE TIMESTAMPTZ;
+13 -11
View File
@@ -51,14 +51,14 @@ use windmill_common::{
JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING,
KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MAVEN_SETTINGS_XML_SETTING,
MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING,
NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING,
OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING,
POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING,
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RESTART_COORDINATION_SETTING,
RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, SAML_METADATA_SETTING,
SCIM_TOKEN_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TEAMS_SETTING,
TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING,
WORKSPACE_REGISTRIES_SETTING,
NPM_CONFIG_REGISTRY_SETTING, NSJAIL_TMPFS_SIZE_MB_SETTING, NUGET_CONFIG_SETTING,
OAUTH_SETTING, OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING,
POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING,
REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING,
RESTART_COORDINATION_SETTING, RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING,
SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING,
TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING,
UV_INDEX_STRATEGY_SETTING, WORKSPACE_REGISTRIES_SETTING,
},
scripts::ScriptLang,
stats_oss::schedule_stats,
@@ -127,9 +127,10 @@ use crate::monitor::{
reload_http_route_workspaced_route_setting, reload_hub_api_secret_setting,
reload_hub_base_url_setting, reload_instance_events_webhook_setting,
reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting,
reload_license_key, reload_npm_config_registry_setting, reload_otel_tracing_proxy_setting,
reload_pip_index_url_setting, reload_retention_period_setting, reload_scim_token_setting,
reload_smtp_config, reload_store_audit_logs_s3_setting, reload_uv_exclude_newer_setting,
reload_license_key, reload_npm_config_registry_setting, reload_nsjail_tmpfs_size_setting,
reload_otel_tracing_proxy_setting, reload_pip_index_url_setting,
reload_retention_period_setting, reload_scim_token_setting, reload_smtp_config,
reload_store_audit_logs_s3_setting, reload_uv_exclude_newer_setting,
reload_uv_index_strategy_setting, reload_worker_config, MonitorIteration,
};
@@ -1782,6 +1783,7 @@ async fn process_notify_event(
STORE_AUDIT_LOGS_S3_SETTING => reload_store_audit_logs_s3_setting(conn).await,
JOB_DEFAULT_TIMEOUT_SECS_SETTING => reload_job_default_timeout_setting(conn).await,
JOB_ISOLATION_SETTING => reload_job_isolation_setting(conn).await,
NSJAIL_TMPFS_SIZE_MB_SETTING => reload_nsjail_tmpfs_size_setting(conn).await,
#[cfg(feature = "parquet")]
OBJECT_STORE_CONFIG_SETTING => {
if !disable_s3_store {
+21 -9
View File
@@ -62,12 +62,12 @@ use windmill_common::{
HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING,
JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING,
MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING,
NUGET_CONFIG_SETTING, OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING,
POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING,
REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING,
RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING,
STORE_AUDIT_LOGS_S3_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING,
UV_INDEX_STRATEGY_SETTING,
NSJAIL_TMPFS_SIZE_MB_SETTING, NUGET_CONFIG_SETTING, OTEL_SETTING,
OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING,
POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING,
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, STORE_AUDIT_LOGS_S3_SETTING,
TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING,
},
indexer::load_indexer_config,
jwt::JWT_SECRET,
@@ -106,9 +106,10 @@ use windmill_worker::{
OtelTracingProxySettings, SameWorkerSender, WorkspaceRegistryMap, BUNFIG_INSTALL_SCOPES,
BUN_INSTALL_MIN_RELEASE_AGE, CARGO_REGISTRIES, INSTANCE_PYTHON_VERSION, JAVA_HOME_DIR,
JOB_DEFAULT_TIMEOUT, JOB_ISOLATION, KEEP_JOB_DIR, MAVEN_REPOS, MAVEN_SETTINGS_XML,
NO_DEFAULT_MAVEN, NPMRC, NPM_CONFIG_REGISTRY, NSJAIL_AVAILABLE, NUGET_CONFIG,
OTEL_TRACING_PROXY_SETTINGS, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, POWERSHELL_REPO_PAT,
POWERSHELL_REPO_URL, UNSHARE_PATH, UV_EXCLUDE_NEWER, UV_INDEX_STRATEGY, WORKSPACE_REGISTRIES,
NO_DEFAULT_MAVEN, NPMRC, NPM_CONFIG_REGISTRY, NSJAIL_AVAILABLE, NSJAIL_TMPFS_SIZE_MB,
NUGET_CONFIG, OTEL_TRACING_PROXY_SETTINGS, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL,
POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, UNSHARE_PATH, UV_EXCLUDE_NEWER, UV_INDEX_STRATEGY,
WORKSPACE_REGISTRIES,
};
#[cfg(feature = "parquet")]
@@ -384,6 +385,7 @@ pub async fn initial_load(
if worker_mode {
reload_job_default_timeout_setting(&conn).await;
reload_job_isolation_setting(&conn).await;
reload_nsjail_tmpfs_size_setting(&conn).await;
reload_extra_pip_index_url_setting(&conn).await;
reload_pip_index_url_setting(&conn).await;
reload_uv_index_strategy_setting(&conn).await;
@@ -1889,6 +1891,16 @@ pub async fn reload_job_default_timeout_setting(conn: &Connection) {
.await;
}
pub async fn reload_nsjail_tmpfs_size_setting(conn: &Connection) {
reload_option_setting_with_tracing(
conn,
NSJAIL_TMPFS_SIZE_MB_SETTING,
"NSJAIL_TMPFS_SIZE_MB",
NSJAIL_TMPFS_SIZE_MB.clone(),
)
.await;
}
pub async fn reload_job_isolation_setting(conn: &Connection) {
let value =
match load_value_from_global_settings_with_conn(conn, JOB_ISOLATION_SETTING, true).await {
+258
View File
@@ -0,0 +1,258 @@
//! Regression test for the app component preview authorization bypass.
//!
//! `POST /api/w/:workspace/apps_u/execute_component/:path` runs in "preview"
//! mode whenever the client supplies `force_viewer_static_fields`. In that
//! mode it accepts request-supplied `raw_code` and enqueues it as a
//! `Viewer`-mode job — i.e. it is the app-editor equivalent of
//! `/jobs/run/preview`. The bug was that this branch did not re-apply the
//! guards `/jobs/run/preview` enforces for arbitrary code execution, so an
//! authenticated Operator (a run-only user who must not be able to create
//! scripts/apps or run preview jobs) could enqueue arbitrary worker code with
//! a single request, escaping the Operator restriction entirely.
//!
//! This test pins down:
//! - an Operator is rejected from preview mode (the core fix; pre-fix this
//! enqueued a job and returned 200),
//! - a regular non-operator member can still run an editor preview (the fix
//! must not over-block the legitimate editor flow),
//! - preview is confined to paths the caller can read (defense-in-depth
//! against scoped tokens / cross-namespace preview), and
//! - run mode (no `force_viewer_static_fields`) is unaffected by the guard.
use serde_json::json;
use sqlx::{Pool, Postgres};
use windmill_test_utils::*;
fn client() -> reqwest::Client {
reqwest::Client::new()
}
fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder {
builder.header("Authorization", format!("Bearer {}", token))
}
/// A preview request: `force_viewer_static_fields` present + inline `raw_code`.
/// This is the exact shape an attacker (or the editor) sends.
fn preview_body(app_path: &str) -> serde_json::Value {
json!({
"args": {},
"component": "comp",
"raw_code": {
"language": "deno",
"content": "export function main() { return \"pwned\"; }",
"path": format!("{}/comp", app_path)
},
"force_viewer_static_fields": {}
})
}
#[sqlx::test(fixtures("base", "app_preview_auth"))]
async fn test_app_preview_authorization(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let base = format!("http://localhost:{port}/api/w/test-workspace/apps_u/execute_component");
// 1. CORE REGRESSION: an Operator sends a preview request in their own
// namespace (so the *only* thing that can reject them is the Operator
// check itself). Pre-fix this returned 200 with an enqueued job UUID;
// post-fix it must be rejected.
let resp = authed(
client().post(format!("{base}/u/operator-user/myapp")),
"OPERATOR_TOKEN",
)
.json(&preview_body("u/operator-user/myapp"))
.send()
.await?;
let status = resp.status();
let body = resp.text().await?;
assert_eq!(
status, 401,
"Operator must be rejected from app preview (got {status}): {body}"
);
assert!(
body.contains("Operators cannot run preview jobs"),
"rejection must be the operator guard, got: {body}"
);
// 2. The fix must NOT over-block the legitimate editor flow: a regular
// non-operator member previewing in their own namespace still works
// (the endpoint returns the enqueued job UUID before any worker runs).
let resp = authed(
client().post(format!("{base}/u/test-user-2/myapp")),
"SECRET_TOKEN_2",
)
.json(&preview_body("u/test-user-2/myapp"))
.send()
.await?;
let status = resp.status();
let body = resp.text().await?;
assert!(
status.is_success(),
"non-operator editor preview must still succeed (got {status}): {body}"
);
assert!(
uuid::Uuid::parse_str(body.trim()).is_ok(),
"successful preview must return a job UUID, got: {body}"
);
// 3. Inline `raw_code` preview is deliberately NOT path-gated: a
// non-operator can already run arbitrary inline code via
// `/jobs/run/preview`, so the app URL path string is irrelevant for the
// inline case. This pins that decision so an over-restrictive path check
// is not re-added for inline previews.
let resp = authed(
client().post(format!("{base}/u/test-user/secretapp")),
"SECRET_TOKEN_2",
)
.json(&preview_body("u/test-user/secretapp"))
.send()
.await?;
let status = resp.status();
let body = resp.text().await?;
assert!(
status.is_success(),
"inline raw_code preview must not be path-gated (got {status}): {body}"
);
assert!(
uuid::Uuid::parse_str(body.trim()).is_ok(),
"inline preview should enqueue a job UUID, got: {body}"
);
// 4. Run mode (no `force_viewer_static_fields`) is unaffected by the new
// preview guard: an Operator hitting a deployed-app path still follows
// the pre-existing policy lookup (here: the app does not exist -> 404),
// proving the guard only gates preview mode.
let resp = authed(
client().post(format!("{base}/u/operator-user/nonexistent")),
"OPERATOR_TOKEN",
)
.json(&json!({
"args": {},
"component": "comp",
"raw_code": {
"language": "deno",
"content": "export function main() { return 1; }",
"path": "u/operator-user/nonexistent/comp"
}
}))
.send()
.await?;
let status = resp.status();
let body = resp.text().await?;
assert_eq!(
status, 404,
"run mode must be unchanged (deployed app lookup -> 404, not the preview guard); got {status}: {body}"
);
// 5. Defense-in-depth: the guard must check the *runnable* being previewed,
// not just the app URL path. A caller pairs an allowed app path
// (`u/test-user-2/myapp`, own namespace) with a `path` pointing at a
// deployed runnable in another user's namespace. Without checking the
// runnable path this would resolve `script/u/test-user/private` with the
// root DB handle and enqueue it; it must be rejected by the path check.
let resp = authed(
client().post(format!("{base}/u/test-user-2/myapp")),
"SECRET_TOKEN_2",
)
.json(&json!({
"args": {},
"component": "comp",
"path": "script/u/test-user/private",
"force_viewer_static_fields": {}
}))
.send()
.await?;
let status = resp.status();
let body = resp.text().await?;
assert_eq!(
status, 400,
"preview targeting a runnable outside the caller's namespace must be rejected even with an allowed app path (got {status}): {body}"
);
// 6. Defense-in-depth: a persisted inline-script preview selects code by the
// caller-controlled `app_script` id. Pairing an allowed app path with an
// id owned by another (private) app must be rejected — without the
// id-ownership check the worker would fetch and run that app's code.
let resp = authed(
client().post(format!("{base}/u/test-user-2/myapp")),
"SECRET_TOKEN_2",
)
.json(&json!({
"args": {},
"component": "comp",
"id": 999777,
"raw_code": {
"language": "deno",
"content": "export function main() { return 1; }",
"path": "u/test-user-2/myapp/comp"
},
"force_viewer_static_fields": {}
}))
.send()
.await?;
let status = resp.status();
let body = resp.text().await?;
assert_eq!(
status, 400,
"preview with an app_script id owned by another app must be rejected (got {status}): {body}"
);
// 7. The id-ownership check must NOT over-block a legitimate persisted
// inline-script preview: an id owned by an app in the caller's own
// namespace passes the guard and enqueues (returns a job UUID).
let resp = authed(
client().post(format!("{base}/u/test-user-2/ownapp")),
"SECRET_TOKEN_2",
)
.json(&json!({
"args": {},
"component": "comp",
"id": 999778,
"raw_code": {
"language": "deno",
"content": "export function main() { return 1; }",
"path": "u/test-user-2/ownapp/comp"
},
"force_viewer_static_fields": {}
}))
.send()
.await?;
let status = resp.status();
let body = resp.text().await?;
assert!(
status.is_success(),
"persisted preview for an app the caller owns must still succeed (got {status}): {body}"
);
assert!(
uuid::Uuid::parse_str(body.trim()).is_ok(),
"successful persisted preview must return a job UUID, got: {body}"
);
// 8. Scope escalation: a token scoped to `apps:run` (but not `jobs:run`)
// can reach this route (it maps to the `apps` scope domain) and is not an
// Operator, but must NOT be able to enqueue arbitrary preview `raw_code`.
// `/jobs/run/preview` requires `jobs:run` for exactly this reason; the
// app preview path must enforce the same. Without the `jobs:run` check
// this enqueues a job (returns a UUID); with it, it is rejected (403).
let resp = authed(
client().post(format!("{base}/u/test-user-2/myapp")),
"APPS_RUN_TOKEN",
)
.json(&preview_body("u/test-user-2/myapp"))
.send()
.await?;
let status = resp.status();
let body = resp.text().await?;
assert_eq!(
status, 403,
"apps:run-scoped token must not escalate to arbitrary preview code (got {status}): {body}"
);
assert!(
body.contains("jobs:run"),
"rejection must be the jobs:run scope gate, got: {body}"
);
Ok(())
}
+34
View File
@@ -0,0 +1,34 @@
-- Fixture for the app component preview authorization regression test.
-- Layered on top of `base` (which provides test-workspace, the admin
-- `test-user`/SECRET_TOKEN, and the non-operator `test-user-2`/SECRET_TOKEN_2).
-- Adds an Operator member so we can assert that Operators cannot reach the
-- arbitrary-code app preview path (`force_viewer_static_fields` + `raw_code`).
INSERT INTO password(email, password_hash, login_type, super_admin, verified, name)
VALUES ('operator@windmill.dev', 'not-a-real-hash', 'password', false, true, 'Operator User');
INSERT INTO usr(workspace_id, email, username, is_admin, operator, role) VALUES
('test-workspace', 'operator@windmill.dev', 'operator-user', false, true, 'Operator');
INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin) VALUES
(encode(sha256('OPERATOR_TOKEN'::bytea), 'hex'), 'OPERATOR_T', 'OPERATOR_TOKEN', 'operator@windmill.dev', 'operator token', false);
-- A non-operator token scoped to `apps:run` but NOT `jobs:run`. It can reach
-- the `apps_u/execute_component` route (route maps to the `apps` scope domain)
-- but must not be able to enqueue arbitrary preview `raw_code`.
INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES
(encode(sha256('APPS_RUN_TOKEN'::bytea), 'hex'), 'APPS_RUN_T', 'APPS_RUN_TOKEN', 'test2@windmill.dev', 'apps:run scoped token', false, '{apps:run}');
-- A private app owned by `test-user` with a persisted inline script. Used to
-- assert that `test-user-2` cannot preview-execute another app's app_script id.
INSERT INTO app (id, workspace_id, path, summary, policy, versions) VALUES
(999001, 'test-workspace', 'u/test-user/private', 'private app', '{}'::jsonb, '{}');
INSERT INTO app_script (id, app, hash, code, code_sha256) VALUES
(999777, 999001, repeat('a', 64), 'export function main(){ return "secret" }', repeat('b', 64));
-- An app owned by `test-user-2` with its own persisted inline script, to assert
-- the id-ownership check does not over-block a legitimate persisted preview.
INSERT INTO app (id, workspace_id, path, summary, policy, versions) VALUES
(999002, 'test-workspace', 'u/test-user-2/ownapp', 'own app', '{}'::jsonb, '{}');
INSERT INTO app_script (id, app, hash, code, code_sha256) VALUES
(999778, 999002, repeat('c', 64), 'export function main(){ return "ok" }', repeat('d', 64));
+4
View File
@@ -1480,6 +1480,9 @@ pub struct FlowWDraft {
pub extra_perms: serde_json::Value,
#[serde(skip_serializing_if = "Option::is_none")]
pub draft: Option<sqlx::types::Json<Box<serde_json::value::RawValue>>>,
/// Timestamp at which the most recent DB draft was created.
#[serde(skip_serializing_if = "Option::is_none")]
pub draft_created_at: Option<chrono::DateTime<chrono::Utc>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub draft_only: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
@@ -1516,6 +1519,7 @@ async fn get_flow_by_path_w_draft(
flow.ws_error_handler_muted,
flow.dedicated_worker,
draft.value AS draft,
draft.created_at AS draft_created_at,
flow.tag,
flow.visible_to_runner_only,
flow.on_behalf_of_email,
+13 -4
View File
@@ -180,13 +180,22 @@ async fn set_job_progress(
// If flow_job_id exists, than we should modify flow_status of corresponding module
// Individual jobs and flows are handled differently
if let Some(flow_job_id) = flow_job_id {
// `v2_job_status` has no workspace_id column and the root db handle
// bypasses RLS (the per-row policy on the table is also inert today —
// `ENABLE ROW LEVEL SECURITY` was never set). Scope the update by
// joining `v2_job` so the URL's workspace_id confines tampering to the
// caller's workspace; without this, an authed member of any workspace
// could overwrite the flow `progress` UI field of a flow in another
// workspace given just the flow UUID.
// TODO: Return error if trying to set completed job?
sqlx::query!(
"UPDATE v2_job_status
SET flow_status = JSONB_SET(flow_status, ARRAY['modules', flow_status->>'step', 'progress'], $1)
WHERE id = $2",
"UPDATE v2_job_status s
SET flow_status = JSONB_SET(s.flow_status, ARRAY['modules', s.flow_status->>'step', 'progress'], $1)
FROM v2_job j
WHERE s.id = $2 AND j.id = s.id AND j.workspace_id = $3",
serde_json::json!(percent.clamp(0, 99)),
flow_job_id
flow_job_id,
w_id,
)
.execute(&db)
.await?;
+5 -1
View File
@@ -95,6 +95,9 @@ pub struct ScriptWDraft<SR> {
pub tag: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub draft: Option<sqlx::types::Json<Box<RawValue>>>,
/// Timestamp at which the most recent DB draft was created.
#[serde(skip_serializing_if = "Option::is_none")]
pub draft_created_at: Option<chrono::DateTime<chrono::Utc>>,
pub schema: Option<Schema>,
#[serde(skip_serializing_if = "Option::is_none")]
pub draft_only: Option<bool>,
@@ -172,6 +175,7 @@ impl ScriptWDraft<ScriptRunnableSettingsHandle> {
kind: self.kind,
tag: self.tag,
draft: self.draft,
draft_created_at: self.draft_created_at,
schema: self.schema,
draft_only: self.draft_only,
envs: self.envs,
@@ -1888,7 +1892,7 @@ async fn get_script_by_path_w_draft(
let mut tx = user_db.begin(&authed).await?;
let script_o = sqlx::query_as::<_, ScriptWDraft<ScriptRunnableSettingsHandle>>(
"SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, runnable_settings_handle, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, has_preprocessor, on_behalf_of_email, assets, modules, debounce_key, debounce_delay_s, labels FROM script LEFT JOIN draft ON
"SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, runnable_settings_handle, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, ws_error_handler_muted, draft.value as draft, draft.created_at as draft_created_at, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, has_preprocessor, on_behalf_of_email, assets, modules, debounce_key, debounce_delay_s, labels FROM script LEFT JOIN draft ON
script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script'
WHERE script.path = $1 AND script.workspace_id = $2
ORDER BY script.created_at DESC LIMIT 1",
+57
View File
@@ -9588,6 +9588,10 @@ paths:
properties:
draft:
$ref: "#/components/schemas/Flow"
draft_created_at:
type: string
format: date-time
description: Timestamp at which the most recent DB draft was created. Used by the frontend's UserDraft staleness check.
/w/{workspace}/flows/exists/{path}:
get:
@@ -9836,6 +9840,13 @@ paths:
- path_autocomplete
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: force
description: |
bypass the server-side cache and re-query the DB, refreshing the
cache. Used right after a deploy so the new path appears immediately.
in: query
schema:
type: boolean
responses:
"200":
description: deduplicated path list, sorted lexicographically
@@ -10712,6 +10723,12 @@ paths:
run_query_params:
type: object
description: Runnable query parameters
temp_script_refs:
type: object
nullable: true
description: "Map of relative-import script path -> temp storage hash. Only honored for inline-script (raw_code) execution so app dev resolves those imports from not-yet-deployed local content."
additionalProperties:
type: string
required:
- args
- component
@@ -19838,6 +19855,12 @@ paths:
properties:
is_alive:
type: boolean
state:
type: string
enum:
- running
- stale
- never_started
last_locked_at:
type: string
format: date-time
@@ -19859,6 +19882,12 @@ paths:
properties:
is_alive:
type: boolean
state:
type: string
enum:
- running
- stale
- never_started
last_locked_at:
type: string
format: date-time
@@ -21064,6 +21093,9 @@ components:
mount_path:
type: string
description: KV v2 secrets engine mount path (e.g., windmill)
kv_secret_path_prefix:
type: string
description: Optional path prefix inserted between the KV data/metadata segment and the workspace id (e.g., "apps/windmill"). When set, secrets are stored at `<mount>/data/<prefix>/<workspace>/<secret>`, allowing a Vault policy scoped to exactly `<mount>/data/<prefix>/*`.
jwt_role:
type: string
description: Vault JWT auth role name for Windmill (optional, if not provided token auth is used)
@@ -21726,6 +21758,10 @@ components:
properties:
draft:
$ref: "#/components/schemas/NewScript"
draft_created_at:
type: string
format: date-time
description: Timestamp at which the most recent DB draft was created. Used by the frontend's UserDraft staleness check.
hash:
type: string
required:
@@ -22793,6 +22829,11 @@ components:
type: string
ws_specific:
type: boolean
edited_at:
type: string
format: date-time
edited_by:
type: string
required:
- workspace_id
- path
@@ -23205,6 +23246,12 @@ components:
description: "Additional script modules keyed by relative file path"
additionalProperties:
$ref: "#/components/schemas/ScriptModule"
temp_script_refs:
type: object
nullable: true
description: "Map of relative-import script path -> temp storage hash so the preview job resolves those imports from not-yet-deployed local content instead of the deployed script"
additionalProperties:
type: string
required:
- args
@@ -26552,6 +26599,12 @@ components:
type: string
restarted_from:
$ref: "#/components/schemas/RestartedFrom"
temp_script_refs:
type: object
nullable: true
description: "Map of relative-import script path -> temp storage hash, propagated to each flow step so inline-script relative imports resolve from not-yet-deployed local content instead of the deployed script"
additionalProperties:
type: string
required:
- value
@@ -26790,6 +26843,10 @@ components:
draft_only:
type: boolean
draft: {}
draft_created_at:
type: string
format: date-time
description: Timestamp at which the most recent DB draft was created. Used by the frontend's UserDraft staleness check.
AppHistory:
type: object
+131 -16
View File
@@ -11,7 +11,7 @@ use crate::{
auth::{get_end_user_email, OptTokened},
db::{ApiAuthed, DB},
jobs::RunJobQuery,
users::{require_owner_of_path, OptAuthed},
users::{require_owner_of_path, require_path_read_access_for_preview, OptAuthed},
utils::{check_scopes, WithStarredInfoQuery},
webhook_util::{WebhookMessage, WebhookShared},
HTTP_CLIENT,
@@ -217,6 +217,9 @@ pub struct AppWithLastVersionAndDraft {
pub draft: Option<sqlx::types::Json<Box<RawValue>>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub draft_only: Option<bool>,
/// Timestamp at which the most recent DB draft was created.
#[serde(skip_serializing_if = "Option::is_none")]
pub draft_created_at: Option<chrono::DateTime<chrono::Utc>>,
}
#[derive(Serialize)]
@@ -642,29 +645,30 @@ async fn get_app_w_draft(
let app_o = sqlx::query_as::<_, AppWithLastVersionAndDraft>(
r#"
SELECT
app.id,
app.path,
app.summary,
app.versions,
app.policy,
SELECT
app.id,
app.path,
app.summary,
app.versions,
app.policy,
app.custom_path,
app.extra_perms,
app.extra_perms,
app_version.value,
app_version.created_at,
app_version.created_at,
app_version.created_by,
app.draft_only,
draft.value AS "draft",
draft.created_at AS "draft_created_at",
app_version.raw_app,
app.labels
FROM app
INNER JOIN app_version
INNER JOIN app_version
ON app_version.id = app.versions[array_upper(app.versions, 1)]
LEFT JOIN draft
ON app.path = draft.path
AND draft.workspace_id = $2
LEFT JOIN draft
ON app.path = draft.path
AND draft.workspace_id = $2
AND draft.typ = 'app'
WHERE app.path = $1
WHERE app.path = $1
AND app.workspace_id = $2
"#,
)
@@ -2039,6 +2043,10 @@ pub struct ExecuteApp {
pub force_viewer_delete_after_secs: Option<i32>,
/// Runnable query parameters (e.g., memory_id for chat-enabled flows)
pub run_query_params: Option<RunJobQuery>,
/// Map of relative-import script path -> temp storage hash. Only honored for
/// inline-script (raw_code, preview) execution so `wmill app dev` resolves
/// those imports from not-yet-deployed local content instead of deployed.
pub temp_script_refs: Option<HashMap<String, String>>,
}
fn digest(code: &str) -> String {
@@ -2113,8 +2121,16 @@ async fn execute_component(
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
Json(payload): Json<ExecuteApp>,
Json(mut payload): Json<ExecuteApp>,
) -> Result<String> {
// Only honor temp_script_refs for the inline-script preview path:
// preview/editor mode (force_viewer_static_fields set, == `is_preview`),
// raw_code present, and no deployed app_script id — i.e. `wmill app dev`.
let temp_script_refs = payload.temp_script_refs.take();
let inject_temp_refs = temp_script_refs.is_some()
&& payload.force_viewer_static_fields.is_some()
&& payload.raw_code.is_some()
&& payload.id.is_none();
match (payload.path.is_some(), payload.raw_code.is_some()) {
(false, false) => {
return Err(Error::BadRequest(
@@ -2138,6 +2154,54 @@ async fn execute_component(
// tag from the deployed policy and ignore the request body.
let is_preview = payload.force_viewer_static_fields.is_some();
// Preview mode runs request-supplied code as a `Viewer`-mode job (the
// app-editor equivalent of `/jobs/run/preview`), so it enforces the same
// guards. Operators must never run preview jobs. `jobs:run` is required
// because this route is reachable with an `apps:run`-scoped token (the
// route maps to the `apps` scope domain), which must not be able to escalate
// to arbitrary code execution. The client-supplied inline `raw_code.tag`
// must stay within the caller's allowed worker tags. A preview can also
// *reference* an existing runnable the caller may not be allowed to read — a
// deployed script/flow via `payload.path` or a persisted `app_script` via
// `payload.id`, both resolved with the root DB handle — so those (and only
// those) are confined to paths the caller can read. Inline `raw_code` is not
// path-gated: a non-operator member can already run arbitrary inline code
// via `/jobs/run/preview`.
if is_preview {
let authed = opt_authed.as_ref().ok_or_else(|| {
Error::NotAuthorized("App component preview requires authentication".to_string())
})?;
if authed.is_operator {
return Err(Error::NotAuthorized(
"Operators cannot run preview jobs for security reasons".to_string(),
));
}
check_scopes(authed, || format!("jobs:run"))?;
if let Some(p) = payload.path.as_deref() {
let runnable_path = p
.strip_prefix("script/")
.or_else(|| p.strip_prefix("flow/"))
.unwrap_or(p);
require_path_read_access_for_preview(authed, &Some(runnable_path.to_string()))?;
}
if let Some(id) = payload.id {
let owner_path = sqlx::query_scalar!(
"SELECT a.path FROM app_script s JOIN app a ON a.id = s.app
WHERE s.id = $1 AND a.workspace_id = $2",
id,
&w_id,
)
.fetch_optional(&db)
.await?
.ok_or_else(|| {
Error::NotAuthorized(format!(
"App script {id} does not belong to an app in this workspace"
))
})?;
require_path_read_access_for_preview(authed, &Some(owner_path))?;
}
}
// Two cases here:
// 1. The component is executed from the editor (i.e. in "preview" mode), then:
// - The policy is set to default (in `Viewer` execution mode).
@@ -2295,7 +2359,7 @@ async fn execute_component(
let resolved_delete_secs =
resolve_delete_after_secs(None, policy_triggerables.delete_after_secs);
let (args, job_id) = build_args(
let (mut args, job_id) = build_args(
policy,
policy_triggerables,
payload.args,
@@ -2306,6 +2370,14 @@ async fn execute_component(
)
.await?;
if inject_temp_refs {
if let Some(refs) = temp_script_refs {
args.extra
.get_or_insert_with(HashMap::new)
.insert("_TEMP_SCRIPT_REFS".to_string(), to_raw_value(&refs));
}
}
let is_flow = payload
.path
.as_ref()
@@ -2341,6 +2413,20 @@ async fn execute_component(
),
_ => unreachable!(),
};
// Preview honors the client-supplied inline tag (`resolved_inline_tag`), so
// — like `/jobs/run/preview` — confine it to worker tags the caller may use
// (a `if_jobs:filter_tags`-restricted token must not escape its filter).
// `is_preview` implies an authed caller (the guard above returns otherwise).
if is_preview {
if let Some(authed) = opt_authed.as_ref() {
crate::jobs::check_tag_available_for_workspace(&db, &w_id, &tag, authed).await?;
}
}
// Identity is already resolved to the requesting user in preview mode (the
// policy is forced to `ExecutionMode::Viewer`, so the job runs as the
// caller). The enqueue stays root-isolated as before — switching the insert
// to user-RLS is not what contains the bypass (the auth guards above are)
// and would add unnecessary breakage risk to the legitimate editor flow.
let tx = PushIsolationLevel::IsolatedRoot(db.clone());
let (email, permissioned_as) = if let Some(on_behalf_of) = on_behalf_of.as_ref() {
@@ -2507,6 +2593,21 @@ async fn upload_s3_file_from_app(
request: axum::extract::Request,
) -> JsonResult<AppUploadFileResponse> {
let policy = if let Some(file_key_regex) = query.force_viewer_file_key_regex {
// `force_viewer_*` lets the caller supply a synthetic upload policy that
// bypasses the deployed app's file_key_regex / resource restrictions.
// It is intended for the app editor's preview path, so it must enforce
// the same guards as `execute_component`'s preview mode (PR #9235):
// authed caller, not an operator, and `apps:write` scope to make sure
// an `apps:run`-scoped token cannot pick its own policy.
let authed = opt_authed.as_ref().ok_or_else(|| {
Error::NotAuthorized("App S3 preview upload requires authentication".to_string())
})?;
if authed.is_operator {
return Err(Error::NotAuthorized(
"Operators cannot run app S3 previews for security reasons".to_string(),
));
}
check_scopes(authed, || format!("apps:write:{}", path.to_path()))?;
Some(Policy {
execution_mode: ExecutionMode::Viewer,
triggerables: None,
@@ -3018,6 +3119,20 @@ async fn download_s3_file_from_app(
let force_viewer_allowed_s3_keys = if let Some(force_viewer_allowed_s3_keys) =
query.force_viewer_allowed_s3_keys.clone()
{
// `force_viewer_allowed_s3_keys` lets the caller supply a synthetic
// allowlist that bypasses the deployed app policy. Apply the same
// preview-mode guard as `execute_component` (PR #9235): authed, not an
// operator, `apps:write` scope so an `apps:run`-scoped token cannot
// pick its own allowlist.
let authed = opt_authed.as_ref().ok_or_else(|| {
Error::NotAuthorized("App S3 preview download requires authentication".to_string())
})?;
if authed.is_operator {
return Err(Error::NotAuthorized(
"Operators cannot run app S3 previews for security reasons".to_string(),
));
}
check_scopes(authed, || format!("apps:write:{}", path))?;
Some(serde_json::from_str::<Vec<S3Key>>(&force_viewer_allowed_s3_keys).unwrap_or_default())
} else {
None
+2 -1
View File
@@ -79,7 +79,8 @@ async fn create_draft(
"INSERT INTO draft
(workspace_id, path, value, typ)
VALUES ($1, $2, $3::text::json, $4)
ON CONFLICT (workspace_id, path, typ) DO UPDATE SET value = EXCLUDED.value",
ON CONFLICT (workspace_id, path, typ)
DO UPDATE SET value = EXCLUDED.value, created_at = now()",
&w_id,
draft.path,
//to preserve key orders
+35 -2
View File
@@ -3633,6 +3633,10 @@ struct Preview {
format: Option<String>,
flow_path: Option<String>,
modules: Option<HashMap<String, ScriptModule>>,
/// Map of relative-import script path -> temp storage hash. When set, the
/// preview job resolves those imports from not-yet-deployed local content
/// (uploaded to raw_script_temp) instead of the deployed script.
temp_script_refs: Option<HashMap<String, String>>,
}
#[cfg(feature = "run_inline")]
@@ -3661,6 +3665,10 @@ struct PreviewFlow {
args: Option<HashMap<String, Box<JsonRawValue>>>,
tag: Option<String>,
restarted_from: Option<RestartedFrom>,
/// Map of relative-import script path -> temp storage hash. Propagated to
/// each flow step so inline-script relative imports resolve from
/// not-yet-deployed local content instead of the deployed script.
temp_script_refs: Option<HashMap<String, String>>,
}
#[derive(Debug, Deserialize)]
@@ -5662,6 +5670,12 @@ async fn run_preview_script(
if let Some(ref modules) = preview.modules {
extra.insert("_MODULES".to_string(), to_raw_value(modules));
}
if let Some(ref temp_script_refs) = preview.temp_script_refs {
extra.insert(
"_TEMP_SCRIPT_REFS".to_string(),
to_raw_value(temp_script_refs),
);
}
let extra = if extra.is_empty() { None } else { Some(extra) };
let push_args = PushArgs { extra, args: &preview_args };
@@ -6010,6 +6024,17 @@ async fn run_bundle_preview_script(
let args = preview.args.unwrap_or_default();
// The bundle's runtime still resolves workspace-path imports
// (`/f/...`) via loader.bun.js, so pass through temp_script_refs the
// same way `run_preview_script` does — otherwise codebase previews
// silently fall back to deployed content for those imports.
let extra = preview.temp_script_refs.as_ref().map(|refs| {
let mut m = HashMap::new();
m.insert("_TEMP_SCRIPT_REFS".to_string(), to_raw_value(refs));
m
});
let push_args = PushArgs { extra, args: &args };
is_tar = match preview.kind {
Some(PreviewKind::Tarbundle) => true,
_ => false,
@@ -6038,7 +6063,7 @@ async fn run_bundle_preview_script(
modules: None,
tag: None,
}),
PushArgs::from(&args),
push_args,
authed.display_username(),
&authed.email,
username_to_permissioned_as(&authed.username),
@@ -6668,6 +6693,14 @@ async fn run_preview_flow_job(
.and_then(|args| args.get("user_message"))
.cloned();
let mut flow_args = raw_flow.args.unwrap_or_default();
if let Some(ref temp_script_refs) = raw_flow.temp_script_refs {
flow_args.insert(
"_TEMP_SCRIPT_REFS".to_string(),
to_raw_value(temp_script_refs),
);
}
let (uuid, mut tx) = push(
&db,
tx,
@@ -6677,7 +6710,7 @@ async fn run_preview_flow_job(
path: raw_flow.path,
restarted_from: raw_flow.restarted_from,
},
PushArgs::from(&raw_flow.args.unwrap_or_default()),
PushArgs::from(&flow_args),
authed.display_username(),
&authed.email,
username_to_permissioned_as(&authed.username),
+18 -6
View File
@@ -12,11 +12,11 @@ use std::{
};
use axum::{
extract::{Extension, Path},
extract::{Extension, Path, Query},
routing::get,
Json, Router,
};
use serde::Serialize;
use serde::{Deserialize, Serialize};
use windmill_common::error::JsonResult;
use crate::db::{ApiAuthed, DB};
@@ -43,16 +43,28 @@ struct ListPathsResponse {
paths: Arc<Vec<String>>,
}
#[derive(Deserialize)]
struct ListPathsQuery {
/// When true, bypass the cached entry and re-query the DB, refreshing the
/// cache. Used by clients that just mutated the workspace (e.g. a deploy)
/// and need the new path reflected immediately.
#[serde(default)]
force: bool,
}
async fn list_paths(
_authed: ApiAuthed,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Query(ListPathsQuery { force }): Query<ListPathsQuery>,
) -> JsonResult<ListPathsResponse> {
if let Some((cached, cached_at)) = PATHS_CACHE.get(&w_id) {
if cached_at.elapsed() < CACHE_TTL {
return Ok(Json(ListPathsResponse { paths: cached }));
if !force {
if let Some((cached, cached_at)) = PATHS_CACHE.get(&w_id) {
if cached_at.elapsed() < CACHE_TTL {
return Ok(Json(ListPathsResponse { paths: cached }));
}
PATHS_CACHE.remove(&w_id);
}
PATHS_CACHE.remove(&w_id);
}
let mut paths: Vec<String> = sqlx::query_scalar!(
@@ -424,6 +424,7 @@ async fn route_job(
.flatten()
.unwrap_or("application/octet-stream".parse().unwrap()),
);
response_headers.insert("x-content-type-options", "nosniff".parse().unwrap());
if !trigger.is_static_website {
response_headers.insert(
"content-disposition",
@@ -443,6 +444,19 @@ async fn route_job(
},
),
);
// For single-file triggers, sandbox any HTML/SVG so it can't
// reach the viewer's session cookie. Allow-scripts/forms/etc.
// keep the opaque origin (cookies still blocked) while
// preserving JS for legitimate HTML payloads. Static-website
// triggers intentionally serve a live web app and cannot be
// sandboxed; restrict write access to those buckets at the
// workspace level.
response_headers.insert(
"content-security-policy",
"sandbox allow-scripts allow-forms allow-popups allow-modals allow-downloads"
.parse()
.unwrap(),
);
}
let body_stream = axum::body::Body::from_stream(s3_object.into_stream());
@@ -53,6 +53,7 @@ pub const EXPOSE_DEBUG_METRICS_SETTING: &str = "expose_debug_metrics";
pub const KEEP_JOB_DIR_SETTING: &str = "keep_job_dir";
pub const REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING: &str = "require_preexisting_user_for_oauth";
pub const JOB_ISOLATION_SETTING: &str = "job_isolation";
pub const NSJAIL_TMPFS_SIZE_MB_SETTING: &str = "nsjail_tmpfs_size_mb";
pub const OBJECT_STORE_CONFIG_SETTING: &str = "object_store_cache_config";
pub const HUB_API_SECRET_SETTING: &str = "hub_api_secret";
@@ -221,6 +221,8 @@ pub struct GlobalSettings {
#[serde(skip_serializing_if = "Option::is_none")]
pub job_default_timeout: Option<i64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub nsjail_tmpfs_size_mb: Option<i64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub bun_install_min_release_age: Option<i64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub uv_exclude_newer: Option<i64>,
@@ -118,6 +118,13 @@ pub struct VaultSettings {
pub address: String,
/// KV v2 mount path (e.g., "windmill")
pub mount_path: String,
/// Optional path prefix inserted between the KV `data`/`metadata` segment
/// and the workspace id, e.g. "apps/windmill". When set, secrets live at
/// `<mount>/data/<prefix>/<workspace>/<secret>`, so a Vault policy can be
/// scoped to exactly `<mount>/data/<prefix>/*`. Surrounding slashes are
/// trimmed.
#[serde(skip_serializing_if = "Option::is_none")]
pub kv_secret_path_prefix: Option<String>,
/// JWT auth role name configured in Vault (used for JWT/OIDC auth)
/// Optional - if not provided, token auth is used
#[serde(skip_serializing_if = "Option::is_none")]
@@ -25,6 +25,7 @@ mod tests {
VaultSettings {
address: "http://127.0.0.1:8200".to_string(),
mount_path: "windmill".to_string(),
kv_secret_path_prefix: None,
jwt_role: Some("windmill-secrets".to_string()),
jwt_mount_path: None,
namespace: None,
+13 -1
View File
@@ -19,6 +19,7 @@ use serde::{Deserialize, Serialize};
lazy_static::lazy_static! {
pub static ref SECRET_SALT: Option<String> = std::env::var("SECRET_SALT").ok();
static ref RESERVED_WM_VAR_NAME: regex::Regex = regex::Regex::new(r"^WM_[A-Z_]+$").unwrap();
}
#[derive(Serialize, Clone)]
@@ -50,6 +51,10 @@ pub struct ListableVariable {
pub labels: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub ws_specific: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub edited_at: Option<chrono::DateTime<Utc>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub edited_by: Option<String>,
}
#[derive(Serialize, Deserialize, sqlx::FromRow)]
@@ -452,7 +457,7 @@ async fn get_cached_workspace_envs(conn: &Connection, w_id: &str) -> Vec<(String
let custom_envs = if let Some(cached_envs) = cached_envs_o {
cached_envs
} else {
let custom_envs = match conn {
let raw_envs = match conn {
Connection::Sql(db) => sqlx::query_as::<_, (String, String)>(
"SELECT name, value FROM workspace_env WHERE workspace_id = $1",
)
@@ -465,6 +470,13 @@ async fn get_cached_workspace_envs(conn: &Connection, w_id: &str) -> Vec<(String
.await
.unwrap_or_default(),
};
// Applied here (not in the SQL branch alone) so agent workers going
// through `Connection::Http` are covered too — drop any name that
// would shadow a built-in `%%WM_*%%` contextual var.
let custom_envs: Vec<(String, String)> = raw_envs
.into_iter()
.filter(|(name, _)| !RESERVED_WM_VAR_NAME.is_match(name))
.collect();
CUSTOM_ENVS_CACHE.insert(
w_id.to_string(),
(chrono::Utc::now().timestamp(), custom_envs.clone()),
@@ -90,6 +90,7 @@ mod tests {
address: std::env::var("VAULT_ADDR")
.unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()),
mount_path: "windmill".to_string(),
kv_secret_path_prefix: None,
jwt_role: None, // Static token mode
jwt_mount_path: None,
namespace: None,
@@ -106,6 +107,7 @@ mod tests {
address: std::env::var("VAULT_ADDR")
.unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()),
mount_path: "windmill".to_string(),
kv_secret_path_prefix: None,
jwt_role: Some("windmill-secrets".to_string()), // JWT mode
jwt_mount_path: None,
namespace: None,
@@ -34,6 +34,7 @@ fn test_vault_settings() -> VaultSettings {
address: std::env::var("VAULT_ADDR")
.unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()),
mount_path: "windmill".to_string(),
kv_secret_path_prefix: None,
jwt_role: Some("windmill-secrets".to_string()),
jwt_mount_path: None,
namespace: None,
@@ -1,3 +1,3 @@
CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release -p windmill_duckdb_ffi_internal
mkdir -p ../target/debug/
cp target/release/libwindmill_duckdb_ffi_internal.* ../target/debug/
cp target/release/libwindmill_duckdb_ffi_internal.* ../target/debug/
@@ -11,6 +11,22 @@ use rust_decimal::{prelude::FromPrimitive, Decimal};
use serde::{Deserialize, Serialize};
use serde_json::value::RawValue;
// Worker passes "" for "no override" — saves an extra C string nullability dance.
// Returns an owned String so the value outlives the raw pointer's lifetime.
fn ptr_to_opt_str(ptr: *const c_char) -> Result<Option<String>, String> {
if ptr.is_null() {
return Ok(None);
}
let s = unsafe { CStr::from_ptr(ptr) }
.to_str()
.map_err(|e| format!("Invalid string in duckdb ffi: {}", e))?;
Ok(if s.is_empty() {
None
} else {
Some(s.to_owned())
})
}
#[derive(Deserialize, Clone, Debug, PartialEq, Default)]
pub struct Arg {
pub name: String,
@@ -34,7 +50,7 @@ pub extern "C" fn get_version() -> c_uint {
// Increment when making breaking changes to the FFI interface.
// The windmill worker will check that the version matches or else refuse to call
// the FFI functions to avoid undefined behavior.
return 1;
return 2;
}
#[unsafe(no_mangle)]
@@ -45,10 +61,16 @@ pub extern "C" fn run_duckdb_ffi(
token: *const c_char,
base_internal_url: *const c_char,
w_id: *const c_char,
memory_limit: *const c_char,
temp_directory: *const c_char,
column_order_ptr: *mut *mut c_char,
collect_last_only: bool,
collect_first_row_only: bool,
) -> *mut c_char {
let resource_limits = match (ptr_to_opt_str(memory_limit), ptr_to_opt_str(temp_directory)) {
(Ok(m), Ok(t)) => Ok(ResourceLimits { memory_limit: m, temp_directory: t }),
(Err(e), _) | (_, Err(e)) => Err(e),
};
let (r, column_order) = match convert_args(
query_block_list,
query_block_list_count,
@@ -57,8 +79,9 @@ pub extern "C" fn run_duckdb_ffi(
base_internal_url,
w_id,
)
.and_then(|args| resource_limits.map(|r| (args, r)))
.and_then(
|(query_block_list, job_args, token, base_internal_url, w_id)| {
|((query_block_list, job_args, token, base_internal_url, w_id), limits)| {
run_duckdb_internal(
query_block_list,
query_block_list_count,
@@ -66,6 +89,7 @@ pub extern "C" fn run_duckdb_ffi(
token,
base_internal_url,
w_id,
limits,
collect_last_only,
collect_first_row_only,
)
@@ -150,7 +174,13 @@ pub extern "C" fn prepare_duckdb_ffi(
token: *const c_char,
base_internal_url: *const c_char,
w_id: *const c_char,
memory_limit: *const c_char,
temp_directory: *const c_char,
) -> *mut c_char {
let resource_limits = match (ptr_to_opt_str(memory_limit), ptr_to_opt_str(temp_directory)) {
(Ok(m), Ok(t)) => Ok(ResourceLimits { memory_limit: m, temp_directory: t }),
(Err(e), _) | (_, Err(e)) => Err(e),
};
let r = match convert_prepare_args(
query_block_list,
query_block_list_count,
@@ -158,9 +188,12 @@ pub extern "C" fn prepare_duckdb_ffi(
base_internal_url,
w_id,
)
.and_then(|(query_block_list, token, base_internal_url, w_id)| {
prepare_duckdb_internal(query_block_list, token, base_internal_url, w_id)
}) {
.and_then(|args| resource_limits.map(|r| (args, r)))
.and_then(
|((query_block_list, token, base_internal_url, w_id), limits)| {
prepare_duckdb_internal(query_block_list, token, base_internal_url, w_id, limits)
},
) {
Ok(result) => result,
Err(err) => {
let err = serde_json::to_string(&err)
@@ -175,12 +208,58 @@ pub extern "C" fn prepare_duckdb_ffi(
})
}
#[derive(Clone, Default)]
struct ResourceLimits {
memory_limit: Option<String>,
temp_directory: Option<String>,
}
fn sql_single_quote(s: &str) -> String {
s.replace('\'', "''")
}
// Bounds memory so DuckDB spills to disk before blowing the cgroup cap and
// getting the worker SIGKILLed. Spill goes to the job dir (when set) so it is
// cleaned up with the job, otherwise DuckDB's default temp_directory is kept.
fn configure_duckdb_resource_limits(
conn: &duckdb::Connection,
limits: &ResourceLimits,
) -> Result<(), String> {
let mut config_sql = String::new();
// jemalloc-specific setting bundled with the Linux DuckDB build. macOS and
// Windows builds may not accept it; gated to avoid breaking those workers.
if cfg!(target_os = "linux") {
config_sql.push_str("SET allocator_background_threads=true;\n");
}
if let Some(mem) = limits.memory_limit.as_deref() {
config_sql.push_str(&format!("SET memory_limit='{}';\n", sql_single_quote(mem)));
}
if let Some(tmp) = limits.temp_directory.as_deref() {
config_sql.push_str(&format!(
"SET temp_directory='{}';\n",
sql_single_quote(tmp)
));
}
if config_sql.is_empty() {
return Ok(());
}
conn.execute_batch(&config_sql).map_err(|e| {
format!(
"Error configuring DuckDB resource limits: {}",
e.to_string()
)
})
}
fn setup_duckdb_connection(
conn: &duckdb::Connection,
token: &str,
base_internal_url: &str,
w_id: &str,
limits: &ResourceLimits,
) -> Result<(), String> {
configure_duckdb_resource_limits(conn, limits)?;
let (s3_access_key, s3_secret_key) = token.rsplit_once('.').unwrap_or(("", token));
let (s3_endpoint_ssl, s3_endpoint) = base_internal_url
.split_once("://")
@@ -249,10 +328,11 @@ fn prepare_duckdb_internal(
token: &str,
base_internal_url: &str,
w_id: &str,
limits: ResourceLimits,
) -> Result<String, String> {
let conn = duckdb::Connection::open_in_memory().map_err(|e| e.to_string())?;
setup_duckdb_connection(&conn, token, base_internal_url, w_id)?;
setup_duckdb_connection(&conn, token, base_internal_url, w_id, &limits)?;
let mut results: Vec<PrepareQueryResult> = vec![];
@@ -379,12 +459,13 @@ fn run_duckdb_internal<'a>(
token: &str,
base_internal_url: &str,
w_id: &str,
limits: ResourceLimits,
collect_last_only: bool,
collect_first_row_only: bool,
) -> Result<(String, Option<Vec<String>>), String> {
let conn = duckdb::Connection::open_in_memory().map_err(|e| e.to_string())?;
setup_duckdb_connection(&conn, token, base_internal_url, w_id)?;
setup_duckdb_connection(&conn, token, base_internal_url, w_id, &limits)?;
let mut results: Vec<Vec<Box<RawValue>>> = vec![];
let mut column_order = None;
+10 -16
View File
@@ -2634,12 +2634,14 @@ mod tests {
// Regression test for WIN-1957: two resources whose values reference each
// other via `$res:` must NOT recurse forever (stack overflow / process
// crash). With the depth guard the resolution terminates with an error.
#[tokio::test]
async fn test_transform_json_value_mutual_resource_recursion_terminates() {
let db_url = std::env::var("DATABASE_URL")
.unwrap_or("postgres://postgres:changeme@localhost:5432/windmill".to_string());
let pool = sqlx::PgPool::connect(&db_url).await.unwrap();
//
// This test needs the real `workspace`/`resource` schema, so it uses
// `#[sqlx::test]` which provisions a migrated ephemeral database per test
// (the bare `DATABASE_URL` database in CI has no migrations applied, which
// previously made the workspace INSERT panic with `relation "workspace"
// does not exist` — WIN-1958).
#[sqlx::test(migrations = "../migrations")]
async fn test_transform_json_value_mutual_resource_recursion_terminates(pool: DB) {
let w_id = format!("dostest{}", Uuid::new_v4().simple());
sqlx::query("INSERT INTO workspace (id, name, owner) VALUES ($1, $1, 'test@windmill.dev')")
@@ -2671,16 +2673,8 @@ mod tests {
)
.await;
// Clean up before asserting so a failed assertion doesn't leave rows.
let _ = sqlx::query("DELETE FROM resource WHERE workspace_id = $1")
.bind(&w_id)
.execute(&pool)
.await;
let _ = sqlx::query("DELETE FROM workspace WHERE id = $1")
.bind(&w_id)
.execute(&pool)
.await;
// The ephemeral test database is dropped automatically, so no manual
// row cleanup is required.
let err = result.expect_err("mutually recursive resources should error, not crash");
assert!(
err.to_string().contains("interpolation depth"),
+12 -5
View File
@@ -134,6 +134,8 @@ async fn list_variables(
"variable.expires_at",
"variable.labels",
"ws_specific.path IS NOT NULL as ws_specific",
"variable.edited_at",
"variable.edited_by",
])
.left()
.join("account")
@@ -216,6 +218,7 @@ async fn get_variable(
"SELECT variable.workspace_id, variable.path, variable.value, variable.is_secret,
variable.description, variable.extra_perms, variable.account, variable.is_oauth,
variable.expires_at, variable.labels,
variable.edited_at, variable.edited_by,
(now() > account.expires_at) as is_expired, account.refresh_error,
resource.path IS NOT NULL as is_linked,
account.refresh_token != '' as is_refreshed,
@@ -441,8 +444,8 @@ async fn create_variable(
sqlx::query!(
"INSERT INTO variable
(workspace_id, path, value, is_secret, description, account, is_oauth, expires_at, labels)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)",
(workspace_id, path, value, is_secret, description, account, is_oauth, expires_at, labels, edited_by)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)",
&w_id,
variable.path,
value,
@@ -451,7 +454,8 @@ async fn create_variable(
variable.account,
variable.is_oauth.unwrap_or(false),
variable.expires_at,
variable.labels.as_deref() as Option<&[String]>
variable.labels.as_deref() as Option<&[String]>,
&authed.username
)
.execute(&mut *tx)
.await?;
@@ -1048,6 +1052,8 @@ async fn update_variable(
}
let npath = if has_sql_updates {
sqlb.set("edited_at", "now()");
sqlb.set_str("edited_by", &authed.username);
sqlb.returning("path");
let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?;
let npath_o: Option<String> = sqlx::query_scalar(&sql).fetch_optional(&mut *tx).await?;
@@ -1078,10 +1084,11 @@ async fn update_variable(
if let Some(nlabels) = &ns.labels {
sqlx::query!(
"UPDATE variable SET labels = $1 WHERE path = $2 AND workspace_id = $3",
"UPDATE variable SET labels = $1, edited_at = now(), edited_by = $4 WHERE path = $2 AND workspace_id = $3",
nlabels as &[String],
&npath,
&w_id
&w_id,
&authed.username
)
.execute(&mut *tx)
.await?;
+2 -1
View File
@@ -13,7 +13,7 @@ default = []
private = ["windmill-worker-volumes/private", "windmill-queue/private", "windmill-common/private", "windmill-dep-map/private", "windmill-runtime-nativets?/private"]
mcp = ["windmill-ai/mcp", "dep:windmill-mcp"]
prometheus = ["dep:prometheus", "windmill-common/prometheus"]
enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker-volumes/enterprise", "windmill-runtime-nativets?/enterprise", "dep:pem", "dep:tokio-util", "dep:opentelemetry-proto", "dep:prost", "dep:hudsucker", "dep:rcgen", "dep:hyper-http-proxy", "dep:hyper-tls", "dep:hyper-util"]
enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker-volumes/enterprise", "windmill-runtime-nativets?/enterprise", "dep:pem", "dep:rsa", "dep:tokio-util", "dep:opentelemetry-proto", "dep:prost", "dep:hudsucker", "dep:rcgen", "dep:hyper-http-proxy", "dep:hyper-tls", "dep:hyper-util"]
mssql = ["dep:tiberius"]
mssql-kerberos = ["mssql", "tiberius/integrated-auth-gssapi"] # Linux/Unix integrated auth
mssql-winauth = ["mssql", "tiberius/winauth"] # Windows integrated auth
@@ -112,6 +112,7 @@ jsonwebtoken.workspace = true
sha2.workspace = true
hmac.workspace = true
pem = { workspace = true, optional = true }
rsa = { workspace = true, optional = true }
urlencoding.workspace = true
nix.workspace = true
bytes.workspace = true
+8
View File
@@ -4,6 +4,11 @@ from importlib.abc import MetaPathFinder, Loader
from importlib.machinery import ModuleSpec, SourceFileLoader
import time
# Injected by backend: maps script path -> temp storage hash so preview jobs
# resolve relative imports from not-yet-deployed local content. Empty ({}) for
# deployed runs.
TEMP_SCRIPT_REFS = TEMP_SCRIPT_REFS_PLACEHOLDER
class WindmillLoader(Loader):
def __init__(self, path):
self.path = path
@@ -48,6 +53,9 @@ class WindmillFinder(MetaPathFinder):
runnable_id = os.environ.get('WM_RUNNABLE_ID')
if runnable_id:
query_params += f"&cache_key={runnable_id}"
temp_hash = TEMP_SCRIPT_REFS.get(script_path) if TEMP_SCRIPT_REFS else None
if temp_hash:
query_params += f"&temp_script_hash={temp_hash}"
url = f"{os.environ.get('BASE_INTERNAL_URL')}/api/w/{os.environ.get('WM_WORKSPACE')}/scripts/raw/p/{script_path}.py{query_params}"
req = urllib.request.Request(url, None, headers)
@@ -90,7 +90,7 @@ mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size=500000000"
options: "size={NSJAIL_TMPFS_SIZE}"
}
@@ -90,7 +90,7 @@ mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size=500000000"
options: "size={NSJAIL_TMPFS_SIZE}"
}
mount {
@@ -53,7 +53,7 @@ mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size=500000000"
options: "size={NSJAIL_TMPFS_SIZE}"
}
mount {
@@ -70,7 +70,7 @@ mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size=800000000"
options: "size={NSJAIL_TMPFS_SIZE}"
}
mount {
@@ -72,7 +72,7 @@ mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size=800000000"
options: "size={NSJAIL_TMPFS_SIZE}"
}
mount {
@@ -64,7 +64,7 @@ mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size=500000000"
options: "size={NSJAIL_TMPFS_SIZE}"
}
mount {
@@ -61,7 +61,7 @@ mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size=500000000"
options: "size={NSJAIL_TMPFS_SIZE}"
}
@@ -54,7 +54,7 @@ mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size=500000000"
options: "size={NSJAIL_TMPFS_SIZE}"
}
@@ -55,7 +55,7 @@ mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size=500000000"
options: "size={NSJAIL_TMPFS_SIZE}"
}
@@ -55,7 +55,7 @@ mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size=800000000"
options: "size={NSJAIL_TMPFS_SIZE}"
}
mount {
@@ -55,7 +55,7 @@ mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size=800000000"
options: "size={NSJAIL_TMPFS_SIZE}"
}
mount {
@@ -68,7 +68,7 @@ mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size=800000000"
options: "size={NSJAIL_TMPFS_SIZE}"
}
mount {
@@ -58,7 +58,7 @@ mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size=500000000"
options: "size={NSJAIL_TMPFS_SIZE}"
}
mount {
@@ -55,7 +55,7 @@ mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size=500000000"
options: "size={NSJAIL_TMPFS_SIZE}"
}
@@ -55,7 +55,7 @@ mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size=500000000"
options: "size={NSJAIL_TMPFS_SIZE}"
}
@@ -54,7 +54,7 @@ mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size=500000000"
options: "size={NSJAIL_TMPFS_SIZE}"
}
@@ -30,8 +30,8 @@ use crate::{
bash_executor::BIN_BASH,
common::{
build_command_with_isolation, check_executor_binary_exists, get_reserved_variables,
read_and_check_result, resolve_nsjail_timeout, start_child_process, transform_json,
OccupancyMetrics,
read_and_check_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes,
start_child_process, transform_json, OccupancyMetrics,
},
handle_child::handle_child,
is_sandboxing_enabled,
@@ -1456,6 +1456,10 @@ mount {{
"{ADDITIONAL_PYTHON_PATHS}",
additional_python_paths_folders.as_str(),
)
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
)
.replace("{TIMEOUT}", &nsjail_timeout),
)?;
} else {
+6 -2
View File
@@ -41,8 +41,8 @@ use crate::handle_child::run_future_with_polling_update_job_poller;
use crate::{
common::{
build_args_map, build_command_with_isolation, get_reserved_variables, read_file,
read_file_content, resolve_nsjail_timeout, start_child_process, OccupancyMetrics,
DEV_CONF_NSJAIL,
read_file_content, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process,
OccupancyMetrics, DEV_CONF_NSJAIL,
},
get_proxy_envs_for_lang,
handle_child::handle_child,
@@ -215,6 +215,10 @@ exit $exit_status
.replace("{SHARED_MOUNT}", shared_mount)
.replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
.replace("#{DEV}", DEV_CONF_NSJAIL)
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
)
.replace("{TIMEOUT}", &nsjail_timeout),
)?;
let mut cmd_args = vec![
+49 -7
View File
@@ -16,8 +16,8 @@ use crate::{
common::{
build_command_with_isolation, create_args_and_out_file, get_reserved_variables,
parse_npm_config, read_file, read_file_content, read_result, resolve_nsjail_timeout,
start_child_process, write_file_binary, MaybeLock, OccupancyMetrics, StreamNotifier,
DEV_CONF_NSJAIL,
resolve_nsjail_tmpfs_size_bytes, start_child_process, write_file_binary, MaybeLock,
OccupancyMetrics, StreamNotifier, DEV_CONF_NSJAIL,
},
get_proxy_envs_for_lang,
handle_child::handle_child,
@@ -28,6 +28,7 @@ use crate::{
};
use windmill_common::{
client::AuthedClient,
jobs::JobKind,
scripts::{id_to_codebase_info, CodebaseInfo, ScriptLang},
utils::WarnAfterExt,
workspace_dependencies::WorkspaceDependenciesPrefetched,
@@ -1160,8 +1161,15 @@ pub async fn prebundle_bun_script(
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
temp_script_refs: &Option<HashMap<String, String>>,
) -> Result<()> {
let (local_path, remote_path) =
compute_bundle_local_and_remote_path(inner_content, lock, script_path, db, w_id).await;
let (local_path, remote_path) = compute_bundle_local_and_remote_path(
inner_content,
lock,
script_path,
db,
w_id,
temp_script_refs,
)
.await;
if exists_in_cache(&local_path, &remote_path).await {
return Ok(());
}
@@ -1252,6 +1260,7 @@ pub async fn compute_bundle_local_and_remote_path(
script_path: &str,
db: Option<&DB>,
w_id: &str,
temp_script_refs: &Option<HashMap<String, String>>,
) -> (String, String) {
let mut input_src = format!("{inner_content}{lock}",);
@@ -1269,6 +1278,18 @@ pub async fn compute_bundle_local_and_remote_path(
}
};
// Keep temp-script-ref (preview) bundles in a distinct cache slot: their
// imports come from not-yet-deployed local content, so they must neither
// reuse a deployed-content bundle nor be saved under the deployed key.
if let Some(refs) = temp_script_refs {
let mut entries: Vec<(&String, &String)> = refs.iter().collect();
entries.sort();
for (path, hash) in entries {
input_src.push_str(path);
input_src.push_str(hash);
}
}
let ws_suffix = crate::workspace_registry_cache_suffix(w_id).await;
input_src.push_str(&ws_suffix);
let hash = windmill_common::utils::calculate_hash(&input_src);
@@ -1333,6 +1354,22 @@ pub async fn handle_bun_job(
) -> error::Result<Box<RawValue>> {
let mut annotation = windmill_common::worker::TypeScriptAnnotations::parse(inner_content);
// Preview jobs may carry _TEMP_SCRIPT_REFS so relative imports resolve from
// not-yet-deployed local content uploaded to raw_script_temp. Extracted up
// front so it reaches both lockfile generation and the runtime loader.
// Gated on JobKind::Preview because job.args includes caller-controlled
// request args; honoring this key on deployed runs would let a caller swap
// import resolution targets in deployed code.
let temp_script_refs: Option<HashMap<String, String>> = if matches!(job.kind, JobKind::Preview)
{
job.args
.as_ref()
.and_then(|x| x.get("_TEMP_SCRIPT_REFS"))
.and_then(|v| serde_json::from_str(v.get()).ok())
} else {
None
};
if annotation.sandbox && NSJAIL_AVAILABLE.is_none() {
return Err(error::Error::ExecutionErr(
"Script has //sandbox annotation but nsjail is not available on this worker. \
@@ -1353,6 +1390,7 @@ pub async fn handle_bun_job(
job.runnable_path(),
Some(db),
&job.workspace_id,
&temp_script_refs,
)
.await
}
@@ -1512,7 +1550,7 @@ pub async fn handle_bun_job(
workspace_dependencies,
annotation.npm,
&mut Some(occupancy_metrics),
&None,
&temp_script_refs,
wac_replay_info.is_some(),
)
.await?;
@@ -1886,7 +1924,7 @@ try {{
} else {
LoaderMode::BunBundle
},
&None,
&temp_script_refs,
)
.await?;
@@ -1903,7 +1941,7 @@ try {{
} else {
LoaderMode::Bun
},
&None,
&temp_script_refs,
)
.await
} else {
@@ -2147,6 +2185,10 @@ try {{
)
.replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
.replace("#{DEV}", DEV_CONF_NSJAIL)
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
)
.replace("{TIMEOUT}", &nsjail_timeout),
)?;
+20 -1
View File
@@ -47,7 +47,9 @@ use windmill_common::{variables, DB};
use tokio::{io::AsyncWriteExt, time::Instant};
use crate::agent_workers::UPDATE_PING_URL;
use crate::{JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, MAX_TIMEOUT_DURATION, PATH_ENV};
use crate::{
JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, MAX_TIMEOUT_DURATION, NSJAIL_TMPFS_SIZE_MB, PATH_ENV,
};
use windmill_common::client::AuthedClient;
/// Additional nsjail config for development. Currently used for nix flake.
@@ -114,6 +116,7 @@ pub async fn create_args_and_out_file(
if let Some(args) = job.args.as_ref() {
if let Some(mut x) = transform_json(client, &job.workspace_id, &args.0, job, conn).await? {
x.remove("_MODULES");
x.remove("_TEMP_SCRIPT_REFS");
write_file(
job_dir,
"args.json",
@@ -122,6 +125,7 @@ pub async fn create_args_and_out_file(
} else {
let mut filtered = args.0.clone();
filtered.remove("_MODULES");
filtered.remove("_TEMP_SCRIPT_REFS");
write_file(
job_dir,
"args.json",
@@ -1004,6 +1008,21 @@ pub async fn resolve_nsjail_timeout(
(duration.as_secs() + 15).to_string()
}
/// Default size (in bytes) of the `/tmp` tmpfs mount inside nsjail sandboxes,
/// used when the `nsjail_tmpfs_size_mb` instance setting is unset.
pub const DEFAULT_NSJAIL_TMPFS_SIZE_BYTES: u64 = 800_000_000;
/// Resolve the tmpfs `size=` value (in bytes, formatted for the nsjail proto)
/// for the `/tmp` tmpfs mount. When the `nsjail_tmpfs_size_mb` instance setting
/// is `None`, `Some(0)`, or negative, falls back to
/// [`DEFAULT_NSJAIL_TMPFS_SIZE_BYTES`].
pub async fn resolve_nsjail_tmpfs_size_bytes() -> String {
match *NSJAIL_TMPFS_SIZE_MB.read().await {
Some(mb) if mb > 0 => ((mb as u64).saturating_mul(1_000_000)).to_string(),
_ => DEFAULT_NSJAIL_TMPFS_SIZE_BYTES.to_string(),
}
}
async fn hash_args(
#[allow(unused)] db: &DB,
#[allow(unused)] client: &AuthedClient,
@@ -27,8 +27,8 @@ use windmill_queue::CanceledBy;
use crate::{
common::{
build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file,
get_reserved_variables, read_result, resolve_nsjail_timeout, start_child_process,
DEV_CONF_NSJAIL,
get_reserved_variables, read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes,
start_child_process, DEV_CONF_NSJAIL,
},
get_proxy_envs_for_lang,
handle_child::handle_child,
@@ -603,6 +603,10 @@ pub async fn handle_csharp_job(
.replace("{SHARED_MOUNT}", shared_mount)
.replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
.replace("#{DEV}", DEV_CONF_NSJAIL)
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
)
.replace("{TIMEOUT}", &nsjail_timeout),
)?;
let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
+89 -2
View File
@@ -11,7 +11,7 @@ use serde_json::{json, Value};
use uuid::Uuid;
use windmill_common::error::{to_anyhow, Error, Result};
use windmill_common::utils::sanitize_string_from_password;
use windmill_common::worker::{Connection, SqlResultCollectionStrategy};
use windmill_common::worker::{get_memory, Connection, SqlResultCollectionStrategy};
use windmill_common::workspaces::{
get_datatable_resource_from_db_unchecked, get_ducklake_from_db_unchecked,
DucklakeCatalogResourceType,
@@ -44,6 +44,7 @@ pub async fn do_duckdb(
#[allow(unused_variables)] column_order_ref: &mut Option<Vec<String>>,
occupancy_metrics: &mut OccupancyMetrics,
parent_runnable_path: Option<String>,
job_dir: &str,
run_inline: bool,
) -> Result<Box<RawValue>> {
let annotations = windmill_common::worker::SqlAnnotations::parse(query);
@@ -160,6 +161,7 @@ pub async fn do_duckdb(
let base_internal_url = client.base_internal_url.clone();
let w_id = job.workspace_id.clone();
let job_dir = job_dir.to_string();
if annotations.prepare {
let result = tokio::task::spawn_blocking(move || {
@@ -168,6 +170,7 @@ pub async fn do_duckdb(
&token,
&base_internal_url,
&w_id,
&job_dir,
)
})
.await
@@ -185,6 +188,7 @@ pub async fn do_duckdb(
&token,
&base_internal_url,
&w_id,
&job_dir,
collection_strategy,
)
})
@@ -259,6 +263,8 @@ struct DuckDbFfiLib {
token: *const c_char,
base_internal_url: *const c_char,
w_id: *const c_char,
memory_limit: *const c_char,
temp_directory: *const c_char,
column_order_ptr: *mut *mut c_char,
collect_last_only: bool,
collect_first_row_only: bool,
@@ -273,6 +279,8 @@ struct DuckDbFfiLib {
token: *const c_char,
base_internal_url: *const c_char,
w_id: *const c_char,
memory_limit: *const c_char,
temp_directory: *const c_char,
) -> *mut c_char,
>,
>,
@@ -319,7 +327,7 @@ impl DuckDbFfiLib {
// Version mismatch should only be possible on Windows agent workers
// We check for it because FFI interface mismatch will cause undefined behavior / crashes
unsafe {
let expected_version: c_uint = 1;
let expected_version: c_uint = 2;
let get_version: Symbol<'static, unsafe extern "C" fn() -> c_uint> =
lib.get(b"get_version")
.map_err(|e| return Error::ExecutionErr(format!("Could not find get_version in the duckdb ffi library. If you are not using docker, consider manually upgrading windmill_duckdb_ffi_lib. {}", e.to_string())))?;
@@ -345,6 +353,35 @@ impl DuckDbFfiLib {
}
}
// 20% headroom for Rust runtime + DuckDB's untracked allocations. Mirrors
// DuckDB's own default ratio, but applied to the worker's cgroup budget
// instead of host RAM.
const DUCKDB_MEMORY_FRACTION: f64 = 0.8;
// Treat cgroup values above 1 PiB as "unlimited" (kernels report page-aligned
// huge numbers when uncapped). get_memory() falls back to host RAM in that
// case, which is exactly what we want to leave to DuckDB's own default.
const CGROUP_UNLIMITED_THRESHOLD: i64 = 1024 * 1024 * 1024 * 1024 * 1024;
// `DUCKDB_MEMORY_LIMIT` env override, else fraction of the worker's cgroup
// memory (as reported by windmill-common), else None (keep DuckDB's default).
fn resolve_duckdb_memory_limit() -> Option<String> {
if let Ok(v) = env::var("DUCKDB_MEMORY_LIMIT") {
let v = v.trim();
if !v.is_empty() {
return Some(v.to_string());
}
}
cgroup_bytes_to_duckdb_memory_limit(get_memory()?)
}
fn cgroup_bytes_to_duckdb_memory_limit(bytes: i64) -> Option<String> {
if bytes <= 0 || bytes >= CGROUP_UNLIMITED_THRESHOLD {
return None;
}
let mib = ((bytes as f64 * DUCKDB_MEMORY_FRACTION) as i64) / (1024 * 1024);
Some(format!("{}MiB", mib.max(64)))
}
// Read backend/windmill-duckdb-ffi-internal/README_DEV.md for details about why we use FFI
fn run_duckdb_ffi_safe<'a>(
query_block_list: impl Iterator<Item = &'a str>,
@@ -353,6 +390,7 @@ fn run_duckdb_ffi_safe<'a>(
token: &str,
base_internal_url: &str,
w_id: &str,
job_dir: &str,
collection_strategy: SqlResultCollectionStrategy,
) -> Result<(Box<RawValue>, Option<Vec<String>>)> {
let query_block_list = query_block_list
@@ -372,6 +410,9 @@ fn run_duckdb_ffi_safe<'a>(
let token = CString::new(token).map_err(to_anyhow)?;
let base_internal_url = CString::new(base_internal_url).map_err(to_anyhow)?;
let w_id = CString::new(w_id).map_err(to_anyhow)?;
let memory_limit =
CString::new(resolve_duckdb_memory_limit().unwrap_or_default()).map_err(to_anyhow)?;
let temp_directory = CString::new(job_dir).map_err(to_anyhow)?;
let run_duckdb_ffi = &DuckDbFfiLib::get_singleton()?.run_duckdb_ffi;
let free_cstr = &DuckDbFfiLib::get_singleton()?.free_cstr;
@@ -384,6 +425,8 @@ fn run_duckdb_ffi_safe<'a>(
token.as_ptr(),
base_internal_url.as_ptr(),
w_id.as_ptr(),
memory_limit.as_ptr(),
temp_directory.as_ptr(),
&mut column_order,
collection_strategy.collect_last_statement_only(query_block_list_count),
collection_strategy.collect_first_row_only(),
@@ -424,6 +467,7 @@ fn prepare_duckdb_ffi_safe<'a>(
token: &str,
base_internal_url: &str,
w_id: &str,
job_dir: &str,
) -> Result<Box<RawValue>> {
let query_block_list = query_block_list
.map(|s| {
@@ -440,6 +484,9 @@ fn prepare_duckdb_ffi_safe<'a>(
let token = CString::new(token).map_err(to_anyhow)?;
let base_internal_url = CString::new(base_internal_url).map_err(to_anyhow)?;
let w_id = CString::new(w_id).map_err(to_anyhow)?;
let memory_limit =
CString::new(resolve_duckdb_memory_limit().unwrap_or_default()).map_err(to_anyhow)?;
let temp_directory = CString::new(job_dir).map_err(to_anyhow)?;
let lib = DuckDbFfiLib::get_singleton()?;
let prepare_fn = lib.prepare_duckdb_ffi.as_ref().ok_or_else(|| {
@@ -456,6 +503,8 @@ fn prepare_duckdb_ffi_safe<'a>(
token.as_ptr(),
base_internal_url.as_ptr(),
w_id.as_ptr(),
memory_limit.as_ptr(),
temp_directory.as_ptr(),
);
let str = CStr::from_ptr(ptr).to_string_lossy().to_string();
free_cstr(ptr);
@@ -783,6 +832,44 @@ pub struct Arg {
mod tests {
use super::*;
#[test]
fn cgroup_bytes_unlimited_or_invalid_returns_none() {
assert_eq!(cgroup_bytes_to_duckdb_memory_limit(0), None);
assert_eq!(cgroup_bytes_to_duckdb_memory_limit(-1), None);
// 1 PiB sentinel: cgroup v1 reports ~i64::MAX when uncapped.
assert_eq!(
cgroup_bytes_to_duckdb_memory_limit(CGROUP_UNLIMITED_THRESHOLD),
None
);
}
#[test]
fn cgroup_bytes_real_values_take_80_percent() {
// 1 GiB -> 80% -> 819 MiB (floored to MiB)
assert_eq!(
cgroup_bytes_to_duckdb_memory_limit(1024 * 1024 * 1024),
Some("819MiB".to_string())
);
// 4 GiB -> 3276 MiB
assert_eq!(
cgroup_bytes_to_duckdb_memory_limit(4 * 1024 * 1024 * 1024),
Some("3276MiB".to_string())
);
}
#[test]
fn cgroup_bytes_tiny_values_floored_to_64mib() {
// Tiny cgroup must not produce a 0/unusable limit.
assert_eq!(
cgroup_bytes_to_duckdb_memory_limit(1024 * 1024),
Some("64MiB".to_string())
);
assert_eq!(
cgroup_bytes_to_duckdb_memory_limit(1),
Some("64MiB".to_string())
);
}
// Tests for parse_attach_db_resource function
#[test]
fn test_parse_attach_db_resource_postgres_res_prefix() {
+6 -2
View File
@@ -22,8 +22,8 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob};
use crate::{
common::{
build_command_with_isolation, capitalize, create_args_and_out_file, get_reserved_variables,
read_result, resolve_nsjail_timeout, start_child_process, OccupancyMetrics,
DEV_CONF_NSJAIL,
read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process,
OccupancyMetrics, DEV_CONF_NSJAIL,
},
handle_child::handle_child,
is_sandboxing_enabled, read_ee_registry, DISABLE_NUSER, GOPRIVATE, GOPROXY, GO_BIN_CACHE_DIR,
@@ -351,6 +351,10 @@ func Run(req Req) (interface{{}}, error){{
.replace("{SHARED_MOUNT}", shared_mount)
.replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
.replace("#{DEV}", DEV_CONF_NSJAIL)
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
)
.replace("{TIMEOUT}", &nsjail_timeout),
)?;
let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
+6 -1
View File
@@ -23,7 +23,8 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob};
use crate::{
common::{
build_command_with_isolation, create_args_and_out_file, get_reserved_variables,
read_result, resolve_nsjail_timeout, start_child_process, OccupancyMetrics,
read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process,
OccupancyMetrics,
},
handle_child, is_sandboxing_enabled, read_ee_registry_bool_with_workspace_override,
read_ee_registry_with_workspace_override,
@@ -669,6 +670,10 @@ async fn run<'a>(
.replace("{SHARED_MOUNT}", &shared_mount)
// .replace("{CACHED_TARGET}", &shared_mount)
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
)
.replace("{TIMEOUT}", &nsjail_timeout),
)?;
let mut cmd = Command::new(NSJAIL_PATH.as_str());
+6 -2
View File
@@ -14,8 +14,8 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob};
use crate::{
common::{
build_command_with_isolation, create_args_and_out_file, get_reserved_variables,
read_result, resolve_nsjail_timeout, start_child_process, OccupancyMetrics,
DEV_CONF_NSJAIL,
read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process,
OccupancyMetrics, DEV_CONF_NSJAIL,
},
get_proxy_envs_for_lang, handle_child, is_sandboxing_enabled, DISABLE_NUSER, NSJAIL_PATH,
PATH_ENV, TRACING_PROXY_CA_CERT_PATH,
@@ -258,6 +258,10 @@ async fn run<'a>(
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
.replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
.replace("#{DEV}", DEV_CONF_NSJAIL)
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
)
.replace("{TIMEOUT}", &nsjail_timeout),
)?;
let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
+6 -2
View File
@@ -20,8 +20,8 @@ use windmill_queue::{append_logs, CanceledBy};
use crate::{
common::{
build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file,
get_reserved_variables, read_result, resolve_nsjail_timeout, start_child_process,
MaybeLock, OccupancyMetrics,
get_reserved_variables, read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes,
start_child_process, MaybeLock, OccupancyMetrics,
},
handle_child::handle_child,
is_sandboxing_enabled, COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH,
@@ -425,6 +425,10 @@ try {{
.replace("{JOB_DIR}", job_dir)
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
.replace("{SHARED_MOUNT}", shared_mount)
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
)
.replace("{TIMEOUT}", &nsjail_timeout),
)?;
+6 -2
View File
@@ -26,8 +26,8 @@ lazy_static::lazy_static! {
use crate::{
common::{
build_args_map, build_command_with_isolation, get_reserved_variables, read_file,
read_file_content, resolve_nsjail_timeout, start_child_process, MaybeLock,
OccupancyMetrics,
read_file_content, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process,
MaybeLock, OccupancyMetrics,
},
handle_child::handle_child,
is_sandboxing_enabled, read_ee_registry_with_workspace_override, DISABLE_NUSER, HOME_ENV,
@@ -682,6 +682,10 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"",
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
.replace("{SHARED_MOUNT}", shared_mount)
.replace("{CACHE_DIR}", &*POWERSHELL_CACHE_DIR)
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
)
.replace("{TIMEOUT}", &nsjail_timeout),
)?;
let cmd_args = vec![
+51 -5
View File
@@ -31,6 +31,7 @@ use windmill_common::{
self,
Error::{self},
},
jobs::JobKind,
scripts::ScriptLang,
utils::calculate_hash,
worker::{
@@ -119,6 +120,18 @@ const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = include_str!("../nsjail/download
const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str = include_str!("../nsjail/run.python3.config.proto");
pub const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py");
/// Render loader.py with the TEMP_SCRIPT_REFS placeholder substituted by a
/// Python dict literal. Preview jobs pass a path -> temp-hash map so relative
/// imports resolve from not-yet-deployed local content; deployed runs pass
/// `None` which renders an empty dict (deployed resolution unchanged).
fn render_relative_python_loader(temp_script_refs: &Option<HashMap<String, String>>) -> String {
let temp_refs_py = temp_script_refs
.as_ref()
.and_then(|m| serde_json::to_string(m).ok())
.unwrap_or_else(|| "{}".to_string());
RELATIVE_PYTHON_LOADER.replace("TEMP_SCRIPT_REFS_PLACEHOLDER", &temp_refs_py)
}
#[cfg(any(feature = "private", test))]
pub fn has_relative_imports(content: &str) -> bool {
RELATIVE_IMPORT_REGEX.is_match(content)
@@ -133,8 +146,8 @@ use windmill_object_store::OBJECT_STORE_SETTINGS;
use crate::{
common::{
build_command_with_isolation, create_args_and_out_file, get_reserved_variables, read_file,
read_result, resolve_nsjail_timeout, start_child_process, OccupancyMetrics, StreamNotifier,
DEV_CONF_NSJAIL,
read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process,
OccupancyMetrics, StreamNotifier, DEV_CONF_NSJAIL,
},
get_proxy_envs_for_lang,
handle_child::handle_child,
@@ -643,6 +656,21 @@ pub async fn handle_python_job(
));
}
// Preview jobs may carry _TEMP_SCRIPT_REFS so relative imports resolve from
// not-yet-deployed local content uploaded to raw_script_temp. Gated on
// JobKind::Preview because job.args includes caller-controlled request
// args; honoring this key on deployed runs would let a caller swap import
// resolution targets in deployed code.
let temp_script_refs: Option<HashMap<String, String>> = if matches!(job.kind, JobKind::Preview)
{
job.args
.as_ref()
.and_then(|x| x.get("_TEMP_SCRIPT_REFS"))
.and_then(|v| serde_json::from_str(v.get()).ok())
} else {
None
};
let (py_version, mut additional_python_paths) = handle_python_deps(
job_dir,
requirements_o,
@@ -658,6 +686,7 @@ pub async fn handle_python_job(
&mut Some(occupancy_metrics),
precomputed_agent_info,
annotations.clone(),
&temp_script_refs,
)
.await?;
@@ -709,6 +738,7 @@ pub async fn handle_python_job(
job.script_entrypoint_override.as_deref(),
inner_content,
&script_path,
&temp_script_refs,
)
.await?;
@@ -993,6 +1023,10 @@ mount {{
)
.replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
.replace("#{DEV}", DEV_CONF_NSJAIL)
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
)
.replace("{TIMEOUT}", &nsjail_timeout),
)?;
} else {
@@ -1532,6 +1566,7 @@ async fn prepare_wrapper(
job_script_entrypoint_override: Option<&str>,
inner_content: &str,
script_path: &str,
temp_script_refs: &Option<HashMap<String, String>>,
) -> error::Result<(
&'static str,
&'static str,
@@ -1571,7 +1606,11 @@ async fn prepare_wrapper(
let _ = write_file(&module_dir, &format!("{last}.py"), inner_content)?;
if relative_imports {
let _ = write_file(job_dir, "loader.py", RELATIVE_PYTHON_LOADER)?;
let _ = write_file(
job_dir,
"loader.py",
&render_relative_python_loader(temp_script_refs),
)?;
}
let sig = windmill_parser_py::parse_python_signature(
@@ -1768,6 +1807,7 @@ pub(crate) async fn handle_python_deps(
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
precomputed_agent_info: Option<PrecomputedAgentInfo>,
annotations: PythonAnnotations,
temp_script_refs: &Option<HashMap<String, String>>,
) -> error::Result<(PyV, Vec<String>)> {
create_dependencies_dir(job_dir).await;
@@ -1796,7 +1836,7 @@ pub(crate) async fn handle_python_deps(
&mut version_specifiers,
&mut locked_v,
&None,
&None, // temp_script_refs: only used during CLI lock generation
temp_script_refs,
))
.await?;
@@ -2007,6 +2047,10 @@ async fn spawn_uv_install(
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
.replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
.replace("#{DEV}", DEV_CONF_NSJAIL)
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
)
.as_str(),
)?;
@@ -3042,7 +3086,8 @@ pub async fn start_worker(
let any_relative_imports = RELATIVE_IMPORT_REGEX.is_match(inner_content);
if any_relative_imports {
let _ = write_file(job_dir, "loader.py", RELATIVE_PYTHON_LOADER)?;
// Dedicated worker runs deployed scripts only — no temp refs.
let _ = write_file(job_dir, "loader.py", &render_relative_python_loader(&None))?;
}
let mut mem_peak: i32 = 0;
@@ -3087,6 +3132,7 @@ pub async fn start_worker(
&mut None,
None,
annotations,
&None, // dedicated worker runs deployed scripts only
)
.await?;
+6 -1
View File
@@ -20,7 +20,8 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob};
use crate::{
common::{
build_command_with_isolation, create_args_and_out_file, get_reserved_variables,
read_result, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL,
read_result, resolve_nsjail_tmpfs_size_bytes, start_child_process, OccupancyMetrics,
DEV_CONF_NSJAIL,
},
get_proxy_envs_for_lang,
handle_child::{self},
@@ -580,6 +581,10 @@ async fn run<'a>(
.replace("{R_CACHE_DIR}", &*R_CACHE_DIR)
.replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
.replace("#{DEV}", DEV_CONF_NSJAIL)
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
)
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()),
)?;
let mut cmd = Command::new(NSJAIL_PATH.as_str());
+5 -2
View File
@@ -23,8 +23,8 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob};
use crate::{
common::{
build_command_with_isolation, create_args_and_out_file, get_reserved_variables,
read_result, resolve_nsjail_timeout, start_child_process, OccupancyMetrics,
DEV_CONF_NSJAIL,
read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process,
OccupancyMetrics, DEV_CONF_NSJAIL,
},
get_proxy_envs_for_lang,
handle_child::{self},
@@ -619,6 +619,7 @@ async fn install<'a>(
envs.clone(),
get_reserved_variables(job, &client.token, conn, parent_runnable_path.clone()).await?,
);
let nsjail_tmpfs_size = resolve_nsjail_tmpfs_size_bytes().await;
par_install_language_dependencies_seq(
InstallDeps::Flat(deps.clone()),
"ruby",
@@ -638,6 +639,7 @@ async fn install<'a>(
.replace("{TARGET}", &dependency.path)
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
.replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
.replace("{NSJAIL_TMPFS_SIZE}", &nsjail_tmpfs_size)
.replace("#{DEV}", DEV_CONF_NSJAIL), // .replace("{BUILD}", &build_dir),
)?;
let mut cmd = Command::new(NSJAIL_PATH.as_str());
@@ -810,6 +812,7 @@ mount {{
.replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
.replace("#{DEV}", DEV_CONF_NSJAIL)
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
.replace("{NSJAIL_TMPFS_SIZE}", &resolve_nsjail_tmpfs_size_bytes().await)
.replace("{TIMEOUT}", &nsjail_timeout),
)?;
let mut cmd = Command::new(NSJAIL_PATH.as_str());
+10 -2
View File
@@ -23,8 +23,8 @@ use windmill_queue::{append_logs, CanceledBy};
use crate::{
common::{
build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file,
get_reserved_variables, read_result, resolve_nsjail_timeout, start_child_process,
OccupancyMetrics, DEV_CONF_NSJAIL,
get_reserved_variables, read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes,
start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL,
},
get_proxy_envs_for_lang,
handle_child::handle_child,
@@ -480,6 +480,10 @@ pub async fn build_rust_crate(
.replace("{RUSTUP_HOME}", RUSTUP_HOME.as_str())
.replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
.replace("#{DEV}", DEV_CONF_NSJAIL)
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
)
.replace("{BUILD}", &build_dir),
)?;
let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
@@ -701,6 +705,10 @@ pub async fn handle_rust_job(
.replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
.replace("#{DEV}", DEV_CONF_NSJAIL)
.replace("{SHARED_MOUNT}", shared_mount)
.replace(
"{NSJAIL_TMPFS_SIZE}",
&resolve_nsjail_tmpfs_size_bytes().await,
)
.replace("{TIMEOUT}", &nsjail_timeout),
)?;
let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
@@ -630,14 +630,50 @@ pub async fn do_snowflake(
)
.to_uppercase();
let public_key = match database.public_key.as_deref() {
Some(key) => pem::parse(key.as_bytes()).map_err(|e| {
Error::ExecutionErr(format!("Failed to parse public key: {}", e.to_string()))
})?,
None => return Err(Error::ExecutionErr("Public key is missing".to_string())),
let public_key_der: Vec<u8> = match database
.public_key
.as_deref()
.map(str::trim)
.filter(|s| !s.is_empty())
{
Some(key) => pem::parse(key.as_bytes())
.map_err(|e| Error::ExecutionErr(format!("Failed to parse public key: {e}")))?
.into_contents(),
None => {
// Derive the public key from the private key — RSA private keys
// contain the public components (n, e).
use rsa::pkcs8::{DecodePrivateKey, EncodePublicKey};
let pk_pem = database
.private_key
.as_deref()
.map(str::trim)
.filter(|s| !s.is_empty())
.ok_or_else(|| {
Error::ExecutionErr(
"Either public_key or private_key must be provided".to_string(),
)
})?;
let rsa_priv = rsa::RsaPrivateKey::from_pkcs8_pem(pk_pem)
.or_else(|_| {
use rsa::pkcs1::DecodeRsaPrivateKey;
rsa::RsaPrivateKey::from_pkcs1_pem(pk_pem)
})
.map_err(|e| {
Error::ExecutionErr(format!(
"Failed to parse private key to derive public key: {e}"
))
})?;
let rsa_pub = rsa::RsaPublicKey::from(&rsa_priv);
rsa_pub
.to_public_key_der()
.map_err(|e| {
Error::ExecutionErr(format!("Failed to encode derived public key: {e}"))
})?
.to_vec()
}
};
let mut public_key_hash = Sha256::new();
public_key_hash.update(public_key.contents());
public_key_hash.update(&public_key_der);
let public_key_fp = engine::general_purpose::STANDARD.encode(public_key_hash.finalize());
+8
View File
@@ -681,6 +681,13 @@ lazy_static::lazy_static! {
pub static ref FLOW_RUNNER_RUNNING: Mutex<bool> = Mutex::new(false);
}
lazy_static::lazy_static! {
/// Optional override for the size of the `/tmp` tmpfs mount in nsjail sandboxes (in megabytes).
/// When `None` (or non-positive), executors fall back to the unified
/// `DEFAULT_NSJAIL_TMPFS_SIZE_BYTES` (800MB).
pub static ref NSJAIL_TMPFS_SIZE_MB: Arc<RwLock<Option<i64>>> = Arc::new(RwLock::new(None));
}
pub fn sleep_queue() -> u64 {
if NATIVE_MODE_RESOLVED.load(std::sync::atomic::Ordering::Relaxed) {
300
@@ -4815,6 +4822,7 @@ pub async fn run_language_executor(
column_order,
occupancy_metrics,
parent_runnable_path,
job_dir,
run_inline,
))
.await;
+17 -1
View File
@@ -4061,7 +4061,7 @@ async fn push_next_flow_job(
_ => nargs,
};
let push_args;
let mut push_args;
let err;
let ov;
@@ -4077,6 +4077,22 @@ async fn push_next_flow_job(
}
};
// Propagate temp script refs from the flow preview job to each step so
// relative imports in inline scripts resolve from not-yet-deployed local
// content (uploaded to raw_script_temp) instead of the deployed script.
// Gated on JobKind::FlowPreview because flow_job.args includes
// caller-controlled request args; honoring this key on deployed flow
// runs would let a caller swap import resolution targets in deployed
// step code.
if matches!(flow_job.kind, JobKind::FlowPreview) {
if let Some(temp_script_refs) = arc_flow_job_args.as_ref().get("_TEMP_SCRIPT_REFS") {
push_args
.extra
.get_or_insert_with(HashMap::new)
.insert("_TEMP_SCRIPT_REFS".to_string(), temp_script_refs.clone());
}
}
tracing::debug!(id = %flow_job.id, root_id = %job_root, "computed args for job {i} of {len}");
let value_with_parallel = module.get_value_with_parallel()?;
+18 -18
View File
@@ -19,18 +19,18 @@
"open": "^10.0.0",
"svelte": "^5.45.2",
"tar-stream": "^3.1.7",
"windmill-parser-wasm-csharp": "*",
"windmill-parser-wasm-go": "*",
"windmill-parser-wasm-java": "*",
"windmill-parser-wasm-nu": "*",
"windmill-parser-wasm-php": "*",
"windmill-parser-wasm-py": "^1.693.1",
"windmill-parser-wasm-py-imports": "^1.693.1",
"windmill-parser-wasm-regex": "*",
"windmill-parser-wasm-ruby": "*",
"windmill-parser-wasm-rust": "*",
"windmill-parser-wasm-ts": "^1.693.1",
"windmill-parser-wasm-yaml": "*",
"windmill-parser-wasm-csharp": "1.510.1",
"windmill-parser-wasm-go": "1.510.1",
"windmill-parser-wasm-java": "1.510.1",
"windmill-parser-wasm-nu": "1.510.1",
"windmill-parser-wasm-php": "1.647.1",
"windmill-parser-wasm-py": "1.693.1",
"windmill-parser-wasm-py-imports": "1.693.1",
"windmill-parser-wasm-regex": "1.692.0",
"windmill-parser-wasm-ruby": "1.526.1",
"windmill-parser-wasm-rust": "1.647.1",
"windmill-parser-wasm-ts": "1.695.0",
"windmill-parser-wasm-yaml": "1.593.0",
"windmill-yaml-validator": "1.1.1",
"ws": "8.18.0",
"yaml": "^2.7.0"
@@ -1438,9 +1438,9 @@
"integrity": "sha512-FC0KbREe2G/sa/9kYIR930wmWw+VL6PvEIqg12J3dsJes3A+0x5JIUPT/jeD+c24DrG0ko/Ub7yDnYs56Bem7g=="
},
"node_modules/windmill-parser-wasm-regex": {
"version": "1.639.0",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.639.0.tgz",
"integrity": "sha512-qvYM4sYxB6M0xrqwBljS2fWqOMk6rp++60TRltJnzZDzVaWQrKjTGwNMmfepGAIWy1OGVKp0SCVERhe2P+O6tQ=="
"version": "1.692.0",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.692.0.tgz",
"integrity": "sha512-BHGTxrinZJ9ef6hFxbKiBqBEr5uqgG/QySOgMA5r1LswO9n/8fyGswr8JcPT2kGaoeoweV6/RQ+RHVaOhosnKw=="
},
"node_modules/windmill-parser-wasm-ruby": {
"version": "1.526.1",
@@ -1453,9 +1453,9 @@
"integrity": "sha512-9yGLYZX2Hn9TdTqGY/5Fp50ftzgUsrfBkSK9vJkKJd5Amyg+yXLBGzd8pz6Org+4uxMenz/16wpsgijvo6uhhQ=="
},
"node_modules/windmill-parser-wasm-ts": {
"version": "1.693.1",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.693.1.tgz",
"integrity": "sha512-xrPgVWwQbOWJKiz68wBDNMrKVOtCY/utyhzSx0kFYIWm/QH/6L8k6LLjo4DOn+PnlBNRXc7qum0sDyB8IuuJYQ=="
"version": "1.695.0",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.695.0.tgz",
"integrity": "sha512-9EFxeRZWmfb7EyhSlcG7dzTTKETPRYAvpRlxxLkhhtI5I219wFgI7kwrMpz4stXHJj/aqBknVv66NQHRstSJmw=="
},
"node_modules/windmill-parser-wasm-yaml": {
"version": "1.593.0",
+49 -1
View File
@@ -28,7 +28,12 @@ import * as wmill from "../../../gen/services.gen.ts";
import { logQueueStatus } from "../../utils/job_polling.ts";
import { resolveWorkspace } from "../../core/context.ts";
import { requireLogin } from "../../core/auth.ts";
import { GLOBAL_CONFIG_OPT } from "../../core/conf.ts";
import {
getWmillYamlPath,
GLOBAL_CONFIG_OPT,
mergeConfigWithConfigFile,
} from "../../core/conf.ts";
import { listSyncCodebases } from "../../utils/codebase.ts";
import { replaceInlineScripts, repopulateFields } from "./app.ts";
import { Runnable } from "./metadata.ts";
import {
@@ -386,6 +391,43 @@ async function dev(opts: DevOptions, appFolder?: string) {
await requireLogin(opts);
const workspaceId = workspace.workspaceId;
// Resolve relative imports in app inline scripts from local (not-yet-deployed)
// content so previews use locally-edited workspace libs instead of deployed.
// Computed here as a startup snapshot; re-run `wmill app dev` to pick up
// later edits to imported workspace scripts. Degrades gracefully (undefined)
// on older backends. The walk must run from the wmill.yaml root so that the
// supported `cd <app>__raw_app && wmill app dev` invocation (cwd is the
// raw_app folder) still sees sibling workspace scripts like `f/lib.ts`.
let appTempRefs: Record<string, string> | undefined = undefined;
{
const wmillYamlPath = getWmillYamlPath();
const workspaceRoot = wmillYamlPath
? path.dirname(wmillYamlPath)
: originalCwd;
const relAppFolder = path.relative(workspaceRoot, targetDir) || ".";
const mergedOpts = await mergeConfigWithConfigFile(opts);
const codebases = await listSyncCodebases(mergedOpts);
const { buildPreviewTempScriptRefs } = await import(
"../generate-metadata/generate-metadata.ts"
);
const savedCwd = process.cwd();
if (workspaceRoot !== savedCwd) {
process.chdir(workspaceRoot);
}
try {
appTempRefs = await buildPreviewTempScriptRefs(
workspace,
mergedOpts as any,
codebases,
{ kind: "app", folder: relAppFolder, rawApp: true },
);
} finally {
if (workspaceRoot !== savedCwd) {
process.chdir(savedCwd);
}
}
}
// Change to target directory for the rest of the command
if (appFolder) {
process.chdir(targetDir);
@@ -941,6 +983,7 @@ async function dev(opts: DevOptions, appFolder?: string) {
appPath,
runnableId,
args,
appTempRefs,
);
log.info(colors.gray(`[backend] Job started: ${uuid}`));
@@ -987,6 +1030,7 @@ async function dev(opts: DevOptions, appFolder?: string) {
appPath,
runnable_id,
v,
appTempRefs,
);
log.info(colors.gray(`[backendAsync] Job started: ${uuid}`));
@@ -1563,6 +1607,7 @@ async function executeRunnable(
appPath: string,
runnableId: string,
args: any,
tempScriptRefs?: Record<string, string>,
): Promise<string> {
const requestBody: any = {
component: runnableId,
@@ -1602,6 +1647,9 @@ async function executeRunnable(
lock: inlineScript.id === undefined ? inlineScript.lock : undefined,
cache_ttl: inlineScript.cache_ttl,
};
if (inlineScript.id === undefined && tempScriptRefs) {
requestBody.temp_script_refs = tempScriptRefs;
}
} else if (
(runnable.type === "path" || runnable.type === "runnableByPath") &&
runnable.runType &&
+59
View File
@@ -0,0 +1,59 @@
import { Command } from "@cliffy/command";
import { Table } from "@cliffy/table";
import * as wmill from "../../../gen/services.gen.ts";
import { requireLogin } from "../../core/auth.ts";
import { resolveWorkspace } from "../../core/context.ts";
import * as log from "../../core/log.ts";
import { GlobalOptions } from "../../types.ts";
import { runCatalogQuery } from "../../utils/catalog.ts";
const DEFAULT_DATATABLE_NAME = "main";
async function list(opts: GlobalOptions & { json?: boolean }) {
if (opts.json) log.setSilent(true);
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
const items = await wmill.listDataTables({
workspace: workspace.workspaceId,
});
if (opts.json) {
console.log(JSON.stringify(items));
} else {
new Table()
.header(["Name", "Resource Type", "Resource Path"])
.padding(2)
.border(true)
.body(items.map((x) => [x.name, x.resource_type, x.resource_path]))
.render();
}
}
async function run(
opts: GlobalOptions & { name?: string; silent?: boolean },
sql: string,
) {
const name = opts.name ?? DEFAULT_DATATABLE_NAME;
await runCatalogQuery(opts, "datatable", name, sql);
}
const command = new Command()
.description("datatable related commands")
.command("list", "list all datatables in the workspace")
.option("--json", "Output as JSON (for piping to jq)")
.action(list as any)
.command("run", "run a SQL query on a datatable")
.arguments("<sql:string>")
.option(
"-n --name <name:string>",
"Datatable name (default: main)",
)
.option(
"-s --silent",
"Output only the final result as JSON. Useful for scripting.",
)
.action(run as any);
export default command;
+59
View File
@@ -0,0 +1,59 @@
import { Command } from "@cliffy/command";
import { Table } from "@cliffy/table";
import * as wmill from "../../../gen/services.gen.ts";
import { requireLogin } from "../../core/auth.ts";
import { resolveWorkspace } from "../../core/context.ts";
import * as log from "../../core/log.ts";
import { GlobalOptions } from "../../types.ts";
import { runCatalogQuery } from "../../utils/catalog.ts";
const DEFAULT_DUCKLAKE_NAME = "main";
async function list(opts: GlobalOptions & { json?: boolean }) {
if (opts.json) log.setSilent(true);
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
const names = await wmill.listDucklakes({
workspace: workspace.workspaceId,
});
if (opts.json) {
console.log(JSON.stringify(names));
} else {
new Table()
.header(["Name"])
.padding(2)
.border(true)
.body(names.map((name) => [name]))
.render();
}
}
async function run(
opts: GlobalOptions & { name?: string; silent?: boolean },
sql: string,
) {
const name = opts.name ?? DEFAULT_DUCKLAKE_NAME;
await runCatalogQuery(opts, "ducklake", name, sql);
}
const command = new Command()
.description("ducklake related commands")
.command("list", "list all ducklakes in the workspace")
.option("--json", "Output as JSON (for piping to jq)")
.action(list as any)
.command("run", "run a SQL query on a ducklake")
.arguments("<sql:string>")
.option(
"-n --name <name:string>",
"Ducklake name (default: main)",
)
.option(
"-s --silent",
"Output only the final result as JSON. Useful for scripting.",
)
.action(run as any);
export default command;
+17
View File
@@ -618,6 +618,22 @@ async function preview(
await replaceAllPathScriptsWithLocal(localFlow.value, localScriptReader, log);
}
// Resolve relative imports in inline scripts from local (not-yet-deployed)
// content so previewing a flow uses locally-edited dependency scripts.
let tempScriptRefs: Record<string, string> | undefined = undefined;
if (useLocalPathScripts) {
const { buildPreviewTempScriptRefs } = await import(
"../generate-metadata/generate-metadata.ts"
);
const resolvedCodebases = (await Promise.resolve(codebases)) as SyncCodebase[];
tempScriptRefs = await buildPreviewTempScriptRefs(
workspace,
opts,
resolvedCodebases,
{ kind: "flow", folder: flowPath }
);
}
const input = opts.data ? await resolve(opts.data) : {};
if (!opts.silent) {
@@ -633,6 +649,7 @@ async function preview(
value: localFlow.value,
path: flowPath.substring(0, flowPath.indexOf(".flow")).replaceAll(SEP, "/"),
args: input,
temp_script_refs: tempScriptRefs,
},
});
@@ -7,6 +7,7 @@ import { SyncOptions, mergeConfigWithConfigFile } from "../../core/conf.ts";
import { resolveWorkspace } from "../../core/context.ts";
import { requireLogin } from "../../core/auth.ts";
import * as log from "../../core/log.ts";
import { Workspace } from "../workspace/workspace.ts";
import {
beginLockfileBatch,
flushLockfileBatch,
@@ -91,6 +92,99 @@ async function walkLocalAppItems(
}));
}
/**
* Build the path -> temp-storage-hash map for a preview target (script, flow,
* or app), so a preview run resolves relative imports from not-yet-deployed
* local content instead of the deployed scripts. Walks all local scripts so
* transitive relative-import targets can be uploaded, then for flow/app adds
* that item's node. Degrades gracefully (returns undefined) on older backends
* without the /raw_temp endpoints.
*/
export async function buildPreviewTempScriptRefs(
workspace: Workspace,
opts: GlobalOptions & SyncOptions & { defaultTs?: "bun" | "deno" },
codebases: SyncCodebase[],
target:
| { kind: "script"; path: string }
| { kind: "flow"; folder: string }
| { kind: "app"; folder: string; rawApp: boolean },
): Promise<Record<string, string> | undefined> {
try {
const rawWorkspaceDependencies = await getRawWorkspaceDependencies(true);
const tree = new DoubleLinkedDependencyTree();
tree.setWorkspaceDeps(rawWorkspaceDependencies);
const ignore = await ignoreF(opts);
for (const e of await walkLocalScripts(codebases, ignore)) {
await generateScriptMetadataInternal(
e,
workspace,
opts,
true, // dryRun: only populate the tree
true, // noStaleMessage
rawWorkspaceDependencies,
codebases,
false,
tree,
);
}
let nodePath: string;
if (target.kind === "script") {
nodePath = scriptPathToRemotePath(target.path);
} else if (target.kind === "flow") {
const folder = target.folder.endsWith(SEP)
? target.folder.slice(0, -1)
: target.folder;
await generateFlowLockInternal(folder, true, workspace, opts, false, true, tree);
nodePath = folder.replaceAll(SEP, "/");
} else {
const folder = target.folder.endsWith(SEP)
? target.folder.slice(0, -1)
: target.folder;
await generateAppLocksInternal(
folder,
target.rawApp,
true,
workspace,
opts,
false,
true,
tree,
);
nodePath = folder.replaceAll(SEP, "/");
}
tree.propagateStaleness();
await uploadScripts(tree, workspace);
const refs = tree.getTempScriptRefs(nodePath);
return refs && Object.keys(refs).length > 0 ? refs : undefined;
} catch (e) {
// Degrade gracefully (preview still runs against deployed versions) but do
// NOT mask the real error: only the missing-/raw_temp-endpoint case is an
// expected old-backend incompatibility — anything else is surfaced verbatim.
const msg = e instanceof Error ? e.message : String(e);
// Narrow: only the missing raw_temp endpoint is the expected old-backend
// signal. A bare 404/"not found" matches far too much (module/command/
// ENOENT "not found", "Script X not found", …) and would mislabel real
// bugs as a backend-too-old issue.
const isOldBackend = /raw_temp|raw_script_temp/i.test(msg);
if (!(opts as { silent?: boolean }).silent) {
log.warn(
colors.yellow(
isOldBackend
? `Backend does not support local-import resolution for preview ` +
`(requires the /raw_temp endpoints); relative imports will use ` +
`deployed script versions.`
: `Failed to resolve local relative imports for preview: ${msg}. ` +
`Falling back to deployed script versions.`,
),
);
}
return undefined;
}
}
/**
* Categorize a flat list of file paths into scripts / flow folders / app
* file paths. Used to derive item lists from a precomputed FS map (e.g.
@@ -0,0 +1,132 @@
import { ProtectionRuleEntry } from "./types.ts";
import { ProtectionRuleset } from "../../../gen/types.gen.ts";
// Reconciliation plan produced by diffing the local protection rules
// against the backend list. `toDelete` holds names present on the backend but
// absent from wmill.yaml (full-reconcile semantics).
export interface ProtectionRulesPlan {
toCreate: ProtectionRuleEntry[];
toUpdate: ProtectionRuleEntry[];
toDelete: string[];
unchanged: string[];
}
function sortedUnique(arr: readonly string[]): string[] {
return [...new Set(arr)].sort();
}
export class ProtectionRulesConverter {
// Canonicalize a single rule so comparisons are insensitive to array order
// and duplicates.
static normalizeEntry(entry: ProtectionRuleEntry): ProtectionRuleEntry {
return {
name: entry.name,
rules: sortedUnique(entry.rules ?? []) as ProtectionRuleEntry["rules"],
bypass_groups: sortedUnique(entry.bypass_groups ?? []),
bypass_users: sortedUnique(entry.bypass_users ?? []),
};
}
// Canonicalize and sort a list of rules by name.
static normalizeList(
entries: ProtectionRuleEntry[] | undefined,
): ProtectionRuleEntry[] {
return (entries ?? [])
.map((e) => ProtectionRulesConverter.normalizeEntry(e))
.sort((a, b) => a.name.localeCompare(b.name));
}
// Convert a backend ProtectionRuleset response into the wmill.yaml shape
// (drops workspace_id, which is implied by the synced workspace).
static fromBackend(rulesets: ProtectionRuleset[]): ProtectionRuleEntry[] {
return ProtectionRulesConverter.normalizeList(
rulesets.map((r) => ({
name: r.name,
rules: [...(r.rules ?? [])],
bypass_groups: [...(r.bypass_groups ?? [])],
bypass_users: [...(r.bypass_users ?? [])],
})),
);
}
static entriesEqual(
a: ProtectionRuleEntry,
b: ProtectionRuleEntry,
): boolean {
const na = ProtectionRulesConverter.normalizeEntry(a);
const nb = ProtectionRulesConverter.normalizeEntry(b);
return (
na.name === nb.name &&
na.rules.length === nb.rules.length &&
na.rules.every((v, i) => v === nb.rules[i]) &&
na.bypass_groups.length === nb.bypass_groups.length &&
na.bypass_groups.every((v, i) => v === nb.bypass_groups[i]) &&
na.bypass_users.length === nb.bypass_users.length &&
na.bypass_users.every((v, i) => v === nb.bypass_users[i])
);
}
static listsEqual(
a: ProtectionRuleEntry[] | undefined,
b: ProtectionRuleEntry[] | undefined,
): boolean {
const na = ProtectionRulesConverter.normalizeList(a);
const nb = ProtectionRulesConverter.normalizeList(b);
if (na.length !== nb.length) return false;
return na.every((entry, i) =>
ProtectionRulesConverter.entriesEqual(entry, nb[i])
);
}
// Compute the create/update/delete plan to make `backend` match `local`.
static computePlan(
local: ProtectionRuleEntry[] | undefined,
backend: ProtectionRuleEntry[] | undefined,
): ProtectionRulesPlan {
const localByName = new Map(
ProtectionRulesConverter.normalizeList(local).map((e) => [e.name, e]),
);
const backendByName = new Map(
ProtectionRulesConverter.normalizeList(backend).map((e) => [e.name, e]),
);
const plan: ProtectionRulesPlan = {
toCreate: [],
toUpdate: [],
toDelete: [],
unchanged: [],
};
for (const [name, entry] of localByName) {
const existing = backendByName.get(name);
if (!existing) {
plan.toCreate.push(entry);
} else if (!ProtectionRulesConverter.entriesEqual(entry, existing)) {
plan.toUpdate.push(entry);
} else {
plan.unchanged.push(name);
}
}
for (const name of backendByName.keys()) {
if (!localByName.has(name)) {
plan.toDelete.push(name);
}
}
plan.toCreate.sort((a, b) => a.name.localeCompare(b.name));
plan.toUpdate.sort((a, b) => a.name.localeCompare(b.name));
plan.toDelete.sort();
plan.unchanged.sort();
return plan;
}
static planHasChanges(plan: ProtectionRulesPlan): boolean {
return (
plan.toCreate.length > 0 ||
plan.toUpdate.length > 0 ||
plan.toDelete.length > 0
);
}
}
+129
View File
@@ -0,0 +1,129 @@
import { existsSync } from "node:fs";
import { writeFile } from "node:fs/promises";
import { dirname, join } from "node:path";
import { stringify as yamlStringify } from "yaml";
import { yamlParseFile } from "../../utils/yaml.ts";
import { yamlOptions } from "../sync/sync.ts";
import {
SyncOptions,
getWmillYamlPath,
getWorkspaceNames,
getEffectiveWorkspaceId,
WorkspaceEntryConfig,
} from "../../core/conf.ts";
import { GlobalOptions } from "../../types.ts";
import { tryResolveBranchWorkspace } from "../../core/context.ts";
import { setClient } from "../../core/client.ts";
import { ProtectionRulesFile } from "./types.ts";
export const PROTECTION_RULES_FILENAME = "protection-rules.yaml";
// protection-rules.yaml lives next to wmill.yaml. wmill.yaml is required: it is
// the single source of truth for which workspaces exist and how to reach them.
export function getProtectionRulesPath(): string | null {
const wmillPath = getWmillYamlPath();
if (!wmillPath) return null;
return join(dirname(wmillPath), PROTECTION_RULES_FILENAME);
}
export async function readProtectionRulesFile(
path: string,
): Promise<ProtectionRulesFile> {
if (!existsSync(path)) return {};
const parsed = (await yamlParseFile(path)) as ProtectionRulesFile | null;
return parsed ?? {};
}
export async function writeProtectionRulesFile(
path: string,
data: ProtectionRulesFile,
): Promise<void> {
// Deterministic key order so diffs/commits stay stable.
const sorted: ProtectionRulesFile = {};
for (const k of Object.keys(data).sort()) sorted[k] = data[k];
await writeFile(path, yamlStringify(sorted, yamlOptions), "utf-8");
}
// Maps a protection-rules.yaml workspace key to its backend workspace id via
// wmill.yaml's `workspaces` block. A key with no matching entry is rejected —
// without it we don't know which backend to talk to.
export class WorkspaceResolver {
private constructor(
private readonly workspaces: Record<string, WorkspaceEntryConfig>,
) {}
static fromConfig(config: SyncOptions): WorkspaceResolver {
const ws = (config.workspaces ?? {}) as Record<
string,
WorkspaceEntryConfig
>;
return new WorkspaceResolver(ws);
}
/** Workspace keys declared in wmill.yaml (excludes reserved keys). */
knownNames(): string[] {
return getWorkspaceNames(this.workspaces as any);
}
has(name: string): boolean {
return this.knownNames().includes(name);
}
/** Backend workspace id (path param) for a key, or throw if unknown. */
backendId(name: string): string {
if (!this.has(name)) {
throw new Error(
`Workspace '${name}' is not defined in wmill.yaml 'workspaces'. ` +
`Add it there (its keys must match protection-rules.yaml).`,
);
}
return getEffectiveWorkspaceId(name, this.workspaces[name]);
}
}
// Point the API client at the backend for a single wmill.yaml workspace key,
// then return the backend workspace id to use as the path param. The backend
// id always comes from the wmill.yaml mapping (the feature's invariant);
// credentials are resolved with the same precedence as every other command:
//
// 1. explicit --base-url + --token -> used as-is (stateless CI; no profile
// or wmill.yaml baseUrl required)
// 2. otherwise, the stored profile matching wmill.yaml workspaces.<ws>
// (its baseUrl + token), with an explicit --token overriding the
// stored token
//
// Throws a clean error if the key is unknown or nothing resolves it — callers
// decide whether to skip (--all) or fail (named arg).
export async function configureClientForWorkspace(
opts: GlobalOptions,
ws: string,
resolver: WorkspaceResolver,
): Promise<string> {
const wsId = resolver.backendId(ws); // throws if not in wmill.yaml
// 1. Explicit credentials — honor them directly, like other commands do.
if (opts.baseUrl) {
if (!opts.token) {
throw new Error(
"When --base-url is set, --token is required for protection-rules.",
);
}
setClient(opts.token, opts.baseUrl.replace(/\/+$/, ""));
return wsId;
}
// 2. Stored-profile resolution. Fresh opts so resolveWorkspace's per-call
// cache can't bleed across keys.
const resolved = await tryResolveBranchWorkspace({ ...opts }, ws);
if (!resolved) {
throw new Error(
`Could not resolve credentials for workspace '${ws}'. Either pass ` +
`--base-url and --token, or ensure wmill.yaml workspaces.${ws} has a ` +
`baseUrl and you've run 'wmill workspace add' for it.`,
);
}
// An explicit --token overrides the stored profile's token.
setClient(opts.token ?? resolved.token, resolved.remote.replace(/\/+$/, ""));
return wsId;
}
@@ -0,0 +1,2 @@
export { pullProtectionRules, pushProtectionRules } from "./protection-rules.ts";
export { default } from "./protection-rules.ts";
@@ -0,0 +1,30 @@
import { Command } from "@cliffy/command";
import { pullProtectionRules } from "./pull.ts";
import { pushProtectionRules } from "./push.ts";
const command = new Command()
.description(
"Sync workspace protection rules between protection-rules.yaml and Windmill. The file is keyed by workspace name; keys must match wmill.yaml 'workspaces'.",
)
.command("pull")
.description(
"Pull protection rules from Windmill into protection-rules.yaml for a workspace",
)
.arguments("[workspace:string]")
.option("--all", "Pull every workspace defined in wmill.yaml")
.option("--dry-run", "Show what would change without writing the file")
.option("--json-output", "Output in JSON format")
.action(pullProtectionRules as any)
.command("push")
.description(
"Push protection rules from protection-rules.yaml to Windmill for a workspace (full reconcile: creates, updates, and deletes)",
)
.arguments("[workspace:string]")
.option("--all", "Push every workspace defined in protection-rules.yaml")
.option("--dry-run", "Show what would change without applying")
.option("--json-output", "Output in JSON format")
.option("--yes", "Skip the confirmation prompt (including deletions)")
.action(pushProtectionRules as any);
export { pullProtectionRules, pushProtectionRules };
export default command;
+143
View File
@@ -0,0 +1,143 @@
import { colors } from "@cliffy/ansi/colors";
import * as log from "../../core/log.ts";
import { GlobalOptions } from "../../types.ts";
import * as wmill from "../../../gen/services.gen.ts";
import { readConfigFile } from "../../core/conf.ts";
import { ProtectionRulesConverter } from "./converter.ts";
import { ProtectionRulesFile } from "./types.ts";
import {
PROTECTION_RULES_FILENAME,
getProtectionRulesPath,
readProtectionRulesFile,
writeProtectionRulesFile,
WorkspaceResolver,
configureClientForWorkspace,
} from "./file.ts";
import { outputResult, fail, displayPlan, structuredPlan } from "./utils.ts";
type PullOpts = GlobalOptions & {
all?: boolean;
dryRun?: boolean;
jsonOutput?: boolean;
};
export async function pullProtectionRules(
opts: PullOpts,
workspaceArg?: string,
) {
// In JSON mode stdout must be exactly one JSON payload. Silence human logs
// (log.info/warn → stdout) here, before anything that logs (readConfigFile,
// workspace resolution). log.error still goes to stderr.
if (opts.jsonOutput) log.setSilent(true);
const prPath = getProtectionRulesPath();
if (!prPath) {
fail(opts, {
error:
"No wmill.yaml found. Run 'wmill init' first — protection-rules.yaml lives next to it.",
});
}
const config = await readConfigFile();
const resolver = WorkspaceResolver.fromConfig(config);
let targets: string[];
if (opts.all) {
targets = resolver.knownNames();
if (targets.length === 0) {
fail(opts, {
error: "No workspaces defined in wmill.yaml 'workspaces' block.",
});
}
} else if (workspaceArg) {
targets = [workspaceArg];
} else {
fail(opts, { error: "Specify a workspace name or use --all." });
}
const file = await readProtectionRulesFile(prPath!);
const perWs: Record<string, any> = {};
let hadError = false;
let anyChange = false;
for (const ws of targets) {
let wsId: string;
try {
wsId = await configureClientForWorkspace(opts, ws, resolver);
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
if (opts.all) {
log.error(colors.red(msg));
hadError = true;
continue;
}
fail(opts, { error: msg });
}
let backend;
try {
backend = ProtectionRulesConverter.fromBackend(
await wmill.listProtectionRules({ workspace: wsId }),
);
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
if (opts.all) {
log.error(colors.red(`[${ws}] failed to fetch: ${msg}`));
hadError = true;
continue;
}
fail(opts, { error: `Failed to fetch protection rules: ${msg}` });
}
const current = ProtectionRulesConverter.normalizeList(file[ws]);
// plan describes how the local file would change to match the backend
const plan = ProtectionRulesConverter.computePlan(backend, current);
if (ProtectionRulesConverter.planHasChanges(plan)) anyChange = true;
perWs[ws] = structuredPlan(plan);
if (!opts.dryRun) {
file[ws] = backend;
} else if (!opts.jsonOutput) {
displayPlan(ws, plan);
}
}
if (opts.dryRun) {
if (opts.jsonOutput) {
console.log(
JSON.stringify({
success: !hadError,
dryRun: true,
partialFailure: hadError,
hasChanges: anyChange,
workspaces: perWs,
}),
);
} else if (!hadError && !anyChange) {
log.info(colors.green("All targeted workspaces are in sync"));
}
if (hadError) process.exit(1);
return;
}
await writeProtectionRulesFile(prPath!, file as ProtectionRulesFile);
const n = Object.keys(perWs).length;
if (hadError) {
// Some --all workspaces failed: status must not say success while we
// exit non-zero.
outputResult(opts, {
success: false,
error: `Pulled ${n} workspace(s) into ${PROTECTION_RULES_FILENAME}, but one or more workspaces failed (see errors above)`,
partialFailure: true,
workspaces: perWs,
});
process.exit(1);
}
outputResult(opts, {
success: true,
message: `Pulled protection rules for ${n} workspace(s) into ${PROTECTION_RULES_FILENAME}`,
workspaces: perWs,
});
}
+262
View File
@@ -0,0 +1,262 @@
import process from "node:process";
import { existsSync } from "node:fs";
import { colors } from "@cliffy/ansi/colors";
import { Confirm } from "@cliffy/prompt/confirm";
import * as log from "../../core/log.ts";
import { GlobalOptions } from "../../types.ts";
import * as wmill from "../../../gen/services.gen.ts";
import { readConfigFile } from "../../core/conf.ts";
import { ProtectionRulesConverter, ProtectionRulesPlan } from "./converter.ts";
import {
getProtectionRulesPath,
readProtectionRulesFile,
WorkspaceResolver,
configureClientForWorkspace,
} from "./file.ts";
import { outputResult, fail, displayPlan, structuredPlan } from "./utils.ts";
type PushOpts = GlobalOptions & {
all?: boolean;
dryRun?: boolean;
jsonOutput?: boolean;
yes?: boolean;
};
interface WsPlan {
ws: string;
wsId: string;
plan: ProtectionRulesPlan;
wipesAll: boolean;
}
export async function pushProtectionRules(
opts: PushOpts,
workspaceArg?: string,
) {
// In JSON mode stdout must be exactly one JSON payload. Silence human logs
// (log.info/warn → stdout, incl. workspace resolution + the empty-list
// delete warning) before anything logs. log.error still goes to stderr.
if (opts.jsonOutput) log.setSilent(true);
const prPath = getProtectionRulesPath();
if (!prPath) {
fail(opts, {
error:
"No wmill.yaml found. Run 'wmill init' first — protection-rules.yaml lives next to it.",
});
}
if (!existsSync(prPath!)) {
fail(opts, {
error:
"No protection-rules.yaml found. Run 'wmill protection-rules pull' first.",
});
}
const config = await readConfigFile();
const resolver = WorkspaceResolver.fromConfig(config);
const file = await readProtectionRulesFile(prPath!);
let targets: string[];
if (opts.all) {
targets = Object.keys(file).sort();
if (targets.length === 0) {
fail(opts, { error: "protection-rules.yaml defines no workspaces." });
}
} else if (workspaceArg) {
if (!(workspaceArg in file)) {
fail(opts, {
error: `Workspace '${workspaceArg}' is not defined in protection-rules.yaml.`,
});
}
targets = [workspaceArg];
} else {
fail(opts, { error: "Specify a workspace name or use --all." });
}
// Phase 1: resolve + diff every target before mutating anything.
const wsPlans: WsPlan[] = [];
let hadError = false;
for (const ws of targets) {
let wsId: string;
try {
wsId = await configureClientForWorkspace(opts, ws, resolver);
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
if (opts.all) {
log.error(colors.red(msg));
hadError = true;
continue;
}
fail(opts, { error: msg });
}
let backend;
try {
backend = ProtectionRulesConverter.fromBackend(
await wmill.listProtectionRules({ workspace: wsId }),
);
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
if (opts.all) {
log.error(colors.red(`[${ws}] failed to fetch: ${msg}`));
hadError = true;
continue;
}
fail(opts, { error: `Failed to fetch protection rules: ${msg}` });
}
const local = ProtectionRulesConverter.normalizeList(file[ws]);
const plan = ProtectionRulesConverter.computePlan(local, backend);
wsPlans.push({
ws,
wsId,
plan,
wipesAll: local.length === 0 && plan.toDelete.length > 0,
});
}
const changed = wsPlans.filter((w) =>
ProtectionRulesConverter.planHasChanges(w.plan)
);
if (opts.jsonOutput && opts.dryRun) {
console.log(
JSON.stringify({
success: !hadError,
dryRun: true,
partialFailure: hadError,
hasChanges: changed.length > 0,
workspaces: Object.fromEntries(
wsPlans.map((w) => [w.ws, structuredPlan(w.plan)]),
),
}),
);
if (hadError) process.exit(1);
return;
}
if (!opts.jsonOutput) {
for (const w of wsPlans) displayPlan(w.ws, w.plan);
}
if (changed.length === 0) {
if (hadError) {
// A workspace failed to resolve/fetch — don't claim success while
// exiting non-zero.
outputResult(opts, {
success: false,
error:
"One or more workspaces failed (see errors above); the rest are in sync",
partialFailure: true,
});
process.exit(1);
}
if (!opts.dryRun) {
outputResult(opts, {
success: true,
message: "No changes to push - all targeted workspaces are in sync",
});
}
return;
}
if (opts.dryRun) {
if (hadError) process.exit(1);
return;
}
// Pushing an empty list wipes a workspace's rules — be loud even with --yes.
for (const w of wsPlans) {
if (w.wipesAll) {
log.warn(
colors.red(
`WARNING: '${w.ws}' has an empty rule list — this DELETES ALL ${w.plan.toDelete.length} backend rule(s) for that workspace.`,
),
);
}
}
const totalDeletes = changed.reduce((n, w) => n + w.plan.toDelete.length, 0);
if (!opts.yes && !!process.stdin.isTTY) {
const confirmed = await Confirm.prompt({
message: totalDeletes > 0
? `Apply these changes? This DELETES ${totalDeletes} protection rule(s) across ${changed.length} workspace(s).`
: `Apply these changes to ${changed.length} workspace(s)?`,
default: totalDeletes === 0,
});
if (!confirmed) {
log.info("Operation cancelled");
return;
}
}
// Phase 2: apply. Track progress so a mid-run failure reports how far it got.
const applied = { created: 0, updated: 0, deleted: 0 };
try {
for (const w of changed) {
// Re-point the client at this workspace (phase 1 left it on the last one).
await configureClientForWorkspace(opts, w.ws, resolver);
for (const entry of w.plan.toCreate) {
const n = ProtectionRulesConverter.normalizeEntry(entry);
await wmill.createProtectionRule({
workspace: w.wsId,
requestBody: {
name: n.name,
rules: n.rules,
bypass_groups: n.bypass_groups,
bypass_users: n.bypass_users,
},
});
applied.created++;
}
for (const entry of w.plan.toUpdate) {
const n = ProtectionRulesConverter.normalizeEntry(entry);
await wmill.updateProtectionRule({
workspace: w.wsId,
ruleName: n.name,
requestBody: {
rules: n.rules,
bypass_groups: n.bypass_groups,
bypass_users: n.bypass_users,
},
});
applied.updated++;
}
for (const name of w.plan.toDelete) {
await wmill.deleteProtectionRule({
workspace: w.wsId,
ruleName: name,
});
applied.deleted++;
}
}
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
fail(opts, {
error:
`Push partially failed after ${applied.created} create, ${applied.updated} update, ` +
`${applied.deleted} delete: ${msg}. Backend is partially reconciled; re-run push to converge.`,
applied,
});
}
if (hadError) {
// Reconcile of resolvable workspaces succeeded, but some --all targets
// failed earlier. Status must reflect the non-zero exit.
outputResult(opts, {
success: false,
error: `Pushed (created ${applied.created}, updated ${applied.updated}, deleted ${applied.deleted}) across ${changed.length} workspace(s), but one or more workspaces failed (see errors above)`,
partialFailure: true,
...applied,
});
process.exit(1);
}
outputResult(opts, {
success: true,
message: `Pushed protection rules (created ${applied.created}, updated ${applied.updated}, deleted ${applied.deleted}) across ${changed.length} workspace(s)`,
...applied,
});
}
@@ -0,0 +1,18 @@
import { ProtectionRuleKind } from "../../../gen/types.gen.ts";
export type { ProtectionRuleKind };
// A single workspace protection ruleset as stored in protection-rules.yaml.
// Mirrors the backend ProtectionRuleset shape minus workspace_id (the workspace
// is the map key).
export interface ProtectionRuleEntry {
name: string;
rules: ProtectionRuleKind[];
bypass_groups: string[];
bypass_users: string[];
}
// protection-rules.yaml is a flat map: workspace name -> its protection rules.
// Workspace names MUST match keys in wmill.yaml's `workspaces` block, which is
// where the backend workspaceId/remote is resolved from.
export type ProtectionRulesFile = Record<string, ProtectionRuleEntry[]>;
@@ -0,0 +1,73 @@
import process from "node:process";
import { colors } from "@cliffy/ansi/colors";
import * as log from "../../core/log.ts";
import { ProtectionRuleEntry } from "./types.ts";
import { ProtectionRulesConverter, ProtectionRulesPlan } from "./converter.ts";
export function outputResult(
opts: { jsonOutput?: boolean },
result: {
success: boolean;
message?: string;
error?: string;
[key: string]: any;
},
): void {
if (opts.jsonOutput) {
console.log(JSON.stringify(result));
} else if (result.success && result.message) {
log.info(colors.green(result.message));
} else if (!result.success && result.error) {
log.error(colors.red(result.error));
}
}
// Report a genuine failure and exit non-zero so CI / scripted callers detect
// it. outputResult alone only logs, which would let a failed (possibly
// partial) reconcile hide behind exit code 0.
export function fail(
opts: { jsonOutput?: boolean },
result: { error: string; [key: string]: any },
): never {
outputResult(opts, { ...result, success: false });
process.exit(1);
}
function describeEntry(entry: ProtectionRuleEntry): string {
const n = ProtectionRulesConverter.normalizeEntry(entry);
const parts = [`rules=[${n.rules.join(", ")}]`];
if (n.bypass_groups.length > 0) {
parts.push(`bypass_groups=[${n.bypass_groups.join(", ")}]`);
}
if (n.bypass_users.length > 0) {
parts.push(`bypass_users=[${n.bypass_users.join(", ")}]`);
}
return parts.join(" ");
}
// Render a reconciliation plan with a per-workspace heading.
export function displayPlan(ws: string, plan: ProtectionRulesPlan): void {
log.info(colors.bold(`workspace ${ws}:`));
for (const e of plan.toCreate) {
log.info(colors.green(` + ${e.name} (${describeEntry(e)})`));
}
for (const e of plan.toUpdate) {
log.info(colors.yellow(` ~ ${e.name} (${describeEntry(e)})`));
}
for (const name of plan.toDelete) {
log.info(colors.red(` - ${name}`));
}
if (!ProtectionRulesConverter.planHasChanges(plan)) {
log.info(colors.green(" in sync"));
}
}
export function structuredPlan(plan: ProtectionRulesPlan) {
return {
create: plan.toCreate.map((e) => e.name),
update: plan.toUpdate.map((e) => e.name),
delete: plan.toDelete,
unchanged: plan.unchanged,
};
}
+29
View File
@@ -1521,6 +1521,33 @@ async function preview(
const codebase =
language == "bun" ? findCodebase(filePath, codebases) : undefined;
// Resolve relative imports from local (not-yet-deployed) content so previewing
// a script that imports other locally-edited scripts uses the local versions
// instead of the deployed ones. Shared with `wmill flow preview` so both
// entry points behave identically; degrades gracefully on older backends.
// Short-circuit when the script has no relative imports: the full-workspace
// dependency walk + diff round-trip is pure overhead in that (common) case.
let tempScriptRefs: Record<string, string> | undefined = undefined;
const { extractRelativeImports } = await import(
"../../utils/relative_imports.ts"
);
const relImports = await extractRelativeImports(
content,
scriptPathToRemotePath(filePath),
language
);
if (relImports.length > 0) {
const { buildPreviewTempScriptRefs } = await import(
"../generate-metadata/generate-metadata.ts"
);
tempScriptRefs = await buildPreviewTempScriptRefs(
workspace,
opts,
codebases,
{ kind: "script", path: filePath }
);
}
let bundledContent: string | Blob | undefined = undefined;
let isTar = false;
@@ -1616,6 +1643,7 @@ async function preview(
language: language,
kind: isTar ? "tarbundle" : "bundle",
format: codebase?.format ?? "cjs",
temp_script_refs: tempScriptRefs,
};
form.append("preview", JSON.stringify(previewPayload));
form.append(
@@ -1683,6 +1711,7 @@ async function preview(
args: input,
language: language as any,
modules: modules ?? undefined,
temp_script_refs: tempScriptRefs,
},
});
+38
View File
@@ -6680,6 +6680,18 @@ Show all available wmill.yaml configuration options
- \`config migrate\` - Migrate wmill.yaml from gitBranches/environments to workspaces format
### datatable
datatable related commands
**Subcommands:**
- \`datatable list\` - list all datatables in the workspace
- \`--json\` - Output as JSON (for piping to jq)
- \`datatable run <sql:string>\` - run a SQL query on a datatable
- \`-n --name <name:string>\` - Datatable name (default: main)
- \`-s --silent\` - Output only the final result as JSON. Useful for scripting.
### dependencies
workspace dependencies related commands
@@ -6709,6 +6721,18 @@ Search Windmill documentation.
**Options:**
- \`--json\` - Output results as JSON.
### ducklake
ducklake related commands
**Subcommands:**
- \`ducklake list\` - list all ducklakes in the workspace
- \`--json\` - Output as JSON (for piping to jq)
- \`ducklake run <sql:string>\` - run a SQL query on a ducklake
- \`-n --name <name:string>\` - Ducklake name (default: main)
- \`-s --silent\` - Output only the final result as JSON. Useful for scripting.
### flow
flow related commands
@@ -6944,6 +6968,20 @@ Validate Windmill flow, schedule, and trigger YAML files in a directory
- \`--locks-required\` - Fail if scripts or flow inline scripts that need locks have no locks
- \`-w, --watch\` - Watch for file changes and re-lint automatically
### protection-rules
**Subcommands:**
- \`protection-rules pull [workspace:string]\` - Pull protection rules from Windmill into protection-rules.yaml for a workspace
- \`--all\` - Pull every workspace defined in wmill.yaml
- \`--dry-run\` - Show what would change without writing the file
- \`--json-output\` - Output in JSON format
- \`protection-rules push [workspace:string]\` - Push protection rules from protection-rules.yaml to Windmill for a workspace (full reconcile: creates, updates, and deletes)
- \`--all\` - Push every workspace defined in protection-rules.yaml
- \`--dry-run\` - Show what would change without applying
- \`--json-output\` - Output in JSON format
- \`--yes\` - Skip the confirmation prompt (including deletions)
### queues
List all queues with their metrics
+9
View File
@@ -21,6 +21,7 @@ import schedule from "./commands/schedule/schedule.ts";
import trigger from "./commands/trigger/trigger.ts";
import sync from "./commands/sync/sync.ts";
import gitsyncSettings from "./commands/gitsync-settings/gitsync-settings.ts";
import protectionRules from "./commands/protection-rules/protection-rules.ts";
import instance from "./commands/instance/instance.ts";
import workerGroups from "./commands/worker-groups/worker-groups.ts";
import lint from "./commands/lint/lint.ts";
@@ -47,6 +48,8 @@ import token from "./commands/token/token.ts";
import generateMetadata from "./commands/generate-metadata/generate-metadata.ts";
import docs from "./commands/docs/docs.ts";
import config from "./commands/config/config.ts";
import datatable from "./commands/datatable/datatable.ts";
import ducklake from "./commands/ducklake/ducklake.ts";
import { fetchVersion } from "./core/context.ts";
export {
@@ -65,10 +68,13 @@ export {
sync,
lint,
gitsyncSettings,
protectionRules,
instance,
dev,
docs,
config,
datatable,
ducklake,
hubPull,
pull,
push,
@@ -185,6 +191,7 @@ const command = new Command()
.command("sync", sync)
.command("lint", lint)
.command("gitsync-settings", gitsyncSettings)
.command("protection-rules", protectionRules)
.command("instance", instance)
.command("worker-groups", workerGroups)
.command("workers", workers)
@@ -198,6 +205,8 @@ const command = new Command()
.command("generate-metadata", generateMetadata)
.command("docs", docs)
.command("config", config)
.command("datatable", datatable)
.command("ducklake", ducklake)
.command("version --version", "Show version information")
.action(async (opts: any) => {
console.log("CLI version: " + VERSION);
+141
View File
@@ -0,0 +1,141 @@
import { colors } from "@cliffy/ansi/colors";
import { Table } from "@cliffy/table";
import * as wmill from "../../gen/services.gen.ts";
import { Preview } from "../../gen/types.gen.ts";
import { requireLogin } from "../core/auth.ts";
import { resolveWorkspace } from "../core/context.ts";
import * as log from "../core/log.ts";
import { GlobalOptions } from "../types.ts";
import { pollJobWithQueueLogging } from "./job_polling.ts";
// Shared building blocks for SQL catalogs (`wmill datatable` and
// `wmill ducklake`). Both expose a SQL surface backed by Windmill script
// previews, but the language and arg shape differ — `buildCatalogQueryPlan`
// centralizes that difference so subcommands stay thin.
export type CatalogKind = "datatable" | "ducklake";
interface CatalogQueryPlan {
language: NonNullable<Preview["language"]>;
content: string;
args: Record<string, unknown>;
}
function buildCatalogQueryPlan(
kind: CatalogKind,
name: string,
sql: string,
): CatalogQueryPlan {
switch (kind) {
case "datatable":
return {
language: "postgresql",
content: sql,
args: { database: `datatable://${name}` },
};
case "ducklake": {
const attach = `ATTACH 'ducklake://${name}' AS dl;\nUSE dl;\n`;
return {
language: "duckdb",
content: attach + sql,
args: {},
};
}
}
}
export async function runCatalogQuery(
opts: GlobalOptions & { silent?: boolean },
kind: CatalogKind,
name: string,
sql: string,
): Promise<void> {
if (opts.silent) {
log.setSilent(true);
}
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
const plan = buildCatalogQueryPlan(kind, name, sql);
log.info(colors.gray(`Running query on ${kind}://${name}`));
const jobId = await wmill.runScriptPreview({
workspace: workspace.workspaceId,
requestBody: {
content: plan.content,
language: plan.language,
args: plan.args,
},
});
const { result, success } = await pollJobWithQueueLogging(
workspace.workspaceId,
jobId,
);
if (!success) {
process.exitCode = 1;
if (opts.silent) {
console.log(JSON.stringify(result));
} else {
log.info(colors.red.bold("Query failed"));
log.info(JSON.stringify(result, null, 2));
}
return;
}
if (opts.silent) {
console.log(JSON.stringify(result));
} else {
renderQueryResult(result);
}
}
/**
* Render a SQL query result. Postgres script previews return rows as an array
* of `{column: value}` objects display those as a table. Anything else
* (DDL output, empty results, scalar payloads) falls back to pretty JSON.
*/
function renderQueryResult(result: unknown): void {
if (Array.isArray(result) && result.length > 0 && result.every(isRecord)) {
const rows = result as Record<string, unknown>[];
const columns = collectColumns(rows);
new Table()
.header(columns)
.padding(2)
.border(true)
.body(rows.map((row) => columns.map((c) => formatCell(row[c]))))
.render();
return;
}
log.info(JSON.stringify(result, null, 2));
}
function isRecord(v: unknown): v is Record<string, unknown> {
return v !== null && typeof v === "object" && !Array.isArray(v);
}
function collectColumns(rows: Record<string, unknown>[]): string[] {
const seen = new Set<string>();
const columns: string[] = [];
for (const row of rows) {
for (const key of Object.keys(row)) {
if (!seen.has(key)) {
seen.add(key);
columns.push(key);
}
}
}
return columns;
}
function formatCell(value: unknown): string {
if (value === null || value === undefined) return "";
if (typeof value === "string") return value;
if (typeof value === "number" || typeof value === "boolean") {
return String(value);
}
return JSON.stringify(value);
}
@@ -0,0 +1,240 @@
/**
* Unit tests for the protection-rules feature: the reconciliation converter,
* the WorkspaceResolver (protection-rules.yaml key -> backend id via
* wmill.yaml), and protection-rules.yaml read/write round-tripping.
*/
import { expect, test, describe } from "bun:test";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { mkdtempSync, rmSync } from "node:fs";
import { ProtectionRulesConverter } from "../src/commands/protection-rules/converter.ts";
import {
WorkspaceResolver,
readProtectionRulesFile,
writeProtectionRulesFile,
} from "../src/commands/protection-rules/file.ts";
import { ProtectionRuleEntry } from "../src/commands/protection-rules/types.ts";
import { SyncOptions } from "../src/core/conf.ts";
const rule = (
name: string,
rules: ProtectionRuleEntry["rules"],
groups: string[] = [],
users: string[] = [],
): ProtectionRuleEntry => ({
name,
rules,
bypass_groups: groups,
bypass_users: users,
});
describe("normalizeEntry", () => {
test("sorts and dedupes rules, groups, users", () => {
const r = rule(
"prod",
["RestrictDeployToDeployers", "DisableDirectDeployment", "DisableDirectDeployment"],
["g/b", "g/a"],
["u/y", "u/x", "u/x"],
);
const n = ProtectionRulesConverter.normalizeEntry(r);
expect(n.rules).toEqual([
"DisableDirectDeployment",
"RestrictDeployToDeployers",
]);
expect(n.bypass_groups).toEqual(["g/a", "g/b"]);
expect(n.bypass_users).toEqual(["u/x", "u/y"]);
});
test("handles missing arrays", () => {
const n = ProtectionRulesConverter.normalizeEntry({
name: "x",
} as unknown as ProtectionRuleEntry);
expect(n.rules).toEqual([]);
expect(n.bypass_groups).toEqual([]);
expect(n.bypass_users).toEqual([]);
});
});
describe("entriesEqual", () => {
test("equal regardless of array order", () => {
const a = rule("p", ["DisableDirectDeployment", "DisableWorkspaceForking"], ["g/a", "g/b"]);
const b = rule("p", ["DisableWorkspaceForking", "DisableDirectDeployment"], ["g/b", "g/a"]);
expect(ProtectionRulesConverter.entriesEqual(a, b)).toBe(true);
});
test("different rules are not equal", () => {
const a = rule("p", ["DisableDirectDeployment"]);
const b = rule("p", ["DisableWorkspaceForking"]);
expect(ProtectionRulesConverter.entriesEqual(a, b)).toBe(false);
});
test("different bypass users are not equal", () => {
const a = rule("p", ["DisableDirectDeployment"], [], ["u/a"]);
const b = rule("p", ["DisableDirectDeployment"], [], ["u/b"]);
expect(ProtectionRulesConverter.entriesEqual(a, b)).toBe(false);
});
});
describe("fromBackend", () => {
test("strips workspace_id and normalizes", () => {
const out = ProtectionRulesConverter.fromBackend([
{
name: "p",
workspace_id: "ws1",
rules: ["DisableWorkspaceForking", "DisableDirectDeployment"],
bypass_groups: ["g/b", "g/a"],
bypass_users: [],
},
]);
expect(out).toEqual([
{
name: "p",
rules: ["DisableDirectDeployment", "DisableWorkspaceForking"],
bypass_groups: ["g/a", "g/b"],
bypass_users: [],
},
]);
});
});
describe("listsEqual", () => {
test("equal regardless of list order", () => {
const a = [rule("a", ["DisableDirectDeployment"]), rule("b", ["DisableWorkspaceForking"])];
const b = [rule("b", ["DisableWorkspaceForking"]), rule("a", ["DisableDirectDeployment"])];
expect(ProtectionRulesConverter.listsEqual(a, b)).toBe(true);
});
test("undefined equals empty", () => {
expect(ProtectionRulesConverter.listsEqual(undefined, [])).toBe(true);
});
test("different length not equal", () => {
expect(
ProtectionRulesConverter.listsEqual([rule("a", [])], []),
).toBe(false);
});
});
describe("computePlan (full reconcile)", () => {
test("creates rules present locally but not on backend", () => {
const plan = ProtectionRulesConverter.computePlan(
[rule("new", ["DisableDirectDeployment"])],
[],
);
expect(plan.toCreate.map((e) => e.name)).toEqual(["new"]);
expect(plan.toUpdate).toEqual([]);
expect(plan.toDelete).toEqual([]);
});
test("deletes backend rules not present locally", () => {
const plan = ProtectionRulesConverter.computePlan(
[],
[rule("stale", ["DisableDirectDeployment"])],
);
expect(plan.toDelete).toEqual(["stale"]);
expect(plan.toCreate).toEqual([]);
});
test("updates rules whose content changed", () => {
const plan = ProtectionRulesConverter.computePlan(
[rule("p", ["DisableDirectDeployment", "DisableWorkspaceForking"])],
[rule("p", ["DisableDirectDeployment"])],
);
expect(plan.toUpdate.map((e) => e.name)).toEqual(["p"]);
expect(plan.toCreate).toEqual([]);
expect(plan.toDelete).toEqual([]);
});
test("unchanged rules are not in create/update/delete", () => {
const same = [rule("p", ["DisableDirectDeployment"], ["g/a"])];
const plan = ProtectionRulesConverter.computePlan(same, [
rule("p", ["DisableDirectDeployment"], ["g/a"]),
]);
expect(ProtectionRulesConverter.planHasChanges(plan)).toBe(false);
expect(plan.unchanged).toEqual(["p"]);
});
test("mixed plan: create + update + delete + unchanged", () => {
const local = [
rule("keep", ["DisableDirectDeployment"]),
rule("change", ["DisableWorkspaceForking"]),
rule("brand-new", ["RestrictDeployToDeployers"]),
];
const backend = [
rule("keep", ["DisableDirectDeployment"]),
rule("change", ["DisableDirectDeployment"]),
rule("gone", ["DisableDirectDeployment"]),
];
const plan = ProtectionRulesConverter.computePlan(local, backend);
expect(plan.toCreate.map((e) => e.name)).toEqual(["brand-new"]);
expect(plan.toUpdate.map((e) => e.name)).toEqual(["change"]);
expect(plan.toDelete).toEqual(["gone"]);
expect(plan.unchanged).toEqual(["keep"]);
expect(ProtectionRulesConverter.planHasChanges(plan)).toBe(true);
});
test("reordered arrays do not produce spurious updates", () => {
const plan = ProtectionRulesConverter.computePlan(
[rule("p", ["DisableWorkspaceForking", "DisableDirectDeployment"], ["g/b", "g/a"])],
[rule("p", ["DisableDirectDeployment", "DisableWorkspaceForking"], ["g/a", "g/b"])],
);
expect(ProtectionRulesConverter.planHasChanges(plan)).toBe(false);
});
});
describe("WorkspaceResolver", () => {
const config: SyncOptions = {
workspaces: {
prod: { workspaceId: "acme-prod" },
dev: {},
commonSpecificItems: { settings: true },
} as any,
};
const r = WorkspaceResolver.fromConfig(config);
test("knownNames excludes reserved keys", () => {
expect(r.knownNames().sort()).toEqual(["dev", "prod"]);
});
test("backendId uses workspaceId when set, else the key name", () => {
expect(r.backendId("prod")).toBe("acme-prod");
expect(r.backendId("dev")).toBe("dev");
});
test("backendId throws for a key absent from wmill.yaml", () => {
expect(() => r.backendId("ghost")).toThrow(/not defined in wmill\.yaml/);
});
test("has reflects membership", () => {
expect(r.has("prod")).toBe(true);
expect(r.has("ghost")).toBe(false);
});
test("empty config resolves to no workspaces", () => {
expect(WorkspaceResolver.fromConfig({}).knownNames()).toEqual([]);
});
});
describe("protection-rules.yaml read/write", () => {
test("round-trips and sorts workspace keys deterministically", async () => {
const dir = mkdtempSync(join(tmpdir(), "prfile-"));
const path = join(dir, "protection-rules.yaml");
try {
expect(await readProtectionRulesFile(path)).toEqual({});
await writeProtectionRulesFile(path, {
prod: [rule("p", ["DisableDirectDeployment"], ["g/a"])],
dev: [],
});
const back = await readProtectionRulesFile(path);
expect(Object.keys(back)).toEqual(["dev", "prod"]);
expect(back.prod).toEqual([
rule("p", ["DisableDirectDeployment"], ["g/a"]),
]);
expect(back.dev).toEqual([]);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
});
+2
View File
@@ -51,6 +51,7 @@
"jszip": "^3.10.1",
"lru-cache": "^11.1.0",
"lucide-svelte": "^0.540.0",
"mdast-util-find-and-replace": "^3.0.2",
"minimatch": "^10.0.1",
"monaco-editor": "npm:@codingame/monaco-vscode-editor-api@=25.0.0",
"monaco-languageclient": "10.6.0",
@@ -71,6 +72,7 @@
"svelte-exmarkdown": "^5.0.0",
"svelte-infinite-loading": "^1.4.0",
"tailwind-merge": "^1.13.2",
"unist-util-visit": "^5.0.0",
"vscode": "npm:@codingame/monaco-vscode-extension-api@=25.0.0",
"vscode-languageclient": "~9.0.1",
"vscode-uri": "~3.1.0",
+2
View File
@@ -124,6 +124,8 @@
"jszip": "^3.10.1",
"lru-cache": "^11.1.0",
"lucide-svelte": "^0.540.0",
"mdast-util-find-and-replace": "^3.0.2",
"unist-util-visit": "^5.0.0",
"minimatch": "^10.0.1",
"monaco-editor": "npm:@codingame/monaco-vscode-editor-api@=25.0.0",
"monaco-languageclient": "10.6.0",
@@ -6,12 +6,10 @@
const {
loading = false,
loadingSave = false,
newFlow = false,
dropdownItems = []
}: {
loading?: boolean
loadingSave?: boolean
newFlow?: boolean
dropdownItems?: Array<{
label: string
onClick: () => void
@@ -33,7 +31,7 @@
unifiedSize="md"
startIcon={{ icon: Save }}
on:click={() => dispatch('save')}
dropdownItems={!newFlow ? dropdownItems : undefined}
{dropdownItems}
tooltipPopover={{
placement: 'bottom-end',
openDelay: dropdownOpen ? 200 : 0,
+19
View File
@@ -1916,6 +1916,25 @@
})
})
// External `code` prop changes should flow into the Monaco editor. The
// `untrack` block reads/writes Monaco without subscribing — only the
// prop read above is tracked — so the editor's own change handler
// (`updateCode`) re-running with the same value short-circuits and we
// don't loop.
$effect(() => {
const next = code ?? ''
const ed = editor
if (!ed) return
untrack(() => {
if (ed.getValue() === next) return
const model = ed.getModel()
if (!model) return
ed.pushUndoStop()
ed.executeEdits('external', [{ range: model.getFullModelRange(), text: next }])
ed.pushUndoStop()
})
})
let isTsWorkerInitialized = resource([() => lang, () => initialized], async () => {
if (lang !== 'typescript' || !initialized) return false
// Use the stable model URI (computed once at mount), not filePath which changes on rename
+252 -137
View File
@@ -1,5 +1,11 @@
<script module lang="ts">
export const EDITOR_BAR_WIDTH_THRESHOLD = 1420
// Below this width even the icon-only helpers cluster is too crowded —
// collapse them into a single "Helpers" dropdown menu.
export const EDITOR_BAR_HELPERS_COMPACT_THRESHOLD = 800
// Tighter threshold for editors embedded inline in narrow panes
// (flow-step editor, raw-app inline script editor).
export const EDITOR_BAR_HELPERS_INLINE_THRESHOLD = 600
function getImportWmillTsStatement(lang: string | undefined) {
if (lang === 'deno') {
@@ -51,7 +57,8 @@
Settings,
Users
} from 'lucide-svelte'
import { capitalize, formatS3Object, toCamel } from '$lib/utils'
import { capitalize, formatS3Object, toCamel, type Item } from '$lib/utils'
import DropdownV2 from './DropdownV2.svelte'
import type { Schema, SchemaProperty, SupportedLanguage } from '$lib/common'
import ScriptVersionHistory from './ScriptVersionHistory.svelte'
import { getResetCode } from '$lib/script_helpers'
@@ -77,6 +84,7 @@
shellcheck: boolean
}
iconOnly?: boolean
compactHelpers?: boolean
validCode?: boolean
kind?: 'script' | 'trigger' | 'approval'
template?:
@@ -112,6 +120,7 @@
editor,
websocketAlive,
iconOnly = false,
compactHelpers = false,
validCode = true,
kind = 'script',
template = 'script',
@@ -237,6 +246,82 @@
let codeViewer: Drawer | undefined = $state()
let codeObj: { language: SupportedLanguage; content: string } | undefined = $state(undefined)
function getHelperItems(): Item[] {
const items: Item[] = []
if (showContextVarPicker && customUi?.contextVar != false) {
items.push({
displayName: 'Context variable',
icon: DollarSign,
action: () => contextualVariablePicker?.openDrawer()
})
}
if (showVarPicker && customUi?.variable != false) {
items.push({
displayName: 'Variable',
icon: DollarSign,
action: () => variablePicker?.openDrawer()
})
}
if (showS3Picker && customUi?.s3object != false) {
items.push({
displayName: 'S3 object',
icon: File,
action: () => s3FilePicker?.open()
})
}
if (showResourcePicker && customUi?.resource != false) {
items.push({
displayName: 'Resource',
icon: Package,
action: () => resourcePicker?.openDrawer()
})
}
if (showGitRepoPicker && customUi?.resource != false) {
items.push({
displayName: 'Git repository',
icon: GitBranch,
action: () => (gitRepoPickerOpen = true)
})
}
if (showResourceTypePicker && customUi?.type != false) {
items.push({
displayName: 'Resource type',
icon: Package,
action: () => resourceTypePicker?.openDrawer()
})
}
if (showDatabasePicker && customUi?.database != false) {
items.push({
displayName: 'Database',
icon: DatabaseIcon,
action: () => databasePicker?.openDrawer()
})
}
if (showDucklakePicker && customUi?.ducklake != false) {
items.push({
displayName: 'Ducklake',
icon: DucklakeIcon,
action: () => ducklakePicker?.openDrawer()
})
}
if (showDataTablePicker && customUi?.dataTable != false) {
items.push({
displayName: 'Data table',
icon: DatabaseIcon,
action: () => dataTablePicker?.openDrawer()
})
}
if (customUi?.reset != false) {
items.push({
displayName: 'Reset content',
icon: RotateCw,
action: () => clearContent(),
separatorTop: items.length > 0
})
}
return items
}
function insertDelegateToGitRepo(resourcePath: string) {
if (!editor) return
@@ -864,153 +949,183 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS
class="rounded-full w-2 h-2 mx-2 {validCode ? 'bg-green-300' : 'bg-red-300'}"
></div>
<div class="flex items-center gap-2">
{#if showContextVarPicker && customUi?.contextVar != false}
<Button
aiId="editor-bar-add-context-variable"
aiDescription="Add context variable"
title="Add context variable"
variant="subtle"
on:click={contextualVariablePicker.openDrawer}
unifiedSize="sm"
startIcon={{ icon: DollarSign }}
{iconOnly}
>+Context var
</Button>
{/if}
{#if showVarPicker && customUi?.variable != false}
<Button
aiId="editor-bar-add-variable"
aiDescription="Add variable"
title="Add variable"
variant="subtle"
on:click={variablePicker.openDrawer}
unifiedSize="sm"
startIcon={{ icon: DollarSign }}
{iconOnly}
>
+Variable
</Button>
{/if}
{#if showS3Picker && customUi?.s3object != false}
<Button
aiId="editor-bar-add-s3-object"
aiDescription="Add S3 Object"
title="Add S3 object"
variant="subtle"
on:click={() => s3FilePicker?.open()}
unifiedSize="sm"
startIcon={{ icon: File }}
{iconOnly}
>+S3 Object
</Button>
{/if}
{#if showResourcePicker && customUi?.resource != false}
<Button
aiId="editor-bar-add-resource"
aiDescription="Add resource"
title="Add resource"
unifiedSize="sm"
variant="subtle"
on:click={resourcePicker.openDrawer}
{iconOnly}
startIcon={{ icon: Package }}
>
+Resource
</Button>
{/if}
{#if showGitRepoPicker && customUi?.resource != false}
<GitRepoPopoverPicker
bind:isOpen={gitRepoPickerOpen}
on:selected={(e) => insertDelegateToGitRepo(e.detail.resourcePath)}
>
{#if compactHelpers}
{#snippet helpersDropdown()}
<DropdownV2 items={getHelperItems} placement="bottom-start">
{#snippet buttonReplacement()}
<Button
nonCaptureEvent
variant="subtle"
unifiedSize="sm"
startIcon={{ icon: Plus }}
title="Helpers"
>
Helpers
</Button>
{/snippet}
</DropdownV2>
{/snippet}
{#if showGitRepoPicker && customUi?.resource != false}
<!-- Wrap the Helpers dropdown so the Git-repo popover anchors to
the visible Helpers button rather than an sr-only placeholder. -->
<GitRepoPopoverPicker
bind:isOpen={gitRepoPickerOpen}
on:selected={(e) => insertDelegateToGitRepo(e.detail.resourcePath)}
>
{@render helpersDropdown()}
</GitRepoPopoverPicker>
{:else}
{@render helpersDropdown()}
{/if}
{:else}
{#if showContextVarPicker && customUi?.contextVar != false}
<Button
aiId="editor-bar-add-git-repo"
aiDescription="Delegate to Git repository"
title="Delegate to Git repository"
aiId="editor-bar-add-context-variable"
aiDescription="Add context variable"
title="Add context variable"
variant="subtle"
on:click={contextualVariablePicker.openDrawer}
unifiedSize="sm"
startIcon={{ icon: DollarSign }}
{iconOnly}
>+Context var
</Button>
{/if}
{#if showVarPicker && customUi?.variable != false}
<Button
aiId="editor-bar-add-variable"
aiDescription="Add variable"
title="Add variable"
variant="subtle"
on:click={variablePicker.openDrawer}
unifiedSize="sm"
startIcon={{ icon: DollarSign }}
{iconOnly}
>
+Variable
</Button>
{/if}
{#if showS3Picker && customUi?.s3object != false}
<Button
aiId="editor-bar-add-s3-object"
aiDescription="Add S3 Object"
title="Add S3 object"
variant="subtle"
on:click={() => s3FilePicker?.open()}
unifiedSize="sm"
startIcon={{ icon: File }}
{iconOnly}
>+S3 Object
</Button>
{/if}
{#if showResourcePicker && customUi?.resource != false}
<Button
aiId="editor-bar-add-resource"
aiDescription="Add resource"
title="Add resource"
unifiedSize="sm"
variant="subtle"
on:click={() => (gitRepoPickerOpen = true)}
on:click={resourcePicker.openDrawer}
{iconOnly}
startIcon={{ icon: GitBranch }}
startIcon={{ icon: Package }}
>
+Git Repo
+Resource
</Button>
</GitRepoPopoverPicker>
{/if}
{/if}
{#if showResourceTypePicker && customUi?.type != false}
<Button
aiId="editor-bar-add-resource-type"
aiDescription="Add resource type"
title="Add resource type"
variant="subtle"
unifiedSize="sm"
on:click={() => resourceTypePicker?.openDrawer()}
{iconOnly}
startIcon={{ icon: Package }}
>
+Type
</Button>
{/if}
{#if showGitRepoPicker && customUi?.resource != false}
<GitRepoPopoverPicker
bind:isOpen={gitRepoPickerOpen}
on:selected={(e) => insertDelegateToGitRepo(e.detail.resourcePath)}
>
<Button
aiId="editor-bar-add-git-repo"
aiDescription="Delegate to Git repository"
title="Delegate to Git repository"
unifiedSize="sm"
variant="subtle"
on:click={() => (gitRepoPickerOpen = true)}
{iconOnly}
startIcon={{ icon: GitBranch }}
>
+Git Repo
</Button>
</GitRepoPopoverPicker>
{/if}
{#if showDatabasePicker && customUi?.database != false}
<Button
aiId="editor-bar-add-database"
aiDescription="Add database"
title="Add database"
variant="subtle"
on:click={() => databasePicker?.openDrawer()}
unifiedSize="sm"
startIcon={{ icon: DatabaseIcon }}
{iconOnly}
>+Database
</Button>
{/if}
{#if showResourceTypePicker && customUi?.type != false}
<Button
aiId="editor-bar-add-resource-type"
aiDescription="Add resource type"
title="Add resource type"
variant="subtle"
unifiedSize="sm"
on:click={() => resourceTypePicker?.openDrawer()}
{iconOnly}
startIcon={{ icon: Package }}
>
+Type
</Button>
{/if}
{#if showDucklakePicker && customUi?.ducklake != false}
<Button
aiId="editor-bar-use-ducklake"
aiDescription="Use Ducklake"
title="Use Ducklake"
variant="subtle"
on:click={() => ducklakePicker?.openDrawer()}
unifiedSize="sm"
startIcon={{ icon: DucklakeIcon }}
{iconOnly}
>+Ducklake
</Button>
{/if}
{#if showDatabasePicker && customUi?.database != false}
<Button
aiId="editor-bar-add-database"
aiDescription="Add database"
title="Add database"
variant="subtle"
on:click={() => databasePicker?.openDrawer()}
unifiedSize="sm"
startIcon={{ icon: DatabaseIcon }}
{iconOnly}
>+Database
</Button>
{/if}
{#if showDataTablePicker && customUi?.dataTable != false}
<Button
aiId="editor-bar-use-datatable"
aiDescription="Use DataTable"
title="Use DataTable"
variant="subtle"
on:click={() => dataTablePicker?.openDrawer()}
unifiedSize="sm"
startIcon={{ icon: DatabaseIcon }}
{iconOnly}
>+Data table
</Button>
{/if}
{#if showDucklakePicker && customUi?.ducklake != false}
<Button
aiId="editor-bar-use-ducklake"
aiDescription="Use Ducklake"
title="Use Ducklake"
variant="subtle"
on:click={() => ducklakePicker?.openDrawer()}
unifiedSize="sm"
startIcon={{ icon: DucklakeIcon }}
{iconOnly}
>+Ducklake
</Button>
{/if}
{#if customUi?.reset != false}
<Button
aiId="editor-bar-reset-content"
aiDescription="Reset content"
title="Reset Content"
unifiedSize="sm"
variant="subtle"
on:click={clearContent}
{iconOnly}
startIcon={{ icon: RotateCw }}
>
Reset
</Button>
{#if showDataTablePicker && customUi?.dataTable != false}
<Button
aiId="editor-bar-use-datatable"
aiDescription="Use DataTable"
title="Use DataTable"
variant="subtle"
on:click={() => dataTablePicker?.openDrawer()}
unifiedSize="sm"
startIcon={{ icon: DatabaseIcon }}
{iconOnly}
>+Data table
</Button>
{/if}
{#if customUi?.reset != false}
<Button
aiId="editor-bar-reset-content"
aiDescription="Reset content"
title="Reset Content"
unifiedSize="sm"
variant="subtle"
on:click={clearContent}
{iconOnly}
startIcon={{ icon: RotateCw }}
>
Reset
</Button>
{/if}
{/if}
{#if customUi?.assistants != false}
@@ -204,7 +204,6 @@
initialPath={snapshotPath ?? path ?? ''}
namePlaceholder={kind}
{kind}
hideFullPath
size="sm"
drawerOffset={4000}
/>

Some files were not shown because too many files have changed in this diff Show More