diff --git a/.claude/hooks/guard-main-branch.sh b/.claude/hooks/guard-main-branch.sh index c7eeea9475..7a3a8189a0 100755 --- a/.claude/hooks/guard-main-branch.sh +++ b/.claude/hooks/guard-main-branch.sh @@ -16,6 +16,23 @@ command="$(echo "$input" | jq -r '.tool_input.command // empty')" if [[ "$command" =~ ^git\ (push|reset|revert|checkout|merge|rebase|commit|add) ]]; then branch="$(git rev-parse --abbrev-ref HEAD 2>/dev/null || true)" if [[ "$branch" == "main" ]]; then - echo "BLOCK: You are on the main branch. Create or switch to a feature branch first." + echo "BLOCK: You are on the main branch. Create or switch to a feature branch first." >&2 + exit 2 + fi +fi + +# Block force-push targeting main from any branch. +if [[ "$command" =~ ^git[[:space:]]+push([[:space:]]|$) ]]; then + has_force=false + if [[ "$command" =~ (--force([[:space:]]|=|$)|--force-with-lease|[[:space:]]-f([[:space:]]|$)) ]]; then + has_force=true + fi + # `+ref` refspec syntax is also a force push. + if [[ "$command" =~ [[:space:]]\+[A-Za-z] ]]; then + has_force=true + fi + if $has_force && [[ "$command" =~ (^|[[:space:]:])\+?main([[:space:]]|$) ]]; then + echo "BLOCK: Force-push to main is not allowed via Claude. Run it yourself if you really mean to." >&2 + exit 2 fi fi diff --git a/.claude/settings.json b/.claude/settings.json index 1ef3704831..ca8d9a898d 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -44,7 +44,25 @@ "Bash(git merge:*)", "Bash(git rebase:*)", "Bash(git add:*)", - "Bash(git commit:*)" + "Bash(git commit:*)", + "Read(/tmp/**)", + "Write(/tmp/**)", + "Edit(/tmp/**)", + "Bash(rm:/tmp/*)", + "Bash(rm:/tmp/**)", + "Bash(rmdir:/tmp/*)", + "Bash(mkdir:/tmp/*)", + "Bash(mkdir:/tmp/**)", + "Bash(cp:/tmp/*)", + "Bash(cp:/tmp/**)", + "Bash(mv:/tmp/*)", + "Bash(mv:/tmp/**)", + "Bash(touch:/tmp/*)", + "Bash(touch:/tmp/**)", + "Bash(chmod:/tmp/*)", + "Bash(chmod:/tmp/**)", + "Bash(tar * /tmp/*)", + "Bash(unzip * /tmp/*)" ], "deny": [ "Read(.env)", @@ -72,7 +90,13 @@ "Bash(chown:*)", "Bash(truncate:*)", "Bash(shred:*)", - "Bash(unlink:*)" + "Bash(unlink:*)", + "mcp__claude_ai_Stripe", + "mcp__claude_ai_Gmail", + "mcp__claude_ai_Google_Calendar", + "mcp__claude_ai_Google_Drive", + "mcp__claude_ai_Slack", + "mcp__claude_ai_Linear" ] }, "enableAllProjectMcpServers": true, diff --git a/.webmux.yaml b/.webmux.yaml index 19a0ea9c30..9c70a10ab1 100644 --- a/.webmux.yaml +++ b/.webmux.yaml @@ -100,10 +100,53 @@ profiles: integrations: github: + autoRemoveOnMerge: true linkedRepos: - repo: windmill-labs/windmill-ee-private alias: ee-private dir: ../windmill-ee-private__worktrees linear: enabled: true + autoCreateWorktrees: true watchTeams: [WIN,GIT] + +oneshot: + systemPrompt: | + You are running in webmux ONESHOT mode. + + # No interactive user + There is NO interactive user — nobody is watching the chat or will respond + to questions, approvals, or status checks. Any message asking the user to + review, approve, confirm, take a look, or "let you know" is wasted output: + it will not be answered. + + # Your job + Take the task to its real conclusion without pausing: + 1. Make the change. + 2. Validate it (run the relevant tests, typecheck, build, or quick + manual check). + 3. Commit. + 4. Push. + 5. Open a pull request. + Only then are you done. + + # Decisions + When something is ambiguous, pick the most reasonable default and proceed. + When you would normally ask "should I X or Y?", just pick one and continue + — note the choice in the PR description if it matters. + + # PR readiness + Default to opening the PR as a draft. If you are highly confident in the + change — the scope is small and well-understood, validation passed + cleanly, and you would not change anything if a reviewer pushed back — + open the PR as ready-for-review directly (omit `--draft` when invoking + `gh pr create`, or call `gh pr ready ` after creation). Err on + the side of draft when validation was partial, the change touches + public APIs or shared infrastructure, or you made a non-obvious judgment + call. + + # Ending your turn + Never end your turn with a question, a suggestion to "take a look", or a + request for approval. Stop only when the PR is open, or when you hit a + technical error you cannot recover from yourself (in which case clearly + state the blocker). diff --git a/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json b/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json index e7ed0aee65..d29a18c691 100644 --- a/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json +++ b/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json @@ -46,11 +46,11 @@ ] }, "nullable": [ - false, - false, - false, - false, - false, + true, + true, + true, + true, + true, true, true ] diff --git a/backend/.sqlx/query-295a88070e1762255cdd7680ba2e7bb2a2ffd66a7c432dd318e73e0f81ea9622.json b/backend/.sqlx/query-295a88070e1762255cdd7680ba2e7bb2a2ffd66a7c432dd318e73e0f81ea9622.json new file mode 100644 index 0000000000..3e47b7b034 --- /dev/null +++ b/backend/.sqlx/query-295a88070e1762255cdd7680ba2e7bb2a2ffd66a7c432dd318e73e0f81ea9622.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO variable\n (workspace_id, path, value, is_secret, description, account, is_oauth, expires_at, labels, edited_by)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Int4", + "Bool", + "Timestamptz", + "TextArray", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "295a88070e1762255cdd7680ba2e7bb2a2ffd66a7c432dd318e73e0f81ea9622" +} diff --git a/backend/.sqlx/query-5104cf045dc9b7b82d0028af11cfb5c2f6fd58e518085caf8e8d189951f7c4d8.json b/backend/.sqlx/query-5104cf045dc9b7b82d0028af11cfb5c2f6fd58e518085caf8e8d189951f7c4d8.json new file mode 100644 index 0000000000..46025de086 --- /dev/null +++ b/backend/.sqlx/query-5104cf045dc9b7b82d0028af11cfb5c2f6fd58e518085caf8e8d189951f7c4d8.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO draft\n (workspace_id, path, value, typ)\n VALUES ($1, $2, $3::text::json, $4)\n ON CONFLICT (workspace_id, path, typ)\n DO UPDATE SET value = EXCLUDED.value, created_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + { + "Custom": { + "name": "draft_type", + "kind": { + "Enum": [ + "script", + "flow", + "app" + ] + } + } + } + ] + }, + "nullable": [] + }, + "hash": "5104cf045dc9b7b82d0028af11cfb5c2f6fd58e518085caf8e8d189951f7c4d8" +} diff --git a/backend/.sqlx/query-5494652553c59b72ca5db4350a8ba3d8bbf1608519b08f2544c64ecfe130c537.json b/backend/.sqlx/query-5494652553c59b72ca5db4350a8ba3d8bbf1608519b08f2544c64ecfe130c537.json new file mode 100644 index 0000000000..cd81ba0052 --- /dev/null +++ b/backend/.sqlx/query-5494652553c59b72ca5db4350a8ba3d8bbf1608519b08f2544c64ecfe130c537.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE variable SET labels = $1, edited_at = now(), edited_by = $4 WHERE path = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "Text", + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "5494652553c59b72ca5db4350a8ba3d8bbf1608519b08f2544c64ecfe130c537" +} diff --git a/backend/.sqlx/query-8816bbe1ea9ea4f359e7a95857d349fa8bd7d23e4589b6936e0d000745cb3f34.json b/backend/.sqlx/query-8816bbe1ea9ea4f359e7a95857d349fa8bd7d23e4589b6936e0d000745cb3f34.json new file mode 100644 index 0000000000..53fe03ce5f --- /dev/null +++ b/backend/.sqlx/query-8816bbe1ea9ea4f359e7a95857d349fa8bd7d23e4589b6936e0d000745cb3f34.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT a.path FROM app_script s JOIN app a ON a.id = s.app\n WHERE s.id = $1 AND a.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Int8", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "8816bbe1ea9ea4f359e7a95857d349fa8bd7d23e4589b6936e0d000745cb3f34" +} diff --git a/backend/.sqlx/query-d4af472614e1b6af8defa40a79d5f584c8b114ef24fdb67a5eefb40ce17acdef.json b/backend/.sqlx/query-d4af472614e1b6af8defa40a79d5f584c8b114ef24fdb67a5eefb40ce17acdef.json new file mode 100644 index 0000000000..23087064d3 --- /dev/null +++ b/backend/.sqlx/query-d4af472614e1b6af8defa40a79d5f584c8b114ef24fdb67a5eefb40ce17acdef.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status s\n SET flow_status = JSONB_SET(s.flow_status, ARRAY['modules', s.flow_status->>'step', 'progress'], $1)\n FROM v2_job j\n WHERE s.id = $2 AND j.id = s.id AND j.workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid", + "Text" + ] + }, + "nullable": [] + }, + "hash": "d4af472614e1b6af8defa40a79d5f584c8b114ef24fdb67a5eefb40ce17acdef" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 1a3efc3e11..b8b0ab8625 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15667,6 +15667,7 @@ dependencies = [ "regex", "reqwest 0.13.1", "reqwest-middleware", + "rsa", "rust_decimal", "serde", "serde_json", diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 7cda45d507..142ca727a7 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -ec3cd353245e1cdf6a290528dbd7f2ac2498386c +daffe7bb81cfcaca666c61de1ee838a44d60ebc2 diff --git a/backend/migrations/20260512200642_add_edited_at_to_variable.down.sql b/backend/migrations/20260512200642_add_edited_at_to_variable.down.sql new file mode 100644 index 0000000000..efacb3376e --- /dev/null +++ b/backend/migrations/20260512200642_add_edited_at_to_variable.down.sql @@ -0,0 +1,3 @@ +ALTER TABLE variable + DROP COLUMN IF EXISTS edited_by, + DROP COLUMN IF EXISTS edited_at; diff --git a/backend/migrations/20260512200642_add_edited_at_to_variable.up.sql b/backend/migrations/20260512200642_add_edited_at_to_variable.up.sql new file mode 100644 index 0000000000..46fc8ad8be --- /dev/null +++ b/backend/migrations/20260512200642_add_edited_at_to_variable.up.sql @@ -0,0 +1,15 @@ +-- Add `edited_at` and `edited_by` so the UI can detect when a variable has +-- been modified remotely while a local autosave was in flight (see the +-- UserDraft staleness check). Mirrors what `resource` already has. +-- +-- Backfill: existing rows get `edited_at = now()` via the column's DEFAULT. +-- All pre-migration variables therefore appear to share a single edit +-- timestamp (the migration time). The staleness check only consumes +-- `edited_at` as an opaque rev string — it doesn't display or sort on it — +-- and only after the user edits a variable forward at least once. So the +-- collision is harmless: no UI flow looks at the pre-migration timestamp +-- before it gets overwritten by a real edit. + +ALTER TABLE variable + ADD COLUMN edited_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), + ADD COLUMN edited_by VARCHAR(50); diff --git a/backend/migrations/20260514233244_convert_draft_created_at_to_timestamptz.down.sql b/backend/migrations/20260514233244_convert_draft_created_at_to_timestamptz.down.sql new file mode 100644 index 0000000000..31a1f44859 --- /dev/null +++ b/backend/migrations/20260514233244_convert_draft_created_at_to_timestamptz.down.sql @@ -0,0 +1,5 @@ +-- Symmetric to the up migration: Postgres's default `TIMESTAMPTZ -> TIMESTAMP` +-- cast strips the timezone by representing the instant in the session's +-- current timezone, mirroring how the original `now()` values were +-- truncated on insert. +ALTER TABLE draft ALTER COLUMN created_at TYPE TIMESTAMP; diff --git a/backend/migrations/20260514233244_convert_draft_created_at_to_timestamptz.up.sql b/backend/migrations/20260514233244_convert_draft_created_at_to_timestamptz.up.sql new file mode 100644 index 0000000000..79f20611d4 --- /dev/null +++ b/backend/migrations/20260514233244_convert_draft_created_at_to_timestamptz.up.sql @@ -0,0 +1,12 @@ +-- `draft.created_at` was originally created as `TIMESTAMP` (no timezone). The +-- new `*WithDraft` API responses surface it as `chrono::DateTime` for the +-- frontend's staleness check, which requires `TIMESTAMPTZ`. +-- +-- We rely on Postgres's default `TIMESTAMP -> TIMESTAMPTZ` cast (no explicit +-- USING), which interprets each existing wall-clock value in the session's +-- current timezone. That's the exact semantics under which the original +-- `INSERT ... DEFAULT now()` values were truncated to TIMESTAMP — so the +-- conversion is a no-op on UTC servers (the common case) and correctly +-- recovers the original instant on non-UTC servers, instead of shifting all +-- pre-migration timestamps by the server's tz offset. +ALTER TABLE draft ALTER COLUMN created_at TYPE TIMESTAMPTZ; diff --git a/backend/src/main.rs b/backend/src/main.rs index d43b9a69bb..89293c379a 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -51,14 +51,14 @@ use windmill_common::{ JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MAVEN_SETTINGS_XML_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING, - NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, - OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, - POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, - REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RESTART_COORDINATION_SETTING, - RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, SAML_METADATA_SETTING, - SCIM_TOKEN_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TEAMS_SETTING, - TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, - WORKSPACE_REGISTRIES_SETTING, + NPM_CONFIG_REGISTRY_SETTING, NSJAIL_TMPFS_SIZE_MB_SETTING, NUGET_CONFIG_SETTING, + OAUTH_SETTING, OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, + POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, + REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, + RESTART_COORDINATION_SETTING, RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, + SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, + TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, + UV_INDEX_STRATEGY_SETTING, WORKSPACE_REGISTRIES_SETTING, }, scripts::ScriptLang, stats_oss::schedule_stats, @@ -127,9 +127,10 @@ use crate::monitor::{ reload_http_route_workspaced_route_setting, reload_hub_api_secret_setting, reload_hub_base_url_setting, reload_instance_events_webhook_setting, reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting, - reload_license_key, reload_npm_config_registry_setting, reload_otel_tracing_proxy_setting, - reload_pip_index_url_setting, reload_retention_period_setting, reload_scim_token_setting, - reload_smtp_config, reload_store_audit_logs_s3_setting, reload_uv_exclude_newer_setting, + reload_license_key, reload_npm_config_registry_setting, reload_nsjail_tmpfs_size_setting, + reload_otel_tracing_proxy_setting, reload_pip_index_url_setting, + reload_retention_period_setting, reload_scim_token_setting, reload_smtp_config, + reload_store_audit_logs_s3_setting, reload_uv_exclude_newer_setting, reload_uv_index_strategy_setting, reload_worker_config, MonitorIteration, }; @@ -1782,6 +1783,7 @@ async fn process_notify_event( STORE_AUDIT_LOGS_S3_SETTING => reload_store_audit_logs_s3_setting(conn).await, JOB_DEFAULT_TIMEOUT_SECS_SETTING => reload_job_default_timeout_setting(conn).await, JOB_ISOLATION_SETTING => reload_job_isolation_setting(conn).await, + NSJAIL_TMPFS_SIZE_MB_SETTING => reload_nsjail_tmpfs_size_setting(conn).await, #[cfg(feature = "parquet")] OBJECT_STORE_CONFIG_SETTING => { if !disable_s3_store { diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index f8ca1e1d22..179c786919 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -62,12 +62,12 @@ use windmill_common::{ HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING, - NUGET_CONFIG_SETTING, OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, - POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, - REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, - RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, - STORE_AUDIT_LOGS_S3_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, - UV_INDEX_STRATEGY_SETTING, + NSJAIL_TMPFS_SIZE_MB_SETTING, NUGET_CONFIG_SETTING, OTEL_SETTING, + OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, + POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, + REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, + SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, STORE_AUDIT_LOGS_S3_SETTING, + TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, }, indexer::load_indexer_config, jwt::JWT_SECRET, @@ -106,9 +106,10 @@ use windmill_worker::{ OtelTracingProxySettings, SameWorkerSender, WorkspaceRegistryMap, BUNFIG_INSTALL_SCOPES, BUN_INSTALL_MIN_RELEASE_AGE, CARGO_REGISTRIES, INSTANCE_PYTHON_VERSION, JAVA_HOME_DIR, JOB_DEFAULT_TIMEOUT, JOB_ISOLATION, KEEP_JOB_DIR, MAVEN_REPOS, MAVEN_SETTINGS_XML, - NO_DEFAULT_MAVEN, NPMRC, NPM_CONFIG_REGISTRY, NSJAIL_AVAILABLE, NUGET_CONFIG, - OTEL_TRACING_PROXY_SETTINGS, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, POWERSHELL_REPO_PAT, - POWERSHELL_REPO_URL, UNSHARE_PATH, UV_EXCLUDE_NEWER, UV_INDEX_STRATEGY, WORKSPACE_REGISTRIES, + NO_DEFAULT_MAVEN, NPMRC, NPM_CONFIG_REGISTRY, NSJAIL_AVAILABLE, NSJAIL_TMPFS_SIZE_MB, + NUGET_CONFIG, OTEL_TRACING_PROXY_SETTINGS, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, + POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, UNSHARE_PATH, UV_EXCLUDE_NEWER, UV_INDEX_STRATEGY, + WORKSPACE_REGISTRIES, }; #[cfg(feature = "parquet")] @@ -384,6 +385,7 @@ pub async fn initial_load( if worker_mode { reload_job_default_timeout_setting(&conn).await; reload_job_isolation_setting(&conn).await; + reload_nsjail_tmpfs_size_setting(&conn).await; reload_extra_pip_index_url_setting(&conn).await; reload_pip_index_url_setting(&conn).await; reload_uv_index_strategy_setting(&conn).await; @@ -1889,6 +1891,16 @@ pub async fn reload_job_default_timeout_setting(conn: &Connection) { .await; } +pub async fn reload_nsjail_tmpfs_size_setting(conn: &Connection) { + reload_option_setting_with_tracing( + conn, + NSJAIL_TMPFS_SIZE_MB_SETTING, + "NSJAIL_TMPFS_SIZE_MB", + NSJAIL_TMPFS_SIZE_MB.clone(), + ) + .await; +} + pub async fn reload_job_isolation_setting(conn: &Connection) { let value = match load_value_from_global_settings_with_conn(conn, JOB_ISOLATION_SETTING, true).await { diff --git a/backend/tests/app_preview_auth.rs b/backend/tests/app_preview_auth.rs new file mode 100644 index 0000000000..4653303392 --- /dev/null +++ b/backend/tests/app_preview_auth.rs @@ -0,0 +1,258 @@ +//! Regression test for the app component preview authorization bypass. +//! +//! `POST /api/w/:workspace/apps_u/execute_component/:path` runs in "preview" +//! mode whenever the client supplies `force_viewer_static_fields`. In that +//! mode it accepts request-supplied `raw_code` and enqueues it as a +//! `Viewer`-mode job — i.e. it is the app-editor equivalent of +//! `/jobs/run/preview`. The bug was that this branch did not re-apply the +//! guards `/jobs/run/preview` enforces for arbitrary code execution, so an +//! authenticated Operator (a run-only user who must not be able to create +//! scripts/apps or run preview jobs) could enqueue arbitrary worker code with +//! a single request, escaping the Operator restriction entirely. +//! +//! This test pins down: +//! - an Operator is rejected from preview mode (the core fix; pre-fix this +//! enqueued a job and returned 200), +//! - a regular non-operator member can still run an editor preview (the fix +//! must not over-block the legitimate editor flow), +//! - preview is confined to paths the caller can read (defense-in-depth +//! against scoped tokens / cross-namespace preview), and +//! - run mode (no `force_viewer_static_fields`) is unaffected by the guard. + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {}", token)) +} + +/// A preview request: `force_viewer_static_fields` present + inline `raw_code`. +/// This is the exact shape an attacker (or the editor) sends. +fn preview_body(app_path: &str) -> serde_json::Value { + json!({ + "args": {}, + "component": "comp", + "raw_code": { + "language": "deno", + "content": "export function main() { return \"pwned\"; }", + "path": format!("{}/comp", app_path) + }, + "force_viewer_static_fields": {} + }) +} + +#[sqlx::test(fixtures("base", "app_preview_auth"))] +async fn test_app_preview_authorization(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/apps_u/execute_component"); + + // 1. CORE REGRESSION: an Operator sends a preview request in their own + // namespace (so the *only* thing that can reject them is the Operator + // check itself). Pre-fix this returned 200 with an enqueued job UUID; + // post-fix it must be rejected. + let resp = authed( + client().post(format!("{base}/u/operator-user/myapp")), + "OPERATOR_TOKEN", + ) + .json(&preview_body("u/operator-user/myapp")) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!( + status, 401, + "Operator must be rejected from app preview (got {status}): {body}" + ); + assert!( + body.contains("Operators cannot run preview jobs"), + "rejection must be the operator guard, got: {body}" + ); + + // 2. The fix must NOT over-block the legitimate editor flow: a regular + // non-operator member previewing in their own namespace still works + // (the endpoint returns the enqueued job UUID before any worker runs). + let resp = authed( + client().post(format!("{base}/u/test-user-2/myapp")), + "SECRET_TOKEN_2", + ) + .json(&preview_body("u/test-user-2/myapp")) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert!( + status.is_success(), + "non-operator editor preview must still succeed (got {status}): {body}" + ); + assert!( + uuid::Uuid::parse_str(body.trim()).is_ok(), + "successful preview must return a job UUID, got: {body}" + ); + + // 3. Inline `raw_code` preview is deliberately NOT path-gated: a + // non-operator can already run arbitrary inline code via + // `/jobs/run/preview`, so the app URL path string is irrelevant for the + // inline case. This pins that decision so an over-restrictive path check + // is not re-added for inline previews. + let resp = authed( + client().post(format!("{base}/u/test-user/secretapp")), + "SECRET_TOKEN_2", + ) + .json(&preview_body("u/test-user/secretapp")) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert!( + status.is_success(), + "inline raw_code preview must not be path-gated (got {status}): {body}" + ); + assert!( + uuid::Uuid::parse_str(body.trim()).is_ok(), + "inline preview should enqueue a job UUID, got: {body}" + ); + + // 4. Run mode (no `force_viewer_static_fields`) is unaffected by the new + // preview guard: an Operator hitting a deployed-app path still follows + // the pre-existing policy lookup (here: the app does not exist -> 404), + // proving the guard only gates preview mode. + let resp = authed( + client().post(format!("{base}/u/operator-user/nonexistent")), + "OPERATOR_TOKEN", + ) + .json(&json!({ + "args": {}, + "component": "comp", + "raw_code": { + "language": "deno", + "content": "export function main() { return 1; }", + "path": "u/operator-user/nonexistent/comp" + } + })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!( + status, 404, + "run mode must be unchanged (deployed app lookup -> 404, not the preview guard); got {status}: {body}" + ); + + // 5. Defense-in-depth: the guard must check the *runnable* being previewed, + // not just the app URL path. A caller pairs an allowed app path + // (`u/test-user-2/myapp`, own namespace) with a `path` pointing at a + // deployed runnable in another user's namespace. Without checking the + // runnable path this would resolve `script/u/test-user/private` with the + // root DB handle and enqueue it; it must be rejected by the path check. + let resp = authed( + client().post(format!("{base}/u/test-user-2/myapp")), + "SECRET_TOKEN_2", + ) + .json(&json!({ + "args": {}, + "component": "comp", + "path": "script/u/test-user/private", + "force_viewer_static_fields": {} + })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!( + status, 400, + "preview targeting a runnable outside the caller's namespace must be rejected even with an allowed app path (got {status}): {body}" + ); + + // 6. Defense-in-depth: a persisted inline-script preview selects code by the + // caller-controlled `app_script` id. Pairing an allowed app path with an + // id owned by another (private) app must be rejected — without the + // id-ownership check the worker would fetch and run that app's code. + let resp = authed( + client().post(format!("{base}/u/test-user-2/myapp")), + "SECRET_TOKEN_2", + ) + .json(&json!({ + "args": {}, + "component": "comp", + "id": 999777, + "raw_code": { + "language": "deno", + "content": "export function main() { return 1; }", + "path": "u/test-user-2/myapp/comp" + }, + "force_viewer_static_fields": {} + })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!( + status, 400, + "preview with an app_script id owned by another app must be rejected (got {status}): {body}" + ); + + // 7. The id-ownership check must NOT over-block a legitimate persisted + // inline-script preview: an id owned by an app in the caller's own + // namespace passes the guard and enqueues (returns a job UUID). + let resp = authed( + client().post(format!("{base}/u/test-user-2/ownapp")), + "SECRET_TOKEN_2", + ) + .json(&json!({ + "args": {}, + "component": "comp", + "id": 999778, + "raw_code": { + "language": "deno", + "content": "export function main() { return 1; }", + "path": "u/test-user-2/ownapp/comp" + }, + "force_viewer_static_fields": {} + })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert!( + status.is_success(), + "persisted preview for an app the caller owns must still succeed (got {status}): {body}" + ); + assert!( + uuid::Uuid::parse_str(body.trim()).is_ok(), + "successful persisted preview must return a job UUID, got: {body}" + ); + + // 8. Scope escalation: a token scoped to `apps:run` (but not `jobs:run`) + // can reach this route (it maps to the `apps` scope domain) and is not an + // Operator, but must NOT be able to enqueue arbitrary preview `raw_code`. + // `/jobs/run/preview` requires `jobs:run` for exactly this reason; the + // app preview path must enforce the same. Without the `jobs:run` check + // this enqueues a job (returns a UUID); with it, it is rejected (403). + let resp = authed( + client().post(format!("{base}/u/test-user-2/myapp")), + "APPS_RUN_TOKEN", + ) + .json(&preview_body("u/test-user-2/myapp")) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!( + status, 403, + "apps:run-scoped token must not escalate to arbitrary preview code (got {status}): {body}" + ); + assert!( + body.contains("jobs:run"), + "rejection must be the jobs:run scope gate, got: {body}" + ); + + Ok(()) +} diff --git a/backend/tests/fixtures/app_preview_auth.sql b/backend/tests/fixtures/app_preview_auth.sql new file mode 100644 index 0000000000..9fcda61c51 --- /dev/null +++ b/backend/tests/fixtures/app_preview_auth.sql @@ -0,0 +1,34 @@ +-- Fixture for the app component preview authorization regression test. +-- Layered on top of `base` (which provides test-workspace, the admin +-- `test-user`/SECRET_TOKEN, and the non-operator `test-user-2`/SECRET_TOKEN_2). +-- Adds an Operator member so we can assert that Operators cannot reach the +-- arbitrary-code app preview path (`force_viewer_static_fields` + `raw_code`). + +INSERT INTO password(email, password_hash, login_type, super_admin, verified, name) + VALUES ('operator@windmill.dev', 'not-a-real-hash', 'password', false, true, 'Operator User'); + +INSERT INTO usr(workspace_id, email, username, is_admin, operator, role) VALUES + ('test-workspace', 'operator@windmill.dev', 'operator-user', false, true, 'Operator'); + +INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin) VALUES + (encode(sha256('OPERATOR_TOKEN'::bytea), 'hex'), 'OPERATOR_T', 'OPERATOR_TOKEN', 'operator@windmill.dev', 'operator token', false); + +-- A non-operator token scoped to `apps:run` but NOT `jobs:run`. It can reach +-- the `apps_u/execute_component` route (route maps to the `apps` scope domain) +-- but must not be able to enqueue arbitrary preview `raw_code`. +INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES + (encode(sha256('APPS_RUN_TOKEN'::bytea), 'hex'), 'APPS_RUN_T', 'APPS_RUN_TOKEN', 'test2@windmill.dev', 'apps:run scoped token', false, '{apps:run}'); + +-- A private app owned by `test-user` with a persisted inline script. Used to +-- assert that `test-user-2` cannot preview-execute another app's app_script id. +INSERT INTO app (id, workspace_id, path, summary, policy, versions) VALUES + (999001, 'test-workspace', 'u/test-user/private', 'private app', '{}'::jsonb, '{}'); +INSERT INTO app_script (id, app, hash, code, code_sha256) VALUES + (999777, 999001, repeat('a', 64), 'export function main(){ return "secret" }', repeat('b', 64)); + +-- An app owned by `test-user-2` with its own persisted inline script, to assert +-- the id-ownership check does not over-block a legitimate persisted preview. +INSERT INTO app (id, workspace_id, path, summary, policy, versions) VALUES + (999002, 'test-workspace', 'u/test-user-2/ownapp', 'own app', '{}'::jsonb, '{}'); +INSERT INTO app_script (id, app, hash, code, code_sha256) VALUES + (999778, 999002, repeat('c', 64), 'export function main(){ return "ok" }', repeat('d', 64)); diff --git a/backend/windmill-api-flows/src/flows.rs b/backend/windmill-api-flows/src/flows.rs index f20511ab14..8a42bce88e 100644 --- a/backend/windmill-api-flows/src/flows.rs +++ b/backend/windmill-api-flows/src/flows.rs @@ -1480,6 +1480,9 @@ pub struct FlowWDraft { pub extra_perms: serde_json::Value, #[serde(skip_serializing_if = "Option::is_none")] pub draft: Option>>, + /// Timestamp at which the most recent DB draft was created. + #[serde(skip_serializing_if = "Option::is_none")] + pub draft_created_at: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub draft_only: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -1516,6 +1519,7 @@ async fn get_flow_by_path_w_draft( flow.ws_error_handler_muted, flow.dedicated_worker, draft.value AS draft, + draft.created_at AS draft_created_at, flow.tag, flow.visible_to_runner_only, flow.on_behalf_of_email, diff --git a/backend/windmill-api-jobs/src/job_metrics.rs b/backend/windmill-api-jobs/src/job_metrics.rs index 59cb23deff..abb95693f5 100644 --- a/backend/windmill-api-jobs/src/job_metrics.rs +++ b/backend/windmill-api-jobs/src/job_metrics.rs @@ -180,13 +180,22 @@ async fn set_job_progress( // If flow_job_id exists, than we should modify flow_status of corresponding module // Individual jobs and flows are handled differently if let Some(flow_job_id) = flow_job_id { + // `v2_job_status` has no workspace_id column and the root db handle + // bypasses RLS (the per-row policy on the table is also inert today — + // `ENABLE ROW LEVEL SECURITY` was never set). Scope the update by + // joining `v2_job` so the URL's workspace_id confines tampering to the + // caller's workspace; without this, an authed member of any workspace + // could overwrite the flow `progress` UI field of a flow in another + // workspace given just the flow UUID. // TODO: Return error if trying to set completed job? sqlx::query!( - "UPDATE v2_job_status - SET flow_status = JSONB_SET(flow_status, ARRAY['modules', flow_status->>'step', 'progress'], $1) - WHERE id = $2", + "UPDATE v2_job_status s + SET flow_status = JSONB_SET(s.flow_status, ARRAY['modules', s.flow_status->>'step', 'progress'], $1) + FROM v2_job j + WHERE s.id = $2 AND j.id = s.id AND j.workspace_id = $3", serde_json::json!(percent.clamp(0, 99)), - flow_job_id + flow_job_id, + w_id, ) .execute(&db) .await?; diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index c297468cce..76952c68ad 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -95,6 +95,9 @@ pub struct ScriptWDraft { pub tag: Option, #[serde(skip_serializing_if = "Option::is_none")] pub draft: Option>>, + /// Timestamp at which the most recent DB draft was created. + #[serde(skip_serializing_if = "Option::is_none")] + pub draft_created_at: Option>, pub schema: Option, #[serde(skip_serializing_if = "Option::is_none")] pub draft_only: Option, @@ -172,6 +175,7 @@ impl ScriptWDraft { kind: self.kind, tag: self.tag, draft: self.draft, + draft_created_at: self.draft_created_at, schema: self.schema, draft_only: self.draft_only, envs: self.envs, @@ -1888,7 +1892,7 @@ async fn get_script_by_path_w_draft( let mut tx = user_db.begin(&authed).await?; let script_o = sqlx::query_as::<_, ScriptWDraft>( - "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, runnable_settings_handle, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, has_preprocessor, on_behalf_of_email, assets, modules, debounce_key, debounce_delay_s, labels FROM script LEFT JOIN draft ON + "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, runnable_settings_handle, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, ws_error_handler_muted, draft.value as draft, draft.created_at as draft_created_at, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, has_preprocessor, on_behalf_of_email, assets, modules, debounce_key, debounce_delay_s, labels FROM script LEFT JOIN draft ON script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script' WHERE script.path = $1 AND script.workspace_id = $2 ORDER BY script.created_at DESC LIMIT 1", diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index a341b98125..e55985d377 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -9588,6 +9588,10 @@ paths: properties: draft: $ref: "#/components/schemas/Flow" + draft_created_at: + type: string + format: date-time + description: Timestamp at which the most recent DB draft was created. Used by the frontend's UserDraft staleness check. /w/{workspace}/flows/exists/{path}: get: @@ -9836,6 +9840,13 @@ paths: - path_autocomplete parameters: - $ref: "#/components/parameters/WorkspaceId" + - name: force + description: | + bypass the server-side cache and re-query the DB, refreshing the + cache. Used right after a deploy so the new path appears immediately. + in: query + schema: + type: boolean responses: "200": description: deduplicated path list, sorted lexicographically @@ -10712,6 +10723,12 @@ paths: run_query_params: type: object description: Runnable query parameters + temp_script_refs: + type: object + nullable: true + description: "Map of relative-import script path -> temp storage hash. Only honored for inline-script (raw_code) execution so app dev resolves those imports from not-yet-deployed local content." + additionalProperties: + type: string required: - args - component @@ -19838,6 +19855,12 @@ paths: properties: is_alive: type: boolean + state: + type: string + enum: + - running + - stale + - never_started last_locked_at: type: string format: date-time @@ -19859,6 +19882,12 @@ paths: properties: is_alive: type: boolean + state: + type: string + enum: + - running + - stale + - never_started last_locked_at: type: string format: date-time @@ -21064,6 +21093,9 @@ components: mount_path: type: string description: KV v2 secrets engine mount path (e.g., windmill) + kv_secret_path_prefix: + type: string + description: Optional path prefix inserted between the KV data/metadata segment and the workspace id (e.g., "apps/windmill"). When set, secrets are stored at `/data///`, allowing a Vault policy scoped to exactly `/data//*`. jwt_role: type: string description: Vault JWT auth role name for Windmill (optional, if not provided token auth is used) @@ -21726,6 +21758,10 @@ components: properties: draft: $ref: "#/components/schemas/NewScript" + draft_created_at: + type: string + format: date-time + description: Timestamp at which the most recent DB draft was created. Used by the frontend's UserDraft staleness check. hash: type: string required: @@ -22793,6 +22829,11 @@ components: type: string ws_specific: type: boolean + edited_at: + type: string + format: date-time + edited_by: + type: string required: - workspace_id - path @@ -23205,6 +23246,12 @@ components: description: "Additional script modules keyed by relative file path" additionalProperties: $ref: "#/components/schemas/ScriptModule" + temp_script_refs: + type: object + nullable: true + description: "Map of relative-import script path -> temp storage hash so the preview job resolves those imports from not-yet-deployed local content instead of the deployed script" + additionalProperties: + type: string required: - args @@ -26552,6 +26599,12 @@ components: type: string restarted_from: $ref: "#/components/schemas/RestartedFrom" + temp_script_refs: + type: object + nullable: true + description: "Map of relative-import script path -> temp storage hash, propagated to each flow step so inline-script relative imports resolve from not-yet-deployed local content instead of the deployed script" + additionalProperties: + type: string required: - value @@ -26790,6 +26843,10 @@ components: draft_only: type: boolean draft: {} + draft_created_at: + type: string + format: date-time + description: Timestamp at which the most recent DB draft was created. Used by the frontend's UserDraft staleness check. AppHistory: type: object diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 9c64bb0da4..c377203f38 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -11,7 +11,7 @@ use crate::{ auth::{get_end_user_email, OptTokened}, db::{ApiAuthed, DB}, jobs::RunJobQuery, - users::{require_owner_of_path, OptAuthed}, + users::{require_owner_of_path, require_path_read_access_for_preview, OptAuthed}, utils::{check_scopes, WithStarredInfoQuery}, webhook_util::{WebhookMessage, WebhookShared}, HTTP_CLIENT, @@ -217,6 +217,9 @@ pub struct AppWithLastVersionAndDraft { pub draft: Option>>, #[serde(skip_serializing_if = "Option::is_none")] pub draft_only: Option, + /// Timestamp at which the most recent DB draft was created. + #[serde(skip_serializing_if = "Option::is_none")] + pub draft_created_at: Option>, } #[derive(Serialize)] @@ -642,29 +645,30 @@ async fn get_app_w_draft( let app_o = sqlx::query_as::<_, AppWithLastVersionAndDraft>( r#" - SELECT - app.id, - app.path, - app.summary, - app.versions, - app.policy, + SELECT + app.id, + app.path, + app.summary, + app.versions, + app.policy, app.custom_path, - app.extra_perms, + app.extra_perms, app_version.value, - app_version.created_at, + app_version.created_at, app_version.created_by, app.draft_only, draft.value AS "draft", + draft.created_at AS "draft_created_at", app_version.raw_app, app.labels FROM app - INNER JOIN app_version + INNER JOIN app_version ON app_version.id = app.versions[array_upper(app.versions, 1)] - LEFT JOIN draft - ON app.path = draft.path - AND draft.workspace_id = $2 + LEFT JOIN draft + ON app.path = draft.path + AND draft.workspace_id = $2 AND draft.typ = 'app' - WHERE app.path = $1 + WHERE app.path = $1 AND app.workspace_id = $2 "#, ) @@ -2039,6 +2043,10 @@ pub struct ExecuteApp { pub force_viewer_delete_after_secs: Option, /// Runnable query parameters (e.g., memory_id for chat-enabled flows) pub run_query_params: Option, + /// Map of relative-import script path -> temp storage hash. Only honored for + /// inline-script (raw_code, preview) execution so `wmill app dev` resolves + /// those imports from not-yet-deployed local content instead of deployed. + pub temp_script_refs: Option>, } fn digest(code: &str) -> String { @@ -2113,8 +2121,16 @@ async fn execute_component( Extension(db): Extension, Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, - Json(payload): Json, + Json(mut payload): Json, ) -> Result { + // Only honor temp_script_refs for the inline-script preview path: + // preview/editor mode (force_viewer_static_fields set, == `is_preview`), + // raw_code present, and no deployed app_script id — i.e. `wmill app dev`. + let temp_script_refs = payload.temp_script_refs.take(); + let inject_temp_refs = temp_script_refs.is_some() + && payload.force_viewer_static_fields.is_some() + && payload.raw_code.is_some() + && payload.id.is_none(); match (payload.path.is_some(), payload.raw_code.is_some()) { (false, false) => { return Err(Error::BadRequest( @@ -2138,6 +2154,54 @@ async fn execute_component( // tag from the deployed policy and ignore the request body. let is_preview = payload.force_viewer_static_fields.is_some(); + // Preview mode runs request-supplied code as a `Viewer`-mode job (the + // app-editor equivalent of `/jobs/run/preview`), so it enforces the same + // guards. Operators must never run preview jobs. `jobs:run` is required + // because this route is reachable with an `apps:run`-scoped token (the + // route maps to the `apps` scope domain), which must not be able to escalate + // to arbitrary code execution. The client-supplied inline `raw_code.tag` + // must stay within the caller's allowed worker tags. A preview can also + // *reference* an existing runnable the caller may not be allowed to read — a + // deployed script/flow via `payload.path` or a persisted `app_script` via + // `payload.id`, both resolved with the root DB handle — so those (and only + // those) are confined to paths the caller can read. Inline `raw_code` is not + // path-gated: a non-operator member can already run arbitrary inline code + // via `/jobs/run/preview`. + if is_preview { + let authed = opt_authed.as_ref().ok_or_else(|| { + Error::NotAuthorized("App component preview requires authentication".to_string()) + })?; + if authed.is_operator { + return Err(Error::NotAuthorized( + "Operators cannot run preview jobs for security reasons".to_string(), + )); + } + check_scopes(authed, || format!("jobs:run"))?; + if let Some(p) = payload.path.as_deref() { + let runnable_path = p + .strip_prefix("script/") + .or_else(|| p.strip_prefix("flow/")) + .unwrap_or(p); + require_path_read_access_for_preview(authed, &Some(runnable_path.to_string()))?; + } + if let Some(id) = payload.id { + let owner_path = sqlx::query_scalar!( + "SELECT a.path FROM app_script s JOIN app a ON a.id = s.app + WHERE s.id = $1 AND a.workspace_id = $2", + id, + &w_id, + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| { + Error::NotAuthorized(format!( + "App script {id} does not belong to an app in this workspace" + )) + })?; + require_path_read_access_for_preview(authed, &Some(owner_path))?; + } + } + // Two cases here: // 1. The component is executed from the editor (i.e. in "preview" mode), then: // - The policy is set to default (in `Viewer` execution mode). @@ -2295,7 +2359,7 @@ async fn execute_component( let resolved_delete_secs = resolve_delete_after_secs(None, policy_triggerables.delete_after_secs); - let (args, job_id) = build_args( + let (mut args, job_id) = build_args( policy, policy_triggerables, payload.args, @@ -2306,6 +2370,14 @@ async fn execute_component( ) .await?; + if inject_temp_refs { + if let Some(refs) = temp_script_refs { + args.extra + .get_or_insert_with(HashMap::new) + .insert("_TEMP_SCRIPT_REFS".to_string(), to_raw_value(&refs)); + } + } + let is_flow = payload .path .as_ref() @@ -2341,6 +2413,20 @@ async fn execute_component( ), _ => unreachable!(), }; + // Preview honors the client-supplied inline tag (`resolved_inline_tag`), so + // — like `/jobs/run/preview` — confine it to worker tags the caller may use + // (a `if_jobs:filter_tags`-restricted token must not escape its filter). + // `is_preview` implies an authed caller (the guard above returns otherwise). + if is_preview { + if let Some(authed) = opt_authed.as_ref() { + crate::jobs::check_tag_available_for_workspace(&db, &w_id, &tag, authed).await?; + } + } + // Identity is already resolved to the requesting user in preview mode (the + // policy is forced to `ExecutionMode::Viewer`, so the job runs as the + // caller). The enqueue stays root-isolated as before — switching the insert + // to user-RLS is not what contains the bypass (the auth guards above are) + // and would add unnecessary breakage risk to the legitimate editor flow. let tx = PushIsolationLevel::IsolatedRoot(db.clone()); let (email, permissioned_as) = if let Some(on_behalf_of) = on_behalf_of.as_ref() { @@ -2507,6 +2593,21 @@ async fn upload_s3_file_from_app( request: axum::extract::Request, ) -> JsonResult { let policy = if let Some(file_key_regex) = query.force_viewer_file_key_regex { + // `force_viewer_*` lets the caller supply a synthetic upload policy that + // bypasses the deployed app's file_key_regex / resource restrictions. + // It is intended for the app editor's preview path, so it must enforce + // the same guards as `execute_component`'s preview mode (PR #9235): + // authed caller, not an operator, and `apps:write` scope to make sure + // an `apps:run`-scoped token cannot pick its own policy. + let authed = opt_authed.as_ref().ok_or_else(|| { + Error::NotAuthorized("App S3 preview upload requires authentication".to_string()) + })?; + if authed.is_operator { + return Err(Error::NotAuthorized( + "Operators cannot run app S3 previews for security reasons".to_string(), + )); + } + check_scopes(authed, || format!("apps:write:{}", path.to_path()))?; Some(Policy { execution_mode: ExecutionMode::Viewer, triggerables: None, @@ -3018,6 +3119,20 @@ async fn download_s3_file_from_app( let force_viewer_allowed_s3_keys = if let Some(force_viewer_allowed_s3_keys) = query.force_viewer_allowed_s3_keys.clone() { + // `force_viewer_allowed_s3_keys` lets the caller supply a synthetic + // allowlist that bypasses the deployed app policy. Apply the same + // preview-mode guard as `execute_component` (PR #9235): authed, not an + // operator, `apps:write` scope so an `apps:run`-scoped token cannot + // pick its own allowlist. + let authed = opt_authed.as_ref().ok_or_else(|| { + Error::NotAuthorized("App S3 preview download requires authentication".to_string()) + })?; + if authed.is_operator { + return Err(Error::NotAuthorized( + "Operators cannot run app S3 previews for security reasons".to_string(), + )); + } + check_scopes(authed, || format!("apps:write:{}", path))?; Some(serde_json::from_str::>(&force_viewer_allowed_s3_keys).unwrap_or_default()) } else { None diff --git a/backend/windmill-api/src/drafts.rs b/backend/windmill-api/src/drafts.rs index 39a68d8f9a..b0cd5f61b0 100644 --- a/backend/windmill-api/src/drafts.rs +++ b/backend/windmill-api/src/drafts.rs @@ -79,7 +79,8 @@ async fn create_draft( "INSERT INTO draft (workspace_id, path, value, typ) VALUES ($1, $2, $3::text::json, $4) - ON CONFLICT (workspace_id, path, typ) DO UPDATE SET value = EXCLUDED.value", + ON CONFLICT (workspace_id, path, typ) + DO UPDATE SET value = EXCLUDED.value, created_at = now()", &w_id, draft.path, //to preserve key orders diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 4d61df606e..cb8d71022f 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -3633,6 +3633,10 @@ struct Preview { format: Option, flow_path: Option, modules: Option>, + /// Map of relative-import script path -> temp storage hash. When set, the + /// preview job resolves those imports from not-yet-deployed local content + /// (uploaded to raw_script_temp) instead of the deployed script. + temp_script_refs: Option>, } #[cfg(feature = "run_inline")] @@ -3661,6 +3665,10 @@ struct PreviewFlow { args: Option>>, tag: Option, restarted_from: Option, + /// Map of relative-import script path -> temp storage hash. Propagated to + /// each flow step so inline-script relative imports resolve from + /// not-yet-deployed local content instead of the deployed script. + temp_script_refs: Option>, } #[derive(Debug, Deserialize)] @@ -5662,6 +5670,12 @@ async fn run_preview_script( if let Some(ref modules) = preview.modules { extra.insert("_MODULES".to_string(), to_raw_value(modules)); } + if let Some(ref temp_script_refs) = preview.temp_script_refs { + extra.insert( + "_TEMP_SCRIPT_REFS".to_string(), + to_raw_value(temp_script_refs), + ); + } let extra = if extra.is_empty() { None } else { Some(extra) }; let push_args = PushArgs { extra, args: &preview_args }; @@ -6010,6 +6024,17 @@ async fn run_bundle_preview_script( let args = preview.args.unwrap_or_default(); + // The bundle's runtime still resolves workspace-path imports + // (`/f/...`) via loader.bun.js, so pass through temp_script_refs the + // same way `run_preview_script` does — otherwise codebase previews + // silently fall back to deployed content for those imports. + let extra = preview.temp_script_refs.as_ref().map(|refs| { + let mut m = HashMap::new(); + m.insert("_TEMP_SCRIPT_REFS".to_string(), to_raw_value(refs)); + m + }); + let push_args = PushArgs { extra, args: &args }; + is_tar = match preview.kind { Some(PreviewKind::Tarbundle) => true, _ => false, @@ -6038,7 +6063,7 @@ async fn run_bundle_preview_script( modules: None, tag: None, }), - PushArgs::from(&args), + push_args, authed.display_username(), &authed.email, username_to_permissioned_as(&authed.username), @@ -6668,6 +6693,14 @@ async fn run_preview_flow_job( .and_then(|args| args.get("user_message")) .cloned(); + let mut flow_args = raw_flow.args.unwrap_or_default(); + if let Some(ref temp_script_refs) = raw_flow.temp_script_refs { + flow_args.insert( + "_TEMP_SCRIPT_REFS".to_string(), + to_raw_value(temp_script_refs), + ); + } + let (uuid, mut tx) = push( &db, tx, @@ -6677,7 +6710,7 @@ async fn run_preview_flow_job( path: raw_flow.path, restarted_from: raw_flow.restarted_from, }, - PushArgs::from(&raw_flow.args.unwrap_or_default()), + PushArgs::from(&flow_args), authed.display_username(), &authed.email, username_to_permissioned_as(&authed.username), diff --git a/backend/windmill-api/src/path_autocomplete.rs b/backend/windmill-api/src/path_autocomplete.rs index 7bb1162051..56bfd62ce5 100644 --- a/backend/windmill-api/src/path_autocomplete.rs +++ b/backend/windmill-api/src/path_autocomplete.rs @@ -12,11 +12,11 @@ use std::{ }; use axum::{ - extract::{Extension, Path}, + extract::{Extension, Path, Query}, routing::get, Json, Router, }; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use windmill_common::error::JsonResult; use crate::db::{ApiAuthed, DB}; @@ -43,16 +43,28 @@ struct ListPathsResponse { paths: Arc>, } +#[derive(Deserialize)] +struct ListPathsQuery { + /// When true, bypass the cached entry and re-query the DB, refreshing the + /// cache. Used by clients that just mutated the workspace (e.g. a deploy) + /// and need the new path reflected immediately. + #[serde(default)] + force: bool, +} + async fn list_paths( _authed: ApiAuthed, Extension(db): Extension, Path(w_id): Path, + Query(ListPathsQuery { force }): Query, ) -> JsonResult { - if let Some((cached, cached_at)) = PATHS_CACHE.get(&w_id) { - if cached_at.elapsed() < CACHE_TTL { - return Ok(Json(ListPathsResponse { paths: cached })); + if !force { + if let Some((cached, cached_at)) = PATHS_CACHE.get(&w_id) { + if cached_at.elapsed() < CACHE_TTL { + return Ok(Json(ListPathsResponse { paths: cached })); + } + PATHS_CACHE.remove(&w_id); } - PATHS_CACHE.remove(&w_id); } let mut paths: Vec = sqlx::query_scalar!( diff --git a/backend/windmill-api/src/triggers/http/handler.rs b/backend/windmill-api/src/triggers/http/handler.rs index ccde55bab1..ffb61e31fa 100644 --- a/backend/windmill-api/src/triggers/http/handler.rs +++ b/backend/windmill-api/src/triggers/http/handler.rs @@ -424,6 +424,7 @@ async fn route_job( .flatten() .unwrap_or("application/octet-stream".parse().unwrap()), ); + response_headers.insert("x-content-type-options", "nosniff".parse().unwrap()); if !trigger.is_static_website { response_headers.insert( "content-disposition", @@ -443,6 +444,19 @@ async fn route_job( }, ), ); + // For single-file triggers, sandbox any HTML/SVG so it can't + // reach the viewer's session cookie. Allow-scripts/forms/etc. + // keep the opaque origin (cookies still blocked) while + // preserving JS for legitimate HTML payloads. Static-website + // triggers intentionally serve a live web app and cannot be + // sandboxed; restrict write access to those buckets at the + // workspace level. + response_headers.insert( + "content-security-policy", + "sandbox allow-scripts allow-forms allow-popups allow-modals allow-downloads" + .parse() + .unwrap(), + ); } let body_stream = axum::body::Body::from_stream(s3_object.into_stream()); diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index ab05a500b7..7306865982 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -53,6 +53,7 @@ pub const EXPOSE_DEBUG_METRICS_SETTING: &str = "expose_debug_metrics"; pub const KEEP_JOB_DIR_SETTING: &str = "keep_job_dir"; pub const REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING: &str = "require_preexisting_user_for_oauth"; pub const JOB_ISOLATION_SETTING: &str = "job_isolation"; +pub const NSJAIL_TMPFS_SIZE_MB_SETTING: &str = "nsjail_tmpfs_size_mb"; pub const OBJECT_STORE_CONFIG_SETTING: &str = "object_store_cache_config"; pub const HUB_API_SECRET_SETTING: &str = "hub_api_secret"; diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index 3b168761e1..eca3acd456 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -221,6 +221,8 @@ pub struct GlobalSettings { #[serde(skip_serializing_if = "Option::is_none")] pub job_default_timeout: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub nsjail_tmpfs_size_mb: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub bun_install_min_release_age: Option, #[serde(skip_serializing_if = "Option::is_none")] pub uv_exclude_newer: Option, diff --git a/backend/windmill-common/src/secret_backend/mod.rs b/backend/windmill-common/src/secret_backend/mod.rs index 36f35f0cf8..71fa2ea999 100644 --- a/backend/windmill-common/src/secret_backend/mod.rs +++ b/backend/windmill-common/src/secret_backend/mod.rs @@ -118,6 +118,13 @@ pub struct VaultSettings { pub address: String, /// KV v2 mount path (e.g., "windmill") pub mount_path: String, + /// Optional path prefix inserted between the KV `data`/`metadata` segment + /// and the workspace id, e.g. "apps/windmill". When set, secrets live at + /// `/data///`, so a Vault policy can be + /// scoped to exactly `/data//*`. Surrounding slashes are + /// trimmed. + #[serde(skip_serializing_if = "Option::is_none")] + pub kv_secret_path_prefix: Option, /// JWT auth role name configured in Vault (used for JWT/OIDC auth) /// Optional - if not provided, token auth is used #[serde(skip_serializing_if = "Option::is_none")] diff --git a/backend/windmill-common/src/secret_backend/tests.rs b/backend/windmill-common/src/secret_backend/tests.rs index 3e2a12382a..8f04a755b7 100644 --- a/backend/windmill-common/src/secret_backend/tests.rs +++ b/backend/windmill-common/src/secret_backend/tests.rs @@ -25,6 +25,7 @@ mod tests { VaultSettings { address: "http://127.0.0.1:8200".to_string(), mount_path: "windmill".to_string(), + kv_secret_path_prefix: None, jwt_role: Some("windmill-secrets".to_string()), jwt_mount_path: None, namespace: None, diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index 57e41bb487..07ac44b755 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -19,6 +19,7 @@ use serde::{Deserialize, Serialize}; lazy_static::lazy_static! { pub static ref SECRET_SALT: Option = std::env::var("SECRET_SALT").ok(); + static ref RESERVED_WM_VAR_NAME: regex::Regex = regex::Regex::new(r"^WM_[A-Z_]+$").unwrap(); } #[derive(Serialize, Clone)] @@ -50,6 +51,10 @@ pub struct ListableVariable { pub labels: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub ws_specific: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub edited_at: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub edited_by: Option, } #[derive(Serialize, Deserialize, sqlx::FromRow)] @@ -452,7 +457,7 @@ async fn get_cached_workspace_envs(conn: &Connection, w_id: &str) -> Vec<(String let custom_envs = if let Some(cached_envs) = cached_envs_o { cached_envs } else { - let custom_envs = match conn { + let raw_envs = match conn { Connection::Sql(db) => sqlx::query_as::<_, (String, String)>( "SELECT name, value FROM workspace_env WHERE workspace_id = $1", ) @@ -465,6 +470,13 @@ async fn get_cached_workspace_envs(conn: &Connection, w_id: &str) -> Vec<(String .await .unwrap_or_default(), }; + // Applied here (not in the SQL branch alone) so agent workers going + // through `Connection::Http` are covered too — drop any name that + // would shadow a built-in `%%WM_*%%` contextual var. + let custom_envs: Vec<(String, String)> = raw_envs + .into_iter() + .filter(|(name, _)| !RESERVED_WM_VAR_NAME.is_match(name)) + .collect(); CUSTOM_ENVS_CACHE.insert( w_id.to_string(), (chrono::Utc::now().timestamp(), custom_envs.clone()), diff --git a/backend/windmill-common/tests/secret_backend_integration.rs b/backend/windmill-common/tests/secret_backend_integration.rs index 350fc297af..ff41f3acf7 100644 --- a/backend/windmill-common/tests/secret_backend_integration.rs +++ b/backend/windmill-common/tests/secret_backend_integration.rs @@ -90,6 +90,7 @@ mod tests { address: std::env::var("VAULT_ADDR") .unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()), mount_path: "windmill".to_string(), + kv_secret_path_prefix: None, jwt_role: None, // Static token mode jwt_mount_path: None, namespace: None, @@ -106,6 +107,7 @@ mod tests { address: std::env::var("VAULT_ADDR") .unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()), mount_path: "windmill".to_string(), + kv_secret_path_prefix: None, jwt_role: Some("windmill-secrets".to_string()), // JWT mode jwt_mount_path: None, namespace: None, diff --git a/backend/windmill-common/tests/secret_backend_migration.rs b/backend/windmill-common/tests/secret_backend_migration.rs index fba27ee260..5a4086fa1b 100644 --- a/backend/windmill-common/tests/secret_backend_migration.rs +++ b/backend/windmill-common/tests/secret_backend_migration.rs @@ -34,6 +34,7 @@ fn test_vault_settings() -> VaultSettings { address: std::env::var("VAULT_ADDR") .unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()), mount_path: "windmill".to_string(), + kv_secret_path_prefix: None, jwt_role: Some("windmill-secrets".to_string()), jwt_mount_path: None, namespace: None, diff --git a/backend/windmill-duckdb-ffi-internal/build_dev.sh b/backend/windmill-duckdb-ffi-internal/build_dev.sh index 1fb13057a2..d4ae2b7583 100755 --- a/backend/windmill-duckdb-ffi-internal/build_dev.sh +++ b/backend/windmill-duckdb-ffi-internal/build_dev.sh @@ -1,3 +1,3 @@ CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release -p windmill_duckdb_ffi_internal mkdir -p ../target/debug/ -cp target/release/libwindmill_duckdb_ffi_internal.* ../target/debug/ \ No newline at end of file +cp target/release/libwindmill_duckdb_ffi_internal.* ../target/debug/ diff --git a/backend/windmill-duckdb-ffi-internal/src/lib.rs b/backend/windmill-duckdb-ffi-internal/src/lib.rs index 2701319d6e..594b0f6df4 100644 --- a/backend/windmill-duckdb-ffi-internal/src/lib.rs +++ b/backend/windmill-duckdb-ffi-internal/src/lib.rs @@ -11,6 +11,22 @@ use rust_decimal::{prelude::FromPrimitive, Decimal}; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; +// Worker passes "" for "no override" — saves an extra C string nullability dance. +// Returns an owned String so the value outlives the raw pointer's lifetime. +fn ptr_to_opt_str(ptr: *const c_char) -> Result, String> { + if ptr.is_null() { + return Ok(None); + } + let s = unsafe { CStr::from_ptr(ptr) } + .to_str() + .map_err(|e| format!("Invalid string in duckdb ffi: {}", e))?; + Ok(if s.is_empty() { + None + } else { + Some(s.to_owned()) + }) +} + #[derive(Deserialize, Clone, Debug, PartialEq, Default)] pub struct Arg { pub name: String, @@ -34,7 +50,7 @@ pub extern "C" fn get_version() -> c_uint { // Increment when making breaking changes to the FFI interface. // The windmill worker will check that the version matches or else refuse to call // the FFI functions to avoid undefined behavior. - return 1; + return 2; } #[unsafe(no_mangle)] @@ -45,10 +61,16 @@ pub extern "C" fn run_duckdb_ffi( token: *const c_char, base_internal_url: *const c_char, w_id: *const c_char, + memory_limit: *const c_char, + temp_directory: *const c_char, column_order_ptr: *mut *mut c_char, collect_last_only: bool, collect_first_row_only: bool, ) -> *mut c_char { + let resource_limits = match (ptr_to_opt_str(memory_limit), ptr_to_opt_str(temp_directory)) { + (Ok(m), Ok(t)) => Ok(ResourceLimits { memory_limit: m, temp_directory: t }), + (Err(e), _) | (_, Err(e)) => Err(e), + }; let (r, column_order) = match convert_args( query_block_list, query_block_list_count, @@ -57,8 +79,9 @@ pub extern "C" fn run_duckdb_ffi( base_internal_url, w_id, ) + .and_then(|args| resource_limits.map(|r| (args, r))) .and_then( - |(query_block_list, job_args, token, base_internal_url, w_id)| { + |((query_block_list, job_args, token, base_internal_url, w_id), limits)| { run_duckdb_internal( query_block_list, query_block_list_count, @@ -66,6 +89,7 @@ pub extern "C" fn run_duckdb_ffi( token, base_internal_url, w_id, + limits, collect_last_only, collect_first_row_only, ) @@ -150,7 +174,13 @@ pub extern "C" fn prepare_duckdb_ffi( token: *const c_char, base_internal_url: *const c_char, w_id: *const c_char, + memory_limit: *const c_char, + temp_directory: *const c_char, ) -> *mut c_char { + let resource_limits = match (ptr_to_opt_str(memory_limit), ptr_to_opt_str(temp_directory)) { + (Ok(m), Ok(t)) => Ok(ResourceLimits { memory_limit: m, temp_directory: t }), + (Err(e), _) | (_, Err(e)) => Err(e), + }; let r = match convert_prepare_args( query_block_list, query_block_list_count, @@ -158,9 +188,12 @@ pub extern "C" fn prepare_duckdb_ffi( base_internal_url, w_id, ) - .and_then(|(query_block_list, token, base_internal_url, w_id)| { - prepare_duckdb_internal(query_block_list, token, base_internal_url, w_id) - }) { + .and_then(|args| resource_limits.map(|r| (args, r))) + .and_then( + |((query_block_list, token, base_internal_url, w_id), limits)| { + prepare_duckdb_internal(query_block_list, token, base_internal_url, w_id, limits) + }, + ) { Ok(result) => result, Err(err) => { let err = serde_json::to_string(&err) @@ -175,12 +208,58 @@ pub extern "C" fn prepare_duckdb_ffi( }) } +#[derive(Clone, Default)] +struct ResourceLimits { + memory_limit: Option, + temp_directory: Option, +} + +fn sql_single_quote(s: &str) -> String { + s.replace('\'', "''") +} + +// Bounds memory so DuckDB spills to disk before blowing the cgroup cap and +// getting the worker SIGKILLed. Spill goes to the job dir (when set) so it is +// cleaned up with the job, otherwise DuckDB's default temp_directory is kept. +fn configure_duckdb_resource_limits( + conn: &duckdb::Connection, + limits: &ResourceLimits, +) -> Result<(), String> { + let mut config_sql = String::new(); + // jemalloc-specific setting bundled with the Linux DuckDB build. macOS and + // Windows builds may not accept it; gated to avoid breaking those workers. + if cfg!(target_os = "linux") { + config_sql.push_str("SET allocator_background_threads=true;\n"); + } + if let Some(mem) = limits.memory_limit.as_deref() { + config_sql.push_str(&format!("SET memory_limit='{}';\n", sql_single_quote(mem))); + } + if let Some(tmp) = limits.temp_directory.as_deref() { + config_sql.push_str(&format!( + "SET temp_directory='{}';\n", + sql_single_quote(tmp) + )); + } + if config_sql.is_empty() { + return Ok(()); + } + conn.execute_batch(&config_sql).map_err(|e| { + format!( + "Error configuring DuckDB resource limits: {}", + e.to_string() + ) + }) +} + fn setup_duckdb_connection( conn: &duckdb::Connection, token: &str, base_internal_url: &str, w_id: &str, + limits: &ResourceLimits, ) -> Result<(), String> { + configure_duckdb_resource_limits(conn, limits)?; + let (s3_access_key, s3_secret_key) = token.rsplit_once('.').unwrap_or(("", token)); let (s3_endpoint_ssl, s3_endpoint) = base_internal_url .split_once("://") @@ -249,10 +328,11 @@ fn prepare_duckdb_internal( token: &str, base_internal_url: &str, w_id: &str, + limits: ResourceLimits, ) -> Result { let conn = duckdb::Connection::open_in_memory().map_err(|e| e.to_string())?; - setup_duckdb_connection(&conn, token, base_internal_url, w_id)?; + setup_duckdb_connection(&conn, token, base_internal_url, w_id, &limits)?; let mut results: Vec = vec![]; @@ -379,12 +459,13 @@ fn run_duckdb_internal<'a>( token: &str, base_internal_url: &str, w_id: &str, + limits: ResourceLimits, collect_last_only: bool, collect_first_row_only: bool, ) -> Result<(String, Option>), String> { let conn = duckdb::Connection::open_in_memory().map_err(|e| e.to_string())?; - setup_duckdb_connection(&conn, token, base_internal_url, w_id)?; + setup_duckdb_connection(&conn, token, base_internal_url, w_id, &limits)?; let mut results: Vec>> = vec![]; let mut column_order = None; diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index b8d0bdcad9..5c17308bbf 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -2634,12 +2634,14 @@ mod tests { // Regression test for WIN-1957: two resources whose values reference each // other via `$res:` must NOT recurse forever (stack overflow / process // crash). With the depth guard the resolution terminates with an error. - #[tokio::test] - async fn test_transform_json_value_mutual_resource_recursion_terminates() { - let db_url = std::env::var("DATABASE_URL") - .unwrap_or("postgres://postgres:changeme@localhost:5432/windmill".to_string()); - let pool = sqlx::PgPool::connect(&db_url).await.unwrap(); - + // + // This test needs the real `workspace`/`resource` schema, so it uses + // `#[sqlx::test]` which provisions a migrated ephemeral database per test + // (the bare `DATABASE_URL` database in CI has no migrations applied, which + // previously made the workspace INSERT panic with `relation "workspace" + // does not exist` — WIN-1958). + #[sqlx::test(migrations = "../migrations")] + async fn test_transform_json_value_mutual_resource_recursion_terminates(pool: DB) { let w_id = format!("dostest{}", Uuid::new_v4().simple()); sqlx::query("INSERT INTO workspace (id, name, owner) VALUES ($1, $1, 'test@windmill.dev')") @@ -2671,16 +2673,8 @@ mod tests { ) .await; - // Clean up before asserting so a failed assertion doesn't leave rows. - let _ = sqlx::query("DELETE FROM resource WHERE workspace_id = $1") - .bind(&w_id) - .execute(&pool) - .await; - let _ = sqlx::query("DELETE FROM workspace WHERE id = $1") - .bind(&w_id) - .execute(&pool) - .await; - + // The ephemeral test database is dropped automatically, so no manual + // row cleanup is required. let err = result.expect_err("mutually recursive resources should error, not crash"); assert!( err.to_string().contains("interpolation depth"), diff --git a/backend/windmill-store/src/variables.rs b/backend/windmill-store/src/variables.rs index c50049fcd6..4c2a0ed670 100644 --- a/backend/windmill-store/src/variables.rs +++ b/backend/windmill-store/src/variables.rs @@ -134,6 +134,8 @@ async fn list_variables( "variable.expires_at", "variable.labels", "ws_specific.path IS NOT NULL as ws_specific", + "variable.edited_at", + "variable.edited_by", ]) .left() .join("account") @@ -216,6 +218,7 @@ async fn get_variable( "SELECT variable.workspace_id, variable.path, variable.value, variable.is_secret, variable.description, variable.extra_perms, variable.account, variable.is_oauth, variable.expires_at, variable.labels, + variable.edited_at, variable.edited_by, (now() > account.expires_at) as is_expired, account.refresh_error, resource.path IS NOT NULL as is_linked, account.refresh_token != '' as is_refreshed, @@ -441,8 +444,8 @@ async fn create_variable( sqlx::query!( "INSERT INTO variable - (workspace_id, path, value, is_secret, description, account, is_oauth, expires_at, labels) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", + (workspace_id, path, value, is_secret, description, account, is_oauth, expires_at, labels, edited_by) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", &w_id, variable.path, value, @@ -451,7 +454,8 @@ async fn create_variable( variable.account, variable.is_oauth.unwrap_or(false), variable.expires_at, - variable.labels.as_deref() as Option<&[String]> + variable.labels.as_deref() as Option<&[String]>, + &authed.username ) .execute(&mut *tx) .await?; @@ -1048,6 +1052,8 @@ async fn update_variable( } let npath = if has_sql_updates { + sqlb.set("edited_at", "now()"); + sqlb.set_str("edited_by", &authed.username); sqlb.returning("path"); let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let npath_o: Option = sqlx::query_scalar(&sql).fetch_optional(&mut *tx).await?; @@ -1078,10 +1084,11 @@ async fn update_variable( if let Some(nlabels) = &ns.labels { sqlx::query!( - "UPDATE variable SET labels = $1 WHERE path = $2 AND workspace_id = $3", + "UPDATE variable SET labels = $1, edited_at = now(), edited_by = $4 WHERE path = $2 AND workspace_id = $3", nlabels as &[String], &npath, - &w_id + &w_id, + &authed.username ) .execute(&mut *tx) .await?; diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 5ebcb0b370..2e75ac3632 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -13,7 +13,7 @@ default = [] private = ["windmill-worker-volumes/private", "windmill-queue/private", "windmill-common/private", "windmill-dep-map/private", "windmill-runtime-nativets?/private"] mcp = ["windmill-ai/mcp", "dep:windmill-mcp"] prometheus = ["dep:prometheus", "windmill-common/prometheus"] -enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker-volumes/enterprise", "windmill-runtime-nativets?/enterprise", "dep:pem", "dep:tokio-util", "dep:opentelemetry-proto", "dep:prost", "dep:hudsucker", "dep:rcgen", "dep:hyper-http-proxy", "dep:hyper-tls", "dep:hyper-util"] +enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker-volumes/enterprise", "windmill-runtime-nativets?/enterprise", "dep:pem", "dep:rsa", "dep:tokio-util", "dep:opentelemetry-proto", "dep:prost", "dep:hudsucker", "dep:rcgen", "dep:hyper-http-proxy", "dep:hyper-tls", "dep:hyper-util"] mssql = ["dep:tiberius"] mssql-kerberos = ["mssql", "tiberius/integrated-auth-gssapi"] # Linux/Unix integrated auth mssql-winauth = ["mssql", "tiberius/winauth"] # Windows integrated auth @@ -112,6 +112,7 @@ jsonwebtoken.workspace = true sha2.workspace = true hmac.workspace = true pem = { workspace = true, optional = true } +rsa = { workspace = true, optional = true } urlencoding.workspace = true nix.workspace = true bytes.workspace = true diff --git a/backend/windmill-worker/loader.py b/backend/windmill-worker/loader.py index c387e5e0fc..d3dc7b8a66 100644 --- a/backend/windmill-worker/loader.py +++ b/backend/windmill-worker/loader.py @@ -4,6 +4,11 @@ from importlib.abc import MetaPathFinder, Loader from importlib.machinery import ModuleSpec, SourceFileLoader import time +# Injected by backend: maps script path -> temp storage hash so preview jobs +# resolve relative imports from not-yet-deployed local content. Empty ({}) for +# deployed runs. +TEMP_SCRIPT_REFS = TEMP_SCRIPT_REFS_PLACEHOLDER + class WindmillLoader(Loader): def __init__(self, path): self.path = path @@ -48,6 +53,9 @@ class WindmillFinder(MetaPathFinder): runnable_id = os.environ.get('WM_RUNNABLE_ID') if runnable_id: query_params += f"&cache_key={runnable_id}" + temp_hash = TEMP_SCRIPT_REFS.get(script_path) if TEMP_SCRIPT_REFS else None + if temp_hash: + query_params += f"&temp_script_hash={temp_hash}" url = f"{os.environ.get('BASE_INTERNAL_URL')}/api/w/{os.environ.get('WM_WORKSPACE')}/scripts/raw/p/{script_path}.py{query_params}" req = urllib.request.Request(url, None, headers) diff --git a/backend/windmill-worker/nsjail/download.py.config.proto b/backend/windmill-worker/nsjail/download.py.config.proto index ee5b3cc402..8880a565c7 100644 --- a/backend/windmill-worker/nsjail/download.py.config.proto +++ b/backend/windmill-worker/nsjail/download.py.config.proto @@ -90,7 +90,7 @@ mount { dst: "/tmp" fstype: "tmpfs" rw: true - options: "size=500000000" + options: "size={NSJAIL_TMPFS_SIZE}" } diff --git a/backend/windmill-worker/nsjail/download.ruby.config.proto b/backend/windmill-worker/nsjail/download.ruby.config.proto index 12a6012c59..98422abb87 100644 --- a/backend/windmill-worker/nsjail/download.ruby.config.proto +++ b/backend/windmill-worker/nsjail/download.ruby.config.proto @@ -90,7 +90,7 @@ mount { dst: "/tmp" fstype: "tmpfs" rw: true - options: "size=500000000" + options: "size={NSJAIL_TMPFS_SIZE}" } mount { diff --git a/backend/windmill-worker/nsjail/download.rust.config.proto b/backend/windmill-worker/nsjail/download.rust.config.proto index d86fdedd36..0884034076 100644 --- a/backend/windmill-worker/nsjail/download.rust.config.proto +++ b/backend/windmill-worker/nsjail/download.rust.config.proto @@ -53,7 +53,7 @@ mount { dst: "/tmp" fstype: "tmpfs" rw: true - options: "size=500000000" + options: "size={NSJAIL_TMPFS_SIZE}" } mount { diff --git a/backend/windmill-worker/nsjail/run.ansible.config.proto b/backend/windmill-worker/nsjail/run.ansible.config.proto index cba55d2892..7df486c53a 100644 --- a/backend/windmill-worker/nsjail/run.ansible.config.proto +++ b/backend/windmill-worker/nsjail/run.ansible.config.proto @@ -70,7 +70,7 @@ mount { dst: "/tmp" fstype: "tmpfs" rw: true - options: "size=800000000" + options: "size={NSJAIL_TMPFS_SIZE}" } mount { diff --git a/backend/windmill-worker/nsjail/run.bash.config.proto b/backend/windmill-worker/nsjail/run.bash.config.proto index 430437a3d7..1136c8d298 100644 --- a/backend/windmill-worker/nsjail/run.bash.config.proto +++ b/backend/windmill-worker/nsjail/run.bash.config.proto @@ -72,7 +72,7 @@ mount { dst: "/tmp" fstype: "tmpfs" rw: true - options: "size=800000000" + options: "size={NSJAIL_TMPFS_SIZE}" } mount { diff --git a/backend/windmill-worker/nsjail/run.bun.config.proto b/backend/windmill-worker/nsjail/run.bun.config.proto index ef4f054097..280ef8a7f9 100644 --- a/backend/windmill-worker/nsjail/run.bun.config.proto +++ b/backend/windmill-worker/nsjail/run.bun.config.proto @@ -64,7 +64,7 @@ mount { dst: "/tmp" fstype: "tmpfs" rw: true - options: "size=500000000" + options: "size={NSJAIL_TMPFS_SIZE}" } mount { diff --git a/backend/windmill-worker/nsjail/run.csharp.config.proto b/backend/windmill-worker/nsjail/run.csharp.config.proto index 1a2bb97897..b58d6c330f 100644 --- a/backend/windmill-worker/nsjail/run.csharp.config.proto +++ b/backend/windmill-worker/nsjail/run.csharp.config.proto @@ -61,7 +61,7 @@ mount { dst: "/tmp" fstype: "tmpfs" rw: true - options: "size=500000000" + options: "size={NSJAIL_TMPFS_SIZE}" } diff --git a/backend/windmill-worker/nsjail/run.go.config.proto b/backend/windmill-worker/nsjail/run.go.config.proto index d5b7cef099..4fec8f3b2d 100644 --- a/backend/windmill-worker/nsjail/run.go.config.proto +++ b/backend/windmill-worker/nsjail/run.go.config.proto @@ -54,7 +54,7 @@ mount { dst: "/tmp" fstype: "tmpfs" rw: true - options: "size=500000000" + options: "size={NSJAIL_TMPFS_SIZE}" } diff --git a/backend/windmill-worker/nsjail/run.java.config.proto b/backend/windmill-worker/nsjail/run.java.config.proto index be42e98682..3b0a635b0c 100644 --- a/backend/windmill-worker/nsjail/run.java.config.proto +++ b/backend/windmill-worker/nsjail/run.java.config.proto @@ -55,7 +55,7 @@ mount { dst: "/tmp" fstype: "tmpfs" rw: true - options: "size=500000000" + options: "size={NSJAIL_TMPFS_SIZE}" } diff --git a/backend/windmill-worker/nsjail/run.nu.config.proto b/backend/windmill-worker/nsjail/run.nu.config.proto index c2c663c613..40a5de69a1 100644 --- a/backend/windmill-worker/nsjail/run.nu.config.proto +++ b/backend/windmill-worker/nsjail/run.nu.config.proto @@ -55,7 +55,7 @@ mount { dst: "/tmp" fstype: "tmpfs" rw: true - options: "size=800000000" + options: "size={NSJAIL_TMPFS_SIZE}" } mount { diff --git a/backend/windmill-worker/nsjail/run.php.config.proto b/backend/windmill-worker/nsjail/run.php.config.proto index 959ac56f51..965ac7cd36 100644 --- a/backend/windmill-worker/nsjail/run.php.config.proto +++ b/backend/windmill-worker/nsjail/run.php.config.proto @@ -55,7 +55,7 @@ mount { dst: "/tmp" fstype: "tmpfs" rw: true - options: "size=800000000" + options: "size={NSJAIL_TMPFS_SIZE}" } mount { diff --git a/backend/windmill-worker/nsjail/run.powershell.config.proto b/backend/windmill-worker/nsjail/run.powershell.config.proto index 3c9dc8dc57..bab869c9c8 100644 --- a/backend/windmill-worker/nsjail/run.powershell.config.proto +++ b/backend/windmill-worker/nsjail/run.powershell.config.proto @@ -68,7 +68,7 @@ mount { dst: "/tmp" fstype: "tmpfs" rw: true - options: "size=800000000" + options: "size={NSJAIL_TMPFS_SIZE}" } mount { diff --git a/backend/windmill-worker/nsjail/run.python3.config.proto b/backend/windmill-worker/nsjail/run.python3.config.proto index 3c272e6d2b..dec9d6bae7 100644 --- a/backend/windmill-worker/nsjail/run.python3.config.proto +++ b/backend/windmill-worker/nsjail/run.python3.config.proto @@ -58,7 +58,7 @@ mount { dst: "/tmp" fstype: "tmpfs" rw: true - options: "size=500000000" + options: "size={NSJAIL_TMPFS_SIZE}" } mount { diff --git a/backend/windmill-worker/nsjail/run.r.config.proto b/backend/windmill-worker/nsjail/run.r.config.proto index 828e7528e0..0f5d71bf5a 100644 --- a/backend/windmill-worker/nsjail/run.r.config.proto +++ b/backend/windmill-worker/nsjail/run.r.config.proto @@ -55,7 +55,7 @@ mount { dst: "/tmp" fstype: "tmpfs" rw: true - options: "size=500000000" + options: "size={NSJAIL_TMPFS_SIZE}" } diff --git a/backend/windmill-worker/nsjail/run.ruby.config.proto b/backend/windmill-worker/nsjail/run.ruby.config.proto index 09d62f5a0f..3b9509cea6 100644 --- a/backend/windmill-worker/nsjail/run.ruby.config.proto +++ b/backend/windmill-worker/nsjail/run.ruby.config.proto @@ -55,7 +55,7 @@ mount { dst: "/tmp" fstype: "tmpfs" rw: true - options: "size=500000000" + options: "size={NSJAIL_TMPFS_SIZE}" } diff --git a/backend/windmill-worker/nsjail/run.rust.config.proto b/backend/windmill-worker/nsjail/run.rust.config.proto index 86ef63bd26..06da7be662 100644 --- a/backend/windmill-worker/nsjail/run.rust.config.proto +++ b/backend/windmill-worker/nsjail/run.rust.config.proto @@ -54,7 +54,7 @@ mount { dst: "/tmp" fstype: "tmpfs" rw: true - options: "size=500000000" + options: "size={NSJAIL_TMPFS_SIZE}" } diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 2608cc6088..f745bc1661 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -30,8 +30,8 @@ use crate::{ bash_executor::BIN_BASH, common::{ build_command_with_isolation, check_executor_binary_exists, get_reserved_variables, - read_and_check_result, resolve_nsjail_timeout, start_child_process, transform_json, - OccupancyMetrics, + read_and_check_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, + start_child_process, transform_json, OccupancyMetrics, }, handle_child::handle_child, is_sandboxing_enabled, @@ -1456,6 +1456,10 @@ mount {{ "{ADDITIONAL_PYTHON_PATHS}", additional_python_paths_folders.as_str(), ) + .replace( + "{NSJAIL_TMPFS_SIZE}", + &resolve_nsjail_tmpfs_size_bytes().await, + ) .replace("{TIMEOUT}", &nsjail_timeout), )?; } else { diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index 31254cf0cf..47f100bd2d 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -41,8 +41,8 @@ use crate::handle_child::run_future_with_polling_update_job_poller; use crate::{ common::{ build_args_map, build_command_with_isolation, get_reserved_variables, read_file, - read_file_content, resolve_nsjail_timeout, start_child_process, OccupancyMetrics, - DEV_CONF_NSJAIL, + read_file_content, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process, + OccupancyMetrics, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, handle_child::handle_child, @@ -215,6 +215,10 @@ exit $exit_status .replace("{SHARED_MOUNT}", shared_mount) .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) + .replace( + "{NSJAIL_TMPFS_SIZE}", + &resolve_nsjail_tmpfs_size_bytes().await, + ) .replace("{TIMEOUT}", &nsjail_timeout), )?; let mut cmd_args = vec![ diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 163de9f7b3..2346cb413f 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -16,8 +16,8 @@ use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, parse_npm_config, read_file, read_file_content, read_result, resolve_nsjail_timeout, - start_child_process, write_file_binary, MaybeLock, OccupancyMetrics, StreamNotifier, - DEV_CONF_NSJAIL, + resolve_nsjail_tmpfs_size_bytes, start_child_process, write_file_binary, MaybeLock, + OccupancyMetrics, StreamNotifier, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, handle_child::handle_child, @@ -28,6 +28,7 @@ use crate::{ }; use windmill_common::{ client::AuthedClient, + jobs::JobKind, scripts::{id_to_codebase_info, CodebaseInfo, ScriptLang}, utils::WarnAfterExt, workspace_dependencies::WorkspaceDependenciesPrefetched, @@ -1160,8 +1161,15 @@ pub async fn prebundle_bun_script( occupancy_metrics: &mut Option<&mut OccupancyMetrics>, temp_script_refs: &Option>, ) -> Result<()> { - let (local_path, remote_path) = - compute_bundle_local_and_remote_path(inner_content, lock, script_path, db, w_id).await; + let (local_path, remote_path) = compute_bundle_local_and_remote_path( + inner_content, + lock, + script_path, + db, + w_id, + temp_script_refs, + ) + .await; if exists_in_cache(&local_path, &remote_path).await { return Ok(()); } @@ -1252,6 +1260,7 @@ pub async fn compute_bundle_local_and_remote_path( script_path: &str, db: Option<&DB>, w_id: &str, + temp_script_refs: &Option>, ) -> (String, String) { let mut input_src = format!("{inner_content}{lock}",); @@ -1269,6 +1278,18 @@ pub async fn compute_bundle_local_and_remote_path( } }; + // Keep temp-script-ref (preview) bundles in a distinct cache slot: their + // imports come from not-yet-deployed local content, so they must neither + // reuse a deployed-content bundle nor be saved under the deployed key. + if let Some(refs) = temp_script_refs { + let mut entries: Vec<(&String, &String)> = refs.iter().collect(); + entries.sort(); + for (path, hash) in entries { + input_src.push_str(path); + input_src.push_str(hash); + } + } + let ws_suffix = crate::workspace_registry_cache_suffix(w_id).await; input_src.push_str(&ws_suffix); let hash = windmill_common::utils::calculate_hash(&input_src); @@ -1333,6 +1354,22 @@ pub async fn handle_bun_job( ) -> error::Result> { let mut annotation = windmill_common::worker::TypeScriptAnnotations::parse(inner_content); + // Preview jobs may carry _TEMP_SCRIPT_REFS so relative imports resolve from + // not-yet-deployed local content uploaded to raw_script_temp. Extracted up + // front so it reaches both lockfile generation and the runtime loader. + // Gated on JobKind::Preview because job.args includes caller-controlled + // request args; honoring this key on deployed runs would let a caller swap + // import resolution targets in deployed code. + let temp_script_refs: Option> = if matches!(job.kind, JobKind::Preview) + { + job.args + .as_ref() + .and_then(|x| x.get("_TEMP_SCRIPT_REFS")) + .and_then(|v| serde_json::from_str(v.get()).ok()) + } else { + None + }; + if annotation.sandbox && NSJAIL_AVAILABLE.is_none() { return Err(error::Error::ExecutionErr( "Script has //sandbox annotation but nsjail is not available on this worker. \ @@ -1353,6 +1390,7 @@ pub async fn handle_bun_job( job.runnable_path(), Some(db), &job.workspace_id, + &temp_script_refs, ) .await } @@ -1512,7 +1550,7 @@ pub async fn handle_bun_job( workspace_dependencies, annotation.npm, &mut Some(occupancy_metrics), - &None, + &temp_script_refs, wac_replay_info.is_some(), ) .await?; @@ -1886,7 +1924,7 @@ try {{ } else { LoaderMode::BunBundle }, - &None, + &temp_script_refs, ) .await?; @@ -1903,7 +1941,7 @@ try {{ } else { LoaderMode::Bun }, - &None, + &temp_script_refs, ) .await } else { @@ -2147,6 +2185,10 @@ try {{ ) .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) + .replace( + "{NSJAIL_TMPFS_SIZE}", + &resolve_nsjail_tmpfs_size_bytes().await, + ) .replace("{TIMEOUT}", &nsjail_timeout), )?; diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 75cd8bf029..95c7063328 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -47,7 +47,9 @@ use windmill_common::{variables, DB}; use tokio::{io::AsyncWriteExt, time::Instant}; use crate::agent_workers::UPDATE_PING_URL; -use crate::{JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, MAX_TIMEOUT_DURATION, PATH_ENV}; +use crate::{ + JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, MAX_TIMEOUT_DURATION, NSJAIL_TMPFS_SIZE_MB, PATH_ENV, +}; use windmill_common::client::AuthedClient; /// Additional nsjail config for development. Currently used for nix flake. @@ -114,6 +116,7 @@ pub async fn create_args_and_out_file( if let Some(args) = job.args.as_ref() { if let Some(mut x) = transform_json(client, &job.workspace_id, &args.0, job, conn).await? { x.remove("_MODULES"); + x.remove("_TEMP_SCRIPT_REFS"); write_file( job_dir, "args.json", @@ -122,6 +125,7 @@ pub async fn create_args_and_out_file( } else { let mut filtered = args.0.clone(); filtered.remove("_MODULES"); + filtered.remove("_TEMP_SCRIPT_REFS"); write_file( job_dir, "args.json", @@ -1004,6 +1008,21 @@ pub async fn resolve_nsjail_timeout( (duration.as_secs() + 15).to_string() } +/// Default size (in bytes) of the `/tmp` tmpfs mount inside nsjail sandboxes, +/// used when the `nsjail_tmpfs_size_mb` instance setting is unset. +pub const DEFAULT_NSJAIL_TMPFS_SIZE_BYTES: u64 = 800_000_000; + +/// Resolve the tmpfs `size=` value (in bytes, formatted for the nsjail proto) +/// for the `/tmp` tmpfs mount. When the `nsjail_tmpfs_size_mb` instance setting +/// is `None`, `Some(0)`, or negative, falls back to +/// [`DEFAULT_NSJAIL_TMPFS_SIZE_BYTES`]. +pub async fn resolve_nsjail_tmpfs_size_bytes() -> String { + match *NSJAIL_TMPFS_SIZE_MB.read().await { + Some(mb) if mb > 0 => ((mb as u64).saturating_mul(1_000_000)).to_string(), + _ => DEFAULT_NSJAIL_TMPFS_SIZE_BYTES.to_string(), + } +} + async fn hash_args( #[allow(unused)] db: &DB, #[allow(unused)] client: &AuthedClient, diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs index 3d09f0d053..cfd9a544a1 100644 --- a/backend/windmill-worker/src/csharp_executor.rs +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -27,8 +27,8 @@ use windmill_queue::CanceledBy; use crate::{ common::{ build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, - get_reserved_variables, read_result, resolve_nsjail_timeout, start_child_process, - DEV_CONF_NSJAIL, + get_reserved_variables, read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, + start_child_process, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, handle_child::handle_child, @@ -603,6 +603,10 @@ pub async fn handle_csharp_job( .replace("{SHARED_MOUNT}", shared_mount) .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) + .replace( + "{NSJAIL_TMPFS_SIZE}", + &resolve_nsjail_tmpfs_size_bytes().await, + ) .replace("{TIMEOUT}", &nsjail_timeout), )?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index d8c40f6c4d..4a019be9ab 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -11,7 +11,7 @@ use serde_json::{json, Value}; use uuid::Uuid; use windmill_common::error::{to_anyhow, Error, Result}; use windmill_common::utils::sanitize_string_from_password; -use windmill_common::worker::{Connection, SqlResultCollectionStrategy}; +use windmill_common::worker::{get_memory, Connection, SqlResultCollectionStrategy}; use windmill_common::workspaces::{ get_datatable_resource_from_db_unchecked, get_ducklake_from_db_unchecked, DucklakeCatalogResourceType, @@ -44,6 +44,7 @@ pub async fn do_duckdb( #[allow(unused_variables)] column_order_ref: &mut Option>, occupancy_metrics: &mut OccupancyMetrics, parent_runnable_path: Option, + job_dir: &str, run_inline: bool, ) -> Result> { let annotations = windmill_common::worker::SqlAnnotations::parse(query); @@ -160,6 +161,7 @@ pub async fn do_duckdb( let base_internal_url = client.base_internal_url.clone(); let w_id = job.workspace_id.clone(); + let job_dir = job_dir.to_string(); if annotations.prepare { let result = tokio::task::spawn_blocking(move || { @@ -168,6 +170,7 @@ pub async fn do_duckdb( &token, &base_internal_url, &w_id, + &job_dir, ) }) .await @@ -185,6 +188,7 @@ pub async fn do_duckdb( &token, &base_internal_url, &w_id, + &job_dir, collection_strategy, ) }) @@ -259,6 +263,8 @@ struct DuckDbFfiLib { token: *const c_char, base_internal_url: *const c_char, w_id: *const c_char, + memory_limit: *const c_char, + temp_directory: *const c_char, column_order_ptr: *mut *mut c_char, collect_last_only: bool, collect_first_row_only: bool, @@ -273,6 +279,8 @@ struct DuckDbFfiLib { token: *const c_char, base_internal_url: *const c_char, w_id: *const c_char, + memory_limit: *const c_char, + temp_directory: *const c_char, ) -> *mut c_char, >, >, @@ -319,7 +327,7 @@ impl DuckDbFfiLib { // Version mismatch should only be possible on Windows agent workers // We check for it because FFI interface mismatch will cause undefined behavior / crashes unsafe { - let expected_version: c_uint = 1; + let expected_version: c_uint = 2; let get_version: Symbol<'static, unsafe extern "C" fn() -> c_uint> = lib.get(b"get_version") .map_err(|e| return Error::ExecutionErr(format!("Could not find get_version in the duckdb ffi library. If you are not using docker, consider manually upgrading windmill_duckdb_ffi_lib. {}", e.to_string())))?; @@ -345,6 +353,35 @@ impl DuckDbFfiLib { } } +// 20% headroom for Rust runtime + DuckDB's untracked allocations. Mirrors +// DuckDB's own default ratio, but applied to the worker's cgroup budget +// instead of host RAM. +const DUCKDB_MEMORY_FRACTION: f64 = 0.8; +// Treat cgroup values above 1 PiB as "unlimited" (kernels report page-aligned +// huge numbers when uncapped). get_memory() falls back to host RAM in that +// case, which is exactly what we want to leave to DuckDB's own default. +const CGROUP_UNLIMITED_THRESHOLD: i64 = 1024 * 1024 * 1024 * 1024 * 1024; + +// `DUCKDB_MEMORY_LIMIT` env override, else fraction of the worker's cgroup +// memory (as reported by windmill-common), else None (keep DuckDB's default). +fn resolve_duckdb_memory_limit() -> Option { + if let Ok(v) = env::var("DUCKDB_MEMORY_LIMIT") { + let v = v.trim(); + if !v.is_empty() { + return Some(v.to_string()); + } + } + cgroup_bytes_to_duckdb_memory_limit(get_memory()?) +} + +fn cgroup_bytes_to_duckdb_memory_limit(bytes: i64) -> Option { + if bytes <= 0 || bytes >= CGROUP_UNLIMITED_THRESHOLD { + return None; + } + let mib = ((bytes as f64 * DUCKDB_MEMORY_FRACTION) as i64) / (1024 * 1024); + Some(format!("{}MiB", mib.max(64))) +} + // Read backend/windmill-duckdb-ffi-internal/README_DEV.md for details about why we use FFI fn run_duckdb_ffi_safe<'a>( query_block_list: impl Iterator, @@ -353,6 +390,7 @@ fn run_duckdb_ffi_safe<'a>( token: &str, base_internal_url: &str, w_id: &str, + job_dir: &str, collection_strategy: SqlResultCollectionStrategy, ) -> Result<(Box, Option>)> { let query_block_list = query_block_list @@ -372,6 +410,9 @@ fn run_duckdb_ffi_safe<'a>( let token = CString::new(token).map_err(to_anyhow)?; let base_internal_url = CString::new(base_internal_url).map_err(to_anyhow)?; let w_id = CString::new(w_id).map_err(to_anyhow)?; + let memory_limit = + CString::new(resolve_duckdb_memory_limit().unwrap_or_default()).map_err(to_anyhow)?; + let temp_directory = CString::new(job_dir).map_err(to_anyhow)?; let run_duckdb_ffi = &DuckDbFfiLib::get_singleton()?.run_duckdb_ffi; let free_cstr = &DuckDbFfiLib::get_singleton()?.free_cstr; @@ -384,6 +425,8 @@ fn run_duckdb_ffi_safe<'a>( token.as_ptr(), base_internal_url.as_ptr(), w_id.as_ptr(), + memory_limit.as_ptr(), + temp_directory.as_ptr(), &mut column_order, collection_strategy.collect_last_statement_only(query_block_list_count), collection_strategy.collect_first_row_only(), @@ -424,6 +467,7 @@ fn prepare_duckdb_ffi_safe<'a>( token: &str, base_internal_url: &str, w_id: &str, + job_dir: &str, ) -> Result> { let query_block_list = query_block_list .map(|s| { @@ -440,6 +484,9 @@ fn prepare_duckdb_ffi_safe<'a>( let token = CString::new(token).map_err(to_anyhow)?; let base_internal_url = CString::new(base_internal_url).map_err(to_anyhow)?; let w_id = CString::new(w_id).map_err(to_anyhow)?; + let memory_limit = + CString::new(resolve_duckdb_memory_limit().unwrap_or_default()).map_err(to_anyhow)?; + let temp_directory = CString::new(job_dir).map_err(to_anyhow)?; let lib = DuckDbFfiLib::get_singleton()?; let prepare_fn = lib.prepare_duckdb_ffi.as_ref().ok_or_else(|| { @@ -456,6 +503,8 @@ fn prepare_duckdb_ffi_safe<'a>( token.as_ptr(), base_internal_url.as_ptr(), w_id.as_ptr(), + memory_limit.as_ptr(), + temp_directory.as_ptr(), ); let str = CStr::from_ptr(ptr).to_string_lossy().to_string(); free_cstr(ptr); @@ -783,6 +832,44 @@ pub struct Arg { mod tests { use super::*; + #[test] + fn cgroup_bytes_unlimited_or_invalid_returns_none() { + assert_eq!(cgroup_bytes_to_duckdb_memory_limit(0), None); + assert_eq!(cgroup_bytes_to_duckdb_memory_limit(-1), None); + // 1 PiB sentinel: cgroup v1 reports ~i64::MAX when uncapped. + assert_eq!( + cgroup_bytes_to_duckdb_memory_limit(CGROUP_UNLIMITED_THRESHOLD), + None + ); + } + + #[test] + fn cgroup_bytes_real_values_take_80_percent() { + // 1 GiB -> 80% -> 819 MiB (floored to MiB) + assert_eq!( + cgroup_bytes_to_duckdb_memory_limit(1024 * 1024 * 1024), + Some("819MiB".to_string()) + ); + // 4 GiB -> 3276 MiB + assert_eq!( + cgroup_bytes_to_duckdb_memory_limit(4 * 1024 * 1024 * 1024), + Some("3276MiB".to_string()) + ); + } + + #[test] + fn cgroup_bytes_tiny_values_floored_to_64mib() { + // Tiny cgroup must not produce a 0/unusable limit. + assert_eq!( + cgroup_bytes_to_duckdb_memory_limit(1024 * 1024), + Some("64MiB".to_string()) + ); + assert_eq!( + cgroup_bytes_to_duckdb_memory_limit(1), + Some("64MiB".to_string()) + ); + } + // Tests for parse_attach_db_resource function #[test] fn test_parse_attach_db_resource_postgres_res_prefix() { diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index b4133a2d0c..ee08aa1b30 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -22,8 +22,8 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ build_command_with_isolation, capitalize, create_args_and_out_file, get_reserved_variables, - read_result, resolve_nsjail_timeout, start_child_process, OccupancyMetrics, - DEV_CONF_NSJAIL, + read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process, + OccupancyMetrics, DEV_CONF_NSJAIL, }, handle_child::handle_child, is_sandboxing_enabled, read_ee_registry, DISABLE_NUSER, GOPRIVATE, GOPROXY, GO_BIN_CACHE_DIR, @@ -351,6 +351,10 @@ func Run(req Req) (interface{{}}, error){{ .replace("{SHARED_MOUNT}", shared_mount) .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) + .replace( + "{NSJAIL_TMPFS_SIZE}", + &resolve_nsjail_tmpfs_size_bytes().await, + ) .replace("{TIMEOUT}", &nsjail_timeout), )?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); diff --git a/backend/windmill-worker/src/java_executor.rs b/backend/windmill-worker/src/java_executor.rs index 2eaeec01f4..f927f921c8 100644 --- a/backend/windmill-worker/src/java_executor.rs +++ b/backend/windmill-worker/src/java_executor.rs @@ -23,7 +23,8 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, - read_result, resolve_nsjail_timeout, start_child_process, OccupancyMetrics, + read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process, + OccupancyMetrics, }, handle_child, is_sandboxing_enabled, read_ee_registry_bool_with_workspace_override, read_ee_registry_with_workspace_override, @@ -669,6 +670,10 @@ async fn run<'a>( .replace("{SHARED_MOUNT}", &shared_mount) // .replace("{CACHED_TARGET}", &shared_mount) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace( + "{NSJAIL_TMPFS_SIZE}", + &resolve_nsjail_tmpfs_size_bytes().await, + ) .replace("{TIMEOUT}", &nsjail_timeout), )?; let mut cmd = Command::new(NSJAIL_PATH.as_str()); diff --git a/backend/windmill-worker/src/nu_executor.rs b/backend/windmill-worker/src/nu_executor.rs index c79f8ba968..d9a55c40c9 100644 --- a/backend/windmill-worker/src/nu_executor.rs +++ b/backend/windmill-worker/src/nu_executor.rs @@ -14,8 +14,8 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, - read_result, resolve_nsjail_timeout, start_child_process, OccupancyMetrics, - DEV_CONF_NSJAIL, + read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process, + OccupancyMetrics, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, handle_child, is_sandboxing_enabled, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, TRACING_PROXY_CA_CERT_PATH, @@ -258,6 +258,10 @@ async fn run<'a>( .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) + .replace( + "{NSJAIL_TMPFS_SIZE}", + &resolve_nsjail_tmpfs_size_bytes().await, + ) .replace("{TIMEOUT}", &nsjail_timeout), )?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index d3d0fb36fb..e0ba015951 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -20,8 +20,8 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ common::{ build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, - get_reserved_variables, read_result, resolve_nsjail_timeout, start_child_process, - MaybeLock, OccupancyMetrics, + get_reserved_variables, read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, + start_child_process, MaybeLock, OccupancyMetrics, }, handle_child::handle_child, is_sandboxing_enabled, COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH, @@ -425,6 +425,10 @@ try {{ .replace("{JOB_DIR}", job_dir) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount) + .replace( + "{NSJAIL_TMPFS_SIZE}", + &resolve_nsjail_tmpfs_size_bytes().await, + ) .replace("{TIMEOUT}", &nsjail_timeout), )?; diff --git a/backend/windmill-worker/src/pwsh_executor.rs b/backend/windmill-worker/src/pwsh_executor.rs index b5b8444f56..5ec6ff6bcd 100644 --- a/backend/windmill-worker/src/pwsh_executor.rs +++ b/backend/windmill-worker/src/pwsh_executor.rs @@ -26,8 +26,8 @@ lazy_static::lazy_static! { use crate::{ common::{ build_args_map, build_command_with_isolation, get_reserved_variables, read_file, - read_file_content, resolve_nsjail_timeout, start_child_process, MaybeLock, - OccupancyMetrics, + read_file_content, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process, + MaybeLock, OccupancyMetrics, }, handle_child::handle_child, is_sandboxing_enabled, read_ee_registry_with_workspace_override, DISABLE_NUSER, HOME_ENV, @@ -682,6 +682,10 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"", .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount) .replace("{CACHE_DIR}", &*POWERSHELL_CACHE_DIR) + .replace( + "{NSJAIL_TMPFS_SIZE}", + &resolve_nsjail_tmpfs_size_bytes().await, + ) .replace("{TIMEOUT}", &nsjail_timeout), )?; let cmd_args = vec![ diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index d3d3de62d6..2023192075 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -31,6 +31,7 @@ use windmill_common::{ self, Error::{self}, }, + jobs::JobKind, scripts::ScriptLang, utils::calculate_hash, worker::{ @@ -119,6 +120,18 @@ const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = include_str!("../nsjail/download const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str = include_str!("../nsjail/run.python3.config.proto"); pub const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py"); +/// Render loader.py with the TEMP_SCRIPT_REFS placeholder substituted by a +/// Python dict literal. Preview jobs pass a path -> temp-hash map so relative +/// imports resolve from not-yet-deployed local content; deployed runs pass +/// `None` which renders an empty dict (deployed resolution unchanged). +fn render_relative_python_loader(temp_script_refs: &Option>) -> String { + let temp_refs_py = temp_script_refs + .as_ref() + .and_then(|m| serde_json::to_string(m).ok()) + .unwrap_or_else(|| "{}".to_string()); + RELATIVE_PYTHON_LOADER.replace("TEMP_SCRIPT_REFS_PLACEHOLDER", &temp_refs_py) +} + #[cfg(any(feature = "private", test))] pub fn has_relative_imports(content: &str) -> bool { RELATIVE_IMPORT_REGEX.is_match(content) @@ -133,8 +146,8 @@ use windmill_object_store::OBJECT_STORE_SETTINGS; use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, read_file, - read_result, resolve_nsjail_timeout, start_child_process, OccupancyMetrics, StreamNotifier, - DEV_CONF_NSJAIL, + read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process, + OccupancyMetrics, StreamNotifier, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, handle_child::handle_child, @@ -643,6 +656,21 @@ pub async fn handle_python_job( )); } + // Preview jobs may carry _TEMP_SCRIPT_REFS so relative imports resolve from + // not-yet-deployed local content uploaded to raw_script_temp. Gated on + // JobKind::Preview because job.args includes caller-controlled request + // args; honoring this key on deployed runs would let a caller swap import + // resolution targets in deployed code. + let temp_script_refs: Option> = if matches!(job.kind, JobKind::Preview) + { + job.args + .as_ref() + .and_then(|x| x.get("_TEMP_SCRIPT_REFS")) + .and_then(|v| serde_json::from_str(v.get()).ok()) + } else { + None + }; + let (py_version, mut additional_python_paths) = handle_python_deps( job_dir, requirements_o, @@ -658,6 +686,7 @@ pub async fn handle_python_job( &mut Some(occupancy_metrics), precomputed_agent_info, annotations.clone(), + &temp_script_refs, ) .await?; @@ -709,6 +738,7 @@ pub async fn handle_python_job( job.script_entrypoint_override.as_deref(), inner_content, &script_path, + &temp_script_refs, ) .await?; @@ -993,6 +1023,10 @@ mount {{ ) .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) + .replace( + "{NSJAIL_TMPFS_SIZE}", + &resolve_nsjail_tmpfs_size_bytes().await, + ) .replace("{TIMEOUT}", &nsjail_timeout), )?; } else { @@ -1532,6 +1566,7 @@ async fn prepare_wrapper( job_script_entrypoint_override: Option<&str>, inner_content: &str, script_path: &str, + temp_script_refs: &Option>, ) -> error::Result<( &'static str, &'static str, @@ -1571,7 +1606,11 @@ async fn prepare_wrapper( let _ = write_file(&module_dir, &format!("{last}.py"), inner_content)?; if relative_imports { - let _ = write_file(job_dir, "loader.py", RELATIVE_PYTHON_LOADER)?; + let _ = write_file( + job_dir, + "loader.py", + &render_relative_python_loader(temp_script_refs), + )?; } let sig = windmill_parser_py::parse_python_signature( @@ -1768,6 +1807,7 @@ pub(crate) async fn handle_python_deps( occupancy_metrics: &mut Option<&mut OccupancyMetrics>, precomputed_agent_info: Option, annotations: PythonAnnotations, + temp_script_refs: &Option>, ) -> error::Result<(PyV, Vec)> { create_dependencies_dir(job_dir).await; @@ -1796,7 +1836,7 @@ pub(crate) async fn handle_python_deps( &mut version_specifiers, &mut locked_v, &None, - &None, // temp_script_refs: only used during CLI lock generation + temp_script_refs, )) .await?; @@ -2007,6 +2047,10 @@ async fn spawn_uv_install( .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) + .replace( + "{NSJAIL_TMPFS_SIZE}", + &resolve_nsjail_tmpfs_size_bytes().await, + ) .as_str(), )?; @@ -3042,7 +3086,8 @@ pub async fn start_worker( let any_relative_imports = RELATIVE_IMPORT_REGEX.is_match(inner_content); if any_relative_imports { - let _ = write_file(job_dir, "loader.py", RELATIVE_PYTHON_LOADER)?; + // Dedicated worker runs deployed scripts only — no temp refs. + let _ = write_file(job_dir, "loader.py", &render_relative_python_loader(&None))?; } let mut mem_peak: i32 = 0; @@ -3087,6 +3132,7 @@ pub async fn start_worker( &mut None, None, annotations, + &None, // dedicated worker runs deployed scripts only ) .await?; diff --git a/backend/windmill-worker/src/r_executor.rs b/backend/windmill-worker/src/r_executor.rs index 70e188dc46..19959ee22c 100644 --- a/backend/windmill-worker/src/r_executor.rs +++ b/backend/windmill-worker/src/r_executor.rs @@ -20,7 +20,8 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, - read_result, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, + read_result, resolve_nsjail_tmpfs_size_bytes, start_child_process, OccupancyMetrics, + DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, handle_child::{self}, @@ -580,6 +581,10 @@ async fn run<'a>( .replace("{R_CACHE_DIR}", &*R_CACHE_DIR) .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) + .replace( + "{NSJAIL_TMPFS_SIZE}", + &resolve_nsjail_tmpfs_size_bytes().await, + ) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), )?; let mut cmd = Command::new(NSJAIL_PATH.as_str()); diff --git a/backend/windmill-worker/src/ruby_executor.rs b/backend/windmill-worker/src/ruby_executor.rs index aff842eb9f..c82d5ad7df 100644 --- a/backend/windmill-worker/src/ruby_executor.rs +++ b/backend/windmill-worker/src/ruby_executor.rs @@ -23,8 +23,8 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, - read_result, resolve_nsjail_timeout, start_child_process, OccupancyMetrics, - DEV_CONF_NSJAIL, + read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, start_child_process, + OccupancyMetrics, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, handle_child::{self}, @@ -619,6 +619,7 @@ async fn install<'a>( envs.clone(), get_reserved_variables(job, &client.token, conn, parent_runnable_path.clone()).await?, ); + let nsjail_tmpfs_size = resolve_nsjail_tmpfs_size_bytes().await; par_install_language_dependencies_seq( InstallDeps::Flat(deps.clone()), "ruby", @@ -638,6 +639,7 @@ async fn install<'a>( .replace("{TARGET}", &dependency.path) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) + .replace("{NSJAIL_TMPFS_SIZE}", &nsjail_tmpfs_size) .replace("#{DEV}", DEV_CONF_NSJAIL), // .replace("{BUILD}", &build_dir), )?; let mut cmd = Command::new(NSJAIL_PATH.as_str()); @@ -810,6 +812,7 @@ mount {{ .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace("{NSJAIL_TMPFS_SIZE}", &resolve_nsjail_tmpfs_size_bytes().await) .replace("{TIMEOUT}", &nsjail_timeout), )?; let mut cmd = Command::new(NSJAIL_PATH.as_str()); diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index dc3acd8b1d..0f404569da 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -23,8 +23,8 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ common::{ build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, - get_reserved_variables, read_result, resolve_nsjail_timeout, start_child_process, - OccupancyMetrics, DEV_CONF_NSJAIL, + get_reserved_variables, read_result, resolve_nsjail_timeout, resolve_nsjail_tmpfs_size_bytes, + start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, handle_child::handle_child, @@ -480,6 +480,10 @@ pub async fn build_rust_crate( .replace("{RUSTUP_HOME}", RUSTUP_HOME.as_str()) .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) + .replace( + "{NSJAIL_TMPFS_SIZE}", + &resolve_nsjail_tmpfs_size_bytes().await, + ) .replace("{BUILD}", &build_dir), )?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); @@ -701,6 +705,10 @@ pub async fn handle_rust_job( .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) .replace("#{DEV}", DEV_CONF_NSJAIL) .replace("{SHARED_MOUNT}", shared_mount) + .replace( + "{NSJAIL_TMPFS_SIZE}", + &resolve_nsjail_tmpfs_size_bytes().await, + ) .replace("{TIMEOUT}", &nsjail_timeout), )?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); diff --git a/backend/windmill-worker/src/snowflake_executor.rs b/backend/windmill-worker/src/snowflake_executor.rs index 839609d9da..35051aee98 100644 --- a/backend/windmill-worker/src/snowflake_executor.rs +++ b/backend/windmill-worker/src/snowflake_executor.rs @@ -630,14 +630,50 @@ pub async fn do_snowflake( ) .to_uppercase(); - let public_key = match database.public_key.as_deref() { - Some(key) => pem::parse(key.as_bytes()).map_err(|e| { - Error::ExecutionErr(format!("Failed to parse public key: {}", e.to_string())) - })?, - None => return Err(Error::ExecutionErr("Public key is missing".to_string())), + let public_key_der: Vec = match database + .public_key + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()) + { + Some(key) => pem::parse(key.as_bytes()) + .map_err(|e| Error::ExecutionErr(format!("Failed to parse public key: {e}")))? + .into_contents(), + None => { + // Derive the public key from the private key — RSA private keys + // contain the public components (n, e). + use rsa::pkcs8::{DecodePrivateKey, EncodePublicKey}; + let pk_pem = database + .private_key + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()) + .ok_or_else(|| { + Error::ExecutionErr( + "Either public_key or private_key must be provided".to_string(), + ) + })?; + let rsa_priv = rsa::RsaPrivateKey::from_pkcs8_pem(pk_pem) + .or_else(|_| { + use rsa::pkcs1::DecodeRsaPrivateKey; + rsa::RsaPrivateKey::from_pkcs1_pem(pk_pem) + }) + .map_err(|e| { + Error::ExecutionErr(format!( + "Failed to parse private key to derive public key: {e}" + )) + })?; + let rsa_pub = rsa::RsaPublicKey::from(&rsa_priv); + rsa_pub + .to_public_key_der() + .map_err(|e| { + Error::ExecutionErr(format!("Failed to encode derived public key: {e}")) + })? + .to_vec() + } }; let mut public_key_hash = Sha256::new(); - public_key_hash.update(public_key.contents()); + public_key_hash.update(&public_key_der); let public_key_fp = engine::general_purpose::STANDARD.encode(public_key_hash.finalize()); diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 81ca846f4d..4c72792ffd 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -681,6 +681,13 @@ lazy_static::lazy_static! { pub static ref FLOW_RUNNER_RUNNING: Mutex = Mutex::new(false); } +lazy_static::lazy_static! { + /// Optional override for the size of the `/tmp` tmpfs mount in nsjail sandboxes (in megabytes). + /// When `None` (or non-positive), executors fall back to the unified + /// `DEFAULT_NSJAIL_TMPFS_SIZE_BYTES` (800MB). + pub static ref NSJAIL_TMPFS_SIZE_MB: Arc>> = Arc::new(RwLock::new(None)); +} + pub fn sleep_queue() -> u64 { if NATIVE_MODE_RESOLVED.load(std::sync::atomic::Ordering::Relaxed) { 300 @@ -4815,6 +4822,7 @@ pub async fn run_language_executor( column_order, occupancy_metrics, parent_runnable_path, + job_dir, run_inline, )) .await; diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 2256513b5c..6e2deb7446 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -4061,7 +4061,7 @@ async fn push_next_flow_job( _ => nargs, }; - let push_args; + let mut push_args; let err; let ov; @@ -4077,6 +4077,22 @@ async fn push_next_flow_job( } }; + // Propagate temp script refs from the flow preview job to each step so + // relative imports in inline scripts resolve from not-yet-deployed local + // content (uploaded to raw_script_temp) instead of the deployed script. + // Gated on JobKind::FlowPreview because flow_job.args includes + // caller-controlled request args; honoring this key on deployed flow + // runs would let a caller swap import resolution targets in deployed + // step code. + if matches!(flow_job.kind, JobKind::FlowPreview) { + if let Some(temp_script_refs) = arc_flow_job_args.as_ref().get("_TEMP_SCRIPT_REFS") { + push_args + .extra + .get_or_insert_with(HashMap::new) + .insert("_TEMP_SCRIPT_REFS".to_string(), temp_script_refs.clone()); + } + } + tracing::debug!(id = %flow_job.id, root_id = %job_root, "computed args for job {i} of {len}"); let value_with_parallel = module.get_value_with_parallel()?; diff --git a/cli/package-lock.json b/cli/package-lock.json index 2033d89833..f14fd32f8d 100644 --- a/cli/package-lock.json +++ b/cli/package-lock.json @@ -19,18 +19,18 @@ "open": "^10.0.0", "svelte": "^5.45.2", "tar-stream": "^3.1.7", - "windmill-parser-wasm-csharp": "*", - "windmill-parser-wasm-go": "*", - "windmill-parser-wasm-java": "*", - "windmill-parser-wasm-nu": "*", - "windmill-parser-wasm-php": "*", - "windmill-parser-wasm-py": "^1.693.1", - "windmill-parser-wasm-py-imports": "^1.693.1", - "windmill-parser-wasm-regex": "*", - "windmill-parser-wasm-ruby": "*", - "windmill-parser-wasm-rust": "*", - "windmill-parser-wasm-ts": "^1.693.1", - "windmill-parser-wasm-yaml": "*", + "windmill-parser-wasm-csharp": "1.510.1", + "windmill-parser-wasm-go": "1.510.1", + "windmill-parser-wasm-java": "1.510.1", + "windmill-parser-wasm-nu": "1.510.1", + "windmill-parser-wasm-php": "1.647.1", + "windmill-parser-wasm-py": "1.693.1", + "windmill-parser-wasm-py-imports": "1.693.1", + "windmill-parser-wasm-regex": "1.692.0", + "windmill-parser-wasm-ruby": "1.526.1", + "windmill-parser-wasm-rust": "1.647.1", + "windmill-parser-wasm-ts": "1.695.0", + "windmill-parser-wasm-yaml": "1.593.0", "windmill-yaml-validator": "1.1.1", "ws": "8.18.0", "yaml": "^2.7.0" @@ -1438,9 +1438,9 @@ "integrity": "sha512-FC0KbREe2G/sa/9kYIR930wmWw+VL6PvEIqg12J3dsJes3A+0x5JIUPT/jeD+c24DrG0ko/Ub7yDnYs56Bem7g==" }, "node_modules/windmill-parser-wasm-regex": { - "version": "1.639.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.639.0.tgz", - "integrity": "sha512-qvYM4sYxB6M0xrqwBljS2fWqOMk6rp++60TRltJnzZDzVaWQrKjTGwNMmfepGAIWy1OGVKp0SCVERhe2P+O6tQ==" + "version": "1.692.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.692.0.tgz", + "integrity": "sha512-BHGTxrinZJ9ef6hFxbKiBqBEr5uqgG/QySOgMA5r1LswO9n/8fyGswr8JcPT2kGaoeoweV6/RQ+RHVaOhosnKw==" }, "node_modules/windmill-parser-wasm-ruby": { "version": "1.526.1", @@ -1453,9 +1453,9 @@ "integrity": "sha512-9yGLYZX2Hn9TdTqGY/5Fp50ftzgUsrfBkSK9vJkKJd5Amyg+yXLBGzd8pz6Org+4uxMenz/16wpsgijvo6uhhQ==" }, "node_modules/windmill-parser-wasm-ts": { - "version": "1.693.1", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.693.1.tgz", - "integrity": "sha512-xrPgVWwQbOWJKiz68wBDNMrKVOtCY/utyhzSx0kFYIWm/QH/6L8k6LLjo4DOn+PnlBNRXc7qum0sDyB8IuuJYQ==" + "version": "1.695.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.695.0.tgz", + "integrity": "sha512-9EFxeRZWmfb7EyhSlcG7dzTTKETPRYAvpRlxxLkhhtI5I219wFgI7kwrMpz4stXHJj/aqBknVv66NQHRstSJmw==" }, "node_modules/windmill-parser-wasm-yaml": { "version": "1.593.0", diff --git a/cli/src/commands/app/dev.ts b/cli/src/commands/app/dev.ts index 7edb793bea..c73a8685c8 100644 --- a/cli/src/commands/app/dev.ts +++ b/cli/src/commands/app/dev.ts @@ -28,7 +28,12 @@ import * as wmill from "../../../gen/services.gen.ts"; import { logQueueStatus } from "../../utils/job_polling.ts"; import { resolveWorkspace } from "../../core/context.ts"; import { requireLogin } from "../../core/auth.ts"; -import { GLOBAL_CONFIG_OPT } from "../../core/conf.ts"; +import { + getWmillYamlPath, + GLOBAL_CONFIG_OPT, + mergeConfigWithConfigFile, +} from "../../core/conf.ts"; +import { listSyncCodebases } from "../../utils/codebase.ts"; import { replaceInlineScripts, repopulateFields } from "./app.ts"; import { Runnable } from "./metadata.ts"; import { @@ -386,6 +391,43 @@ async function dev(opts: DevOptions, appFolder?: string) { await requireLogin(opts); const workspaceId = workspace.workspaceId; + // Resolve relative imports in app inline scripts from local (not-yet-deployed) + // content so previews use locally-edited workspace libs instead of deployed. + // Computed here as a startup snapshot; re-run `wmill app dev` to pick up + // later edits to imported workspace scripts. Degrades gracefully (undefined) + // on older backends. The walk must run from the wmill.yaml root so that the + // supported `cd __raw_app && wmill app dev` invocation (cwd is the + // raw_app folder) still sees sibling workspace scripts like `f/lib.ts`. + let appTempRefs: Record | undefined = undefined; + { + const wmillYamlPath = getWmillYamlPath(); + const workspaceRoot = wmillYamlPath + ? path.dirname(wmillYamlPath) + : originalCwd; + const relAppFolder = path.relative(workspaceRoot, targetDir) || "."; + const mergedOpts = await mergeConfigWithConfigFile(opts); + const codebases = await listSyncCodebases(mergedOpts); + const { buildPreviewTempScriptRefs } = await import( + "../generate-metadata/generate-metadata.ts" + ); + const savedCwd = process.cwd(); + if (workspaceRoot !== savedCwd) { + process.chdir(workspaceRoot); + } + try { + appTempRefs = await buildPreviewTempScriptRefs( + workspace, + mergedOpts as any, + codebases, + { kind: "app", folder: relAppFolder, rawApp: true }, + ); + } finally { + if (workspaceRoot !== savedCwd) { + process.chdir(savedCwd); + } + } + } + // Change to target directory for the rest of the command if (appFolder) { process.chdir(targetDir); @@ -941,6 +983,7 @@ async function dev(opts: DevOptions, appFolder?: string) { appPath, runnableId, args, + appTempRefs, ); log.info(colors.gray(`[backend] Job started: ${uuid}`)); @@ -987,6 +1030,7 @@ async function dev(opts: DevOptions, appFolder?: string) { appPath, runnable_id, v, + appTempRefs, ); log.info(colors.gray(`[backendAsync] Job started: ${uuid}`)); @@ -1563,6 +1607,7 @@ async function executeRunnable( appPath: string, runnableId: string, args: any, + tempScriptRefs?: Record, ): Promise { const requestBody: any = { component: runnableId, @@ -1602,6 +1647,9 @@ async function executeRunnable( lock: inlineScript.id === undefined ? inlineScript.lock : undefined, cache_ttl: inlineScript.cache_ttl, }; + if (inlineScript.id === undefined && tempScriptRefs) { + requestBody.temp_script_refs = tempScriptRefs; + } } else if ( (runnable.type === "path" || runnable.type === "runnableByPath") && runnable.runType && diff --git a/cli/src/commands/datatable/datatable.ts b/cli/src/commands/datatable/datatable.ts new file mode 100644 index 0000000000..36ae612b5c --- /dev/null +++ b/cli/src/commands/datatable/datatable.ts @@ -0,0 +1,59 @@ +import { Command } from "@cliffy/command"; +import { Table } from "@cliffy/table"; + +import * as wmill from "../../../gen/services.gen.ts"; +import { requireLogin } from "../../core/auth.ts"; +import { resolveWorkspace } from "../../core/context.ts"; +import * as log from "../../core/log.ts"; +import { GlobalOptions } from "../../types.ts"; +import { runCatalogQuery } from "../../utils/catalog.ts"; + +const DEFAULT_DATATABLE_NAME = "main"; + +async function list(opts: GlobalOptions & { json?: boolean }) { + if (opts.json) log.setSilent(true); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + const items = await wmill.listDataTables({ + workspace: workspace.workspaceId, + }); + + if (opts.json) { + console.log(JSON.stringify(items)); + } else { + new Table() + .header(["Name", "Resource Type", "Resource Path"]) + .padding(2) + .border(true) + .body(items.map((x) => [x.name, x.resource_type, x.resource_path])) + .render(); + } +} + +async function run( + opts: GlobalOptions & { name?: string; silent?: boolean }, + sql: string, +) { + const name = opts.name ?? DEFAULT_DATATABLE_NAME; + await runCatalogQuery(opts, "datatable", name, sql); +} + +const command = new Command() + .description("datatable related commands") + .command("list", "list all datatables in the workspace") + .option("--json", "Output as JSON (for piping to jq)") + .action(list as any) + .command("run", "run a SQL query on a datatable") + .arguments("") + .option( + "-n --name ", + "Datatable name (default: main)", + ) + .option( + "-s --silent", + "Output only the final result as JSON. Useful for scripting.", + ) + .action(run as any); + +export default command; diff --git a/cli/src/commands/ducklake/ducklake.ts b/cli/src/commands/ducklake/ducklake.ts new file mode 100644 index 0000000000..7ba9af5424 --- /dev/null +++ b/cli/src/commands/ducklake/ducklake.ts @@ -0,0 +1,59 @@ +import { Command } from "@cliffy/command"; +import { Table } from "@cliffy/table"; + +import * as wmill from "../../../gen/services.gen.ts"; +import { requireLogin } from "../../core/auth.ts"; +import { resolveWorkspace } from "../../core/context.ts"; +import * as log from "../../core/log.ts"; +import { GlobalOptions } from "../../types.ts"; +import { runCatalogQuery } from "../../utils/catalog.ts"; + +const DEFAULT_DUCKLAKE_NAME = "main"; + +async function list(opts: GlobalOptions & { json?: boolean }) { + if (opts.json) log.setSilent(true); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + const names = await wmill.listDucklakes({ + workspace: workspace.workspaceId, + }); + + if (opts.json) { + console.log(JSON.stringify(names)); + } else { + new Table() + .header(["Name"]) + .padding(2) + .border(true) + .body(names.map((name) => [name])) + .render(); + } +} + +async function run( + opts: GlobalOptions & { name?: string; silent?: boolean }, + sql: string, +) { + const name = opts.name ?? DEFAULT_DUCKLAKE_NAME; + await runCatalogQuery(opts, "ducklake", name, sql); +} + +const command = new Command() + .description("ducklake related commands") + .command("list", "list all ducklakes in the workspace") + .option("--json", "Output as JSON (for piping to jq)") + .action(list as any) + .command("run", "run a SQL query on a ducklake") + .arguments("") + .option( + "-n --name ", + "Ducklake name (default: main)", + ) + .option( + "-s --silent", + "Output only the final result as JSON. Useful for scripting.", + ) + .action(run as any); + +export default command; diff --git a/cli/src/commands/flow/flow.ts b/cli/src/commands/flow/flow.ts index 7f437b8005..03a8d14d51 100644 --- a/cli/src/commands/flow/flow.ts +++ b/cli/src/commands/flow/flow.ts @@ -618,6 +618,22 @@ async function preview( await replaceAllPathScriptsWithLocal(localFlow.value, localScriptReader, log); } + // Resolve relative imports in inline scripts from local (not-yet-deployed) + // content so previewing a flow uses locally-edited dependency scripts. + let tempScriptRefs: Record | undefined = undefined; + if (useLocalPathScripts) { + const { buildPreviewTempScriptRefs } = await import( + "../generate-metadata/generate-metadata.ts" + ); + const resolvedCodebases = (await Promise.resolve(codebases)) as SyncCodebase[]; + tempScriptRefs = await buildPreviewTempScriptRefs( + workspace, + opts, + resolvedCodebases, + { kind: "flow", folder: flowPath } + ); + } + const input = opts.data ? await resolve(opts.data) : {}; if (!opts.silent) { @@ -633,6 +649,7 @@ async function preview( value: localFlow.value, path: flowPath.substring(0, flowPath.indexOf(".flow")).replaceAll(SEP, "/"), args: input, + temp_script_refs: tempScriptRefs, }, }); diff --git a/cli/src/commands/generate-metadata/generate-metadata.ts b/cli/src/commands/generate-metadata/generate-metadata.ts index 1de173e582..76c82ed39f 100644 --- a/cli/src/commands/generate-metadata/generate-metadata.ts +++ b/cli/src/commands/generate-metadata/generate-metadata.ts @@ -7,6 +7,7 @@ import { SyncOptions, mergeConfigWithConfigFile } from "../../core/conf.ts"; import { resolveWorkspace } from "../../core/context.ts"; import { requireLogin } from "../../core/auth.ts"; import * as log from "../../core/log.ts"; +import { Workspace } from "../workspace/workspace.ts"; import { beginLockfileBatch, flushLockfileBatch, @@ -91,6 +92,99 @@ async function walkLocalAppItems( })); } +/** + * Build the path -> temp-storage-hash map for a preview target (script, flow, + * or app), so a preview run resolves relative imports from not-yet-deployed + * local content instead of the deployed scripts. Walks all local scripts so + * transitive relative-import targets can be uploaded, then for flow/app adds + * that item's node. Degrades gracefully (returns undefined) on older backends + * without the /raw_temp endpoints. + */ +export async function buildPreviewTempScriptRefs( + workspace: Workspace, + opts: GlobalOptions & SyncOptions & { defaultTs?: "bun" | "deno" }, + codebases: SyncCodebase[], + target: + | { kind: "script"; path: string } + | { kind: "flow"; folder: string } + | { kind: "app"; folder: string; rawApp: boolean }, +): Promise | undefined> { + try { + const rawWorkspaceDependencies = await getRawWorkspaceDependencies(true); + const tree = new DoubleLinkedDependencyTree(); + tree.setWorkspaceDeps(rawWorkspaceDependencies); + const ignore = await ignoreF(opts); + + for (const e of await walkLocalScripts(codebases, ignore)) { + await generateScriptMetadataInternal( + e, + workspace, + opts, + true, // dryRun: only populate the tree + true, // noStaleMessage + rawWorkspaceDependencies, + codebases, + false, + tree, + ); + } + + let nodePath: string; + if (target.kind === "script") { + nodePath = scriptPathToRemotePath(target.path); + } else if (target.kind === "flow") { + const folder = target.folder.endsWith(SEP) + ? target.folder.slice(0, -1) + : target.folder; + await generateFlowLockInternal(folder, true, workspace, opts, false, true, tree); + nodePath = folder.replaceAll(SEP, "/"); + } else { + const folder = target.folder.endsWith(SEP) + ? target.folder.slice(0, -1) + : target.folder; + await generateAppLocksInternal( + folder, + target.rawApp, + true, + workspace, + opts, + false, + true, + tree, + ); + nodePath = folder.replaceAll(SEP, "/"); + } + + tree.propagateStaleness(); + await uploadScripts(tree, workspace); + const refs = tree.getTempScriptRefs(nodePath); + return refs && Object.keys(refs).length > 0 ? refs : undefined; + } catch (e) { + // Degrade gracefully (preview still runs against deployed versions) but do + // NOT mask the real error: only the missing-/raw_temp-endpoint case is an + // expected old-backend incompatibility — anything else is surfaced verbatim. + const msg = e instanceof Error ? e.message : String(e); + // Narrow: only the missing raw_temp endpoint is the expected old-backend + // signal. A bare 404/"not found" matches far too much (module/command/ + // ENOENT "not found", "Script X not found", …) and would mislabel real + // bugs as a backend-too-old issue. + const isOldBackend = /raw_temp|raw_script_temp/i.test(msg); + if (!(opts as { silent?: boolean }).silent) { + log.warn( + colors.yellow( + isOldBackend + ? `Backend does not support local-import resolution for preview ` + + `(requires the /raw_temp endpoints); relative imports will use ` + + `deployed script versions.` + : `Failed to resolve local relative imports for preview: ${msg}. ` + + `Falling back to deployed script versions.`, + ), + ); + } + return undefined; + } +} + /** * Categorize a flat list of file paths into scripts / flow folders / app * file paths. Used to derive item lists from a precomputed FS map (e.g. diff --git a/cli/src/commands/protection-rules/converter.ts b/cli/src/commands/protection-rules/converter.ts new file mode 100644 index 0000000000..0d3ece5e36 --- /dev/null +++ b/cli/src/commands/protection-rules/converter.ts @@ -0,0 +1,132 @@ +import { ProtectionRuleEntry } from "./types.ts"; +import { ProtectionRuleset } from "../../../gen/types.gen.ts"; + +// Reconciliation plan produced by diffing the local protection rules +// against the backend list. `toDelete` holds names present on the backend but +// absent from wmill.yaml (full-reconcile semantics). +export interface ProtectionRulesPlan { + toCreate: ProtectionRuleEntry[]; + toUpdate: ProtectionRuleEntry[]; + toDelete: string[]; + unchanged: string[]; +} + +function sortedUnique(arr: readonly string[]): string[] { + return [...new Set(arr)].sort(); +} + +export class ProtectionRulesConverter { + // Canonicalize a single rule so comparisons are insensitive to array order + // and duplicates. + static normalizeEntry(entry: ProtectionRuleEntry): ProtectionRuleEntry { + return { + name: entry.name, + rules: sortedUnique(entry.rules ?? []) as ProtectionRuleEntry["rules"], + bypass_groups: sortedUnique(entry.bypass_groups ?? []), + bypass_users: sortedUnique(entry.bypass_users ?? []), + }; + } + + // Canonicalize and sort a list of rules by name. + static normalizeList( + entries: ProtectionRuleEntry[] | undefined, + ): ProtectionRuleEntry[] { + return (entries ?? []) + .map((e) => ProtectionRulesConverter.normalizeEntry(e)) + .sort((a, b) => a.name.localeCompare(b.name)); + } + + // Convert a backend ProtectionRuleset response into the wmill.yaml shape + // (drops workspace_id, which is implied by the synced workspace). + static fromBackend(rulesets: ProtectionRuleset[]): ProtectionRuleEntry[] { + return ProtectionRulesConverter.normalizeList( + rulesets.map((r) => ({ + name: r.name, + rules: [...(r.rules ?? [])], + bypass_groups: [...(r.bypass_groups ?? [])], + bypass_users: [...(r.bypass_users ?? [])], + })), + ); + } + + static entriesEqual( + a: ProtectionRuleEntry, + b: ProtectionRuleEntry, + ): boolean { + const na = ProtectionRulesConverter.normalizeEntry(a); + const nb = ProtectionRulesConverter.normalizeEntry(b); + return ( + na.name === nb.name && + na.rules.length === nb.rules.length && + na.rules.every((v, i) => v === nb.rules[i]) && + na.bypass_groups.length === nb.bypass_groups.length && + na.bypass_groups.every((v, i) => v === nb.bypass_groups[i]) && + na.bypass_users.length === nb.bypass_users.length && + na.bypass_users.every((v, i) => v === nb.bypass_users[i]) + ); + } + + static listsEqual( + a: ProtectionRuleEntry[] | undefined, + b: ProtectionRuleEntry[] | undefined, + ): boolean { + const na = ProtectionRulesConverter.normalizeList(a); + const nb = ProtectionRulesConverter.normalizeList(b); + if (na.length !== nb.length) return false; + return na.every((entry, i) => + ProtectionRulesConverter.entriesEqual(entry, nb[i]) + ); + } + + // Compute the create/update/delete plan to make `backend` match `local`. + static computePlan( + local: ProtectionRuleEntry[] | undefined, + backend: ProtectionRuleEntry[] | undefined, + ): ProtectionRulesPlan { + const localByName = new Map( + ProtectionRulesConverter.normalizeList(local).map((e) => [e.name, e]), + ); + const backendByName = new Map( + ProtectionRulesConverter.normalizeList(backend).map((e) => [e.name, e]), + ); + + const plan: ProtectionRulesPlan = { + toCreate: [], + toUpdate: [], + toDelete: [], + unchanged: [], + }; + + for (const [name, entry] of localByName) { + const existing = backendByName.get(name); + if (!existing) { + plan.toCreate.push(entry); + } else if (!ProtectionRulesConverter.entriesEqual(entry, existing)) { + plan.toUpdate.push(entry); + } else { + plan.unchanged.push(name); + } + } + + for (const name of backendByName.keys()) { + if (!localByName.has(name)) { + plan.toDelete.push(name); + } + } + + plan.toCreate.sort((a, b) => a.name.localeCompare(b.name)); + plan.toUpdate.sort((a, b) => a.name.localeCompare(b.name)); + plan.toDelete.sort(); + plan.unchanged.sort(); + + return plan; + } + + static planHasChanges(plan: ProtectionRulesPlan): boolean { + return ( + plan.toCreate.length > 0 || + plan.toUpdate.length > 0 || + plan.toDelete.length > 0 + ); + } +} diff --git a/cli/src/commands/protection-rules/file.ts b/cli/src/commands/protection-rules/file.ts new file mode 100644 index 0000000000..4b2742d4f6 --- /dev/null +++ b/cli/src/commands/protection-rules/file.ts @@ -0,0 +1,129 @@ +import { existsSync } from "node:fs"; +import { writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { stringify as yamlStringify } from "yaml"; + +import { yamlParseFile } from "../../utils/yaml.ts"; +import { yamlOptions } from "../sync/sync.ts"; +import { + SyncOptions, + getWmillYamlPath, + getWorkspaceNames, + getEffectiveWorkspaceId, + WorkspaceEntryConfig, +} from "../../core/conf.ts"; +import { GlobalOptions } from "../../types.ts"; +import { tryResolveBranchWorkspace } from "../../core/context.ts"; +import { setClient } from "../../core/client.ts"; +import { ProtectionRulesFile } from "./types.ts"; + +export const PROTECTION_RULES_FILENAME = "protection-rules.yaml"; + +// protection-rules.yaml lives next to wmill.yaml. wmill.yaml is required: it is +// the single source of truth for which workspaces exist and how to reach them. +export function getProtectionRulesPath(): string | null { + const wmillPath = getWmillYamlPath(); + if (!wmillPath) return null; + return join(dirname(wmillPath), PROTECTION_RULES_FILENAME); +} + +export async function readProtectionRulesFile( + path: string, +): Promise { + if (!existsSync(path)) return {}; + const parsed = (await yamlParseFile(path)) as ProtectionRulesFile | null; + return parsed ?? {}; +} + +export async function writeProtectionRulesFile( + path: string, + data: ProtectionRulesFile, +): Promise { + // Deterministic key order so diffs/commits stay stable. + const sorted: ProtectionRulesFile = {}; + for (const k of Object.keys(data).sort()) sorted[k] = data[k]; + await writeFile(path, yamlStringify(sorted, yamlOptions), "utf-8"); +} + +// Maps a protection-rules.yaml workspace key to its backend workspace id via +// wmill.yaml's `workspaces` block. A key with no matching entry is rejected — +// without it we don't know which backend to talk to. +export class WorkspaceResolver { + private constructor( + private readonly workspaces: Record, + ) {} + + static fromConfig(config: SyncOptions): WorkspaceResolver { + const ws = (config.workspaces ?? {}) as Record< + string, + WorkspaceEntryConfig + >; + return new WorkspaceResolver(ws); + } + + /** Workspace keys declared in wmill.yaml (excludes reserved keys). */ + knownNames(): string[] { + return getWorkspaceNames(this.workspaces as any); + } + + has(name: string): boolean { + return this.knownNames().includes(name); + } + + /** Backend workspace id (path param) for a key, or throw if unknown. */ + backendId(name: string): string { + if (!this.has(name)) { + throw new Error( + `Workspace '${name}' is not defined in wmill.yaml 'workspaces'. ` + + `Add it there (its keys must match protection-rules.yaml).`, + ); + } + return getEffectiveWorkspaceId(name, this.workspaces[name]); + } +} + +// Point the API client at the backend for a single wmill.yaml workspace key, +// then return the backend workspace id to use as the path param. The backend +// id always comes from the wmill.yaml mapping (the feature's invariant); +// credentials are resolved with the same precedence as every other command: +// +// 1. explicit --base-url + --token -> used as-is (stateless CI; no profile +// or wmill.yaml baseUrl required) +// 2. otherwise, the stored profile matching wmill.yaml workspaces. +// (its baseUrl + token), with an explicit --token overriding the +// stored token +// +// Throws a clean error if the key is unknown or nothing resolves it — callers +// decide whether to skip (--all) or fail (named arg). +export async function configureClientForWorkspace( + opts: GlobalOptions, + ws: string, + resolver: WorkspaceResolver, +): Promise { + const wsId = resolver.backendId(ws); // throws if not in wmill.yaml + + // 1. Explicit credentials — honor them directly, like other commands do. + if (opts.baseUrl) { + if (!opts.token) { + throw new Error( + "When --base-url is set, --token is required for protection-rules.", + ); + } + setClient(opts.token, opts.baseUrl.replace(/\/+$/, "")); + return wsId; + } + + // 2. Stored-profile resolution. Fresh opts so resolveWorkspace's per-call + // cache can't bleed across keys. + const resolved = await tryResolveBranchWorkspace({ ...opts }, ws); + if (!resolved) { + throw new Error( + `Could not resolve credentials for workspace '${ws}'. Either pass ` + + `--base-url and --token, or ensure wmill.yaml workspaces.${ws} has a ` + + `baseUrl and you've run 'wmill workspace add' for it.`, + ); + } + // An explicit --token overrides the stored profile's token. + setClient(opts.token ?? resolved.token, resolved.remote.replace(/\/+$/, "")); + return wsId; +} diff --git a/cli/src/commands/protection-rules/index.ts b/cli/src/commands/protection-rules/index.ts new file mode 100644 index 0000000000..8ea8c47a1c --- /dev/null +++ b/cli/src/commands/protection-rules/index.ts @@ -0,0 +1,2 @@ +export { pullProtectionRules, pushProtectionRules } from "./protection-rules.ts"; +export { default } from "./protection-rules.ts"; diff --git a/cli/src/commands/protection-rules/protection-rules.ts b/cli/src/commands/protection-rules/protection-rules.ts new file mode 100644 index 0000000000..932a956333 --- /dev/null +++ b/cli/src/commands/protection-rules/protection-rules.ts @@ -0,0 +1,30 @@ +import { Command } from "@cliffy/command"; +import { pullProtectionRules } from "./pull.ts"; +import { pushProtectionRules } from "./push.ts"; + +const command = new Command() + .description( + "Sync workspace protection rules between protection-rules.yaml and Windmill. The file is keyed by workspace name; keys must match wmill.yaml 'workspaces'.", + ) + .command("pull") + .description( + "Pull protection rules from Windmill into protection-rules.yaml for a workspace", + ) + .arguments("[workspace:string]") + .option("--all", "Pull every workspace defined in wmill.yaml") + .option("--dry-run", "Show what would change without writing the file") + .option("--json-output", "Output in JSON format") + .action(pullProtectionRules as any) + .command("push") + .description( + "Push protection rules from protection-rules.yaml to Windmill for a workspace (full reconcile: creates, updates, and deletes)", + ) + .arguments("[workspace:string]") + .option("--all", "Push every workspace defined in protection-rules.yaml") + .option("--dry-run", "Show what would change without applying") + .option("--json-output", "Output in JSON format") + .option("--yes", "Skip the confirmation prompt (including deletions)") + .action(pushProtectionRules as any); + +export { pullProtectionRules, pushProtectionRules }; +export default command; diff --git a/cli/src/commands/protection-rules/pull.ts b/cli/src/commands/protection-rules/pull.ts new file mode 100644 index 0000000000..9086800422 --- /dev/null +++ b/cli/src/commands/protection-rules/pull.ts @@ -0,0 +1,143 @@ +import { colors } from "@cliffy/ansi/colors"; + +import * as log from "../../core/log.ts"; +import { GlobalOptions } from "../../types.ts"; +import * as wmill from "../../../gen/services.gen.ts"; +import { readConfigFile } from "../../core/conf.ts"; + +import { ProtectionRulesConverter } from "./converter.ts"; +import { ProtectionRulesFile } from "./types.ts"; +import { + PROTECTION_RULES_FILENAME, + getProtectionRulesPath, + readProtectionRulesFile, + writeProtectionRulesFile, + WorkspaceResolver, + configureClientForWorkspace, +} from "./file.ts"; +import { outputResult, fail, displayPlan, structuredPlan } from "./utils.ts"; + +type PullOpts = GlobalOptions & { + all?: boolean; + dryRun?: boolean; + jsonOutput?: boolean; +}; + +export async function pullProtectionRules( + opts: PullOpts, + workspaceArg?: string, +) { + // In JSON mode stdout must be exactly one JSON payload. Silence human logs + // (log.info/warn → stdout) here, before anything that logs (readConfigFile, + // workspace resolution). log.error still goes to stderr. + if (opts.jsonOutput) log.setSilent(true); + + const prPath = getProtectionRulesPath(); + if (!prPath) { + fail(opts, { + error: + "No wmill.yaml found. Run 'wmill init' first — protection-rules.yaml lives next to it.", + }); + } + + const config = await readConfigFile(); + const resolver = WorkspaceResolver.fromConfig(config); + + let targets: string[]; + if (opts.all) { + targets = resolver.knownNames(); + if (targets.length === 0) { + fail(opts, { + error: "No workspaces defined in wmill.yaml 'workspaces' block.", + }); + } + } else if (workspaceArg) { + targets = [workspaceArg]; + } else { + fail(opts, { error: "Specify a workspace name or use --all." }); + } + + const file = await readProtectionRulesFile(prPath!); + const perWs: Record = {}; + let hadError = false; + let anyChange = false; + + for (const ws of targets) { + let wsId: string; + try { + wsId = await configureClientForWorkspace(opts, ws, resolver); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + if (opts.all) { + log.error(colors.red(msg)); + hadError = true; + continue; + } + fail(opts, { error: msg }); + } + + let backend; + try { + backend = ProtectionRulesConverter.fromBackend( + await wmill.listProtectionRules({ workspace: wsId }), + ); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + if (opts.all) { + log.error(colors.red(`[${ws}] failed to fetch: ${msg}`)); + hadError = true; + continue; + } + fail(opts, { error: `Failed to fetch protection rules: ${msg}` }); + } + + const current = ProtectionRulesConverter.normalizeList(file[ws]); + // plan describes how the local file would change to match the backend + const plan = ProtectionRulesConverter.computePlan(backend, current); + if (ProtectionRulesConverter.planHasChanges(plan)) anyChange = true; + perWs[ws] = structuredPlan(plan); + + if (!opts.dryRun) { + file[ws] = backend; + } else if (!opts.jsonOutput) { + displayPlan(ws, plan); + } + } + + if (opts.dryRun) { + if (opts.jsonOutput) { + console.log( + JSON.stringify({ + success: !hadError, + dryRun: true, + partialFailure: hadError, + hasChanges: anyChange, + workspaces: perWs, + }), + ); + } else if (!hadError && !anyChange) { + log.info(colors.green("All targeted workspaces are in sync")); + } + if (hadError) process.exit(1); + return; + } + + await writeProtectionRulesFile(prPath!, file as ProtectionRulesFile); + const n = Object.keys(perWs).length; + if (hadError) { + // Some --all workspaces failed: status must not say success while we + // exit non-zero. + outputResult(opts, { + success: false, + error: `Pulled ${n} workspace(s) into ${PROTECTION_RULES_FILENAME}, but one or more workspaces failed (see errors above)`, + partialFailure: true, + workspaces: perWs, + }); + process.exit(1); + } + outputResult(opts, { + success: true, + message: `Pulled protection rules for ${n} workspace(s) into ${PROTECTION_RULES_FILENAME}`, + workspaces: perWs, + }); +} diff --git a/cli/src/commands/protection-rules/push.ts b/cli/src/commands/protection-rules/push.ts new file mode 100644 index 0000000000..2ee253eaab --- /dev/null +++ b/cli/src/commands/protection-rules/push.ts @@ -0,0 +1,262 @@ +import process from "node:process"; +import { existsSync } from "node:fs"; + +import { colors } from "@cliffy/ansi/colors"; +import { Confirm } from "@cliffy/prompt/confirm"; + +import * as log from "../../core/log.ts"; +import { GlobalOptions } from "../../types.ts"; +import * as wmill from "../../../gen/services.gen.ts"; +import { readConfigFile } from "../../core/conf.ts"; + +import { ProtectionRulesConverter, ProtectionRulesPlan } from "./converter.ts"; +import { + getProtectionRulesPath, + readProtectionRulesFile, + WorkspaceResolver, + configureClientForWorkspace, +} from "./file.ts"; +import { outputResult, fail, displayPlan, structuredPlan } from "./utils.ts"; + +type PushOpts = GlobalOptions & { + all?: boolean; + dryRun?: boolean; + jsonOutput?: boolean; + yes?: boolean; +}; + +interface WsPlan { + ws: string; + wsId: string; + plan: ProtectionRulesPlan; + wipesAll: boolean; +} + +export async function pushProtectionRules( + opts: PushOpts, + workspaceArg?: string, +) { + // In JSON mode stdout must be exactly one JSON payload. Silence human logs + // (log.info/warn → stdout, incl. workspace resolution + the empty-list + // delete warning) before anything logs. log.error still goes to stderr. + if (opts.jsonOutput) log.setSilent(true); + + const prPath = getProtectionRulesPath(); + if (!prPath) { + fail(opts, { + error: + "No wmill.yaml found. Run 'wmill init' first — protection-rules.yaml lives next to it.", + }); + } + if (!existsSync(prPath!)) { + fail(opts, { + error: + "No protection-rules.yaml found. Run 'wmill protection-rules pull' first.", + }); + } + + const config = await readConfigFile(); + const resolver = WorkspaceResolver.fromConfig(config); + const file = await readProtectionRulesFile(prPath!); + + let targets: string[]; + if (opts.all) { + targets = Object.keys(file).sort(); + if (targets.length === 0) { + fail(opts, { error: "protection-rules.yaml defines no workspaces." }); + } + } else if (workspaceArg) { + if (!(workspaceArg in file)) { + fail(opts, { + error: `Workspace '${workspaceArg}' is not defined in protection-rules.yaml.`, + }); + } + targets = [workspaceArg]; + } else { + fail(opts, { error: "Specify a workspace name or use --all." }); + } + + // Phase 1: resolve + diff every target before mutating anything. + const wsPlans: WsPlan[] = []; + let hadError = false; + for (const ws of targets) { + let wsId: string; + try { + wsId = await configureClientForWorkspace(opts, ws, resolver); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + if (opts.all) { + log.error(colors.red(msg)); + hadError = true; + continue; + } + fail(opts, { error: msg }); + } + + let backend; + try { + backend = ProtectionRulesConverter.fromBackend( + await wmill.listProtectionRules({ workspace: wsId }), + ); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + if (opts.all) { + log.error(colors.red(`[${ws}] failed to fetch: ${msg}`)); + hadError = true; + continue; + } + fail(opts, { error: `Failed to fetch protection rules: ${msg}` }); + } + + const local = ProtectionRulesConverter.normalizeList(file[ws]); + const plan = ProtectionRulesConverter.computePlan(local, backend); + wsPlans.push({ + ws, + wsId, + plan, + wipesAll: local.length === 0 && plan.toDelete.length > 0, + }); + } + + const changed = wsPlans.filter((w) => + ProtectionRulesConverter.planHasChanges(w.plan) + ); + + if (opts.jsonOutput && opts.dryRun) { + console.log( + JSON.stringify({ + success: !hadError, + dryRun: true, + partialFailure: hadError, + hasChanges: changed.length > 0, + workspaces: Object.fromEntries( + wsPlans.map((w) => [w.ws, structuredPlan(w.plan)]), + ), + }), + ); + if (hadError) process.exit(1); + return; + } + + if (!opts.jsonOutput) { + for (const w of wsPlans) displayPlan(w.ws, w.plan); + } + + if (changed.length === 0) { + if (hadError) { + // A workspace failed to resolve/fetch — don't claim success while + // exiting non-zero. + outputResult(opts, { + success: false, + error: + "One or more workspaces failed (see errors above); the rest are in sync", + partialFailure: true, + }); + process.exit(1); + } + if (!opts.dryRun) { + outputResult(opts, { + success: true, + message: "No changes to push - all targeted workspaces are in sync", + }); + } + return; + } + + if (opts.dryRun) { + if (hadError) process.exit(1); + return; + } + + // Pushing an empty list wipes a workspace's rules — be loud even with --yes. + for (const w of wsPlans) { + if (w.wipesAll) { + log.warn( + colors.red( + `WARNING: '${w.ws}' has an empty rule list — this DELETES ALL ${w.plan.toDelete.length} backend rule(s) for that workspace.`, + ), + ); + } + } + + const totalDeletes = changed.reduce((n, w) => n + w.plan.toDelete.length, 0); + if (!opts.yes && !!process.stdin.isTTY) { + const confirmed = await Confirm.prompt({ + message: totalDeletes > 0 + ? `Apply these changes? This DELETES ${totalDeletes} protection rule(s) across ${changed.length} workspace(s).` + : `Apply these changes to ${changed.length} workspace(s)?`, + default: totalDeletes === 0, + }); + if (!confirmed) { + log.info("Operation cancelled"); + return; + } + } + + // Phase 2: apply. Track progress so a mid-run failure reports how far it got. + const applied = { created: 0, updated: 0, deleted: 0 }; + try { + for (const w of changed) { + // Re-point the client at this workspace (phase 1 left it on the last one). + await configureClientForWorkspace(opts, w.ws, resolver); + for (const entry of w.plan.toCreate) { + const n = ProtectionRulesConverter.normalizeEntry(entry); + await wmill.createProtectionRule({ + workspace: w.wsId, + requestBody: { + name: n.name, + rules: n.rules, + bypass_groups: n.bypass_groups, + bypass_users: n.bypass_users, + }, + }); + applied.created++; + } + for (const entry of w.plan.toUpdate) { + const n = ProtectionRulesConverter.normalizeEntry(entry); + await wmill.updateProtectionRule({ + workspace: w.wsId, + ruleName: n.name, + requestBody: { + rules: n.rules, + bypass_groups: n.bypass_groups, + bypass_users: n.bypass_users, + }, + }); + applied.updated++; + } + for (const name of w.plan.toDelete) { + await wmill.deleteProtectionRule({ + workspace: w.wsId, + ruleName: name, + }); + applied.deleted++; + } + } + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + fail(opts, { + error: + `Push partially failed after ${applied.created} create, ${applied.updated} update, ` + + `${applied.deleted} delete: ${msg}. Backend is partially reconciled; re-run push to converge.`, + applied, + }); + } + + if (hadError) { + // Reconcile of resolvable workspaces succeeded, but some --all targets + // failed earlier. Status must reflect the non-zero exit. + outputResult(opts, { + success: false, + error: `Pushed (created ${applied.created}, updated ${applied.updated}, deleted ${applied.deleted}) across ${changed.length} workspace(s), but one or more workspaces failed (see errors above)`, + partialFailure: true, + ...applied, + }); + process.exit(1); + } + outputResult(opts, { + success: true, + message: `Pushed protection rules (created ${applied.created}, updated ${applied.updated}, deleted ${applied.deleted}) across ${changed.length} workspace(s)`, + ...applied, + }); +} diff --git a/cli/src/commands/protection-rules/types.ts b/cli/src/commands/protection-rules/types.ts new file mode 100644 index 0000000000..0256f24e8f --- /dev/null +++ b/cli/src/commands/protection-rules/types.ts @@ -0,0 +1,18 @@ +import { ProtectionRuleKind } from "../../../gen/types.gen.ts"; + +export type { ProtectionRuleKind }; + +// A single workspace protection ruleset as stored in protection-rules.yaml. +// Mirrors the backend ProtectionRuleset shape minus workspace_id (the workspace +// is the map key). +export interface ProtectionRuleEntry { + name: string; + rules: ProtectionRuleKind[]; + bypass_groups: string[]; + bypass_users: string[]; +} + +// protection-rules.yaml is a flat map: workspace name -> its protection rules. +// Workspace names MUST match keys in wmill.yaml's `workspaces` block, which is +// where the backend workspaceId/remote is resolved from. +export type ProtectionRulesFile = Record; diff --git a/cli/src/commands/protection-rules/utils.ts b/cli/src/commands/protection-rules/utils.ts new file mode 100644 index 0000000000..9809e5808c --- /dev/null +++ b/cli/src/commands/protection-rules/utils.ts @@ -0,0 +1,73 @@ +import process from "node:process"; + +import { colors } from "@cliffy/ansi/colors"; +import * as log from "../../core/log.ts"; +import { ProtectionRuleEntry } from "./types.ts"; +import { ProtectionRulesConverter, ProtectionRulesPlan } from "./converter.ts"; + +export function outputResult( + opts: { jsonOutput?: boolean }, + result: { + success: boolean; + message?: string; + error?: string; + [key: string]: any; + }, +): void { + if (opts.jsonOutput) { + console.log(JSON.stringify(result)); + } else if (result.success && result.message) { + log.info(colors.green(result.message)); + } else if (!result.success && result.error) { + log.error(colors.red(result.error)); + } +} + +// Report a genuine failure and exit non-zero so CI / scripted callers detect +// it. outputResult alone only logs, which would let a failed (possibly +// partial) reconcile hide behind exit code 0. +export function fail( + opts: { jsonOutput?: boolean }, + result: { error: string; [key: string]: any }, +): never { + outputResult(opts, { ...result, success: false }); + process.exit(1); +} + +function describeEntry(entry: ProtectionRuleEntry): string { + const n = ProtectionRulesConverter.normalizeEntry(entry); + const parts = [`rules=[${n.rules.join(", ")}]`]; + if (n.bypass_groups.length > 0) { + parts.push(`bypass_groups=[${n.bypass_groups.join(", ")}]`); + } + if (n.bypass_users.length > 0) { + parts.push(`bypass_users=[${n.bypass_users.join(", ")}]`); + } + return parts.join(" "); +} + +// Render a reconciliation plan with a per-workspace heading. +export function displayPlan(ws: string, plan: ProtectionRulesPlan): void { + log.info(colors.bold(`workspace ${ws}:`)); + for (const e of plan.toCreate) { + log.info(colors.green(` + ${e.name} (${describeEntry(e)})`)); + } + for (const e of plan.toUpdate) { + log.info(colors.yellow(` ~ ${e.name} (${describeEntry(e)})`)); + } + for (const name of plan.toDelete) { + log.info(colors.red(` - ${name}`)); + } + if (!ProtectionRulesConverter.planHasChanges(plan)) { + log.info(colors.green(" in sync")); + } +} + +export function structuredPlan(plan: ProtectionRulesPlan) { + return { + create: plan.toCreate.map((e) => e.name), + update: plan.toUpdate.map((e) => e.name), + delete: plan.toDelete, + unchanged: plan.unchanged, + }; +} diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index b9d9c6ee60..808456f151 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -1521,6 +1521,33 @@ async function preview( const codebase = language == "bun" ? findCodebase(filePath, codebases) : undefined; + // Resolve relative imports from local (not-yet-deployed) content so previewing + // a script that imports other locally-edited scripts uses the local versions + // instead of the deployed ones. Shared with `wmill flow preview` so both + // entry points behave identically; degrades gracefully on older backends. + // Short-circuit when the script has no relative imports: the full-workspace + // dependency walk + diff round-trip is pure overhead in that (common) case. + let tempScriptRefs: Record | undefined = undefined; + const { extractRelativeImports } = await import( + "../../utils/relative_imports.ts" + ); + const relImports = await extractRelativeImports( + content, + scriptPathToRemotePath(filePath), + language + ); + if (relImports.length > 0) { + const { buildPreviewTempScriptRefs } = await import( + "../generate-metadata/generate-metadata.ts" + ); + tempScriptRefs = await buildPreviewTempScriptRefs( + workspace, + opts, + codebases, + { kind: "script", path: filePath } + ); + } + let bundledContent: string | Blob | undefined = undefined; let isTar = false; @@ -1616,6 +1643,7 @@ async function preview( language: language, kind: isTar ? "tarbundle" : "bundle", format: codebase?.format ?? "cjs", + temp_script_refs: tempScriptRefs, }; form.append("preview", JSON.stringify(previewPayload)); form.append( @@ -1683,6 +1711,7 @@ async function preview( args: input, language: language as any, modules: modules ?? undefined, + temp_script_refs: tempScriptRefs, }, }); diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 2b0962e09d..3ef4f913c5 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -6680,6 +6680,18 @@ Show all available wmill.yaml configuration options - \`config migrate\` - Migrate wmill.yaml from gitBranches/environments to workspaces format +### datatable + +datatable related commands + +**Subcommands:** + +- \`datatable list\` - list all datatables in the workspace + - \`--json\` - Output as JSON (for piping to jq) +- \`datatable run \` - run a SQL query on a datatable + - \`-n --name \` - Datatable name (default: main) + - \`-s --silent\` - Output only the final result as JSON. Useful for scripting. + ### dependencies workspace dependencies related commands @@ -6709,6 +6721,18 @@ Search Windmill documentation. **Options:** - \`--json\` - Output results as JSON. +### ducklake + +ducklake related commands + +**Subcommands:** + +- \`ducklake list\` - list all ducklakes in the workspace + - \`--json\` - Output as JSON (for piping to jq) +- \`ducklake run \` - run a SQL query on a ducklake + - \`-n --name \` - Ducklake name (default: main) + - \`-s --silent\` - Output only the final result as JSON. Useful for scripting. + ### flow flow related commands @@ -6944,6 +6968,20 @@ Validate Windmill flow, schedule, and trigger YAML files in a directory - \`--locks-required\` - Fail if scripts or flow inline scripts that need locks have no locks - \`-w, --watch\` - Watch for file changes and re-lint automatically +### protection-rules + +**Subcommands:** + +- \`protection-rules pull [workspace:string]\` - Pull protection rules from Windmill into protection-rules.yaml for a workspace + - \`--all\` - Pull every workspace defined in wmill.yaml + - \`--dry-run\` - Show what would change without writing the file + - \`--json-output\` - Output in JSON format +- \`protection-rules push [workspace:string]\` - Push protection rules from protection-rules.yaml to Windmill for a workspace (full reconcile: creates, updates, and deletes) + - \`--all\` - Push every workspace defined in protection-rules.yaml + - \`--dry-run\` - Show what would change without applying + - \`--json-output\` - Output in JSON format + - \`--yes\` - Skip the confirmation prompt (including deletions) + ### queues List all queues with their metrics diff --git a/cli/src/main.ts b/cli/src/main.ts index dd078c6c9c..34ecf989d4 100755 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -21,6 +21,7 @@ import schedule from "./commands/schedule/schedule.ts"; import trigger from "./commands/trigger/trigger.ts"; import sync from "./commands/sync/sync.ts"; import gitsyncSettings from "./commands/gitsync-settings/gitsync-settings.ts"; +import protectionRules from "./commands/protection-rules/protection-rules.ts"; import instance from "./commands/instance/instance.ts"; import workerGroups from "./commands/worker-groups/worker-groups.ts"; import lint from "./commands/lint/lint.ts"; @@ -47,6 +48,8 @@ import token from "./commands/token/token.ts"; import generateMetadata from "./commands/generate-metadata/generate-metadata.ts"; import docs from "./commands/docs/docs.ts"; import config from "./commands/config/config.ts"; +import datatable from "./commands/datatable/datatable.ts"; +import ducklake from "./commands/ducklake/ducklake.ts"; import { fetchVersion } from "./core/context.ts"; export { @@ -65,10 +68,13 @@ export { sync, lint, gitsyncSettings, + protectionRules, instance, dev, docs, config, + datatable, + ducklake, hubPull, pull, push, @@ -185,6 +191,7 @@ const command = new Command() .command("sync", sync) .command("lint", lint) .command("gitsync-settings", gitsyncSettings) + .command("protection-rules", protectionRules) .command("instance", instance) .command("worker-groups", workerGroups) .command("workers", workers) @@ -198,6 +205,8 @@ const command = new Command() .command("generate-metadata", generateMetadata) .command("docs", docs) .command("config", config) + .command("datatable", datatable) + .command("ducklake", ducklake) .command("version --version", "Show version information") .action(async (opts: any) => { console.log("CLI version: " + VERSION); diff --git a/cli/src/utils/catalog.ts b/cli/src/utils/catalog.ts new file mode 100644 index 0000000000..920bb65f6b --- /dev/null +++ b/cli/src/utils/catalog.ts @@ -0,0 +1,141 @@ +import { colors } from "@cliffy/ansi/colors"; +import { Table } from "@cliffy/table"; + +import * as wmill from "../../gen/services.gen.ts"; +import { Preview } from "../../gen/types.gen.ts"; +import { requireLogin } from "../core/auth.ts"; +import { resolveWorkspace } from "../core/context.ts"; +import * as log from "../core/log.ts"; +import { GlobalOptions } from "../types.ts"; +import { pollJobWithQueueLogging } from "./job_polling.ts"; + +// Shared building blocks for SQL catalogs (`wmill datatable` and +// `wmill ducklake`). Both expose a SQL surface backed by Windmill script +// previews, but the language and arg shape differ — `buildCatalogQueryPlan` +// centralizes that difference so subcommands stay thin. + +export type CatalogKind = "datatable" | "ducklake"; + +interface CatalogQueryPlan { + language: NonNullable; + content: string; + args: Record; +} + +function buildCatalogQueryPlan( + kind: CatalogKind, + name: string, + sql: string, +): CatalogQueryPlan { + switch (kind) { + case "datatable": + return { + language: "postgresql", + content: sql, + args: { database: `datatable://${name}` }, + }; + case "ducklake": { + const attach = `ATTACH 'ducklake://${name}' AS dl;\nUSE dl;\n`; + return { + language: "duckdb", + content: attach + sql, + args: {}, + }; + } + } +} + +export async function runCatalogQuery( + opts: GlobalOptions & { silent?: boolean }, + kind: CatalogKind, + name: string, + sql: string, +): Promise { + if (opts.silent) { + log.setSilent(true); + } + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + const plan = buildCatalogQueryPlan(kind, name, sql); + + log.info(colors.gray(`Running query on ${kind}://${name}`)); + + const jobId = await wmill.runScriptPreview({ + workspace: workspace.workspaceId, + requestBody: { + content: plan.content, + language: plan.language, + args: plan.args, + }, + }); + + const { result, success } = await pollJobWithQueueLogging( + workspace.workspaceId, + jobId, + ); + + if (!success) { + process.exitCode = 1; + if (opts.silent) { + console.log(JSON.stringify(result)); + } else { + log.info(colors.red.bold("Query failed")); + log.info(JSON.stringify(result, null, 2)); + } + return; + } + + if (opts.silent) { + console.log(JSON.stringify(result)); + } else { + renderQueryResult(result); + } +} + +/** + * Render a SQL query result. Postgres script previews return rows as an array + * of `{column: value}` objects — display those as a table. Anything else + * (DDL output, empty results, scalar payloads) falls back to pretty JSON. + */ +function renderQueryResult(result: unknown): void { + if (Array.isArray(result) && result.length > 0 && result.every(isRecord)) { + const rows = result as Record[]; + const columns = collectColumns(rows); + new Table() + .header(columns) + .padding(2) + .border(true) + .body(rows.map((row) => columns.map((c) => formatCell(row[c])))) + .render(); + return; + } + log.info(JSON.stringify(result, null, 2)); +} + +function isRecord(v: unknown): v is Record { + return v !== null && typeof v === "object" && !Array.isArray(v); +} + +function collectColumns(rows: Record[]): string[] { + const seen = new Set(); + const columns: string[] = []; + for (const row of rows) { + for (const key of Object.keys(row)) { + if (!seen.has(key)) { + seen.add(key); + columns.push(key); + } + } + } + return columns; +} + +function formatCell(value: unknown): string { + if (value === null || value === undefined) return ""; + if (typeof value === "string") return value; + if (typeof value === "number" || typeof value === "boolean") { + return String(value); + } + return JSON.stringify(value); +} diff --git a/cli/test/protection_rules_converter_unit.test.ts b/cli/test/protection_rules_converter_unit.test.ts new file mode 100644 index 0000000000..c024cdf502 --- /dev/null +++ b/cli/test/protection_rules_converter_unit.test.ts @@ -0,0 +1,240 @@ +/** + * Unit tests for the protection-rules feature: the reconciliation converter, + * the WorkspaceResolver (protection-rules.yaml key -> backend id via + * wmill.yaml), and protection-rules.yaml read/write round-tripping. + */ + +import { expect, test, describe } from "bun:test"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { mkdtempSync, rmSync } from "node:fs"; +import { ProtectionRulesConverter } from "../src/commands/protection-rules/converter.ts"; +import { + WorkspaceResolver, + readProtectionRulesFile, + writeProtectionRulesFile, +} from "../src/commands/protection-rules/file.ts"; +import { ProtectionRuleEntry } from "../src/commands/protection-rules/types.ts"; +import { SyncOptions } from "../src/core/conf.ts"; + +const rule = ( + name: string, + rules: ProtectionRuleEntry["rules"], + groups: string[] = [], + users: string[] = [], +): ProtectionRuleEntry => ({ + name, + rules, + bypass_groups: groups, + bypass_users: users, +}); + +describe("normalizeEntry", () => { + test("sorts and dedupes rules, groups, users", () => { + const r = rule( + "prod", + ["RestrictDeployToDeployers", "DisableDirectDeployment", "DisableDirectDeployment"], + ["g/b", "g/a"], + ["u/y", "u/x", "u/x"], + ); + const n = ProtectionRulesConverter.normalizeEntry(r); + expect(n.rules).toEqual([ + "DisableDirectDeployment", + "RestrictDeployToDeployers", + ]); + expect(n.bypass_groups).toEqual(["g/a", "g/b"]); + expect(n.bypass_users).toEqual(["u/x", "u/y"]); + }); + + test("handles missing arrays", () => { + const n = ProtectionRulesConverter.normalizeEntry({ + name: "x", + } as unknown as ProtectionRuleEntry); + expect(n.rules).toEqual([]); + expect(n.bypass_groups).toEqual([]); + expect(n.bypass_users).toEqual([]); + }); +}); + +describe("entriesEqual", () => { + test("equal regardless of array order", () => { + const a = rule("p", ["DisableDirectDeployment", "DisableWorkspaceForking"], ["g/a", "g/b"]); + const b = rule("p", ["DisableWorkspaceForking", "DisableDirectDeployment"], ["g/b", "g/a"]); + expect(ProtectionRulesConverter.entriesEqual(a, b)).toBe(true); + }); + + test("different rules are not equal", () => { + const a = rule("p", ["DisableDirectDeployment"]); + const b = rule("p", ["DisableWorkspaceForking"]); + expect(ProtectionRulesConverter.entriesEqual(a, b)).toBe(false); + }); + + test("different bypass users are not equal", () => { + const a = rule("p", ["DisableDirectDeployment"], [], ["u/a"]); + const b = rule("p", ["DisableDirectDeployment"], [], ["u/b"]); + expect(ProtectionRulesConverter.entriesEqual(a, b)).toBe(false); + }); +}); + +describe("fromBackend", () => { + test("strips workspace_id and normalizes", () => { + const out = ProtectionRulesConverter.fromBackend([ + { + name: "p", + workspace_id: "ws1", + rules: ["DisableWorkspaceForking", "DisableDirectDeployment"], + bypass_groups: ["g/b", "g/a"], + bypass_users: [], + }, + ]); + expect(out).toEqual([ + { + name: "p", + rules: ["DisableDirectDeployment", "DisableWorkspaceForking"], + bypass_groups: ["g/a", "g/b"], + bypass_users: [], + }, + ]); + }); +}); + +describe("listsEqual", () => { + test("equal regardless of list order", () => { + const a = [rule("a", ["DisableDirectDeployment"]), rule("b", ["DisableWorkspaceForking"])]; + const b = [rule("b", ["DisableWorkspaceForking"]), rule("a", ["DisableDirectDeployment"])]; + expect(ProtectionRulesConverter.listsEqual(a, b)).toBe(true); + }); + + test("undefined equals empty", () => { + expect(ProtectionRulesConverter.listsEqual(undefined, [])).toBe(true); + }); + + test("different length not equal", () => { + expect( + ProtectionRulesConverter.listsEqual([rule("a", [])], []), + ).toBe(false); + }); +}); + +describe("computePlan (full reconcile)", () => { + test("creates rules present locally but not on backend", () => { + const plan = ProtectionRulesConverter.computePlan( + [rule("new", ["DisableDirectDeployment"])], + [], + ); + expect(plan.toCreate.map((e) => e.name)).toEqual(["new"]); + expect(plan.toUpdate).toEqual([]); + expect(plan.toDelete).toEqual([]); + }); + + test("deletes backend rules not present locally", () => { + const plan = ProtectionRulesConverter.computePlan( + [], + [rule("stale", ["DisableDirectDeployment"])], + ); + expect(plan.toDelete).toEqual(["stale"]); + expect(plan.toCreate).toEqual([]); + }); + + test("updates rules whose content changed", () => { + const plan = ProtectionRulesConverter.computePlan( + [rule("p", ["DisableDirectDeployment", "DisableWorkspaceForking"])], + [rule("p", ["DisableDirectDeployment"])], + ); + expect(plan.toUpdate.map((e) => e.name)).toEqual(["p"]); + expect(plan.toCreate).toEqual([]); + expect(plan.toDelete).toEqual([]); + }); + + test("unchanged rules are not in create/update/delete", () => { + const same = [rule("p", ["DisableDirectDeployment"], ["g/a"])]; + const plan = ProtectionRulesConverter.computePlan(same, [ + rule("p", ["DisableDirectDeployment"], ["g/a"]), + ]); + expect(ProtectionRulesConverter.planHasChanges(plan)).toBe(false); + expect(plan.unchanged).toEqual(["p"]); + }); + + test("mixed plan: create + update + delete + unchanged", () => { + const local = [ + rule("keep", ["DisableDirectDeployment"]), + rule("change", ["DisableWorkspaceForking"]), + rule("brand-new", ["RestrictDeployToDeployers"]), + ]; + const backend = [ + rule("keep", ["DisableDirectDeployment"]), + rule("change", ["DisableDirectDeployment"]), + rule("gone", ["DisableDirectDeployment"]), + ]; + const plan = ProtectionRulesConverter.computePlan(local, backend); + expect(plan.toCreate.map((e) => e.name)).toEqual(["brand-new"]); + expect(plan.toUpdate.map((e) => e.name)).toEqual(["change"]); + expect(plan.toDelete).toEqual(["gone"]); + expect(plan.unchanged).toEqual(["keep"]); + expect(ProtectionRulesConverter.planHasChanges(plan)).toBe(true); + }); + + test("reordered arrays do not produce spurious updates", () => { + const plan = ProtectionRulesConverter.computePlan( + [rule("p", ["DisableWorkspaceForking", "DisableDirectDeployment"], ["g/b", "g/a"])], + [rule("p", ["DisableDirectDeployment", "DisableWorkspaceForking"], ["g/a", "g/b"])], + ); + expect(ProtectionRulesConverter.planHasChanges(plan)).toBe(false); + }); +}); + +describe("WorkspaceResolver", () => { + const config: SyncOptions = { + workspaces: { + prod: { workspaceId: "acme-prod" }, + dev: {}, + commonSpecificItems: { settings: true }, + } as any, + }; + const r = WorkspaceResolver.fromConfig(config); + + test("knownNames excludes reserved keys", () => { + expect(r.knownNames().sort()).toEqual(["dev", "prod"]); + }); + + test("backendId uses workspaceId when set, else the key name", () => { + expect(r.backendId("prod")).toBe("acme-prod"); + expect(r.backendId("dev")).toBe("dev"); + }); + + test("backendId throws for a key absent from wmill.yaml", () => { + expect(() => r.backendId("ghost")).toThrow(/not defined in wmill\.yaml/); + }); + + test("has reflects membership", () => { + expect(r.has("prod")).toBe(true); + expect(r.has("ghost")).toBe(false); + }); + + test("empty config resolves to no workspaces", () => { + expect(WorkspaceResolver.fromConfig({}).knownNames()).toEqual([]); + }); +}); + +describe("protection-rules.yaml read/write", () => { + test("round-trips and sorts workspace keys deterministically", async () => { + const dir = mkdtempSync(join(tmpdir(), "prfile-")); + const path = join(dir, "protection-rules.yaml"); + try { + expect(await readProtectionRulesFile(path)).toEqual({}); + + await writeProtectionRulesFile(path, { + prod: [rule("p", ["DisableDirectDeployment"], ["g/a"])], + dev: [], + }); + const back = await readProtectionRulesFile(path); + expect(Object.keys(back)).toEqual(["dev", "prod"]); + expect(back.prod).toEqual([ + rule("p", ["DisableDirectDeployment"], ["g/a"]), + ]); + expect(back.dev).toEqual([]); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 7f2c10aa6c..2ee59752f4 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -51,6 +51,7 @@ "jszip": "^3.10.1", "lru-cache": "^11.1.0", "lucide-svelte": "^0.540.0", + "mdast-util-find-and-replace": "^3.0.2", "minimatch": "^10.0.1", "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@=25.0.0", "monaco-languageclient": "10.6.0", @@ -71,6 +72,7 @@ "svelte-exmarkdown": "^5.0.0", "svelte-infinite-loading": "^1.4.0", "tailwind-merge": "^1.13.2", + "unist-util-visit": "^5.0.0", "vscode": "npm:@codingame/monaco-vscode-extension-api@=25.0.0", "vscode-languageclient": "~9.0.1", "vscode-uri": "~3.1.0", diff --git a/frontend/package.json b/frontend/package.json index c379388fd6..c7f0c548e0 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -124,6 +124,8 @@ "jszip": "^3.10.1", "lru-cache": "^11.1.0", "lucide-svelte": "^0.540.0", + "mdast-util-find-and-replace": "^3.0.2", + "unist-util-visit": "^5.0.0", "minimatch": "^10.0.1", "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@=25.0.0", "monaco-languageclient": "10.6.0", diff --git a/frontend/src/lib/components/DeployButton.svelte b/frontend/src/lib/components/DeployButton.svelte index 16ccc04cd2..d8828d515a 100644 --- a/frontend/src/lib/components/DeployButton.svelte +++ b/frontend/src/lib/components/DeployButton.svelte @@ -6,12 +6,10 @@ const { loading = false, loadingSave = false, - newFlow = false, dropdownItems = [] }: { loading?: boolean loadingSave?: boolean - newFlow?: boolean dropdownItems?: Array<{ label: string onClick: () => void @@ -33,7 +31,7 @@ unifiedSize="md" startIcon={{ icon: Save }} on:click={() => dispatch('save')} - dropdownItems={!newFlow ? dropdownItems : undefined} + {dropdownItems} tooltipPopover={{ placement: 'bottom-end', openDelay: dropdownOpen ? 200 : 0, diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index 5f2d65ab2d..03f4c04642 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -1916,6 +1916,25 @@ }) }) + // External `code` prop changes should flow into the Monaco editor. The + // `untrack` block reads/writes Monaco without subscribing — only the + // prop read above is tracked — so the editor's own change handler + // (`updateCode`) re-running with the same value short-circuits and we + // don't loop. + $effect(() => { + const next = code ?? '' + const ed = editor + if (!ed) return + untrack(() => { + if (ed.getValue() === next) return + const model = ed.getModel() + if (!model) return + ed.pushUndoStop() + ed.executeEdits('external', [{ range: model.getFullModelRange(), text: next }]) + ed.pushUndoStop() + }) + }) + let isTsWorkerInitialized = resource([() => lang, () => initialized], async () => { if (lang !== 'typescript' || !initialized) return false // Use the stable model URI (computed once at mount), not filePath which changes on rename diff --git a/frontend/src/lib/components/EditorBar.svelte b/frontend/src/lib/components/EditorBar.svelte index b6addb6176..23bec9c76c 100644 --- a/frontend/src/lib/components/EditorBar.svelte +++ b/frontend/src/lib/components/EditorBar.svelte @@ -1,5 +1,11 @@
-
+
{#if meta != undefined} + {@const nameDisabled = disabled || disableEditing} {#if !hideUser}
- { - setDirty() - const kind = e.detail - if (meta) { - if (kind === 'folder') { + { - currentTarget.select() - }} - /> - -
-
- {/if} - {#if pathUsageInFlowsPromise || pathUsageInAppsPromise || pathUsageInScriptsPromise} @@ -618,9 +599,3 @@ {/if}
- - diff --git a/frontend/src/lib/components/PathNameAutocomplete.svelte b/frontend/src/lib/components/PathNameAutocomplete.svelte index fa27f4dc3b..62e8d141db 100644 --- a/frontend/src/lib/components/PathNameAutocomplete.svelte +++ b/frontend/src/lib/components/PathNameAutocomplete.svelte @@ -19,18 +19,31 @@ * the same page reuse a single fetch per workspace. */ const pathListCache = new Map() + /** Workspaces whose next fetch must bypass the server-side cache. Set by + * invalidateWorkspacePaths (e.g. after a deploy) and cleared once a forced + * fetch succeeds, so a just-created path shows up immediately instead of + * after the backend's 60s TTL. */ + const forceNextFetch = new Set() + export async function fetchWorkspacePaths(workspace: string): Promise { + const force = forceNextFetch.has(workspace) const now = Date.now() const existing = pathListCache.get(workspace) - if (existing) { + // When forcing, ignore any cached/in-flight entry — it may predate the + // deploy (or have been written by a fetch that hit the stale backend + // cache) and would otherwise mask the new path. + if (existing && !force) { if (existing.paths && now - existing.at < PATH_LIST_TTL_MS) return existing.paths if (existing.pending) return existing.pending } const pending = (async () => { try { - const res = await PathAutocompleteService.listPathAutocompletePaths({ workspace }) + const res = await PathAutocompleteService.listPathAutocompletePaths({ workspace, force }) const paths = res.paths ?? [] pathListCache.set(workspace, { at: Date.now(), paths, pending: null }) + // Only clear the force flag once a forced fetch has actually + // landed fresh data, so a failed retry still forces. + forceNextFetch.delete(workspace) return paths } catch (_e) { pathListCache.delete(workspace) @@ -43,6 +56,7 @@ export function invalidateWorkspacePaths(workspace: string) { pathListCache.delete(workspace) + forceNextFetch.add(workspace) } /** Derive the set of path segments that exist directly under a given folder @@ -72,6 +86,7 @@ + +
{#if otherDirty.length > 0} diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index 4df73ce7a2..710ce9bd5d 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -4,7 +4,6 @@ const bubble = createBubbler() import { DraftService, - type NewScript, ScriptService, type NewScriptWithDraft, type Script, @@ -35,7 +34,6 @@ cleanValueProperties, emptySchema, emptyString, - encodeState, generateRandomString, orderedJsonStringify, readFieldsRecursively, @@ -43,6 +41,7 @@ type Value } from '$lib/utils' import Path from './Path.svelte' + import { invalidateWorkspacePaths } from './PathNameAutocomplete.svelte' import ScriptEditor from './ScriptEditor.svelte' import { Alert, Button, Drawer, SecondsInput, Tab, TabContent, Tabs } from './common' import LanguageIcon from './common/languageIcons/LanguageIcon.svelte' @@ -52,7 +51,21 @@ import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte' import ErrorHandlerToggleButton from '$lib/components/details/ErrorHandlerToggleButton.svelte' - import { Bug, CheckCircle, Code, Plus, Rocket, Save, Settings, Shuffle, X } from 'lucide-svelte' + import { + Bug, + CheckCircle, + Code, + EllipsisVertical, + Plus, + Rocket, + Save, + Settings, + Shuffle, + Tag, + X + } from 'lucide-svelte' + import DropdownV2 from './DropdownV2.svelte' + import { isMac, type Item } from '$lib/utils' import { sendUserToast } from '$lib/toast' import { isCloudHosted } from '$lib/cloud' import Awareness from './Awareness.svelte' @@ -110,7 +123,6 @@ savedScript = $bindable(undefined), searchParams = new URLSearchParams(), disableHistoryChange = false, - replaceStateFn = (url) => window.history.replaceState(null, '', url), customUi = {}, savedPrimarySchedule = undefined, functionExports = undefined, @@ -147,6 +159,36 @@ let deployedValue: Value | undefined = $state(undefined) // Value to diff against let deployedBy: string | undefined = $state(undefined) // Author let confirmCallback: () => void = $state(() => {}) // What happens when user clicks `override` in warning + + // Top-bar responsive collapse — container width, not viewport. + let topbarWidth = $state(0) + const compactTopbar = $derived(topbarWidth > 0 && topbarWidth < 720) + const mod = isMac() ? '⌘' : 'Ctrl+' + + function getCompactMenuItems(): Item[] { + const hasTags = ($workerTags?.length ?? 0) > 0 + return [ + { + displayName: 'Save draft', + icon: Save, + action: () => saveDraft(), + shortcut: `${mod}S`, + disabled: initialPath != '' && !savedScript + }, + ...(customUi?.topBar?.tagEdit != false && hasTags + ? [ + { + displayName: 'Worker tag', + icon: Tag, + action: () => { + selectedTab = 'runtime' + metadataOpen = true + } + } + ] + : []) + ] + } let open: boolean = $state(false) // Is confirmation modal open let args: Record = $state(untrack(() => initialArgs)) // Test args input let selectedInputTab: 'main' | 'preprocessor' | 'diagram' = $state('main') @@ -258,15 +300,11 @@ // Add triggers context store const triggersState = $state( - new Triggers( - [ - { type: 'webhook', path: '', isDraft: false }, - { type: 'default_email', path: '', isDraft: false }, - ...(script.draft_triggers ?? []) - ], - undefined, - saveSessionDraft - ) + new Triggers([ + { type: 'webhook', path: '', isDraft: false }, + { type: 'default_email', path: '', isDraft: false }, + ...(script.draft_triggers ?? []) + ]) ) const captureOn = writable(undefined) @@ -330,28 +368,6 @@ let loadingSave = $state(false) let loadingDraft = $state(false) - let timeout2: number | undefined = undefined - function encodeScriptState(script: NewScript) { - untrack(() => timeout2 && clearTimeout(timeout2)) - timeout2 = setTimeout(() => { - replaceStateFn( - '#' + - encodeState({ - ...script, - draft_triggers: structuredClone(triggersState.getDraftTriggersSnapshot()) - }) - ) - }, 500) - } - - let timeout: number | undefined = undefined - function saveSessionDraft() { - timeout && clearTimeout(timeout) - timeout = setTimeout(() => { - encodeScriptState(script) - }, 500) - } - if (script.content == '') { if (template === 'wac_python') { script.modules = { @@ -514,11 +530,6 @@ loadingSave = true try { - try { - localStorage.removeItem(script.path) - } catch (e) { - console.error('error interacting with local storage', e) - } script.schema = script.schema ?? emptySchema() try { const result = await inferArgs( @@ -582,6 +593,10 @@ } }) + // New/updated path now exists server-side — drop the autocomplete + // cache so it shows up immediately instead of after the 60s TTL. + invalidateWorkspacePaths($workspaceStore!) + if (!initialPath) { await CaptureService.moveCapturesAndConfigs({ workspace: $workspaceStore!, @@ -654,11 +669,6 @@ loadingDraft = true try { - try { - localStorage.removeItem(script.path) - } catch (e) { - console.error('error interacting with local storage', e) - } script.schema = script.schema ?? emptySchema() try { const result = await inferArgs( @@ -1037,7 +1047,15 @@ }) $effect(() => { readFieldsRecursively(script) - !disableHistoryChange && encodeScriptState(script) + }) + // Mirror the draft triggers (held in a separate `triggersState` $state) + // back into `script.draft_triggers` so the UserDraft autosave — which + // deep-tracks `script` — picks them up. Pre-PR ScriptBuilder ran its own + // localStorage autosave that explicitly snapshotted triggersState; the + // switch to a unified UserDraft handle dropped that bridge. + $effect(() => { + readFieldsRecursively(triggersState.triggers) + script.draft_triggers = triggersState.getDraftTriggersSnapshot() }) loadWorkerTags() @@ -1937,9 +1955,9 @@
-
+
-
+
+ {/if} + {/snippet} + {#if compactTopbar} + + {#snippet buttonReplacement()} + {/if} - handleEditScript(false, detail)} /> diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index f64158cc78..b53a984fae 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -20,10 +20,13 @@ import SchemaForm from './SchemaForm.svelte' import PowerShellCommonParams from './PowerShellCommonParams.svelte' import LogPanel from './scriptEditor/LogPanel.svelte' - import EditorBar, { EDITOR_BAR_WIDTH_THRESHOLD } from './EditorBar.svelte' + import EditorBar, { + EDITOR_BAR_WIDTH_THRESHOLD, + EDITOR_BAR_HELPERS_COMPACT_THRESHOLD + } from './EditorBar.svelte' import JobLoader from './JobLoader.svelte' import JobProgressBar from '$lib/components/jobs/JobProgressBar.svelte' - import { createEventDispatcher, onDestroy, onMount, untrack } from 'svelte' + import { createEventDispatcher, getContext, onDestroy, onMount, untrack } from 'svelte' import { Button } from './common' import SplitPanesWrapper from './splitPanes/SplitPanesWrapper.svelte' import WindmillIcon from './icons/WindmillIcon.svelte' @@ -84,6 +87,15 @@ import { getStringError } from './copilot/chat/utils' import type { ScriptOptions } from './copilot/chat/ContextManager.svelte' import { aiChatManager, AIMode } from './copilot/chat/AIChatManager.svelte' + + // Forward-looking hook for the upcoming session-pane feature: that PR will + // `setContext('aiChatManager', ...)` from the session wrapper so this editor + // can detect it and hide its own AI/VS Code controls. On main today nothing + // sets the context, so `inSessionPane` is always false and these buttons + // render normally — keep the check anyway to avoid re-touching this file + // when the session-pane PR lands. Untyped getContext to avoid coupling to + // the AIChatManager class export (which lives on the chat-visuals PR). + const inSessionPane = !!getContext('aiChatManager') import { triggerableByAI } from '$lib/actions/triggerableByAI.svelte' import AssetsDropdownButton from './assets/AssetsDropdownButton.svelte' import { canHavePreprocessor } from '$lib/script_helpers' @@ -608,6 +620,10 @@ let debugBreakpoints = new SvelteSet() let breakpointDecorations: string[] = $state([]) let currentLineDecoration: string[] = $state([]) + let hoverBreakpointDecoration: string[] = $state([]) + // Line currently showing the ghost breakpoint, used to short-circuit redundant + // deltaDecorations calls on every mousemove event. + let hoverBreakpointLine: number | null = null // Get the DAP server URL based on language const dapServerUrl = $derived(getDebugServerUrl((lang || 'python3') as DebugLanguage)) const debugFilePath = $derived(`/tmp/script${getDebugFileExtension(lang || '')}`) @@ -640,6 +656,13 @@ stickiness: 1 } + // Ghost breakpoint shown while hovering the gutter on a line without a breakpoint + const hoverBreakpointDecorationType: meditor.IModelDecorationOptions = { + glyphMarginClassName: 'debug-breakpoint-glyph-hover', + glyphMarginHoverMessage: { value: 'Click to add a breakpoint' }, + stickiness: 1 + } + const currentLineDecorationType = { isWholeLine: true, className: 'debug-current-line', @@ -886,6 +909,35 @@ debugBreakpoints.add(line) } updateBreakpointDecorations() + clearHoverBreakpointDecoration() + } + + function updateHoverBreakpointDecoration(line: number): void { + if (hoverBreakpointLine === line) return + const monacoEditor = editor?.getEditor?.() + if (!monacoEditor) return + + const decorations = [ + { + range: { startLineNumber: line, startColumn: 1, endLineNumber: line, endColumn: 1 }, + options: hoverBreakpointDecorationType + } + ] + + const oldDecorations = untrack(() => hoverBreakpointDecoration) + hoverBreakpointDecoration = monacoEditor.deltaDecorations(oldDecorations, decorations) + hoverBreakpointLine = line + } + + function clearHoverBreakpointDecoration(): void { + if (hoverBreakpointLine === null) return + const monacoEditor = editor?.getEditor?.() + if (!monacoEditor) return + const oldDecorations = untrack(() => hoverBreakpointDecoration) + if (oldDecorations.length > 0) { + hoverBreakpointDecoration = monacoEditor.deltaDecorations(oldDecorations, []) + } + hoverBreakpointLine = null } function updateBreakpointDecorations(): void { @@ -1115,8 +1167,7 @@ // Add click handler for glyph margin (breakpoint toggle) const mouseDownDisposable = monacoEditor.onMouseDown((e) => { - // MouseTargetType.GUTTER_GLYPH_MARGIN = 2 - if (e.target.type === 2) { + if (e.target.type === meditor.MouseTargetType.GUTTER_GLYPH_MARGIN) { const line = e.target.position?.lineNumber if (line) { toggleBreakpoint(line) @@ -1124,6 +1175,27 @@ } }) + // Show a ghost breakpoint while hovering anywhere in the gutter on an empty line. + // Hover area is intentionally wider than the click target — clicks still only + // toggle when landing on the glyph margin itself, but the ghost helps users find it. + const mouseMoveDisposable = monacoEditor.onMouseMove((e) => { + const t = e.target.type + const isGutter = + t === meditor.MouseTargetType.GUTTER_GLYPH_MARGIN || + t === meditor.MouseTargetType.GUTTER_LINE_NUMBERS || + t === meditor.MouseTargetType.GUTTER_LINE_DECORATIONS + const line = e.target.position?.lineNumber + if (isGutter && line && !debugBreakpoints.has(line)) { + updateHoverBreakpointDecoration(line) + } else { + clearHoverBreakpointDecoration() + } + }) + + const mouseLeaveDisposable = monacoEditor.onMouseLeave(() => { + clearHoverBreakpointDecoration() + }) + // Add F9 keyboard shortcut for toggling breakpoint at cursor monacoEditor.addCommand(120, () => { // KeyCode.F9 = 120 @@ -1156,6 +1228,9 @@ return () => { mouseDownDisposable.dispose() + mouseMoveDisposable.dispose() + mouseLeaveDisposable.dispose() + clearHoverBreakpointDecoration() // Disable glyph margin when exiting debug mode monacoEditor.updateOptions({ glyphMargin: false }) } @@ -1314,18 +1389,35 @@ setContext('disableTooltips', untrack(() => customUi)?.disableTooltips === true) - let codePanelSize = $state(70) - let testPanelSize = $state(30) - let storedTestPanelSize = untrack(() => testPanelSize) + // Pixel minimum width for the test pane while open. The pane's `minSize` + // is a percentage, so we derive it from the live splitpane container + // width (Svelte wires a ResizeObserver for bind:clientWidth). + let splitContainerWidth = $state(0) + const TEST_PANE_MIN_PX = 400 + const testPaneMinPercent = $derived( + splitContainerWidth > 0 ? Math.min(80, (TEST_PANE_MIN_PX / splitContainerWidth) * 100) : 0 + ) + + // Raw user-controlled test size (what the splitter wrote, or what the + // toggle set). The size we actually pass to is clamped to the + // dynamic minimum below — so when the editor shrinks, the displayed test + // pane grows to honor the new minimum without needing an effect. The code + // pane's size is purely derived from it (100 - test). + let rawTestPanelSize = $state(30) + let storedTestPanelSize = untrack(() => rawTestPanelSize) + const testPanelSize = $derived( + rawTestPanelSize === 0 ? 0 : Math.max(rawTestPanelSize, testPaneMinPercent) + ) + const codePanelSize = $derived(100 - testPanelSize) function toggleTestPanel() { if (testPanelSize > 0) { - storedTestPanelSize = testPanelSize - codePanelSize += testPanelSize - testPanelSize = 0 + // Store the raw (unclamped) preference so reopening on a wider screen + // restores the user's intent, not the pixel-min that inflated the pane. + storedTestPanelSize = rawTestPanelSize + rawTestPanelSize = 0 } else { - codePanelSize -= storedTestPanelSize - testPanelSize = storedTestPanelSize + rawTestPanelSize = Math.max(storedTestPanelSize, testPaneMinPercent) } } @@ -1348,8 +1440,7 @@ const delta = want - argsHeightBonus if (delta === 0) return if (testPanelSize > 0) { - testPanelSize += delta - codePanelSize -= delta + rawTestPanelSize += delta } else { // preview collapsed — bake the bonus into the size it // restores to so it lands correctly on expand. @@ -1462,6 +1553,7 @@ {collabMode} {validCode} iconOnly={width < EDITOR_BAR_WIDTH_THRESHOLD} + compactHelpers={width < EDITOR_BAR_HELPERS_COMPACT_THRESHOLD} on:collabPopup={() => (showCollabPopup = true)} {editor} {lang} @@ -1484,14 +1576,15 @@ {/snippet} {/if} - {#if !noSyncFromGithub && customUi?.editorBar?.useVsCode != false} + {#if !noSyncFromGithub && customUi?.editorBar?.useVsCode != false && !inSessionPane}
- - - {#if lang === 'ansible' && ansibleAlternativeExecutionMode != null} - - - - {@render editorContent()} - - -
-
-

File Browser

+
+ + + {#if lang === 'ansible' && ansibleAlternativeExecutionMode != null} + + + + {@render editorContent()} + + +
+
+

File Browser

+
+
- -
- - - {:else} - - {@render editorContent()} - {/if} - - -
- {#if showTabs} -
- - - {#if hasPreprocessor} -
- -
- {/if} - {#if isWac} -
- -
- {/if} - -
- {/if} - - {#if debugMode && isDebuggableScript} -
- -
- {/if} - - {#if selectedTab === 'diagram'} -
- -
+ + {:else} - {#if previewLayout !== 'bottom'} -
-
-
- {#if !(debugMode && isDebuggableScript)} -
-
- {#if testIsLoading} - - {:else} - {@const disableTriggerButton = - customUi?.previewPanel?.disableTriggerButton === true} - - {#if !disableTriggerButton} - - {/if} - {/if} + + {@render editorContent()} + {/if} + + testPanelSize, (v) => (rawTestPanelSize = v)} + minSize={testPaneMinPercent} + class={customUi?.previewPanel?.hideArgs ? '!overflow-visible' : ''} + > +
+ {#if showTabs} +
+ + + {#if hasPreprocessor} +
+
- {#if lastRecording} -
- {/if} -
- {#if customUi?.previewPanel?.disableJsonView !== true} - {/if} - recordAndTest() - } - ]} - /> -
+ {#if isWac} +
+ +
+ {/if} +
{/if} - {#if customUi?.previewPanel?.hideArgs} - -
-
- {#if testIsLoading} - - {:else if (customUi?.previewPanel?.downstreamSubscribers ?? 0) > 0} - - {@const downstream = customUi!.previewPanel!.downstreamSubscribers!} -
- - - {#snippet trigger()} - - - {/snippet} - {#snippet content({ close })} -
- -
- {/snippet} - -
- {:else} - - {/if} -
- {#if customUi?.previewPanel?.argsAboveLogs && schema?.properties && Object.keys(schema.properties).length > 0} -
- {#key argsRender} - - {/key} -
- {/if} -
- { - historyTabActive = tab === 'history' - if (historyTabActive) { - loadPastTests() - } - }} - previewIsLoading={debugMode - ? $debugState.running && !$debugState.stopped - : testIsLoading} - {editor} - {diffEditor} - args={activeModuleTab !== null ? testPanelArgs : args} - {showCaptures} - customUi={customUi?.previewPanel} - showCustomResultPanel={showDebugPanel} - > - {#if scriptProgress && !debugMode} - - {/if} - -
-
- {:else} - {#key previewLayout} - - - {#if previewLayout === 'bottom' && !(debugMode && isDebuggableScript)} -
- {#if testIsLoading} - - {:else} - - {/if} -
- {/if} - {#if jsonView} -
- { - if (e.detail) { - if (activeModuleTab !== null) { - testPanelArgs = e.detail - } else { - args = e.detail - } - } - }} - updateOnBlur={false} - placeholder={`Write args as JSON.

Example:

{
  "foo": "12"
}`} - /> + {/snippet} + {#snippet content({ close })} +
+ + +
+ {/snippet} +
{:else} -
-
- {#key argsRender} - {#if activeModuleTab !== null} - - {:else} - - {/if} - {/key} - {#if showPsCommonParams} -
- -
- {/if} -
-
+ {/if} - - +
+ {#if customUi?.previewPanel?.argsAboveLogs && schema?.properties && Object.keys(schema.properties).length > 0} +
+ {#key argsRender} + + {/key} +
+ {/if} +
{ + historyTabActive = tab === 'history' + if (historyTabActive) { + loadPastTests() + } + }} previewIsLoading={debugMode ? $debugState.running && !$debugState.stopped : testIsLoading} @@ -1993,7 +1947,6 @@ showCustomResultPanel={showDebugPanel} > {#if scriptProgress && !debugMode} - {/if} - {#snippet capturesTab()} -
- +
+
+ {:else} + {#key previewLayout} + + + {#if previewLayout === 'bottom' && !(debugMode && isDebuggableScript)} +
+ {#if testIsLoading} + + {:else} + + {/if} +
+ {/if} + {#if jsonView} +
+ { + if (e.detail) { + if (activeModuleTab !== null) { + testPanelArgs = e.detail + } else { + args = e.detail + } + } + }} + updateOnBlur={false} + placeholder={`Write args as JSON.

Example:

{
  "foo": "12"
}`} />
- {/snippet} - {#snippet customResultPanel()} - - {/snippet} - - - - {/key} + {:else} +
+
+ {#key argsRender} + {#if activeModuleTab !== null} + + {:else} + + {/if} + {/key} + {#if showPsCommonParams} +
+ +
+ {/if} +
+
+ {/if} + + + + {#if scriptProgress && !debugMode} + + + {/if} + {#snippet capturesTab()} +
+ +
+ {/snippet} + {#snippet customResultPanel()} + + {/snippet} + + + + {/key} + {/if} {/if} - {/if} -
- - +
+ + +
{#snippet addModuleForm(close: () => void)} @@ -2224,14 +2326,15 @@
{/if}
-
+
{#if assets?.length} {/if} + {#if isDebuggableScript && customUi?.editorBar?.debug != false} @@ -2268,8 +2369,9 @@
+ {#if searchError} +
+ + {searchError} + +
+ {/if} {#if allLogs == undefined}
{:else if Object.keys(allLogs).length == 0}
No logs Search only covers a recent time window, configurable in instance settings - under Indexer.Search only covers a recent time window, configurable in instance settings under + Indexer.
{:else if minTs && maxTs} diff --git a/frontend/src/lib/components/SummaryPathDisplay.svelte b/frontend/src/lib/components/SummaryPathDisplay.svelte index b9cbd2886d..64343bf7eb 100644 --- a/frontend/src/lib/components/SummaryPathDisplay.svelte +++ b/frontend/src/lib/components/SummaryPathDisplay.svelte @@ -150,7 +150,6 @@ initialPath={path ?? ''} namePlaceholder={kind} {kind} - hideFullPath size="sm" drawerOffset={4000} /> @@ -199,7 +198,6 @@ initialPath={path ?? ''} namePlaceholder={kind} {kind} - hideFullPath size="sm" drawerOffset={4000} /> diff --git a/frontend/src/lib/components/Toast.svelte b/frontend/src/lib/components/Toast.svelte index 034887235e..0dd1b91026 100644 --- a/frontend/src/lib/components/Toast.svelte +++ b/frontend/src/lib/components/Toast.svelte @@ -82,16 +82,24 @@ } }) - let color = classes[untrack(() => type)] + // Defensive: a miscall like `sendUserToast(msg, err)` passes a non- + // AlertType as `type`. Without a fallback the `classes[type]` lookup + // returns undefined and `color.descriptionClass` throws — and because + // the toast renders inside the root layout, that crashes the whole + // page instead of just dropping one toast. Coerce anything unknown to + // 'error' (a bad type almost always accompanies an error path). + const safeType: ToastType = untrack(() => (type in classes ? type : 'error')) + + let color = classes[safeType] let containerClass = { success: 'toast-success', error: 'toast-error', info: 'toast-info', warning: 'toast-warning' - }[untrack(() => type)] + }[safeType] - let Icon = $derived(icons[type]) + let Icon = icons[safeType] let showMore = $state(false) const MAX_MSG_LEN = 160 diff --git a/frontend/src/lib/components/VariableEditor.svelte b/frontend/src/lib/components/VariableEditor.svelte index addbfc095f..962009e63d 100644 --- a/frontend/src/lib/components/VariableEditor.svelte +++ b/frontend/src/lib/components/VariableEditor.svelte @@ -10,11 +10,15 @@ import { canWrite } from '$lib/utils' import { Save } from 'lucide-svelte' import VariableForm from './VariableForm.svelte' + import { invalidateWorkspacePaths } from './PathNameAutocomplete.svelte' import WsSpecificVersions from './WsSpecificVersions.svelte' import { resource } from 'runed' import { deepEqual } from 'fast-equals' import { getUserExt } from '$lib/user' import type { UserExt } from '$lib/stores' + import { UserDraft, checkStaleness, type UserDraftHandle } from '$lib/userDraft.svelte' + import { notifyRestoredFromLocal } from '$lib/userDraftToast' + import LocalDraftStaleModal from './common/confirmationModal/LocalDraftStaleModal.svelte' const dispatch = createEventDispatcher() @@ -27,13 +31,79 @@ let editPath: string | undefined = $state(undefined) - let states: Record = $state({}) + // Per-workspace handles are driven by `useMany`. We track the workspace + // IDs (and their seeded defaults) in a parallel `$state` array; on every + // mutation `useMany` reconciles, acquiring entries for new workspaces and + // releasing them on component teardown. `states` indexes the resulting + // handles by workspace ID for ergonomic lookup downstream. + let workspaceSpecs = $state>([]) let initialStates: Record = $state({}) let existedInitially: Record = $state({}) let extraPerms: Record> = $state({}) let perWsUser: Record = $state({}) let selected: string | undefined = $state(undefined) let pathError = $state('') + // Backend `edited_at` per workspace — the rev the staleness check + // compares the local autosave's recorded rev against. Variables have + // no DB-draft concept, so only `remoteRev` is ever populated. + let fetchedRev: Record = $state({}) + + // Local-draft staleness modal: opened when the backend variable moved + // on (someone else edited it) since the local autosave was written. + let staleModalOpen = $state(false) + let pendingStale: { ws: string; backend: VariableState } | undefined = undefined + + function onStaleLoadLatest(): void { + if (!pendingStale) { + staleModalOpen = false + return + } + const { ws, backend } = pendingStale + UserDraft.discard('variable', editPath ?? '', backend, { workspace: ws }) + initialStates[ws] = $state.snapshot(backend) as VariableState + pendingStale = undefined + staleModalOpen = false + } + + function onStaleKeepDraft(): void { + if (pendingStale) { + const { ws } = pendingStale + UserDraft.saveMeta( + 'variable', + editPath ?? '', + { remoteRev: fetchedRev[ws] }, + { workspace: ws } + ) + } + pendingStale = undefined + staleModalOpen = false + } + + const handlesArray = UserDraft.useMany(() => + workspaceSpecs.map((s) => ({ + itemKind: 'variable' as const, + path: editPath ?? '', + workspace: s.ws, + defaultValue: s.defaultValue + })) + ) + const states = $derived.by(() => { + const out: Record> = {} + for (let i = 0; i < workspaceSpecs.length; i++) { + const handle = handlesArray[i] + if (handle) out[workspaceSpecs[i].ws] = handle + } + return out + }) + + /** Register a workspace so `useMany` acquires (or reuses) its handle. + * `defaultValue` is what the handle reports when no autosave is persisted; + * an existing autosave always wins. The default itself never round-trips + * to localStorage — only the user's first real edit triggers a write. */ + function ensureHandle(ws: string, defaultValue: VariableState): void { + if (workspaceSpecs.some((s) => s.ws === ws)) return + workspaceSpecs.push({ ws, defaultValue }) + } let drawer: Drawer | undefined = $state() let form: VariableForm | undefined = $state() @@ -47,7 +117,7 @@ const MAX_VARIABLE_LENGTH = 10000 const edit = $derived(editPath !== undefined) const initialPath = $derived(editPath ?? '') - const current = $derived(selected ? states[selected] : undefined) + const current = $derived(selected ? states[selected]?.draft : undefined) const can_write = $derived.by(() => { if (!selected || !edit) return true const perms = extraPerms[selected] @@ -55,7 +125,7 @@ return canWrite(editPath ?? '', perms, perWsUser[selected] ?? $userStore) }) const dirtyWorkspaces = $derived( - Object.keys(states).filter((ws) => !deepEqual(states[ws], initialStates[ws])) + Object.keys(states).filter((ws) => !deepEqual(states[ws].draft, initialStates[ws])) ) const anyDirty = $derived(dirtyWorkspaces.length > 0) const otherDirty = $derived( @@ -64,7 +134,10 @@ : dirtyWorkspaces ) const dirtyValid = $derived( - dirtyWorkspaces.every((ws) => states[ws].variable.value.length <= MAX_VARIABLE_LENGTH) + dirtyWorkspaces.every((ws) => { + const v = states[ws].draft + return !!v && v.variable.value.length <= MAX_VARIABLE_LENGTH + }) ) const dirtyCanWrite = $derived( dirtyWorkspaces.every((ws) => { @@ -84,6 +157,7 @@ VariableService.getVariable({ workspace: ws, path: p, decryptSecret: false }), getUserExt(ws) ]).then(([v, user]) => { + fetchedRev[ws] = v.edited_at const s: VariableState = { path: v.path, variable: { @@ -94,7 +168,29 @@ labels: v.labels ?? undefined, wsSpecific: v.ws_specific ?? false } - states[ws] = s + // See ResourceEditor for the same pattern: a backend that + // moved on since the autosave was written → staleness modal; + // otherwise just a "showing your local autosave" toast with + // a "Reset to deployed" escape. + const persisted = UserDraft.get('variable', p, { workspace: ws }) + const previousMeta = UserDraft.getMeta('variable', p, { workspace: ws }) + if (persisted !== undefined && !deepEqual(persisted, s)) { + const cause = checkStaleness(previousMeta, v.edited_at) + if (cause) { + pendingStale = { ws, backend: s } + staleModalOpen = true + } else { + if (previousMeta.remoteRev === undefined && previousMeta.remoteDraftRev === undefined) { + UserDraft.saveMeta('variable', p, { remoteRev: v.edited_at }, { workspace: ws }) + } + notifyRestoredFromLocal(false, true, { + onResetToDeployed: () => { + UserDraft.discard('variable', p, s, { workspace: ws }) + } + }) + } + } + ensureHandle(ws, s) initialStates[ws] = structuredClone(s) existedInitially[ws] = true extraPerms[ws] = v.extra_perms ?? {} @@ -103,8 +199,26 @@ }) }) + // Seed the staleness rev once a real autosave appears (see + // ResourceEditor for the rationale). Self-limiting via the + // meta-already-set guard. + $effect(() => { + for (const ws of Object.keys(states)) { + const h = states[ws] + const rev = fetchedRev[ws] + const baseline = initialStates[ws] + if (!h || rev === undefined || baseline === undefined) continue + const draft = h.draft + if (draft === undefined || deepEqual(draft, baseline)) continue + if (h.meta.remoteRev !== undefined || h.meta.remoteDraftRev !== undefined) continue + untrack(() => h.setMeta({ remoteRev: rev })) + } + }) + function reset() { - states = {} + // Clearing workspaceSpecs triggers useMany's reconcile to release + // every acquired entry. The $derived `states` then collapses to {}. + workspaceSpecs = [] initialStates = {} existedInitially = {} extraPerms = {} @@ -122,7 +236,7 @@ labels: undefined, wsSpecific: false } - states[ws] = s + ensureHandle(ws, s) initialStates[ws] = structuredClone(s) existedInitially[ws] = false selected = ws @@ -143,7 +257,7 @@ path: editPath, decryptSecret: true }) - const s = states[selected] + const s = states[selected]?.draft const ini = initialStates[selected] if (s) s.variable.value = getV.value ?? '' if (ini) ini.variable.value = getV.value ?? '' @@ -154,7 +268,7 @@ const dirty = dirtyWorkspaces try { for (const ws of dirty) { - const s = states[ws] + const s = states[ws].draft! const ini = initialStates[ws] if (existedInitially[ws]) { await VariableService.updateVariable({ @@ -186,6 +300,11 @@ } }) } + // Saved on the backend — drop the local autosave for this workspace. + UserDraft.remove('variable', editPath ?? '', { workspace: ws }) + // Path now exists server-side — drop the autocomplete cache so + // it shows up immediately instead of after the 60s TTL. + invalidateWorkspacePaths(ws) } sendUserToast(edit ? `Updated variable in ${dirty.length} workspace(s)` : `Created variable`) dispatch('create') @@ -196,6 +315,13 @@ } + + window.history.replaceState(null, '', path), gotoFn = (path: string, opt?: Record) => window.history.pushState(null, '', path), unsavedConfirmationModal, - onSavedNewAppPath + onSavedNewAppPath, + initialRevs }: AppEditorProps = $props() migrateApp(untrack(() => app)) - const stateApp = $state(untrack(() => app)) + const appDraftPath = newApp ? '' : (path ?? '') + const appDraftHandle = UserDraft.use('app', appDraftPath) + // Prefer the persisted autosave over the prop when both exist (e.g. + // /apps/add reload: the route always initializes `app` to an empty + // template, but the user's last session is sitting in LS under the + // empty-path entry). The route is responsible for wiping the entry + // (`UserDraft.remove`) when it wants to force a fresh start — + // `?nodraft=true`, template/hub loads, etc. + const stateApp = $state(untrack(() => appDraftHandle.draft ?? app)) const appStore = writable(stateApp) + // Captured once on mount: the load-time revs are only used as the + // seed meta on the very first persist of this entry. After that the + // handle's own meta wins. + const capturedInitialRevs = untrack(() => initialRevs) + // `useLocalStorageValue`'s `saveInitialValue: false` skips the first + // `set val` that DIFFERS from the loaded LS state — meant to absorb a + // route's "load baseline" write. In AppEditor's $effect-mirror pattern + // the loaded baseline always matches LS (stateApp is initialised from + // the handle's draft), so the skip slot survives until the user's + // FIRST edit and silently swallows it. Consume the slot up-front with + // a wipe-then-restore pair: the wipe sets state.val = undefined + // in-memory (the consumption side-effect of skipNextWrite, which + // suppresses the localStorage delete the wipe would otherwise schedule), + // and the restore immediately puts the value+meta back. Net effect: LS + // gets re-written once on mount and user edits persist normally. + let firstMirror = true + $effect(() => { + readFieldsRecursively(stateApp) + untrack(() => { + // Resolve the meta to attach BEFORE the wipe — the wipe clears + // in-memory meta and would otherwise force-seed `initialRevs` + // even when the handle had real meta. + const currentMeta = appDraftHandle.meta + const hasMeta = + currentMeta.remoteRev !== undefined || currentMeta.remoteDraftRev !== undefined + const meta: UserDraftMeta = hasMeta ? currentMeta : (capturedInitialRevs ?? {}) + if (firstMirror) { + firstMirror = false + appDraftHandle.setDraftAndMeta(undefined, {}) + } + appDraftHandle.setDraftAndMeta(stateApp, meta) + }) + }) const selectedComponent = writable(undefined) // $: selectedComponent.subscribe((s) => { @@ -166,7 +209,7 @@ runnableComponents: writable({}), appPath: writablePath, workspace: $workspaceStore ?? '', - onchange: () => saveFrontendDraft(), + onchange: undefined, isEditor: true, jobs: writable([]), staticExporter: writable({}), @@ -219,19 +262,6 @@ stylePanel: () => StylePanel }) - let timeout: number | undefined = undefined - - function saveFrontendDraft() { - timeout && clearTimeout(timeout) - timeout = setTimeout(() => { - try { - localStorage.setItem(path != '' ? `app-${path}` : 'app', encodeState($appStore)) - } catch (err) { - console.error('Error storing frontend draft in localStorage', err) - } - }, 500) - } - function hashchange(e: HashChangeEvent) { context.hash = e.newURL.split('#')[1] context = context @@ -754,9 +784,6 @@ $effect(() => { path && untrack(() => onPathChange()) }) - $effect(() => { - $appStore && untrack(() => saveFrontendDraft()) - }) $effect(() => { context.mode = $mode == 'dnd' ? 'editor' : 'viewer' }) diff --git a/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte b/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte index bc7261b389..7a55f28860 100644 --- a/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte @@ -5,13 +5,20 @@ import Toggle from '$lib/components/Toggle.svelte' import { AppService, DraftService, type Policy } from '$lib/gen' import { redo, undo } from '$lib/history.svelte' - import { enterpriseLicense, userStore, workspaceStore } from '$lib/stores' + import { UserDraft } from '$lib/userDraft.svelte' + import { enterpriseLicense, tutorialsToDo, userStore, workspaceStore } from '$lib/stores' import { isMac, type Item, userPathPrefix } from '$lib/utils' + import { resetAllTodos, skipAllTodos } from '$lib/tutorialUtils' + import { getTutorialIndex } from '$lib/tutorials/config' import { random_adj } from '$lib/components/random_positive_adjetive' import { AlignHorizontalSpaceAround, BellOff, + BookOpen, Bug, + CheckCheck, + CheckCircle, + Circle, DiffIcon, Expand, FileJson, @@ -19,6 +26,7 @@ FormInput, History, Laptop2, + RefreshCw, Save, Smartphone, FileClock, @@ -57,6 +65,7 @@ import EditorHeader from '$lib/components/EditorHeader.svelte' import { editPathFor, invalidate as invalidatePicker } from '$lib/components/workspacePicker' + import { invalidateWorkspacePaths } from '$lib/components/PathNameAutocomplete.svelte' import { goto } from '$app/navigation' import HideButton from './settingsPanel/HideButton.svelte' import DeployOverrideConfirmationModal from '$lib/components/common/confirmationModal/DeployOverrideConfirmationModal.svelte' @@ -181,6 +190,10 @@ let deploymentMsg = $state('') let preserveOnBehalfOf = $state(false) + // Top-bar responsive collapse — container width, not viewport. + let topbarWidth = $state(0) + const compactTopbar = $derived(topbarWidth > 0 && topbarWidth < 720) + function closeSaveDrawer() { saveDrawerOpen = false } @@ -204,6 +217,9 @@ preserve_on_behalf_of: preserveOnBehalfOf || undefined } }) + // New path now exists server-side — drop the autocomplete cache so + // it shows up immediately instead of after the 60s TTL. + invalidateWorkspacePaths($workspaceStore!) savedApp = { summary: $summary, value: structuredClone($state.snapshot($app)), @@ -213,11 +229,7 @@ } closeSaveDrawer() sendUserToast('App deployed successfully') - try { - localStorage.removeItem(`app-${path}`) - } catch (e) { - console.error('error interacting with local storage', e) - } + UserDraft.remove('app', path) onSavedNewAppPath?.(path) } catch (e) { sendUserToast('Error creating app', e) @@ -302,6 +314,7 @@ } }) invalidatePicker($workspaceStore!, 'app') + invalidateWorkspacePaths($workspaceStore!) savedApp = { summary: $summary, value: structuredClone($state.snapshot($app)), @@ -317,12 +330,8 @@ closeSaveDrawer() sendUserToast('App deployed successfully') + UserDraft.remove('app', $appPath) if ($appPath !== npath) { - try { - localStorage.removeItem(`app-${appPath}`) - } catch (e) { - console.error('error interacting with local storage', e) - } onSavedNewAppPath?.(npath) } } @@ -394,6 +403,10 @@ } draftDrawerOpen = false + // The initial draft was promoted to a real path on the backend — + // drop the autosave keyed on the prior (possibly empty) path so + // a future "+ App" click opens on a clean slate. + UserDraft.remove('app', $appPath) onSavedNewAppPath?.(newEditedPath) } catch (e) { sendUserToast('Error saving initial draft', e) @@ -484,11 +497,7 @@ } sendUserToast('Draft saved') - try { - localStorage.removeItem(`app-${path}`) - } catch (e) { - console.error('error interacting with local storage', e) - } + UserDraft.remove('app', path) loading.saveDraft = false if (newApp || savedApp.draft_only) { onSavedNewAppPath?.(newEditedPath || path) @@ -586,6 +595,28 @@ } let moreItems = $derived([ + ...(compactTopbar + ? [ + { + displayName: 'Save draft', + icon: Save, + action: () => saveDraft(), + shortcut: `${mod}S`, + disabled: !newApp && !savedApp + }, + { + displayName: `Debug runs (${$jobs?.length > 99 ? '99+' : ($jobs?.length ?? 0)})`, + icon: Bug, + action: () => { + if (selectedJobId == undefined && $jobs.length > 0) { + selectedJobId = $jobs[$jobs.length - 1] + } + $jobsDrawerOpen = true + }, + separatorBottom: true + } + ] + : []), { displayName: 'Undo', icon: Undo, @@ -699,6 +730,40 @@ action: () => { appExport?.open(toStatic($app, $staticExporter, $summary).app) } + }, + { + displayName: 'Tutorials', + icon: BookOpen, + separatorTop: true, + submenuItems: [ + { + displayName: 'Background runnables', + action: () => appEditorTutorial?.runTutorialById('backgroundrunnables'), + icon: $tutorialsToDo.includes(getTutorialIndex('backgroundrunnables')) + ? Circle + : CheckCircle, + iconColor: $tutorialsToDo.includes(getTutorialIndex('backgroundrunnables')) + ? undefined + : 'green' + }, + { + displayName: 'Connection', + action: () => appEditorTutorial?.runTutorialById('connection'), + icon: $tutorialsToDo.includes(getTutorialIndex('connection')) ? Circle : CheckCircle, + iconColor: $tutorialsToDo.includes(getTutorialIndex('connection')) ? undefined : 'green' + }, + { + displayName: 'Reset tutorials', + action: () => resetAllTodos(), + icon: RefreshCw, + separatorTop: true + }, + { + displayName: 'Skip tutorials', + action: () => skipAllTodos(), + icon: CheckCheck + } + ] } ]) as Item[] @@ -932,9 +997,10 @@
-
+
goto(editPathFor(item))} /> -
+
{#if $app} {/if}
- +
+ + {#if $tutorialsToDo.includes(getTutorialIndex('backgroundrunnables')) || $tutorialsToDo.includes(getTutorialIndex('connection'))} + + {/if} +
- diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/common/PanelSection.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/common/PanelSection.svelte index 4fa88a021b..eda78badd8 100644 --- a/frontend/src/lib/components/apps/editor/settingsPanel/common/PanelSection.svelte +++ b/frontend/src/lib/components/apps/editor/settingsPanel/common/PanelSection.svelte @@ -1,6 +1,12 @@ + + + + + +{#if open} + +{/if} diff --git a/frontend/src/lib/components/copilot/chat/AIChat.svelte b/frontend/src/lib/components/copilot/chat/AIChat.svelte index 007675d0bf..07ccd1efb1 100644 --- a/frontend/src/lib/components/copilot/chat/AIChat.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChat.svelte @@ -120,7 +120,6 @@ loadPastChat={(id) => { aiChatManager.loadPastChat(id) }} - cancel={aiChatManager.cancel} askAi={aiChatManager.askAi} {headerLeft} hasDiff={aiChatManager.scriptEditorOptions && diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index 9081283d09..5967a75d30 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -1,18 +1,21 @@
-
-
- {@render headerLeft?.()} -

Chat

-
-
- - {#snippet trigger()} - + {#if !hideHeader} +
+
+ {@render headerLeft?.()} +

Chat

+
+
+ + {#snippet trigger()}
- - {/snippet} - -
-
+ {/if} {#if messages.length === 0} - You can use {getModifierKey()}L to open or close this chat, and {getModifierKey()}K in the - script editor to modify selected lines. + {#if emptyHint} + {@render emptyHint()} + {:else} + You can use {getModifierKey()}L to open or close this chat, and {getModifierKey()}K in the + script editor to modify selected lines. + {/if} {/if} {#if messages.length > 0} -
{ - aiChatManager.disableAutomaticScroll() - }} - > -
- {#each messages as message, messageIndex (messageIndex)} - - {/each} - {#if aiChatManager.loading && !aiChatManager.currentReply && !isLastMessageTool} -
- -
- {/if} +
+
+
+ {#each messages as message, messageIndex (messageIndex)} + + {/each} + {#if showTypingIndicator} +
+ {#if waitingForUserAction} + + + Waiting for your input + + {:else} + + {/if} +
+ {/if} +
+ {#if showScrollToLatest} +
+
+ {/if}
{/if} -
0} class="relative"> - {#if aiChatManager.loading} -
- -
- {:else if aiChatManager.flowAiChatHelpers?.hasPendingChanges()} +
+ {#if aiChatManager.flowAiChatHelpers?.hasPendingChanges()}
- +
+ +
{/if} -
+
+ {#if inputPreface} + {@render inputPreface()} + {/if} -
- {#if aiChatManager.mode === 'script' && hasDiff} - - {/if} +
+
+ {#if showContextPicker && !disabled} + + {#snippet trigger()} +
+ @ +
+ {/snippet} + {#snippet content({ close })} + {#if aiChatManager.mode === AIMode.APP} + { + void aiChatInput?.addContextToSelection(element) + close() + }} + /> + {:else} + { + void aiChatInput?.addContextToSelection(element) + close() + }} + onSelectWorkspaceItem={(element) => { + void aiChatInput?.addContextToSelection(element) + close() + }} + /> + {/if} + {/snippet} + + {/if} + {#if aiChatManager.mode === 'script' && hasDiff} + + {/if} +
{#if disabled}
{:else}
- + {#if !hideModeSelector} + + {/if} {#if aiChatManager.mode === AIMode.APP} {/if} @@ -344,8 +519,8 @@ {#each suggestions as suggestion (suggestion)}
+ + diff --git a/frontend/src/lib/components/copilot/chat/AIChatInlineWidget.svelte b/frontend/src/lib/components/copilot/chat/AIChatInlineWidget.svelte index 3b0b40d0e1..8e57dde1fe 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatInlineWidget.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatInlineWidget.svelte @@ -3,7 +3,7 @@ import AIChatInput from './AIChatInput.svelte' import { aiChatManager, AIMode } from './AIChatManager.svelte' import type { Selection } from 'monaco-editor' - import LoadingIcon from '$lib/components/apps/svelte-select/lib/LoadingIcon.svelte' + import ChatTypingIndicator from './ChatTypingIndicator.svelte' import { sendUserToast } from '$lib/toast' import { onDestroy } from 'svelte' import type { AIChatEditorHandler } from './monaco-adapter' @@ -183,20 +183,19 @@ }) -{#snippet bottomRightSnippet()} - {#if processing} - - {:else if aiChatManager.pendingNewCode} - - {getModifierKey()}↓ to apply - - {:else} -
- {/if} +{#snippet pendingApplyHint()} + + {getModifierKey()}↓ to apply + {/snippet} {#if show} -
+
+ {#if processing} +
+ +
+ {/if} aiChatManager.cancelInlineRequest('user pressed stop')} disabled={processing} />
diff --git a/frontend/src/lib/components/copilot/chat/AIChatInput.svelte b/frontend/src/lib/components/copilot/chat/AIChatInput.svelte index f0fc826243..7626b42380 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatInput.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatInput.svelte @@ -1,19 +1,21 @@ -
- {#if aiChatManager.mode === AIMode.SCRIPT || aiChatManager.mode === AIMode.FLOW || aiChatManager.mode === AIMode.GLOBAL} - {#if showContext} -
- - {#snippet trigger()} -
@
- {/snippet} - {#snippet content({ close })} - { - void addContextToSelection(element) - close() - }} - onSelectWorkspaceItem={(element) => { - void addContextToSelection(element) - close() - }} - /> - {/snippet} - - {#each selectedContext as element (element.type + '-' + element.title)} - { - selectedContext = selectedContext?.filter( - (c) => c.type !== element.type || c.title !== element.title - ) - }} - /> - {/each} -
- {/if} - void addContextToSelection(contextElement)} - onSendRequest={() => { - if (disabled) { - return - } +{#snippet sendStopButton()} + {@const isLoading = loading ?? aiChatManager.loading} + {@const sendDisabled = disabled || instructions.trim().length === 0} +